Package com.facebook.hiveio.common

Examples of com.facebook.hiveio.common.ProgressReporter

464
465
466
467
468
469
470
471
472
473
474
    // CHECKSTYLE: resume LineLengthCheck
    HadoopUtils.setWorkOutputDir(taskAttemptContext);
    JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration());
    int fileId = CREATED_FILES_COUNTER.incrementAndGet();
    String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId);
    Reporter reporter = new ProgressReporter(taskAttemptContext);
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        baseOutputFormat.getRecordWriter(null, jobConf, name, reporter);
    LOG.info("getBaseRecordWriter: Created new {} with file {}", baseWriter, name);
    return baseWriter;
  }
View Full Code Here
127
128
129
130
131
132
133
134
   * @throws IOException I/O errors
   */
  public org.apache.hadoop.mapred.RecordReader<WritableComparable, Writable>
  getBaseRecordReader(JobConf jobConf, Progressable progressable)
    throws IOException {
    Reporter reporter = new ProgressReporter(progressable);
    return baseInputFormat.getRecordReader(baseSplit, jobConf, reporter);
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
    long elapsedTime = System.currentTimeMillis() - startTime;
    if (elapsedTime > writeTimeoutMs) {
      if (LOG.isInfoEnabled()) {
        LOG.info("write: Write taking too long ({}s), creating new file to write to", elapsedTime);
      }
      baseWriter.close(new ProgressReporter(taskAttemptContext));
      baseWriter = HiveApiOutputFormat.getBaseRecordWriter(taskAttemptContext, baseOutputFormat);
    }
  }
View Full Code Here
127
128
129
130
131
132
133
134
   * @throws IOException I/O errors
   */
  public org.apache.hadoop.mapred.RecordReader<WritableComparable, Writable>
  getBaseRecordReader(JobConf jobConf, Progressable progressable)
    throws IOException {
    Reporter reporter = new ProgressReporter(progressable);
    return baseInputFormat.getRecordReader(baseSplit, jobConf, reporter);
  }
View Full Code Here
368
369
370
371
372
373
374
375
    int fileId = CREATED_FILES_COUNTER.incrementAndGet();
    String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId);
    if (LOG.isInfoEnabled()) {
      LOG.info("getBaseRecordWriter: Created new with file {}", name);
    }
    Reporter reporter = new ProgressReporter(taskAttemptContext);
    return baseOutputFormat.getRecordWriter(null, jobConf, name, reporter);
  }
View Full Code Here
130
131
132
133
134
135
136
  }

  @Override
  public void close(TaskAttemptContext context)
    throws IOException, InterruptedException {
    baseWriter.close(new ProgressReporter(context));
  }
View Full Code Here
364
365
366
367
368
369
370
371
372
373
374
    // CHECKSTYLE: resume LineLengthCheck
    HadoopUtils.setWorkOutputDir(taskAttemptContext);
    JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration());
    int fileId = CREATED_FILES_COUNTER.incrementAndGet();
    String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId);
    Reporter reporter = new ProgressReporter(taskAttemptContext);
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        baseOutputFormat.getRecordWriter(null, jobConf, name, reporter);
    LOG.info("getBaseRecordWriter: Created new {} with file {}", baseWriter, name);
    return baseWriter;
  }
View Full Code Here
131
132
133
134
135
136
137
  }

  @Override
  public void close(TaskAttemptContext context)
    throws IOException, InterruptedException {
    baseWriter.close(new ProgressReporter(context));
  }
View Full Code Here
498
499
500
501
502
503
504
505
506
507
508
    // CHECKSTYLE: resume LineLengthCheck
    HadoopUtils.setWorkOutputDir(taskAttemptContext);
    JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration());
    int fileId = CREATED_FILES_COUNTER.incrementAndGet();
    String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId);
    Reporter reporter = new ProgressReporter(taskAttemptContext);
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        baseOutputFormat.getRecordWriter(null, jobConf, name, reporter);
    LOG.info("getBaseRecordWriter: Created new {} with file {}", baseWriter, name);
    return baseWriter;
  }
View Full Code Here
405
406
407
408
409
410
411
412
413
414
415
    // CHECKSTYLE: resume LineLengthCheck
    HadoopUtils.setWorkOutputDir(taskAttemptContext);
    JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration());
    int fileId = CREATED_FILES_COUNTER.incrementAndGet();
    String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId);
    Reporter reporter = new ProgressReporter(taskAttemptContext);
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        baseOutputFormat.getRecordWriter(null, jobConf, name, reporter);
    LOG.info("getBaseRecordWriter: Created new {} with file {}", baseWriter, name);
    return baseWriter;
  }
View Full Code Here
TOP

Related Classes of com.facebook.hiveio.common.ProgressReporter

  • com.facebook.hiveio.input.HInputSplit
  • com.facebook.hiveio.output.HiveApiOutputFormat
  • com.facebook.hiveio.output.RecordWriterImpl
  • com.facebook.hiveio.output.ResettableRecordWriterImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.