Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.TaskAttemptContext

434
435
436
437
438
439
440
441
      Schema valSchema, String baseOutputPath) throws IOException,
      InterruptedException {
    checkBaseOutputPath(baseOutputPath);
    Job job = new Job(context.getConfiguration());
    setSchema(job, keySchema, valSchema);
    TaskAttemptContext taskContext = createTaskAttemptContext(job.getConfiguration(), context.getTaskAttemptID());
    getRecordWriter(taskContext, baseOutputPath).write(key, value);
  }
View Full Code Here
457
458
459
460
461
462
463
464
465
466
467
   checkNamedOutputName(context, namedOutput, false);
   checkBaseOutputPath(baseOutputPath);
   if (!namedOutputs.contains(namedOutput)) {
      throw new IllegalArgumentException("Undefined named output '" + namedOutput + "'");
   }
   TaskAttemptContext taskContext = getContext(namedOutput);
   RecordWriter recordWriter = getRecordWriter(taskContext, baseOutputPath);
   long position = -1;
   if (recordWriter instanceof Syncable) {
      Syncable syncableWriter = (Syncable) recordWriter;
      position = syncableWriter.sync();
View Full Code Here
524
525
526
527
528
529
530
531
532
533
534
   // Create a taskAttemptContext for the named output with
   // output format and output key/value types put in the context
  @SuppressWarnings("deprecation")
  private TaskAttemptContext getContext(String nameOutput) throws IOException {

    TaskAttemptContext taskContext = taskContexts.get(nameOutput);

    if (taskContext != null) {
      return taskContext;
    }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
    Job job = new Job();
    AvroJob.setInputKeySchema(job, Schema.create(Schema.Type.STRING));
    Configuration conf = job.getConfiguration();

    FileSplit inputSplit = createMock(FileSplit.class);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);
    expect(context.getConfiguration()).andReturn(conf).anyTimes();

    replay(inputSplit);
    replay(context);

    AvroKeyInputFormat inputFormat = new AvroKeyInputFormat();
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
  public void testWrite() throws IOException {
    Schema writerSchema = Schema.create(Schema.Type.INT);
    GenericData dataModel = new ReflectData();
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);

    // Write an avro container file with two records: 1 and 2.
    AvroKeyRecordWriter<Integer> recordWriter = new AvroKeyRecordWriter<Integer>(
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
  public void testSycnableWrite() throws IOException {
    Schema writerSchema = Schema.create(Schema.Type.INT);
    GenericData dataModel = new ReflectData();
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    FileOutputStream outputStream = new FileOutputStream(new File("target/temp.avro"));
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);

    // Write an avro container file with two records: 1 and 2.
    AvroKeyRecordWriter<Integer> recordWriter = new AvroKeyRecordWriter<Integer>(
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    // Configure a mock task attempt context.
    Job job = new Job(conf);
    job.getConfiguration().set("mapred.output.dir", mTempDir.getRoot().getPath());
    Schema writerSchema = Schema.create(Schema.Type.INT);
    AvroJob.setOutputKeySchema(job, writerSchema);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);
    expect(context.getConfiguration())
        .andReturn(job.getConfiguration()).anyTimes();
    expect(context.getTaskAttemptID())
        .andReturn(new TaskAttemptID("id", 1, true, 1, 1))
        .anyTimes();

    // Create a mock record writer.
    @SuppressWarnings("unchecked")
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
public class TestAvroKeyValueRecordWriter {
  @Test
  public void testWriteRecords() throws IOException {
    Job job = new Job();
    AvroJob.setOutputValueSchema(job, TextStats.SCHEMA$);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);

    AvroDatumConverterFactory factory = new AvroDatumConverterFactory(job.getConfiguration());
    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
  }
  @Test public void testUsingReflection() throws Exception {
    Job job = new Job();
    Schema schema = ReflectData.get().getSchema(R1.class);
    AvroJob.setOutputValueSchema(job, schema);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);
    replay(context);

    R1 record = new R1();
    record.attribute = "test";
    AvroValue<R1> avroValue = new AvroValue<R1>(record);
View Full Code Here
401
402
403
404
405
406
407
408
    checkBaseOutputPath(baseOutputPath);
    if (!namedOutputs.contains(namedOutput)) {
      throw new IllegalArgumentException("Undefined named output '" +
        namedOutput + "'");
    }
    TaskAttemptContext taskContext = getContext(namedOutput);
    getRecordWriter(taskContext, baseOutputPath).write(key, value);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.TaskAttemptContext

  • co.gridport.kafka.hadoop.KafkaOutputFormat
  • com.asakusafw.runtime.compatibility.JobCompatibility
  • com.datasalt.pangool.tuplemr.mapred.lib.output.PangoolMultipleOutputs
  • com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat
  • com.facebook.giraph.hive.input.benchmark.InputBenchmark
  • com.facebook.giraph.hive.tailer.Tailer
  • com.facebook.hiveio.benchmark.InputBenchmark
  • com.facebook.hiveio.input.HiveInput$RecordIterator
  • com.facebook.hiveio.output.FaultyCheckOutputSpecsTest
  • com.hadoop.mapreduce.TestLzoTextInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.