Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.TaskAttemptContext

88
89
90
91
92
93
94
95
96
97
98
    JobContext job = ContextFactory.createJobContext();
    ChunkInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations("A", "B", "C", "D"));
    ChunkInputFormat.setMockInstance(job.getConfiguration(), "instance1");
    ChunkInputFormat cif = new ChunkInputFormat();
    RangeInputSplit ris = new RangeInputSplit();
    TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job.getConfiguration());
    RecordReader<List<Entry<Key,Value>>,InputStream> rr = cif.createRecordReader(ris, tac);
    rr.initialize(ris, tac);
   
    assertTrue(rr.nextKeyValue());
    List<Entry<Key,Value>> info = rr.getCurrentKey();
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
    JobContext job = ContextFactory.createJobContext();
    ChunkInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations("A", "B", "C", "D"));
    ChunkInputFormat.setMockInstance(job.getConfiguration(), "instance2");
    ChunkInputFormat cif = new ChunkInputFormat();
    RangeInputSplit ris = new RangeInputSplit();
    TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job.getConfiguration());
    RecordReader<List<Entry<Key,Value>>,InputStream> crr = cif.createRecordReader(ris, tac);
    crr.initialize(ris, tac);
   
    assertTrue(crr.nextKeyValue());
    InputStream cis = crr.getCurrentValue();
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
    JobContext job = ContextFactory.createJobContext();
    ChunkInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations("A", "B", "C", "D"));
    ChunkInputFormat.setMockInstance(job.getConfiguration(), "instance3");
    ChunkInputFormat cif = new ChunkInputFormat();
    RangeInputSplit ris = new RangeInputSplit();
    TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job.getConfiguration());
    RecordReader<List<Entry<Key,Value>>,InputStream> crr = cif.createRecordReader(ris, tac);
    crr.initialize(ris, tac);
   
    assertTrue(crr.nextKeyValue());
    List<Entry<Key,Value>> info = crr.getCurrentKey();
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    // Configure a mock task attempt context.
    Job job = new Job(conf);
    job.getConfiguration().set("mapred.output.dir", mTempDir.getRoot().getPath());
    Schema writerSchema = Schema.create(Schema.Type.INT);
    AvroJob.setOutputKeySchema(job, writerSchema);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);
    expect(context.getConfiguration())
        .andReturn(job.getConfiguration()).anyTimes();
    expect(context.getTaskAttemptID())
        .andReturn(TaskAttemptID.forName("attempt_200707121733_0001_m_000000_0"))
        .anyTimes();
    expect(context.getNumReduceTasks()).andReturn(1);

    // Create a mock record writer.
    @SuppressWarnings("unchecked")
    RecordWriter<AvroKey<Integer>, NullWritable> expectedRecordWriter
        = createMock(RecordWriter.class);
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
public class TestAvroKeyValueRecordWriter {
  @Test
  public void testWriteRecords() throws IOException {
    Job job = new Job();
    AvroJob.setOutputValueSchema(job, TextStats.SCHEMA$);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);

    AvroDatumConverterFactory factory = new AvroDatumConverterFactory(job.getConfiguration());
    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
  }
  @Test public void testUsingReflection() throws Exception {
    Job job = new Job();
    Schema schema = ReflectData.get().getSchema(R1.class);
    AvroJob.setOutputValueSchema(job, schema);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);
    replay(context);

    R1 record = new R1();
    record.attribute = "test";
    AvroValue<R1> avroValue = new AvroValue<R1>(record);
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
  @Test
  public void testSyncableWriteRecords() throws IOException {
    Job job = new Job();
    AvroJob.setOutputValueSchema(job, TextStats.SCHEMA$);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);

    AvroDatumConverterFactory factory = new AvroDatumConverterFactory(job.getConfiguration());
    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
    expect(inputSplit.getPath()).andReturn(new Path("/path/to/an/avro/file")).anyTimes();
    expect(inputSplit.getStart()).andReturn(0L).anyTimes();
    expect(inputSplit.getLength()).andReturn(avroFileInput.length()).anyTimes();

    // Create a mock task attempt context for this record reader.
    TaskAttemptContext context = createMock(TaskAttemptContext.class);
    expect(context.getConfiguration()).andReturn(conf).anyTimes();

    // Initialize the record reader.
    replay(inputSplit);
    replay(context);
    recordReader.initialize(inputSplit, context);
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
    expect(inputSplit.getPath()).andReturn(new Path("/path/to/an/avro/file")).anyTimes();
    expect(inputSplit.getStart()).andReturn(0L).anyTimes();
    expect(inputSplit.getLength()).andReturn(avroFileInput.length()).anyTimes();

    // Create a mock task attempt context for this record reader.
    TaskAttemptContext context = createMock(TaskAttemptContext.class);
    expect(context.getConfiguration()).andReturn(conf).anyTimes();

    // Initialize the record reader.
    replay(inputSplit);
    replay(context);
    recordReader.initialize(inputSplit, context);
View Full Code Here
394
395
396
397
398
399
400
401
    checkBaseOutputPath(baseOutputPath);
    if (!namedOutputs.contains(namedOutput)) {
      throw new IllegalArgumentException("Undefined named output '" +
        namedOutput + "'");
    }
    TaskAttemptContext taskContext = getContext(namedOutput);
    getRecordWriter(taskContext, baseOutputPath).write(key, value);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.TaskAttemptContext

  • co.gridport.kafka.hadoop.KafkaOutputFormat
  • com.asakusafw.runtime.compatibility.JobCompatibility
  • com.datasalt.pangool.tuplemr.mapred.lib.output.PangoolMultipleOutputs
  • com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat
  • com.facebook.giraph.hive.input.benchmark.InputBenchmark
  • com.facebook.giraph.hive.tailer.Tailer
  • com.facebook.hiveio.benchmark.InputBenchmark
  • com.facebook.hiveio.input.HiveInput$RecordIterator
  • com.facebook.hiveio.output.FaultyCheckOutputSpecsTest
  • com.hadoop.mapreduce.TestLzoTextInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.