Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.TaskAttemptContext

346
347
348
349
350
351
352
353
354
355
356
    //
    // Old Hadoop API
    //
    public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException
    {
        TaskAttemptContext tac = new TaskAttemptContext(jobConf, new TaskAttemptID());
        List<org.apache.hadoop.mapreduce.InputSplit> newInputSplits = this.getSplits(tac);
        org.apache.hadoop.mapred.InputSplit[] oldInputSplits = new org.apache.hadoop.mapred.InputSplit[newInputSplits.size()];
        for (int i = 0; i < newInputSplits.size(); i++)
            oldInputSplits[i] = (ColumnFamilySplit)newInputSplits.get(i);
        return oldInputSplits;
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
256
      // validate input split
      InputSplit split = splits.get(i);
      Assert.assertTrue(split instanceof TableSnapshotRegionSplit);

      // validate record reader
      TaskAttemptContext taskAttemptContext = mock(TaskAttemptContext.class);
      when(taskAttemptContext.getConfiguration()).thenReturn(job.getConfiguration());
      RecordReader<ImmutableBytesWritable, Result> rr = tsif.createRecordReader(split, taskAttemptContext);
      rr.initialize(split, taskAttemptContext);

      // validate we can read all the data back
      while (rr.nextKeyValue()) {
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
    JobContext job = ContextFactory.createJobContext();
    AccumuloRowInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations());
    AccumuloRowInputFormat.setMockInstance(job.getConfiguration(), "instance1");
    AccumuloRowInputFormat crif = new AccumuloRowInputFormat();
    RangeInputSplit ris = new RangeInputSplit();
    TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
    RecordReader<Text,PeekingIterator<Entry<Key,Value>>> rr = crif.createRecordReader(ris, tac);
    rr.initialize(ris, tac);
   
    assertTrue(rr.nextKeyValue());
    assertEquals(new Text("row1"), rr.getCurrentKey());
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
    AccumuloOutputFormat output = new AccumuloOutputFormat();

    TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
    for (InputSplit split : splits) {
      TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
      RecordReader<Key,Value> reader = input.createRecordReader(split, tac);
      RecordWriter<Text,Mutation> writer = output.getRecordWriter(tac);
      Mapper<Key,Value,Text,Mutation>.Context context = ContextFactory.createMapContext(mapper, tac, reader, writer, split);
      reader.initialize(split, context);
      mapper.run(context);
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
   
    // when running a mapreduce, you won't need to instantiate the output
    // format and record writer
    // mapreduce will do that for you, and you will just use
    // output.collect(tableName, mutation)
    TaskAttemptContext context = ContextFactory.createTaskAttemptContext(job);
    RecordWriter<Text,Mutation> rw = new AccumuloOutputFormat().getRecordWriter(context);
   
    Text colf = new Text("colfam");
    System.out.println("writing ...");
    for (int i = 0; i < 10000; i++) {
View Full Code Here
325
326
327
328
329
330
331
332
333
334
335
      Assert.assertArrayEquals(new byte[0], risplit.getPassword());
      Assert.assertEquals("testtable", risplit.getTable());
      Assert.assertEquals(new Authorizations(), risplit.getAuths());
      Assert.assertEquals("testmapinstance", risplit.getInstanceName());

      TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
      RecordReader<Key,Value> reader = input.createRecordReader(split, tac);
      Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, reader, null, split);
      reader.initialize(split, context);
      mapper.run(context);
    }
View Full Code Here
351
352
353
354
355
356
357
358
359
360
361
    JobContext job = ContextFactory.createJobContext();
    AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable2", new Authorizations());
    AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
    AccumuloInputFormat input = new AccumuloInputFormat();
    RangeInputSplit ris = new RangeInputSplit();
    TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
    RecordReader<Key,Value> rr = input.createRecordReader(ris, tac);
    rr.initialize(ris, tac);

    TestMapper mapper = new TestMapper();
    Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, rr, null, ris);
View Full Code Here
384
385
386
387
388
389
390
391
392
393
394
    AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
    final String regex = ".*1.*";
    AccumuloInputFormat.setRegex(job, org.apache.accumulo.core.client.mapreduce.InputFormatBase.RegexType.ROW, regex);
    AccumuloInputFormat input = new AccumuloInputFormat();
    RangeInputSplit ris = new RangeInputSplit();
    TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
    RecordReader<Key,Value> rr = input.createRecordReader(ris, tac);
    rr.initialize(ris, tac);

    Pattern p = Pattern.compile(regex);
    while (rr.nextKeyValue()) {
View Full Code Here
481
482
483
484
485
486
487
488
489
490
491
    TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();

    RangeInputSplit emptySplit = new RangeInputSplit();

    // Using an empty split should fall back to the information in the Job's Configuration
    TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
    RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, tac);
    Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, reader, null, emptySplit);
    reader.initialize(emptySplit, context);
    mapper.run(context);
  }
View Full Code Here
519
520
521
522
523
524
525
526
527
528
529
    RangeInputSplit emptySplit = new RangeInputSplit();
    emptySplit.setUsername("root");
    emptySplit.setPassword("anythingelse".getBytes());

    // Using an empty split should fall back to the information in the Job's Configuration
    TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
    RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, tac);
    Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, reader, null, emptySplit);
    reader.initialize(emptySplit, context);
    mapper.run(context);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.TaskAttemptContext

  • co.gridport.kafka.hadoop.KafkaOutputFormat
  • com.asakusafw.runtime.compatibility.JobCompatibility
  • com.datasalt.pangool.tuplemr.mapred.lib.output.PangoolMultipleOutputs
  • com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat
  • com.facebook.giraph.hive.input.benchmark.InputBenchmark
  • com.facebook.giraph.hive.tailer.Tailer
  • com.facebook.hiveio.benchmark.InputBenchmark
  • com.facebook.hiveio.input.HiveInput$RecordIterator
  • com.facebook.hiveio.output.FaultyCheckOutputSpecsTest
  • com.hadoop.mapreduce.TestLzoTextInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.