Examples of org.apache.hadoop.mapred.lib.MultipleOutputs

org.apache.hadoop.mapred.lib.MultipleOutputs

nes additional single text based output 'text' for the job MultipleOutputs.addNamedOutput(conf, "text", TextOutputFormat.class, LongWritable.class, Text.class); // Defines additional multi sequencefile based output 'sequence' for the // job MultipleOutputs.addMultiNamedOutput(conf, "seq", SequenceFileOutputFormat.class, LongWritable.class, Text.class); ... JobClient jc = new JobClient(); RunningJob job = jc.submitJob(conf); ...

Job configuration usage pattern is:

 public class MOReduce implements Reducer<WritableComparable, Writable> { private MultipleOutputs mos; public void configure(JobConf conf) { ... mos = new MultipleOutputs(conf); } public void reduce(WritableComparable key, Iterator<Writable> values, OutputCollector output, Reporter reporter) throws IOException { ... mos.getCollector("text", reporter).collect(key, new Text("Hello")); mos.getCollector("seq", "A", reporter).collect(key, new Text("Bye")); mos.getCollector("seq", "B", reporter).collect(key, new Text("Chau")); ... } public void close() throws IOException { mos.close(); ... } }

@deprecated Use {@link org.apache.hadoop.mapreduce.lib.output.MultipleOutputs} instead

      }


      Validate.isTrue(rhatInput.hasNext(), "Empty R-hat input!");


      closeables.addFirst(rhatInput);
      outputs = new MultipleOutputs(new JobConf(conf));
      closeables.addFirst(new IOUtils.MultipleOutputsCloseableAdapter(outputs));


      qr = new QRLastStep(qhatInput, rhatInput, blockNum);
      closeables.addFirst(qr);
      /*

View Full Code Here

                                              xiPathStr));
        }
      }


      if (outputBBt || xi != null) {
        outputs = new MultipleOutputs(new JobConf(conf));
        closeables.addFirst(new IOUtils.MultipleOutputsCloseableAdapter(outputs));
      }


    }

View Full Code Here

      String sbPathStr = conf.get(PROP_SB_PATH);
      if (sbPathStr != null) {
        sb = SSVDHelper.loadAndSumUpVectors(new Path(sbPathStr), conf);
      }


      outputs = new MultipleOutputs(new JobConf(conf));
      closeables.addFirst(new Closeable() {
        @Override
        public void close() throws IOException {
          outputs.close();
        }

View Full Code Here

    int p = Integer.parseInt(jobConf.get(PROP_P));
    kp = k + p;


    yLookahead = Lists.newArrayListWithCapacity(kp);
    qSolver = new GivensThinSolver(r, kp);
    outputs = new MultipleOutputs(new JobConf(jobConf));
    closeables.addFirst(new Closeable() {
      @Override
      public void close() throws IOException {
        outputs.close();
      }

View Full Code Here

      }


      Validate.isTrue(rhatInput.hasNext(), "Empty R-hat input!");


      closeables.addFirst(rhatInput);
      outputs = new MultipleOutputs(new JobConf(conf));
      closeables.addFirst(new IOUtils.MultipleOutputsCloseableAdapter(outputs));


      qr = new QRLastStep(qhatInput, rhatInput, blockNum);
      closeables.addFirst(qr);
      /*

View Full Code Here

      if (xiPathStr != null) {
        xi = SSVDHelper.loadAndSumUpVectors(new Path(xiPathStr), conf);
      }


      if (outputBBt || xi != null) {
        outputs = new MultipleOutputs(new JobConf(conf));
        closeables.addFirst(new IOUtils.MultipleOutputsCloseableAdapter(outputs));
      }


    }

View Full Code Here

    @Override
    protected void setup(Context context) throws IOException {
      randomSelectionPercent =
          context.getConfiguration().getFloat(RANDOM_SELECTION_PCT, 0);
      multipleOutputs =
          new MultipleOutputs(new JobConf(context.getConfiguration()));
      trainingCollector = multipleOutputs.getCollector(TRAINING_TAG, null);
      testCollector = multipleOutputs.getCollector(TEST_TAG, null);
    }

View Full Code Here

      _granularity = KafkaETLCommons.getGranularity(_props);


      _topic = KafkaETLCommons.getTopic(_props);
      System.out.println("topic=" + _topic);


      _mos = new MultipleOutputs(conf);


      _ignoreErrors = _props.getBoolean(KafkaETLCommons.IGNORE_ERRORS,
          false);


    } catch (Exception e) {

View Full Code Here

       _props = KafkaETLUtils.getPropsFromJob(job);
       _contextList = new ArrayList<KafkaETLContext>();
       _job = job;
       _reporter = reporter;
       _contextIndex = -1;
       _mos = new MultipleOutputs(job);
       try {
           _limit = _props.getInt("kafka.request.limit", -1);
           
           /*get attemp id*/
           String taskId = _job.get("mapred.task.id");

View Full Code Here

  private boolean hashRawVid;


  @Override
  public void configure(JobConf job) {
    super.configure(job);
    this.mos = new MultipleOutputs(job);
    this.hashRawVid = job.getBoolean("hashRawVid", true);
  }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapred.lib.MultipleOutputs

cc.mrlda.DocumentMapper

cc.mrlda.ParseCorpus$TokenizeMapper

cc.mrlda.polylda.DocumentMapper

cc.mrlda.polylda.TermReducer

cc.mrlda.TermReducer

com.intel.hadoop.graphbuilder.idnormalize.mapreduce.SortDictReducer

kafka.etl.KafkaETLMapper

kafka.etl.KafkaETLRecordReader

mrdp.ch6.ChainMapperDriver$UserIdBinningMapper

org.apache.hadoop.fs.FileSystem

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.