Examples of org.apache.hadoop.mapred.lib.MultipleOutputs

org.apache.hadoop.mapred.lib.MultipleOutputs

nes additional single text based output 'text' for the job MultipleOutputs.addNamedOutput(conf, "text", TextOutputFormat.class, LongWritable.class, Text.class); // Defines additional multi sequencefile based output 'sequence' for the // job MultipleOutputs.addMultiNamedOutput(conf, "seq", SequenceFileOutputFormat.class, LongWritable.class, Text.class); ... JobClient jc = new JobClient(); RunningJob job = jc.submitJob(conf); ...

Job configuration usage pattern is:

 public class MOReduce implements Reducer<WritableComparable, Writable> { private MultipleOutputs mos; public void configure(JobConf conf) { ... mos = new MultipleOutputs(conf); } public void reduce(WritableComparable key, Iterator<Writable> values, OutputCollector output, Reporter reporter) throws IOException { ... mos.getCollector("text", reporter).collect(key, new Text("Hello")); mos.getCollector("seq", "A", reporter).collect(key, new Text("Bye")); mos.getCollector("seq", "B", reporter).collect(key, new Text("Chau")); ... } public void close() throws IOException { mos.close(); ... } }

@deprecated Use {@link org.apache.hadoop.mapreduce.lib.output.MultipleOutputs} instead

    @Override
    protected void setup(Context context) throws IOException {
      randomSelectionPercent =
          context.getConfiguration().getFloat(RANDOM_SELECTION_PCT, 0);
      multipleOutputs =
          new MultipleOutputs(new JobConf(context.getConfiguration()));
      trainingCollector = multipleOutputs.getCollector(TRAINING_TAG, null);
      testCollector = multipleOutputs.getCollector(TEST_TAG, null);
    }

View Full Code Here

      }


      Validate.isTrue(rhatInput.hasNext(), "Empty R-hat input!");


      closeables.addFirst(rhatInput);
      outputs = new MultipleOutputs(new JobConf(context.getConfiguration()));
      closeables.addFirst(new IOUtils.MultipleOutputsCloseableAdapter(outputs));


      qr = new QRLastStep(qhatInput, rhatInput, blockNum);
      closeables.addFirst(qr);
      /*

View Full Code Here


        Validate.isTrue(k > 0, "invalid k parameter");
        Validate.isTrue(p >= 0, "invalid p parameter");
        mBBt = new UpperTriangular(k + p);


        outputs = new MultipleOutputs(new JobConf(context.getConfiguration()));
        closeables.addFirst(new IOUtils.MultipleOutputsCloseableAdapter(outputs));


      }
    }

View Full Code Here

      int p = Integer.parseInt(context.getConfiguration().get(PROP_P));
      kp = k + p;
      long omegaSeed = Long.parseLong(context.getConfiguration().get(PROP_OMEGA_SEED));
      omega = new Omega(omegaSeed, k, p);


      outputs = new MultipleOutputs(new JobConf(context.getConfiguration()));
      closeables.addFirst(new Closeable() {
        @Override
        public void close() throws IOException {
          outputs.close();
        }

View Full Code Here


    private MultipleOutputs mos = null;


    @Override
    public void configure(JobConf conf) {
      mos = new MultipleOutputs(conf);
    }

View Full Code Here

    @Override
    protected void setup(Context context) throws IOException {
      randomSelectionPercent =
          context.getConfiguration().getFloat(RANDOM_SELECTION_PCT, 0);
      multipleOutputs =
          new MultipleOutputs(new JobConf(context.getConfiguration()));
      trainingCollector = multipleOutputs.getCollector(TRAINING_TAG, null);
      testCollector = multipleOutputs.getCollector(TEST_TAG, null);
    }

View Full Code Here

      }


      Validate.isTrue(rhatInput.hasNext(), "Empty R-hat input!");


      closeables.addFirst(rhatInput);
      outputs = new MultipleOutputs(new JobConf(conf));
      closeables.addFirst(new IOUtils.MultipleOutputsCloseableAdapter(outputs));


      qr = new QRLastStep(qhatInput, rhatInput, blockNum);
      closeables.addFirst(qr);
      /*

View Full Code Here

                                              xiPathStr));
        }
      }


      if (outputBBt || xi != null) {
        outputs = new MultipleOutputs(new JobConf(conf));
        closeables.addFirst(new IOUtils.MultipleOutputsCloseableAdapter(outputs));
      }


    }

View Full Code Here

    int p = Integer.parseInt(jobConf.get(PROP_P));
    kp = k + p;


    yLookahead = Lists.newArrayListWithCapacity(kp);
    qSolver = new GivensThinSolver(r, kp);
    outputs = new MultipleOutputs(new JobConf(jobConf));
    closeables.addFirst(new Closeable() {
      @Override
      public void close() throws IOException {
        outputs.close();
      }

View Full Code Here

      String sbPathStr = conf.get(PROP_SB_PATH);
      if (sbPathStr != null) {
        sb = SSVDHelper.loadAndSumUpVectors(new Path(sbPathStr), conf);
      }


      outputs = new MultipleOutputs(new JobConf(conf));
      closeables.addFirst(new Closeable() {
        @Override
        public void close() throws IOException {
          outputs.close();
        }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapred.lib.MultipleOutputs

cc.mrlda.DocumentMapper

cc.mrlda.ParseCorpus$TokenizeMapper

cc.mrlda.polylda.DocumentMapper

cc.mrlda.polylda.TermReducer

cc.mrlda.TermReducer

com.intel.hadoop.graphbuilder.idnormalize.mapreduce.SortDictReducer

kafka.etl.KafkaETLMapper

kafka.etl.KafkaETLRecordReader

mrdp.ch6.ChainMapperDriver$UserIdBinningMapper

org.apache.hadoop.fs.FileSystem

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.