Examples of org.apache.hadoop.io.NullWritable

org.apache.hadoop.io.NullWritable
Singleton Writable with no data.

      fs.delete(in, true);


    final SequenceFile.Writer centerWriter = SequenceFile.createWriter(fs,
        conf, center, VectorWritable.class, NullWritable.class,
        CompressionType.NONE);
    final NullWritable value = NullWritable.get();


    final SequenceFile.Writer dataWriter = SequenceFile.createWriter(fs, conf,
        in, VectorWritable.class, NullWritable.class, CompressionType.NONE);


    Random r = new Random();

View Full Code Here

    if (fs.exists(dst)) {
      fs.delete(dst, false);
    }
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, 
      conf, dst, job.getMapOutputKeyClass(), NullWritable.class);
    NullWritable nullValue = NullWritable.get();
    float stepSize = samples.length / (float) numPartitions;
    int last = -1;
    for(int i = 1; i < numPartitions; ++i) {
      int k = Math.round(stepSize * i);
      while (last >= k && comparator.compare(samples[last], samples[k]) == 0) {

View Full Code Here

 
  TextOutputFormat<?, ?> theOutputFormat = new TextOutputFormat();
  RecordWriter theRecordWriter = theOutputFormat
      .getRecordWriter(tContext);
  
  NullWritable nullWritable = NullWritable.get();
  try {
    theRecordWriter.write(key2, val2);
    theRecordWriter.write(null, nullWritable);
    theRecordWriter.write(null, val2);
    theRecordWriter.write(nullWritable, val1);

View Full Code Here

    
    TextOutputFormat<?, ?> theOutputFormat = new TextOutputFormat();
    RecordWriter theRecordWriter = theOutputFormat
        .getRecordWriter(tContext);
    
    NullWritable nullWritable = NullWritable.get();
    try {
      theRecordWriter.write(key1, val1);
      theRecordWriter.write(null, nullWritable);
      theRecordWriter.write(null, val1);
      theRecordWriter.write(nullWritable, val2);

View Full Code Here

        InMemRecordReader reader = (InMemRecordReader) inputFormat.getRecordReader(
            split, conf, null);


        for (int tree = 0; tree < split.getNbTrees(); tree++) {
          IntWritable key = reader.createKey();
          NullWritable value = reader.createValue();


          // reader.next() should return true until there is no tree left
          assertEquals(tree < split.getNbTrees(), reader.next(key, value));
          
          assertEquals(split.getFirstId() + tree, key.get());

View Full Code Here

    // read
    final SequenceFile.Reader reader = new Reader(FileSystem.getLocal(
      baseTmpDir.getDefaultConfiguration()), new Path(outputFile.toString()),
        baseTmpDir.getDefaultConfiguration());
    final List<String> actual = Lists.newArrayList();
    final NullWritable key = NullWritable.get();
    final Text value = new Text();
    while (reader.next(key, value)) {
      actual.add(value.toString());
    }
    reader.close();

View Full Code Here

    assertEquals("Progress should be zero before any records are read",
        0.0f, recordReader.getProgress(), 0.0f);


    // Some variables to hold the records.
    AvroKey<CharSequence> key;
    NullWritable value;


    // Read the first record.
    assertTrue("Expected at least one record", recordReader.nextKeyValue());
    key = recordReader.getCurrentKey();
    value = recordReader.getCurrentValue();

View Full Code Here


    
    AvroRecordReader<Pair<Integer, Long>> recordReader = new AvroRecordReader<Pair<Integer, Long>>(job, fileSplit);
    
    AvroWrapper<Pair<Integer, Long>> inputPair = new AvroWrapper<Pair<Integer, Long>>(null);
    NullWritable ignore = NullWritable.get();
    
    long sumOfCounts = 0;
    long numOfCounts = 0;
    while(recordReader.next(inputPair, ignore)) {
      Assert.assertEquals((Integer)inputPair.datum().get(0), defaultRank);

View Full Code Here

    String dir = System.getProperty("test.dir", ".") + "/mapred";
    Path inputPath = new Path(dir + "/out" + "/myavro1-r-00000.avro");
    FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath);
    FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);
    AvroWrapper<Utf8> inputPair = new AvroWrapper<Utf8>(null);
    NullWritable ignore = NullWritable.get();
    AvroRecordReader<Utf8> recordReader = new AvroRecordReader<Utf8>(job, fileSplit);
    long sumOfCounts = 0;
    long numOfCounts = 0;
    while(recordReader.next(inputPair, ignore)) {
        sumOfCounts += Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}","").trim());

View Full Code Here

    assertEquals(1, splits.length);


    // read the whole file
    org.apache.hadoop.mapred.RecordReader<NullWritable, OrcLazyRow> reader =
        in.getRecordReader(splits[0], conf, Reporter.NULL);
    NullWritable key = reader.createKey();
    OrcLazyRow value = (OrcLazyRow) reader.createValue();
    List<? extends StructField> fields =inspector.getAllStructFieldRefs();
    StringObjectInspector strInspector = (StringObjectInspector)
        fields.get(0).getFieldObjectInspector();
    assertEquals(true, reader.next(key, value));

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.NullWritable

bulkimport.BulkImportJobExample$VerboseInputSampler

ca.ulaval.ift.graal.pagerank.mapreduce.TopKMapper

cc.bf.ReadNgram

com.alexholmes.hadooputils.combine.avro.mapred.CombineAvroInputFormatTest

com.asakusafw.testtools.TestDataHolder

com.ebay.erl.mobius.core.datajoin.EvenlyPartitioner

com.facebook.hive.orc.TestInputOutputFormat

com.thinkaurelius.faunus.FaunusVertexTest

com.twitter.maple.tap.MemorySourceTap$MemorySourceScheme

elephantdb.cascading.ElephantScheme

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.