Examples of org.apache.hadoop.mapred.RecordWriter

org.apache.hadoop.mapred.RecordWriter
RecordWriter writes the output <key, value> pairs to an output file.
RecordWriter implementations write the job outputs to the {@link FileSystem}. @see OutputFormat

              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    conf.setInt("mapred.max.split.size", 50);
    RecordWriter writer =
        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(1,2,3), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(4,5,6), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(7,8,9), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    SearchArgument sarg =
        SearchArgumentFactory.newBuilder()
            .startAnd()
            .lessThan("z", new Integer(0))

View Full Code Here

              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    conf.setInt("mapred.max.split.size", 50);
    RecordWriter writer =
        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new SimpleRow(null), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new SimpleRow(null), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new SimpleRow(null), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    SearchArgument sarg =
        SearchArgumentFactory.newBuilder()
            .startAnd()
            .lessThan("z", new String("foo"))

View Full Code Here

          ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    RecordWriter writer =
        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(1,2,3), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(4,5,6), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(7,8,9), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    properties.setProperty("columns", "z,r");
    properties.setProperty("columns.types", "int:struct<x:int,y:int>");
    serde.initialize(conf, properties);
    inspector = (StructObjectInspector) serde.getObjectInspector();

View Full Code Here

  // MultithreaderMapRunner.
  private synchronized RecordWriter getRecordWriter(String namedOutput,
                                                    String baseFileName,
                                                    final Reporter reporter,Schema schema)
    throws IOException {
    RecordWriter writer = recordWriters.get(baseFileName);
    if (writer == null) {
      if (countersEnabled && reporter == null) {
        throw new IllegalArgumentException(
          "Counters are enabled, Reporter cannot be NULL");
      }

View Full Code Here

      checkTokenName(multiName);
    }


    String baseFileName = (multi) ? namedOutput + "_" + multiName : baseOutputFileName;


    final RecordWriter writer =
      getRecordWriter(namedOutput, baseFileName, reporter,schema);


    return new AvroCollector() {
   
      @SuppressWarnings({"unchecked"})
      public void collect(Object key) throws IOException{
       AvroWrapper wrapper = new AvroWrapper(key);
       writer.write(wrapper, NullWritable.get());
      }
      
      public void collect(Object key,Object value) throws IOException
      {
        writer.write(key,value);
      }  
    
    };
  }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapred.RecordWriter

com.facebook.hive.orc.TestInputOutputFormat

com.flaptor.hounder.crawler.Nutch9Fetcher

org.apache.avro.mapred.AvroMultipleOutputs

org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat

org.apache.hadoop.mapred.lib.NullOutputFormat

org.apache.hive.hcatalog.mapreduce.DynamicPartitionFileRecordWriterContainer

org.apache.hive.hcatalog.mapreduce.FileRecordWriterContainer

org.apache.nutch.fetcher.FetcherOutputFormat

org.apache.nutch.segment.SegmentMerger$SegmentOutputFormat

org.apache.nutch.segment.SegmentReader$TextOutputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.