Examples of org.apache.avro.file.DataFileWriter$BufferedFileOutputStream

org.apache.avro.file.DataFileWriter
Thrown by {@link #append(Object)} when an exception occurs while writing adatum to the buffer. When this is thrown, the file is unaltered and may continue to be appended to.

  
  private void ingestAndVerifyAvro(Schema schema, GenericData.Record... records) throws IOException {
    deleteAllDocuments();
    
    GenericDatumWriter datum = new GenericDatumWriter(schema);
    DataFileWriter writer = new DataFileWriter(datum);
    writer.setMeta("Meta-Key0", "Meta-Value0");
    writer.setMeta("Meta-Key1", "Meta-Value1");
    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    writer.create(schema, bout);
    for (GenericData.Record record : records) {
      writer.append(record);
    }
    writer.flush();
    writer.close();


    DataFileReader<GenericData.Record> reader = new DataFileReader(new ReadAvroContainerBuilder.ForwardOnlySeekableInputStream(new ByteArrayInputStream(bout.toByteArray())), new GenericDatumReader());
    Schema schema2 = reader.getSchema();
    assertEquals(schema, schema2);
    for (GenericData.Record record : records) {

View Full Code Here

  /**
   */
  void emitData(File outData, Schema schema, int numRecords, Instantiator inster) throws IOException, InstantiationException {
    DatumWriter dout = new ReflectDatumWriter(schema);


    DataFileWriter out = new DataFileWriter(dout);
    out = out.create(schema, outData);
    try {
      for (int i = 0; i < numRecords; i++) {
        out.append(inster.create());
      }
      //encoder.flush();
    } finally {
      out.close();
    }
  }

View Full Code Here

        encoder.flush();
      } finally {
        out.close();
      }
    } else {
      DataFileWriter out = new DataFileWriter(dout);
      try {
        out.create(schema, outfile);
        for (int i = 0; i < numRecords; i++) {
          TestRecord tr = new TestRecord();
          out.append(tr);
        }
      } finally {
        out.close();
      }
    }
  }

View Full Code Here


    GenerateRandomData grd = new GenerateRandomData();
    Schema schema = Schema.parse(inputSchemaFile);


    GenericDatumWriter datum = new GenericDatumWriter(schema);
    DataFileWriter out = new DataFileWriter(datum);
    out.create(schema, outputDataFile);
    try {
      for (int i = 0; i < numToEmit; i++) {
        out.append((GenericData.Record) grd.generateData(schema));
      }
    } finally {
      out.close();
    }
  }

View Full Code Here

      int numGoodParses = 0;
      int lineno = 0;
      Schema schema = typeTree.getAvroSchema();


      GenericDatumWriter gdWriter = new GenericDatumWriter(schema);
      DataFileWriter outData = new DataFileWriter(gdWriter);
      outData = outData.create(schema, fs2.create(avroDataFile));


      try {
        in = new BufferedReader(new InputStreamReader(fs.open(p)));
        try {
          String str = in.readLine();
          while (str != null) {
            GenericContainer gct = typeTree.parse(str);
            if (gct != null) {
              numGoodParses++;
              outData.append(gct);
            } else {
              if (verbose) {
                System.err.println("unparsed line: '" + str + "'");
              }
            }
            str = in.readLine();
            lineno++;
          }      
        } finally {
          in.close();
        }
      } finally {
        outData.close();
      }
      if (verbose) {
        System.err.println();
        System.err.println("Total # input lines: " + lineno);
        System.err.println("Total # lines parsed correctly: " + numGoodParses);

View Full Code Here

  protected void processFile(
      final File file,
      final String typeName,
      final StageToHdfsPlugin<?> plugin,
      final StageRunData runData ) {
    final DataFileWriter writer = runData.getWriter(
        typeName,
        plugin);
    if (writer != null) {
      final Object[] objs = plugin.toHdfsObjects(file);
      for (final Object obj : objs) {
        try {
          writer.append(obj);
        }
        catch (final IOException e) {
          LOGGER.error(
              "Cannot append data to seequence file",
              e);

View Full Code Here

  private synchronized DataFileWriter getDataWriterCreateIfNull(
      final String typeName,
      final StageToHdfsPlugin plugin ) {
    if (!cachedWriters.containsKey(typeName)) {
      FSDataOutputStream out = null;
      final DataFileWriter dfw = new DataFileWriter(
          new GenericDatumWriter());
      cachedWriters.put(
          typeName,
          dfw);
      dfw.setCodec(CodecFactory.snappyCodec());
      try {
        // TODO: we should probably clean up the type name to make it
        // HDFS path safe in case there are invalid characters
        // also, if a file already exists do we want to delete it or
        // append to it?
        out = fs.create(new Path(
            hdfsBaseDirectory,
            typeName));
        dfw.create(
            plugin.getAvroSchemaForHdfsType(),
            out);


      }
      catch (final IOException e) {

View Full Code Here

0 1

TOP

Related Classes of org.apache.avro.file.DataFileWriter$BufferedFileOutputStream

com.cloudera.cdk.morphline.avro.AvroMorphlineTest

com.cloudera.cdk.morphline.avro.WriteAvroToByteArrayBuilder$WriteAvroToByteArray

com.cloudera.recordbreaker.learnstructure.LearnStructure

com.cloudera.recordbreaker.learnstructure.test.GenerateRandomData

com.cloudera.recordbreaker.learnstructure.test.GenerateTestAvro

mil.nga.giat.geowave.ingest.hdfs.StageRunData

mil.nga.giat.geowave.ingest.hdfs.StageToHdfsDriver

org.apache.avro.file.DataFileStream.DataBlock

org.apache.avro.io.BinaryEncoder

org.apache.avro.io.EncoderFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.