Package org.apache.avro.generic

Examples of org.apache.avro.generic.GenericDatumWriter


                    String name, Progressable prog)
    throws IOException {

    Schema schema = AvroJob.getOutputSchema(job);
   
    final DataFileWriter writer = new DataFileWriter(new GenericDatumWriter());

    if (FileOutputFormat.getCompressOutput(job)) {
      int level = job.getInt(AvroOutputFormat.DEFLATE_LEVEL_KEY,
                             AvroOutputFormat.DEFAULT_DEFLATE_LEVEL);
      writer.setCodec(CodecFactory.deflateCodec(level));
View Full Code Here


  }
 
  private void ingestAndVerifyAvro(Schema schema, GenericData.Record... records) throws IOException {
    deleteAllDocuments();
   
    GenericDatumWriter datum = new GenericDatumWriter(schema);
    DataFileWriter writer = new DataFileWriter(datum);
    writer.setMeta("Meta-Key0", "Meta-Value0");
    writer.setMeta("Meta-Key1", "Meta-Value1");
    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    writer.create(schema, bout);
    for (GenericData.Record record : records) {
      writer.append(record);
    }
    writer.flush();
    writer.close();

    DataFileReader<GenericData.Record> reader = new DataFileReader(new ReadAvroContainerBuilder.ForwardOnlySeekableInputStream(new ByteArrayInputStream(bout.toByteArray())), new GenericDatumReader());
    Schema schema2 = reader.getSchema();
    assertEquals(schema, schema2);
    for (GenericData.Record record : records) {
      assertTrue(reader.hasNext());
      GenericData.Record record2 = reader.next();
      assertEquals(record, record2);
    }
    assertFalse(reader.hasNext());
    reader.close();

    Record event = new Record();
    event.getFields().put(Fields.ATTACHMENT_BODY, new ByteArrayInputStream(bout.toByteArray()));
    morphline = createMorphline("test-morphlines/readAvroContainer");
    deleteAllDocuments();
    assertTrue(load(event));
    assertEquals(records.length, queryResultSetSize("*:*"));
       
    GenericDatumWriter datumWriter = new GenericDatumWriter(schema);
    bout = new ByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().binaryEncoder(bout, null);
    for (GenericData.Record record : records) {
      datumWriter.write(record, encoder);
    }
    encoder.flush();

    Decoder decoder = DecoderFactory.get().binaryDecoder(new ByteArrayInputStream(bout.toByteArray()), null);
    DatumReader<GenericData.Record> datumReader = new GenericDatumReader<GenericData.Record>(schema);
View Full Code Here

    if (morphlineConfigFile.contains("Json")) {
      encoder = EncoderFactory.get().jsonEncoder(schema, bout);
    } else {
      encoder = EncoderFactory.get().binaryEncoder(bout, null);
    }
    GenericDatumWriter datumWriter = new GenericDatumWriter(schema);
    for (GenericData.Record record : expecteds) {
      datumWriter.write(record, encoder);
    }
    encoder.flush();

    morphline = createMorphline(morphlineConfigFile);
    for (int j = 0; j < 3; j++) { // also test reuse of objects and low level avro buffers
View Full Code Here

      if (morphlineConfigFile.contains("Json")) {
        encoder = EncoderFactory.get().jsonEncoder(schema, bout);
      } else {
        encoder = EncoderFactory.get().binaryEncoder(bout, null);
      }
      GenericDatumWriter datumWriter = new GenericDatumWriter(schema);
      for (GenericData.Record record : expecteds) {
        datumWriter.write(record, encoder);
      }
      encoder.flush();
      bytes = bout.toByteArray();
    }
View Full Code Here

            Preconditions.checkNotNull(attachment);
            GenericContainer datum = (GenericContainer) attachment;
            schema = getSchema(datum, schema);
            assert schema != null;
            if (dataFileWriter == null) { // init
              GenericDatumWriter datumWriter = new GenericDatumWriter(schema);
              dataFileWriter = new DataFileWriter(datumWriter);
              if (codecFactory != null) {
                dataFileWriter.setCodec(codecFactory);
              }
              for (Map.Entry<String,String> entry : metadata.entrySet()) {
View Full Code Here

    }
     
    @SuppressWarnings("unchecked")
    private void writeContainerless(Record src, OutputStream dst) {
      try {
        GenericDatumWriter datumWriter = new GenericDatumWriter();
        Encoder encoder = null;
        Schema schema = null;
        for (Object attachment : src.get(Fields.ATTACHMENT_BODY)) {
          Preconditions.checkNotNull(attachment);
          GenericContainer datum = (GenericContainer) attachment;
          schema = getSchema(datum, schema);
          assert schema != null;
          datumWriter.setSchema(schema);
          if (encoder == null) { // init
            if (format == Format.containerlessJSON) {
              encoder = EncoderFactory.get().jsonEncoder(schema, dst);
            } else {
              encoder = EncoderFactory.get().binaryEncoder(dst, null);
            }         
            assert encoder != null;
          }
          datumWriter.write(datum, encoder);
        }
        encoder.flush();
      } catch (IOException e) {
        throw new MorphlineRuntimeException(e);
      }
View Full Code Here

          for (Object attachment : src.get(Fields.ATTACHMENT_BODY)) {
            Preconditions.checkNotNull(attachment);
            GenericContainer datum = (GenericContainer) attachment;
            schema = getSchema(datum, schema);
            if (dataFileWriter == null) { // init
              GenericDatumWriter datumWriter = new GenericDatumWriter(schema);
              dataFileWriter = new DataFileWriter(datumWriter);
              if (codecFactory != null) {
                dataFileWriter.setCodec(codecFactory);
              }
              for (Map.Entry<String,String> entry : metadata.entrySet()) {
View Full Code Here

      }
    }
     
    private void writeContainerless(Record src, OutputStream dst) {
      try {
        GenericDatumWriter datumWriter = new GenericDatumWriter();
        Encoder encoder = null;
        Schema schema = null;
        for (Object attachment : src.get(Fields.ATTACHMENT_BODY)) {
          Preconditions.checkNotNull(attachment);
          GenericContainer datum = (GenericContainer) attachment;
          schema = getSchema(datum, schema);
          datumWriter.setSchema(schema);
          if (encoder == null) { // init
            if (format == Format.containerlessJSON) {
              encoder = EncoderFactory.get().jsonEncoder(schema, dst);
            } else {
              encoder = EncoderFactory.get().binaryEncoder(dst, null);
            }         
          }
          datumWriter.write(datum, encoder);
        }
        encoder.flush();
      } catch (IOException e) {
        throw new MorphlineRuntimeException(e);
      }
View Full Code Here

  }
 
  private void ingestAndVerifyAvro(Schema schema, GenericData.Record... records) throws IOException {
    deleteAllDocuments();
   
    GenericDatumWriter datum = new GenericDatumWriter(schema);
    DataFileWriter writer = new DataFileWriter(datum);
    writer.setMeta("Meta-Key0", "Meta-Value0");
    writer.setMeta("Meta-Key1", "Meta-Value1");
    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    writer.create(schema, bout);
    for (GenericData.Record record : records) {
      writer.append(record);
    }
    writer.flush();
    writer.close();

    DataFileReader<GenericData.Record> reader = new DataFileReader(new ReadAvroContainerBuilder.ForwardOnlySeekableInputStream(new ByteArrayInputStream(bout.toByteArray())), new GenericDatumReader());
    Schema schema2 = reader.getSchema();
    assertEquals(schema, schema2);
    for (GenericData.Record record : records) {
      assertTrue(reader.hasNext());
      GenericData.Record record2 = reader.next();
      assertEquals(record, record2);
    }
    assertFalse(reader.hasNext());
    reader.close();

    Record event = new Record();
    event.getFields().put(Fields.ATTACHMENT_BODY, new ByteArrayInputStream(bout.toByteArray()));
    morphline = createMorphline("test-morphlines/readAvroContainer");
    deleteAllDocuments();
    assertTrue(load(event));
    assertEquals(records.length, queryResultSetSize("*:*"));
       
    GenericDatumWriter datumWriter = new GenericDatumWriter(schema);
    bout = new ByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().binaryEncoder(bout, null);
    for (GenericData.Record record : records) {
      datumWriter.write(record, encoder);
    }
    encoder.flush();

    Decoder decoder = DecoderFactory.get().binaryDecoder(new ByteArrayInputStream(bout.toByteArray()), null);
    DatumReader<GenericData.Record> datumReader = new GenericDatumReader<GenericData.Record>(schema);
View Full Code Here

    if (morphlineConfigFile.contains("Json")) {
      encoder = EncoderFactory.get().jsonEncoder(schema, bout);
    } else {
      encoder = EncoderFactory.get().binaryEncoder(bout, null);
    }
    GenericDatumWriter datumWriter = new GenericDatumWriter(schema);
    for (GenericData.Record record : expecteds) {
      datumWriter.write(record, encoder);
    }
    encoder.flush();

    morphline = createMorphline(morphlineConfigFile);
    for (int j = 0; j < 3; j++) { // also test reuse of objects and low level avro buffers
View Full Code Here

TOP

Related Classes of org.apache.avro.generic.GenericDatumWriter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.