Package org.apache.avro.file

Examples of org.apache.avro.file.DataFileWriter.create()

242
243
244
245
246
247
248
249
250
251
252
    if (avroType == null) {
      throw new IllegalStateException("Can't write a non-typed Avro collection");
    }
    DatumWriter datumWriter = Avros.newWriter((AvroType)recordCollection.getPType());
    DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
    dataFileWriter.create(avroType.getSchema(), outputStream);

    for (Object record : recordCollection.materialize()) {
      dataFileWriter.append(avroType.getOutputMapFn().map(record));
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
      writer.setCodec(CodecFactory.deflateCodec(level));
    }

    Path path =
      FileOutputFormat.getTaskOutputPath(job, name+AvroOutputFormat.EXT);
    writer.create(schema, path.getFileSystem(job).create(path));

    return new RecordWriter<TetherData, NullWritable>() {
        public void write(TetherData datum, NullWritable ignore)
          throws IOException {
          writer.appendEncoded(datum.buffer());
View Full Code Here
239
240
241
242
243
244
245
246
247
248
249
    }

    GenericDatumWriter genericDatumWriter = new GenericDatumWriter(schema);

    DataFileWriter dataFileWriter = new DataFileWriter(genericDatumWriter);
    dataFileWriter.create(schema, outputStream);

    for (Object record : genericRecords) {
      dataFileWriter.append(record);
    }
View Full Code Here
246
247
248
249
250
251
252
253
254
255
256
    if (avroType == null) {
      throw new IllegalStateException("Can't write a non-typed Avro collection");
    }
    DatumWriter datumWriter = Avros.newWriter((AvroType)recordCollection.getPType());
    DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
    dataFileWriter.create(avroType.getSchema(), outputStream);

    for (Object record : recordCollection.materialize()) {
      dataFileWriter.append(avroType.getOutputMapFn().map(record));
    }
View Full Code Here
247
248
249
250
251
252
253
254
255
256
257
    if (avroType == null) {
      throw new IllegalStateException("Can't write a non-typed Avro collection");
    }
    DatumWriter datumWriter = Avros.newWriter((AvroType)recordCollection.getPType());
    DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
    dataFileWriter.create(avroType.getSchema(), outputStream);

    for (Object record : recordCollection.materialize()) {
      dataFileWriter.append(avroType.getOutputMapFn().map(record));
    }
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
    }

    GenericDatumWriter genericDatumWriter = new GenericDatumWriter(schema);

    DataFileWriter dataFileWriter = new DataFileWriter(genericDatumWriter);
    dataFileWriter.create(schema, outputStream);

    for (Object record : genericRecords) {
      dataFileWriter.append(record);
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
      writer.setCodec(CodecFactory.deflateCodec(level));
    }

    Path path =
      FileOutputFormat.getTaskOutputPath(job, name+AvroOutputFormat.EXT);
    writer.create(schema, path.getFileSystem(job).create(path));

    return new RecordWriter<TetherData, NullWritable>() {
        public void write(TetherData datum, NullWritable ignore)
          throws IOException {
          writer.appendEncoded(datum.buffer());
View Full Code Here
698
699
700
701
702
703
704
705
706
707
708
    GenericDatumWriter datum = new GenericDatumWriter(schema);
    DataFileWriter writer = new DataFileWriter(datum);
    writer.setMeta("Meta-Key0", "Meta-Value0");
    writer.setMeta("Meta-Key1", "Meta-Value1");
    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    writer.create(schema, bout);
    for (GenericData.Record record : records) {
      writer.append(record);
    }
    writer.flush();
    writer.close();
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
                dataFileWriter.setCodec(codecFactory);
              }
              for (Map.Entry<String,String> entry : metadata.entrySet()) {
                dataFileWriter.setMeta(entry.getKey(), entry.getValue());             
              }
              dataFileWriter.create(schema, dst);
            }
            dataFileWriter.append(datum);
          }
          if (dataFileWriter != null) {
            dataFileWriter.flush();
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
                dataFileWriter.setCodec(codecFactory);
              }
              for (Map.Entry<String,String> entry : metadata.entrySet()) {
                dataFileWriter.setMeta(entry.getKey(), entry.getValue());             
              }
              dataFileWriter.create(schema, dst);
            }
            dataFileWriter.append(datum);
          }
          if (dataFileWriter != null) {
            dataFileWriter.flush();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.