Package org.apache.crunch.types.avro

Examples of org.apache.crunch.types.avro.AvroType

236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeAvroFile(FSDataOutputStream outputStream, PCollection recordCollection) throws IOException {

    AvroType avroType = (AvroType)recordCollection.getPType();
    if (avroType == null) {
      throw new IllegalStateException("Can't write a non-typed Avro collection");
    }
    DatumWriter datumWriter = Avros.newWriter((AvroType)recordCollection.getPType());
    DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
    dataFileWriter.create(avroType.getSchema(), outputStream);

    for (Object record : recordCollection.materialize()) {
      dataFileWriter.append(avroType.getOutputMapFn().map(record));
    }

    dataFileWriter.close();
    outputStream.close();
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
    }
    return rdd;
  }

  private AvroSerDe getAvroSerde(PType ptype, Configuration conf) {
    AvroType at = (AvroType) ptype;
    Map<String, String> props = AvroMode.fromType(at).withFactoryFromConfiguration(conf).getModeProperties();
    return new AvroSerDe(at, props);
  }
View Full Code Here
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeAvroFile(FSDataOutputStream outputStream, PCollection recordCollection) throws IOException {

    AvroType avroType = (AvroType)recordCollection.getPType();
    if (avroType == null) {
      throw new IllegalStateException("Can't write a non-typed Avro collection");
    }
    DatumWriter datumWriter = Avros.newWriter((AvroType)recordCollection.getPType());
    DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
    dataFileWriter.create(avroType.getSchema(), outputStream);

    for (Object record : recordCollection.materialize()) {
      dataFileWriter.append(avroType.getOutputMapFn().map(record));
    }

    dataFileWriter.close();
    outputStream.close();
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
    }
    return rdd;
  }

  private AvroSerDe getAvroSerde(PType ptype, Configuration conf) {
    AvroType at = (AvroType) ptype;
    Map<String, String> props = AvroMode.fromType(at).withFactoryFromConfiguration(conf).getModeProperties();
    return new AvroSerDe(at, props);
  }
View Full Code Here
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeAvroFile(FSDataOutputStream outputStream, PCollection recordCollection) throws IOException {

    AvroType avroType = (AvroType)recordCollection.getPType();
    if (avroType == null) {
      throw new IllegalStateException("Can't write a non-typed Avro collection");
    }
    DatumWriter datumWriter = Avros.newWriter((AvroType)recordCollection.getPType());
    DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
    dataFileWriter.create(avroType.getSchema(), outputStream);

    for (Object record : recordCollection.materialize()) {
      dataFileWriter.append(avroType.getOutputMapFn().map(record));
    }

    dataFileWriter.close();
    outputStream.close();
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.types.avro.AvroType

  • org.apache.commons.lang.builder.HashCodeBuilder
  • org.apache.crunch.impl.mem.MemPipeline
  • org.apache.crunch.impl.spark.collect.PGroupedTableImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.