Package org.apache.crunch.types.avro

Examples of org.apache.crunch.types.avro.AvroType


  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeAvroFile(FSDataOutputStream outputStream, PCollection recordCollection) throws IOException {

    AvroType avroType = (AvroType)recordCollection.getPType();
    if (avroType == null) {
      throw new IllegalStateException("Can't write a non-typed Avro collection");
    }
    DatumWriter datumWriter = Avros.newWriter((AvroType)recordCollection.getPType());
    DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
    dataFileWriter.create(avroType.getSchema(), outputStream);

    for (Object record : recordCollection.materialize()) {
      dataFileWriter.append(avroType.getOutputMapFn().map(record));
    }

    dataFileWriter.close();
    outputStream.close();
  }
View Full Code Here


    }
    return rdd;
  }

  private AvroSerDe getAvroSerde(PType ptype, Configuration conf) {
    AvroType at = (AvroType) ptype;
    Map<String, String> props = AvroMode.fromType(at).withFactoryFromConfiguration(conf).getModeProperties();
    return new AvroSerDe(at, props);
  }
View Full Code Here

  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeAvroFile(FSDataOutputStream outputStream, PCollection recordCollection) throws IOException {

    AvroType avroType = (AvroType)recordCollection.getPType();
    if (avroType == null) {
      throw new IllegalStateException("Can't write a non-typed Avro collection");
    }
    DatumWriter datumWriter = Avros.newWriter((AvroType)recordCollection.getPType());
    DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
    dataFileWriter.create(avroType.getSchema(), outputStream);

    for (Object record : recordCollection.materialize()) {
      dataFileWriter.append(avroType.getOutputMapFn().map(record));
    }

    dataFileWriter.close();
    outputStream.close();
  }
View Full Code Here

    }
    return rdd;
  }

  private AvroSerDe getAvroSerde(PType ptype, Configuration conf) {
    AvroType at = (AvroType) ptype;
    Map<String, String> props = AvroMode.fromType(at).withFactoryFromConfiguration(conf).getModeProperties();
    return new AvroSerDe(at, props);
  }
View Full Code Here

  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeAvroFile(FSDataOutputStream outputStream, PCollection recordCollection) throws IOException {

    AvroType avroType = (AvroType)recordCollection.getPType();
    if (avroType == null) {
      throw new IllegalStateException("Can't write a non-typed Avro collection");
    }
    DatumWriter datumWriter = Avros.newWriter((AvroType)recordCollection.getPType());
    DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
    dataFileWriter.create(avroType.getSchema(), outputStream);

    for (Object record : recordCollection.materialize()) {
      dataFileWriter.append(avroType.getOutputMapFn().map(record));
    }

    dataFileWriter.close();
    outputStream.close();
  }
View Full Code Here

TOP

Related Classes of org.apache.crunch.types.avro.AvroType

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.