Examples of org.apache.crunch.types.PType

org.apache.crunch.types.PType
A {@code PType} defines a mapping between a data type that is used in a Crunch pipeline and aserialization and storage format that is used to read/write data from/to HDFS. Every {@link PCollection} has an associated {@code PType} that tells Crunch how to read/write data fromthat {@code PCollection}.

    writer.close();
  }


  private void writeSequenceFileFromPCollection(final FileSystem fs, final Path path,
      final PCollection collection) throws IOException {
    final PType pType = collection.getPType();
    final Converter converter = pType.getConverter();
    final Class valueClass = converter.getValueClass();


    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path,
        NullWritable.class, valueClass);


    for (final Object o : collection.materialize()) {
      final Object value = pType.getOutputMapFn().map(o);
      writer.append(NullWritable.get(), value);
    }


    writer.close();
  }

View Full Code Here

    writer.close();
  }
  
  private void writeSequenceFileFromPCollection(final FileSystem fs, final Path path,
      final PCollection collection) throws IOException {
    final PType pType = collection.getPType();
    final Converter converter = pType.getConverter();
    final Class valueClass = converter.getValueClass();


    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path,
        NullWritable.class, valueClass);


    for (final Object o : collection.materialize()) {
      final Object value = pType.getOutputMapFn().map(o);
      writer.append(NullWritable.get(), value);
    }


    writer.close();
  }

View Full Code Here

    writer.close();
  }


  private void writeSequenceFileFromPCollection(final FileSystem fs, final Path path,
      final PCollection collection) throws IOException {
    final PType pType = collection.getPType();
    final Converter converter = pType.getConverter();
    final Class valueClass = converter.getValueClass();


    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path,
        NullWritable.class, valueClass);


    for (final Object o : collection.materialize()) {
      final Object value = pType.getOutputMapFn().map(o);
      writer.append(NullWritable.get(), value);
    }


    writer.close();
  }

View Full Code Here

    writer.close();
  }


  private void writeSequenceFileFromPCollection(final FileSystem fs, final Path path,
      final PCollection collection) throws IOException {
    final PType pType = collection.getPType();
    final Converter converter = pType.getConverter();
    final Class valueClass = converter.getValueClass();


    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path,
        NullWritable.class, valueClass);


    for (final Object o : collection.materialize()) {
      final Object value = pType.getOutputMapFn().map(o);
      writer.append(NullWritable.get(), value);
    }


    writer.close();
  }

View Full Code Here

    writer.close();
  }
  
  private void writeSequenceFileFromPCollection(final FileSystem fs, final Path path,
      final PCollection collection) throws IOException {
    final PType pType = collection.getPType();
    final Converter converter = pType.getConverter();
    final Class valueClass = converter.getValueClass();


    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path,
        NullWritable.class, valueClass);


    for (final Object o : collection.materialize()) {
      final Object value = pType.getOutputMapFn().map(o);
      writer.append(NullWritable.get(), value);
    }


    writer.close();
  }

View Full Code Here

TOP

Related Classes of org.apache.crunch.types.PType

org.apache.crunch.impl.mem.MemPipeline

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.