Package org.apache.crunch.types

Examples of org.apache.crunch.types.Converter

76
77
78
79
80
81
82
83
84
85
    return this;
  }

  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = ptype.getConverter();
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    configureForMapReduce(job, keyClass, valueClass, formatBundle, outputPath, name);
  }
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
286
  }
 
  private void writeSequenceFileFromPCollection(final FileSystem fs, final Path path,
      final PCollection collection) throws IOException {
    final PType pType = collection.getPType();
    final Converter converter = pType.getConverter();
    final Class valueClass = converter.getValueClass();

    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path,
        NullWritable.class, valueClass);

    for (final Object o : collection.materialize()) {
View Full Code Here
42
43
44
45
46
47
48
49
50
51
    this.fileNamingScheme = fileNamingScheme;
  }

  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = ptype.getConverter();
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    configureForMapReduce(job, keyClass, valueClass, outputFormatClass, outputPath, name);
  }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
    return "Text(" + path + ")";
  }

  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = ptype.getConverter();
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    configureForMapReduce(job, keyClass, valueClass, getOutputFormat(ptype), outputPath, name);
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
    this.fileNamingScheme = fileNamingScheme;
  }

  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = ptype.getConverter();
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    configureForMapReduce(job, keyClass, valueClass, outputFormatClass, outputPath, name);
  }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
    return "Text(" + path + ")";
  }

  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = ptype.getConverter();
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    configureForMapReduce(job, keyClass, valueClass, getOutputFormat(ptype), outputPath, name);
  }
View Full Code Here
62
63
64
65
66
67
68
69
70
    DoFn<?, ?> fn = ptype.getOutputMapFn();
    return new DoNode(fn, name, ptype, NO_CHILDREN, ptype.getGroupingConverter(), null);
  }

  public static <S> DoNode createOutputNode(String name, PType<S> ptype) {
    Converter outputConverter = ptype.getConverter();
    DoFn<?, ?> fn = ptype.getOutputMapFn();
    return new DoNode(fn, name, ptype, NO_CHILDREN, outputConverter, null);
  }
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
    fn.configure(conf);
    for (DoNode child : children) {
      childRTNodes.add(child.toRTNode(false, conf, nodeContext));
    }

    Converter inputConverter = null;
    if (inputNode) {
      if (nodeContext == NodeContext.MAP) {
        inputConverter = ptype.getConverter();
      } else {
        inputConverter = ((PGroupedTableType<?, ?>) ptype).getGroupingConverter();
View Full Code Here
175
176
177
178
179
180
181
182
183
184
185
186
  @SuppressWarnings("unchecked")
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {

    Preconditions.checkNotNull(name, "Output name should not be null"); // see CRUNCH-82

    Converter converter = getConverter(ptype);
    Class<?> keyClass = converter.getKeyClass();
    Class<?> valueClass = Void.class;

    CrunchOutputs.addNamedOutput(job, name, formatBundle, keyClass, valueClass);
    job.setOutputFormatClass(formatBundle.getFormatClass());
    formatBundle.configure(job.getConfiguration());
View Full Code Here
62
63
64
65
66
67
68
69
70
    DoFn<?, ?> fn = ptype.getOutputMapFn();
    return new DoNode(fn, name, ptype, NO_CHILDREN, ptype.getGroupingConverter(), null);
  }

  public static <S> DoNode createOutputNode(String name, PType<S> ptype) {
    Converter outputConverter = ptype.getConverter();
    DoFn<?, ?> fn = ptype.getOutputMapFn();
    return new DoNode(fn, name, ptype, NO_CHILDREN, outputConverter, null);
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.types.Converter

  • com.twitter.elephantbird.crunch.LzoProtobufTarget
  • com.twitter.elephantbird.crunch.LzoThriftTarget
  • org.apache.crunch.impl.mem.MemPipeline
  • org.apache.crunch.impl.mr.plan.DoNode
  • org.apache.crunch.impl.spark.collect.InputCollection
  • org.apache.crunch.impl.spark.collect.InputTable
  • org.apache.crunch.impl.spark.SparkRuntime
  • org.apache.crunch.io.impl.FileTargetImpl
  • org.apache.crunch.io.text.TextFileTarget
  • org.kitesdk.data.crunch.DatasetTarget
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.