Package org.apache.crunch.impl.spark.fn

Examples of org.apache.crunch.impl.spark.fn.PairMapFunction

233
234
235
236
237
238
239
240
241
242
243
              outRDD = ((JavaRDD) rdd)
                  .map(new MapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            } else {
              outRDD = ((JavaPairRDD) rdd)
                  .map(new PairMapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            }

            try {
              Job job = new Job(conf);
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    }

    JavaPairRDD<ByteArray, List<byte[]>> groupedRDD;
    if (groupingOptions.getPartitionerClass() != null) {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .mapToPair(
              new PartitionedMapOutputFunction(keySerde, valueSerde, ptype, groupingOptions.getPartitionerClass(),
              numPartitions, runtime.getRuntimeContext()))
          .groupByKey(new SparkPartitioner(numPartitions));
    } else {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .mapToPair(new MapOutputFunction(keySerde, valueSerde))
          .groupByKey(numPartitions);
    }

    if (groupingOptions.requireSortedKeys() || groupingOptions.getSortComparatorClass() != null) {
View Full Code Here
307
308
309
310
311
312
313
314
315
316
317
              outRDD = ((JavaRDD) rdd)
                  .map(new MapFunction(c.applyPTypeTransforms() ? ptype.getOutputMapFn() : ident, ctxt))
                  .mapToPair(new OutputConverterFunction(c));
            } else {
              outRDD = ((JavaPairRDD) rdd)
                  .map(new PairMapFunction(c.applyPTypeTransforms() ? ptype.getOutputMapFn() : ident, ctxt))
                  .mapToPair(new OutputConverterFunction(c));
            }
            try {
              Job job = new Job(conf);
              if (t instanceof PathTarget) {
View Full Code Here
54
55
56
57
58
59
60
61
62
63
          converter.getValueClass());
      input.rdd().setName(source.toString());
      MapFn mapFn = converter.applyPTypeTransforms() ? source.getType().getInputMapFn() : IdentityFn.getInstance();
      return input
          .map(new InputConverterFunction(source.getConverter()))
          .map(new PairMapFunction(mapFn, runtime.getRuntimeContext()));
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
238
239
240
241
242
243
244
245
246
247
248
              outRDD = ((JavaRDD) rdd)
                  .map(new MapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            } else {
              outRDD = ((JavaPairRDD) rdd)
                  .map(new PairMapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            }
            try {
              Job job = new Job(conf);
              if (t instanceof PathTarget) {
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
    }

    JavaPairRDD<ByteArray, List<byte[]>> groupedRDD;
    if (groupingOptions.getPartitionerClass() != null) {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .map(new PartitionedMapOutputFunction(keySerde, valueSerde, ptype, groupingOptions.getPartitionerClass(),
              numPartitions, runtime.getRuntimeContext()))
          .groupByKey(new SparkPartitioner(numPartitions));
    } else {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .map(new MapOutputFunction(keySerde, valueSerde))
          .groupByKey(numPartitions);
    }

    if (groupingOptions.requireSortedKeys() || groupingOptions.getSortComparatorClass() != null) {
View Full Code Here
304
305
306
307
308
309
310
311
312
313
314
              outRDD = ((JavaRDD) rdd)
                  .map(new MapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            } else {
              outRDD = ((JavaPairRDD) rdd)
                  .map(new PairMapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            }
            try {
              Job job = new Job(conf);
              if (t instanceof PathTarget) {
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
    }

    JavaPairRDD<ByteArray, List<byte[]>> groupedRDD;
    if (groupingOptions.getPartitionerClass() != null) {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .map(new PartitionedMapOutputFunction(keySerde, valueSerde, ptype, groupingOptions.getPartitionerClass(),
              numPartitions, runtime.getRuntimeContext()))
          .groupByKey(new SparkPartitioner(numPartitions));
    } else {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .map(new MapOutputFunction(keySerde, valueSerde))
          .groupByKey(numPartitions);
    }

    if (groupingOptions.requireSortedKeys() || groupingOptions.getSortComparatorClass() != null) {
View Full Code Here
TOP

Related Classes of org.apache.crunch.impl.spark.fn.PairMapFunction

  • org.apache.crunch.impl.spark.collect.InputTable
  • org.apache.crunch.impl.spark.collect.PGroupedTableImpl
  • org.apache.crunch.impl.spark.SparkRuntime
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.