Package org.apache.crunch.impl.spark.fn

Examples of org.apache.crunch.impl.spark.fn.MapFunction

229
230
231
232
233
234
235
236
237
238
239
          if (t instanceof MapReduceTarget) { //TODO: check this earlier
            Converter c = t.getConverter(ptype);
            JavaPairRDD<?, ?> outRDD;
            if (rdd instanceof JavaRDD) {
              outRDD = ((JavaRDD) rdd)
                  .map(new MapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            } else {
              outRDD = ((JavaPairRDD) rdd)
                  .map(new PairMapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
View Full Code Here
55
56
57
58
59
60
61
62
63
64
          converter.getValueClass());
      input.rdd().setName(source.toString());
      MapFn mapFn = converter.applyPTypeTransforms() ? source.getType().getInputMapFn() : IdentityFn.getInstance();
      return input
          .map(new InputConverterFunction(source.getConverter()))
          .map(new MapFunction(mapFn, runtime.getRuntimeContext()));
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
303
304
305
306
307
308
309
310
311
312
313
            Converter c = t.getConverter(ptype);
            IdentityFn ident = IdentityFn.getInstance();
            JavaPairRDD<?, ?> outRDD;
            if (rdd instanceof JavaRDD) {
              outRDD = ((JavaRDD) rdd)
                  .map(new MapFunction(c.applyPTypeTransforms() ? ptype.getOutputMapFn() : ident, ctxt))
                  .mapToPair(new OutputConverterFunction(c));
            } else {
              outRDD = ((JavaPairRDD) rdd)
                  .map(new PairMapFunction(c.applyPTypeTransforms() ? ptype.getOutputMapFn() : ident, ctxt))
                  .mapToPair(new OutputConverterFunction(c));
View Full Code Here
234
235
236
237
238
239
240
241
242
243
244
          if (t instanceof MapReduceTarget) { //TODO: check this earlier
            Converter c = t.getConverter(ptype);
            JavaPairRDD<?, ?> outRDD;
            if (rdd instanceof JavaRDD) {
              outRDD = ((JavaRDD) rdd)
                  .map(new MapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            } else {
              outRDD = ((JavaPairRDD) rdd)
                  .map(new PairMapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
View Full Code Here
51
52
53
54
55
56
57
58
59
60
          source.getConverter().getKeyClass(),
          source.getConverter().getValueClass());
      input.rdd().setName(source.toString());
      return input
          .map(new InputConverterFunction(source.getConverter()))
          .map(new MapFunction(source.getType().getInputMapFn(), runtime.getRuntimeContext()));
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
300
301
302
303
304
305
306
307
308
309
310
          if (t instanceof MapReduceTarget) { //TODO: check this earlier
            Converter c = t.getConverter(ptype);
            JavaPairRDD<?, ?> outRDD;
            if (rdd instanceof JavaRDD) {
              outRDD = ((JavaRDD) rdd)
                  .map(new MapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            } else {
              outRDD = ((JavaPairRDD) rdd)
                  .map(new PairMapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
View Full Code Here
50
51
52
53
54
55
56
57
58
59
          source.getConverter().getKeyClass(),
          source.getConverter().getValueClass());
      input.rdd().setName(source.toString());
      return input
          .map(new InputConverterFunction(source.getConverter()))
          .map(new MapFunction(source.getType().getInputMapFn(), runtime.getRuntimeContext()));
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.impl.spark.fn.MapFunction

  • org.apache.crunch.impl.spark.collect.InputCollection
  • org.apache.crunch.impl.spark.SparkRuntime
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.