Package org.apache.crunch.impl.spark.fn

Examples of org.apache.crunch.impl.spark.fn.PairFlatMapDoFn

57
58
59
60
61
62
63
64
65
66
    for (int i = 0; i < rdds.length; i++) {
      if (parents.get(i) instanceof PTableBase) {
        rdds[i] = (JavaPairRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
      } else {
        JavaRDD rdd = (JavaRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
        rdds[i] = rdd.mapPartitionsToPair(new PairFlatMapDoFn(IdentityFn.getInstance(), runtime.getRuntimeContext()));
      }
    }
    return runtime.getSparkContext().union(rdds);
  }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
      runtime.setCombineFn((CombineFn) combineFn);
    }
    JavaRDDLike<?, ?> parentRDD = ((SparkCollection) getOnlyParent()).getJavaRDDLike(runtime);
    fn.configure(runtime.getConfiguration());
    if (parentRDD instanceof JavaRDD) {
      return ((JavaRDD) parentRDD).mapPartitions(new PairFlatMapDoFn(fn, runtime.getRuntimeContext()));
    } else {
      return ((JavaPairRDD) parentRDD).mapPartitions(new PairFlatMapPairDoFn(fn, runtime.getRuntimeContext()));
    }
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
    for (int i = 0; i < rdds.length; i++) {
      if (parents.get(i) instanceof PTableBase) {
        rdds[i] = (JavaPairRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
      } else {
        JavaRDD rdd = (JavaRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
        rdds[i] = rdd.mapPartitions(new PairFlatMapDoFn(IdentityFn.getInstance(), runtime.getRuntimeContext()));
      }
    }
    return runtime.getSparkContext().union(rdds);
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.impl.spark.fn.PairFlatMapDoFn

  • org.apache.crunch.impl.spark.collect.DoTable
  • org.apache.crunch.impl.spark.collect.UnionTable
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.