Package org.apache.crunch.impl.spark.serde

Examples of org.apache.crunch.impl.spark.serde.SerDe

84
85
86
87
88
89
90
91
92
93
94
    JavaPairRDD<K, V> parentRDD = (JavaPairRDD<K, V>) ((SparkCollection)getOnlyParent()).getJavaRDDLike(runtime);
    if (combineFn != null) {
      parentRDD = parentRDD.mapPartitionsToPair(
          new CombineMapsideFunction<K, V>(combineFn, runtime.getRuntimeContext()));
    }
    SerDe keySerde, valueSerde;
    PTableType<K, V> parentType = ptype.getTableType();
    if (parentType instanceof AvroType) {
      keySerde = getAvroSerde(parentType.getKeyType(), runtime.getConfiguration());
      valueSerde = getAvroSerde(parentType.getValueType(), runtime.getConfiguration());
    } else {
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
  private JavaRDDLike<?, ?> getJavaRDDLikeInternal(SparkRuntime runtime, CombineFn<K, V> combineFn) {
    JavaPairRDD<K, V> parentRDD = (JavaPairRDD<K, V>) ((SparkCollection)getOnlyParent()).getJavaRDDLike(runtime);
    if (combineFn != null) {
      parentRDD = parentRDD.mapPartitions(new CombineMapsideFunction<K, V>(combineFn, runtime.getRuntimeContext()));
    }
    SerDe keySerde, valueSerde;
    PTableType<K, V> parentType = ptype.getTableType();
    if (parentType instanceof AvroType) {
      keySerde = new AvroSerDe((AvroType) parentType.getKeyType());
      valueSerde = new AvroSerDe((AvroType) parentType.getValueType());
    } else {
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
  private JavaRDDLike<?, ?> getJavaRDDLikeInternal(SparkRuntime runtime, CombineFn<K, V> combineFn) {
    JavaPairRDD<K, V> parentRDD = (JavaPairRDD<K, V>) ((SparkCollection)getOnlyParent()).getJavaRDDLike(runtime);
    if (combineFn != null) {
      parentRDD = parentRDD.mapPartitions(new CombineMapsideFunction<K, V>(combineFn, runtime.getRuntimeContext()));
    }
    SerDe keySerde, valueSerde;
    PTableType<K, V> parentType = ptype.getTableType();
    if (parentType instanceof AvroType) {
      keySerde = getAvroSerde(parentType.getKeyType(), runtime.getConfiguration());
      valueSerde = getAvroSerde(parentType.getValueType(), runtime.getConfiguration());
    } else {
View Full Code Here
TOP

Related Classes of org.apache.crunch.impl.spark.serde.SerDe

  • org.apache.crunch.impl.spark.collect.PGroupedTableImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.