Package org.apache.crunch.types

Examples of org.apache.crunch.types.PTypeFamily.pairs()

39
40
41
42
43
44
45
46
47
48
49
  public static <K, U, V> PTable<K, Pair<Collection<U>, Collection<V>>> cogroup(PTable<K, U> left, PTable<K, V> right) {
    PTypeFamily ptf = left.getTypeFamily();
    PType<K> keyType = left.getPTableType().getKeyType();
    PType<U> leftType = left.getPTableType().getValueType();
    PType<V> rightType = right.getPTableType().getValueType();
    PType<Pair<U, V>> itype = ptf.pairs(leftType, rightType);

    PTable<K, Pair<U, V>> cgLeft = left.parallelDo("coGroupTag1", new CogroupFn1<K, U, V>(),
        ptf.tableOf(keyType, itype));
    PTable<K, Pair<U, V>> cgRight = right.parallelDo("coGroupTag2", new CogroupFn2<K, U, V>(),
        ptf.tableOf(keyType, itype));
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
    PTable<K, Pair<U, V>> cgRight = right.parallelDo("coGroupTag2", new CogroupFn2<K, U, V>(),
        ptf.tableOf(keyType, itype));

    PTable<K, Pair<U, V>> both = cgLeft.union(cgRight);

    PType<Pair<Collection<U>, Collection<V>>> otype = ptf.pairs(ptf.collections(leftType), ptf.collections(rightType));
    return both.groupByKey().parallelDo("cogroup",
        new PostGroupFn<K, U, V>(leftType, rightType), ptf.tableOf(keyType, otype));
  }

  private static class CogroupFn1<K, V, U> extends MapValuesFn<K, V, Pair<V, U>> {
View Full Code Here
167
168
169
170
171
172
173
174
175
176
177
      } else {
        TupleFactory tf = null;
        switch (cols.length) {
        case 2:
          tf = TupleFactory.PAIR;
          keyPType = ptf.pairs(pt.get(cols[0]), pt.get(cols[1]));
          break;
        case 3:
          tf = TupleFactory.TUPLE3;
          keyPType = ptf.triples(pt.get(cols[0]), pt.get(cols[1]), pt.get(cols[2]));
          break;
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
              rightReadable, right.getPTableType(), includeUnmatchedLeftValues);
    ParallelDoOptions options = ParallelDoOptions.builder()
        .sourceTargets(rightReadable.getSourceTargets())
        .build();
    return left.parallelDo("mapjoin", mapJoinDoFn,
        tf.tableOf(left.getKeyType(), tf.pairs(left.getValueType(), right.getValueType())),
        options);
  }

  static class MapsideJoinDoFn<K, U, V> extends DoFn<Pair<K, U>, Pair<K, Pair<U, V>>> {
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
    PTypeFamily rtf = right.getTypeFamily();

    PTable<Pair<Integer, Integer>, Pair<K1, U>> leftCross = left.parallelDo(new GFCross<Pair<K1, U>>(0, parallelism),
        ltf.tableOf(ltf.pairs(ltf.ints(), ltf.ints()), ltf.pairs(left.getKeyType(), left.getValueType())));
    PTable<Pair<Integer, Integer>, Pair<K2, V>> rightCross = right.parallelDo(new GFCross<Pair<K2, V>>(1, parallelism),
        rtf.tableOf(rtf.pairs(rtf.ints(), rtf.ints()), rtf.pairs(right.getKeyType(), right.getValueType())));

    PTable<Pair<Integer, Integer>, Pair<Pair<K1, U>, Pair<K2, V>>> cg = leftCross.join(rightCross);

    PTypeFamily ctf = cg.getTypeFamily();
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
    PTypeFamily rtf = right.getTypeFamily();

    PTable<Pair<Integer, Integer>, Pair<K1, U>> leftCross = left.parallelDo(new GFCross<Pair<K1, U>>(0, parallelism),
        ltf.tableOf(ltf.pairs(ltf.ints(), ltf.ints()), ltf.pairs(left.getKeyType(), left.getValueType())));
    PTable<Pair<Integer, Integer>, Pair<K2, V>> rightCross = right.parallelDo(new GFCross<Pair<K2, V>>(1, parallelism),
        rtf.tableOf(rtf.pairs(rtf.ints(), rtf.ints()), rtf.pairs(right.getKeyType(), right.getValueType())));

    PTable<Pair<Integer, Integer>, Pair<Pair<K1, U>, Pair<K2, V>>> cg = leftCross.join(rightCross);

    PTypeFamily ctf = cg.getTypeFamily();
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
            Pair<Pair<K1, U>, Pair<K2, V>> valuePair = input.second();
            return Pair.of(Pair.of(valuePair.first().first(), valuePair.second().first()),
                Pair.of(valuePair.first().second(), valuePair.second().second()));
          }
        },
        ctf.tableOf(ctf.pairs(left.getKeyType(), right.getKeyType()),
            ctf.pairs(left.getValueType(), right.getValueType())));
  }

  /**
   * Performs a full cross join on the specified {@link PCollection}s (using the
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
            return Pair.of(Pair.of(valuePair.first().first(), valuePair.second().first()),
                Pair.of(valuePair.first().second(), valuePair.second().second()));
          }
        },
        ctf.tableOf(ctf.pairs(left.getKeyType(), right.getKeyType()),
            ctf.pairs(left.getValueType(), right.getValueType())));
  }

  /**
   * Performs a full cross join on the specified {@link PCollection}s (using the
   * same strategy as Pig's CROSS operator).
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
      throw new Error();

    PTable<Pair<Integer, Integer>, U> leftCross = left.parallelDo(new GFCross<U>(0, parallelism),
        ltf.tableOf(ltf.pairs(ltf.ints(), ltf.ints()), left.getPType()));
    PTable<Pair<Integer, Integer>, V> rightCross = right.parallelDo(new GFCross<V>(1, parallelism),
        rtf.tableOf(rtf.pairs(rtf.ints(), rtf.ints()), right.getPType()));

    PTable<Pair<Integer, Integer>, Pair<U, V>> cg = leftCross.join(rightCross);

    PTypeFamily ctf = cg.getTypeFamily();
View Full Code Here
208
209
210
211
212
213
214
215
216
    return cg.parallelDo("Extract second element", new MapFn<Pair<Pair<Integer, Integer>, Pair<U, V>>, Pair<U, V>>() {
      @Override
      public Pair<U, V> map(Pair<Pair<Integer, Integer>, Pair<U, V>> input) {
        return input.second();
      }
    }, ctf.pairs(left.getPType(), right.getPType()));
  }

}
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.