List of tableOf() Examples

Examples of tableOf()

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

    PType<U> leftType = left.getPTableType().getValueType();
    PType<V> rightType = right.getPTableType().getValueType();
    PType<Pair<U, V>> itype = ptf.pairs(leftType, rightType);


    PTable<K, Pair<U, V>> cgLeft = left.parallelDo("coGroupTag1", new CogroupFn1<K, U, V>(),
        ptf.tableOf(keyType, itype));
    PTable<K, Pair<U, V>> cgRight = right.parallelDo("coGroupTag2", new CogroupFn2<K, U, V>(),
        ptf.tableOf(keyType, itype));


    PTable<K, Pair<U, V>> both = cgLeft.union(cgRight);

View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

    PType<Pair<U, V>> itype = ptf.pairs(leftType, rightType);


    PTable<K, Pair<U, V>> cgLeft = left.parallelDo("coGroupTag1", new CogroupFn1<K, U, V>(),
        ptf.tableOf(keyType, itype));
    PTable<K, Pair<U, V>> cgRight = right.parallelDo("coGroupTag2", new CogroupFn2<K, U, V>(),
        ptf.tableOf(keyType, itype));


    PTable<K, Pair<U, V>> both = cgLeft.union(cgRight);


    PType<Pair<Collection<U>, Collection<V>>> otype = ptf.pairs(ptf.collections(leftType), ptf.collections(rightType));
    return both.groupByKey().parallelDo("cogroup",

View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()


    PTable<K, Pair<U, V>> both = cgLeft.union(cgRight);


    PType<Pair<Collection<U>, Collection<V>>> otype = ptf.pairs(ptf.collections(leftType), ptf.collections(rightType));
    return both.groupByKey().parallelDo("cogroup", 
        new PostGroupFn<K, U, V>(leftType, rightType), ptf.tableOf(keyType, otype));
  }


  private static class CogroupFn1<K, V, U> extends MapValuesFn<K, V, Pair<V, U>> {
    @Override
    public Pair<V, U> map(V v) {

View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

        PageRankData prd = input.second();
        for (String link : prd.urls) {
          emitter.emit(Pair.of(link, prd.propagatedScore()));
        }
      }
    }, ptf.tableOf(ptf.strings(), ptf.floats()));


    return input.cogroup(outbound).mapValues(
        new MapFn<Pair<Collection<PageRankData>, Collection<Float>>, PageRankData>() {
          @Override
          public PageRankData map(Pair<Collection<PageRankData>, Collection<Float>> input) {

View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

  }


  private static <T> PTable<String, BloomFilter> createFilterTable(PCollection<T> collection, BloomFilterFn<T> filterFn) {
    PTypeFamily tf = collection.getTypeFamily();
    PTable<String, BloomFilter> table = collection.parallelDo(filterFn,
        tf.tableOf(tf.strings(), Writables.writables(BloomFilter.class)));
    return table.groupByKey(1).combineValues(new BloomFilterAggregator());
  }




  @SuppressWarnings("serial")

View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

   * @param valueType The {@code PType} for the value of the SequenceFile entry
   * @return A new {@code SourceTable<K, V>} instance
   */
  public static <K, V> TableSource<K, V> sequenceFile(Path path, PType<K> keyType, PType<V> valueType) {
    PTypeFamily ptf = keyType.getFamily();
    return new SeqFileTableSource<K, V>(path, ptf.tableOf(keyType, valueType));
  }


  /**
   * Creates a {@code TableSource<K, V>} instance for the SequenceFile(s) at the given {@code Path}s.
   *

View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

   * @param valueType The {@code PType} for the value of the SequenceFile entry
   * @return A new {@code SourceTable<K, V>} instance
   */
  public static <K, V> TableSource<K, V> sequenceFile(List<Path> paths, PType<K> keyType, PType<V> valueType) {
    PTypeFamily ptf = keyType.getFamily();
    return new SeqFileTableSource<K, V>(paths, ptf.tableOf(keyType, valueType));
  }


  /**
   * Creates a {@code Source<String>} instance for the text file(s) at the given path name.
   *

View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

  public static <S> PCollection<S> distinct(PCollection<S> input, int flushEvery) {
    Preconditions.checkArgument(flushEvery > 0);
    PType<S> pt = input.getPType();
    PTypeFamily ptf = pt.getFamily();
    return input
        .parallelDo("pre-distinct", new PreDistinctFn<S>(flushEvery, pt), ptf.tableOf(pt, ptf.nulls()))
        .groupByKey()
        .parallelDo("post-distinct", new PostDistinctFn<S>(), pt);
  }
  
  /**

View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

              rightReadable, right.getPTableType(), includeUnmatchedLeftValues);
    ParallelDoOptions options = ParallelDoOptions.builder()
        .sourceTargets(rightReadable.getSourceTargets())
        .build();
    return left.parallelDo("mapjoin", mapJoinDoFn,
        tf.tableOf(left.getKeyType(), tf.pairs(left.getValueType(), right.getValueType())),
        options);
  }


  static class MapsideJoinDoFn<K, U, V> extends DoFn<Pair<K, U>, Pair<K, Pair<U, V>>> {

View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

    PTypeFamily rtf = right.getTypeFamily();


    PTable<Pair<Integer, Integer>, Pair<K1, U>> leftCross = left.parallelDo(new GFCross<Pair<K1, U>>(0, parallelism),
        ltf.tableOf(ltf.pairs(ltf.ints(), ltf.ints()), ltf.pairs(left.getKeyType(), left.getValueType())));
    PTable<Pair<Integer, Integer>, Pair<K2, V>> rightCross = right.parallelDo(new GFCross<Pair<K2, V>>(1, parallelism),
        rtf.tableOf(rtf.pairs(rtf.ints(), rtf.ints()), rtf.pairs(right.getKeyType(), right.getValueType())));


    PTable<Pair<Integer, Integer>, Pair<Pair<K1, U>, Pair<K2, V>>> cg = leftCross.join(rightCross);


    PTypeFamily ctf = cg.getTypeFamily();

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.