Package org.apache.crunch

Examples of org.apache.crunch.GroupingOptions$Builder


      }
    }, tableOf(HBaseTypes.keyValues(), nulls()));
    List<KeyValue> splitPoints = getSplitPoints(table);
    Path partitionFile = new Path(((DistributedPipeline) kvs.getPipeline()).createTempPath(), "partition");
    writePartitionInfo(conf, partitionFile, splitPoints);
    GroupingOptions options = GroupingOptions.builder()
        .partitionerClass(TotalOrderPartitioner.class)
        .sortComparatorClass(KeyValueComparator.class)
        .conf(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString())
        .numReducers(splitPoints.size() + 1)
        .build();
View Full Code Here


      }
    }, tableOf(HBaseTypes.keyValues(), nulls()));
    List<KeyValue> splitPoints = getSplitPoints(table);
    Path partitionFile = new Path(((DistributedPipeline) kvs.getPipeline()).createTempPath(), "partition");
    writePartitionInfo(conf, partitionFile, splitPoints);
    GroupingOptions options = GroupingOptions.builder()
        .partitionerClass(TotalOrderPartitioner.class)
        .sortComparatorClass(KeyValueComparator.class)
        .conf(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString())
        .numReducers(splitPoints.size() + 1)
        .build();
View Full Code Here

      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
      }
    }, type);
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, order);
    return pt.groupByKey(options).ungroup().keys();
  }
View Full Code Here

   *
   * @return a {@code PTable} representing the sorted collection.
   */
  public static <K, V> PTable<K, V> sort(PTable<K, V> table, int numReducers, Order key) {
    Configuration conf = table.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(table, conf, numReducers, key);
    return table.groupByKey(options).ungroup();
  }
View Full Code Here

      ColumnOrder... columnOrders) {
    PType<T> pType = collection.getPType();
    KeyExtraction<T> ke = new KeyExtraction<T>(pType, columnOrders);
    PTable<Object, T> pt = collection.by(ke.getByFn(), ke.getKeyType());
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, columnOrders);
    return pt.groupByKey(options).ungroup().values();
  }
View Full Code Here

      }
    }, tableOf(writables(KeyValue.class), nulls()));
    List <KeyValue> splitPoints = getSplitPoints(table);
    Path partitionFile = new Path(((DistributedPipeline) kvs.getPipeline()).createTempPath(), "partition");
    writePartitionInfo(conf, partitionFile, splitPoints);
    GroupingOptions options = GroupingOptions.builder()
        .partitionerClass(TotalOrderPartitioner.class)
        .conf(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString())
        .numReducers(splitPoints.size() + 1)
        .sortComparatorClass(KeyValueComparator.class)
        .build();
View Full Code Here

      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
      }
    }, type);
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, order);
    return pt.groupByKey(options).ungroup().keys();
  }
View Full Code Here

   *
   * @return a {@code PTable} representing the sorted collection.
   */
  public static <K, V> PTable<K, V> sort(PTable<K, V> table, int numReducers, Order key) {
    Configuration conf = table.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(table, conf, numReducers, key);
    return table.groupByKey(options).ungroup();
  }
View Full Code Here

      ColumnOrder... columnOrders) {
    PType<T> pType = collection.getPType();
    SortFns.KeyExtraction<T> ke = new SortFns.KeyExtraction<T>(pType, columnOrders);
    PTable<Object, T> pt = collection.by(ke.getByFn(), ke.getKeyType());
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, columnOrders);
    return pt.groupByKey(options).ungroup().values();
  }
View Full Code Here

      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
      }
    }, type);
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, order);
    return pt.groupByKey(options).ungroup().keys();
  }
View Full Code Here

TOP

Related Classes of org.apache.crunch.GroupingOptions$Builder

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.