Examples of com.google.appengine.tools.mapreduce.inputs.ConsecutiveLongInput

Package com.google.appengine.tools.mapreduce.inputs

Examples of com.google.appengine.tools.mapreduce.inputs.ConsecutiveLongInput

com.google.appengine.tools.mapreduce.inputs.ConsecutiveLongInput
Produces all longs in a given interval. @author ohler@google.com (Christian Ohler)


  private MapSpecification<Long, Entity, Void> getCreationJobSpec(int bytesPerEntity, int entities,
      int shardCount) {
    // [START mapSpec]
    MapSpecification<Long, Entity, Void> spec = new MapSpecification.Builder<>(
        new ConsecutiveLongInput(0, entities, shardCount),
        new EntityCreator(datastoreType, bytesPerEntity),
        new DatastoreOutput())
        .setJobName("Create MapReduce entities")
        .build();
    // [END mapSpec]

View Full Code Here

        new SizeSegmentedGoogleCloudStorageFileOutput(BUCKET, 30, fileNamePattern, mimeType);
    MarshallingOutput<String, GoogleCloudStorageFileSet> op =
        new MarshallingOutput<>(output, Marshallers.getStringMarshaller());


    MapSpecification<Long, String, GoogleCloudStorageFileSet> spec = new MapSpecification.Builder<>(
        new ConsecutiveLongInput(0, 100, 1), new MapOnly(), op).setJobName("Test job").build();


    runTest(spec, new Verifier<GoogleCloudStorageFileSet>() {
      @Override
      public void verify(MapReduceResult<GoogleCloudStorageFileSet> result) throws Exception {
        List<GcsFilename> files = result.getOutputResult().getFiles();

View Full Code Here

    });
  }


  @Test
  public void testSomeShardsEmpty() throws Exception {
    runTest(new MapReduceSpecification.Builder<>(new ConsecutiveLongInput(0, 5, 10),
        new Mod37Mapper(), ValueProjectionReducer.<String, Long>create(),
        new InMemoryOutput<Long>())
        .setKeyMarshaller(Marshallers.getStringMarshaller())
        .setValueMarshaller(Marshallers.getLongMarshaller())
        .setJobName("Empty test MR")

View Full Code Here

  }


  @Test
  public void testMoreReducersThanFanout() throws Exception {
    MapReduceSpecification<Long, String, Long, Long, List<List<Long>>> spec =
        new MapReduceSpecification.Builder<>(new ConsecutiveLongInput(0, 50000, 10),
            new Mod37Mapper(), ValueProjectionReducer.<String, Long>create(),
            new InMemoryOutput<Long>())
            .setKeyMarshaller(Marshallers.getStringMarshaller())
            .setValueMarshaller(Marshallers.getLongMarshaller())
            .setJobName("Empty test MR")

View Full Code Here

  public void testSomeNumbers() throws Exception {
    MapReduceSpecification.Builder<Long, String, Long, KeyValue<String, List<Long>>,
        List<List<KeyValue<String, List<Long>>>>> mrSpecBuilder =
        new MapReduceSpecification.Builder<>();
    mrSpecBuilder.setJobName("Test MR");
    mrSpecBuilder.setInput(new ConsecutiveLongInput(-10000, 10000, 10));
    mrSpecBuilder.setMapper(new Mod37Mapper());
    mrSpecBuilder.setKeyMarshaller(Marshallers.getStringMarshaller());
    mrSpecBuilder.setValueMarshaller(Marshallers.getLongMarshaller());
    mrSpecBuilder.setReducer(new TestReducer());
    mrSpecBuilder.setOutput(new InMemoryOutput<KeyValue<String, List<Long>>>());

View Full Code Here

  @Test
  public void testReduceOnlyLooksAtKeys() throws Exception {
    MapReduceSpecification.Builder<Long, String, Long, String, List<List<String>>> builder =
        new MapReduceSpecification.Builder<>();
    builder.setJobName("Test MR");
    builder.setInput(new ConsecutiveLongInput(-10000, 10000, 10));
    builder.setMapper(new Mod37Mapper());
    builder.setKeyMarshaller(Marshallers.getStringMarshaller());
    builder.setValueMarshaller(Marshallers.getLongMarshaller());
    builder.setReducer(KeyProjectionReducer.<String, Long>create());
    builder.setOutput(new InMemoryOutput<String>());

View Full Code Here

  @Test
  public void testSlicingJob() throws Exception {
    MapReduceSpecification.Builder<Long, String, Long, String, List<List<String>>> builder =
        new MapReduceSpecification.Builder<>();
    builder.setJobName("Test MR");
    builder.setInput(new ConsecutiveLongInput(-100, 100, 10));
    builder.setMapper(new Mod37Mapper());
    builder.setKeyMarshaller(Marshallers.getStringMarshaller());
    builder.setValueMarshaller(Marshallers.getLongMarshaller());
    builder.setReducer(KeyProjectionReducer.<String, Long>create());
    builder.setOutput(new InMemoryOutput<String>());

View Full Code Here

  }


  @Test
  public void testSideOutput() throws Exception {


    runTest(new MapReduceSpecification.Builder<>(new ConsecutiveLongInput(0, 6, 6),
        new SideOutputMapper(), KeyProjectionReducer.<String, Void>create(),
        new InMemoryOutput<String>()).setKeyMarshaller(Marshallers.getStringMarshaller())
        .setValueMarshaller(Marshallers.getVoidMarshaller()).setJobName("Test MR").build(),
        new Verifier<List<List<String>>>() {
          @Override

View Full Code Here

              new BigQueryMarshallerByType<SampleTable>(SampleTable.class), BUCKET,
              getJobKey().getName());


      int numShards = (rowCount / MAX_ROWS_PER_WRITER) + 1;
      MapSpecification<Long, SampleTable, BigQueryStoreResult<GoogleCloudStorageFileSet>> spec =
          new MapSpecification.Builder<>(new ConsecutiveLongInput(0, rowCount, numShards),
              new RandomBigQueryDataCreator(), bqStore).setJobName("Load random data into BigQuery")
              .build();
      return spec;
    }

View Full Code Here


  // [START createMapReduceSpec]
  public static MapReduceSpecification<Long, Integer, Integer, ArrayList<Integer>,
      GoogleCloudStorageFileSet> createMapReduceSpec(String bucket, long start, long limit,
          int shards) {
    ConsecutiveLongInput input = new ConsecutiveLongInput(start, limit, shards);
    Mapper<Long, Integer, Integer> mapper = new SeedToRandomMapper();
    Marshaller<Integer> intermediateKeyMarshaller = Marshallers.getIntegerMarshaller();
    Marshaller<Integer> intermediateValueMarshaller = Marshallers.getIntegerMarshaller();
    Reducer<Integer, Integer, ArrayList<Integer>> reducer = new CollisionFindingReducer();
    Marshaller<ArrayList<Integer>> outputMarshaller = Marshallers.getSerializationMarshaller();

View Full Code Here

0 1

TOP

Related Classes of com.google.appengine.tools.mapreduce.inputs.ConsecutiveLongInput

com.google.appengine.demos.mapreduce.bigqueryload.RandomDataLoadServlet$BigQueryLoadJob

com.google.appengine.demos.mapreduce.entitycount.ChainedMapReduceJob

com.google.appengine.demos.mapreduce.entitycount.Servlet

com.google.appengine.demos.mapreduce.randomcollisions.CollisionFindingServlet

com.google.appengine.tck.mapreduce.MapReduceTest

com.google.appengine.tools.mapreduce.EndToEndTest

com.google.appengine.tools.mapreduce.impl.InMemoryShuffleJobTest

java.util.NoSuchElementException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.