Package com.google.appengine.tools.mapreduce.inputs

Examples of com.google.appengine.tools.mapreduce.inputs.DatastoreInput

139
140
141
142
143
144
145
146
147
148
149
  private MapReduceSpecification<Entity, String, Long, KeyValue<String, Long>,
      List<List<KeyValue<String, Long>>>> getCountJobSpec(int mapShardCount, int reduceShardCount) {
    Query query =
        new Query(datastoreType).setFilter(new FilterPredicate("foo", FilterOperator.EQUAL, "bar"));

    return new MapReduceSpecification.Builder<>(new DatastoreInput(query, mapShardCount),
        new CountMapper(), new CountReducer(), new InMemoryOutput<KeyValue<String, Long>>())
        .setKeyMarshaller(Marshallers.getStringMarshaller())
        .setValueMarshaller(Marshallers.getLongMarshaller())
        .setJobName("MapReduceTest count")
        .setNumReducers(reduceShardCount)
View Full Code Here
904
905
906
907
908
909
910
911
912
913
914
    final DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService();
    // Datastore restriction: id cannot be zero.
    for (long i = 1; i <= 100; ++i) {
      datastoreService.put(new Entity(KeyFactory.createKey("Test", i)));
    }
    runTest(new MapReduceSpecification.Builder<>(new DatastoreInput("Test", 5), new TestMapper(),
        new TestReducer(), new InMemoryOutput<KeyValue<String, List<Long>>>())
        .setKeyMarshaller(Marshallers.getStringMarshaller())
        .setValueMarshaller(Marshallers.getLongMarshaller()).setJobName("Test MR").build(),
        new Verifier<List<List<KeyValue<String, List<Long>>>>>() {
          @Override
View Full Code Here
953
954
955
956
957
958
959
960
961
962
963
        });
  }

  @Test
  public void testNoData() throws Exception {
    runTest(new MapReduceSpecification.Builder<>(new DatastoreInput("Test", 2), new TestMapper(),
        NoReducer.<String, Long, Void>create(), new NoOutput<Void, Void>())
        .setKeyMarshaller(Marshallers.getStringMarshaller())
        .setValueMarshaller(Marshallers.getLongMarshaller()).setJobName("Test MR").build(),
        new Verifier<Void>() {
          @Override
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
  @Test
  public void testOutputInOrder() throws Exception {
    MapReduceSpecification.Builder<Entity, String, Long, Long, Boolean> mrSpecBuilder =
        new MapReduceSpecification.Builder<>();
    mrSpecBuilder.setJobName("Test MR").setInput(new DatastoreInput("Test", 2))
        .setMapper(new TestMapper()).setKeyMarshaller(Marshallers.getStringMarshaller())
        .setValueMarshaller(Marshallers.getLongMarshaller())
        .setReducer(ValueProjectionReducer.<String, Long>create())
        .setOutput(new CustomOutput())
        .setNumReducers(17);
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
  }

  private String startStatsJob(int mapShardCount, int reduceShardCount) {
    // (I, K, V, O, R)
    // Input<I>, Input<Entity>, so I=Entity
    Input input = new DatastoreInput("Reading", mapShardCount);

    // Mapper<I,K,V>, Mapper<Entity, String, String>, so I=Entity, K=String, V=String
    Mapper<Entity, String, String> mapper = new ReadingMapper();

    // Reducer<K,V,O>, Reducer<String, String, ReadingHistory>, so K=String, V=String,
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
  private String startStatsJob(int mapShardCount, int reduceShardCount) {
    return MapReduceJob.start(
        MapReduceSpecification.of(
            "MapReduceTest stats",
            new DatastoreInput("MapReduceTest", mapShardCount),
            new CountMapper(),
            Marshallers.getStringMarshaller(),
            Marshallers.getLongMarshaller(),
            new CountReducer(),
            new InMemoryOutput<KeyValue<String, Long>>(reduceShardCount)),
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
              datastoreService.put(new Entity(KeyFactory.createKey("Test", i)));
            }
          }
        },
        MapReduceSpecification.of("Test MR",
            new DatastoreInput("Test", 5),
            new TestMapper(),
            Marshallers.getStringMarshaller(),
            Marshallers.getLongMarshaller(),
            new TestReducer(),
            new InMemoryOutput<KeyValue<String, List<Long>>>(1)),
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
        new Preparer() {
          @Override public void prepare() throws Exception {
          }
        },
        MapReduceSpecification.of("Test MR",
            new DatastoreInput("Test", 2),
            new TestMapper(),
            Marshallers.getStringMarshaller(),
            Marshallers.getLongMarshaller(),
            NoReducer.<String, Long, Void>create(),
            NoOutput.<Void, Void>create(1)),
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
        int mapShardCount = 1;

        builder = new MapReduceSpecification.Builder();
        builder.setJobName("MapReduceTest stats");
        builder.setInput(new DatastoreInput("MapReduceTest", mapShardCount));
        builder.setMapper(new CountMapper());
        builder.setKeyMarshaller(Marshallers.getStringMarshaller());
        builder.setValueMarshaller(Marshallers.getLongMarshaller());
        builder.setReducer(new CountReducer());
        builder.setOutput(new InMemoryOutput<KeyValue<String, Long>>());
View Full Code Here
TOP

Related Classes of com.google.appengine.tools.mapreduce.inputs.DatastoreInput

  • com.google.appengine.api.datastore.DatastoreService
  • com.google.appengine.api.datastore.Key
  • com.google.appengine.api.datastore.Query
  • com.google.appengine.demos.mapreduce.entitycount.ChainedMapReduceJob
  • com.google.appengine.demos.mapreduce.entitycount.Servlet
  • com.google.appengine.tck.mapreduce.MapReduceTest
  • com.google.appengine.tools.mapreduce.CustomOutputTest
  • com.google.appengine.tools.mapreduce.EndToEndTest
  • com.openhouseautomation.mapreduce.ReadingServletMR
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.