Package com.google.appengine.tools.mapreduce.impl

Examples of com.google.appengine.tools.mapreduce.impl.FilesByShard

214
215
216
217
218
219
220
221
222
223
224
225
    public Value<MapReduceResult<FilesByShard>> run(MapReduceResult<FilesByShard> mapResult) {

      Context context = new BaseContext(mrJobId);
      int mapShards = findMaxFilesPerShard(mapResult.getOutputResult());
      int reduceShards = mrSpec.getNumReducers();
      FilesByShard filesByShard = mapResult.getOutputResult();
      filesByShard.splitShards(Math.max(mapShards, reduceShards));
      GoogleCloudStorageSortInput input = new GoogleCloudStorageSortInput(filesByShard);
      ((Input<?>) input).setContext(context);
      List<? extends InputReader<KeyValue<ByteBuffer, ByteBuffer>>> readers = input.createReaders();
      Output<KeyValue<ByteBuffer, List<ByteBuffer>>, FilesByShard> output =
          new GoogleCloudStorageSortOutput(settings.getBucketName(), mrJobId,
View Full Code Here
302
303
304
305
306
307
308
309
310
311
312
     */
    @Override
    public Value<MapReduceResult<FilesByShard>> run(MapReduceResult<FilesByShard> priorResult) {

      Context context = new BaseContext(mrJobId);
      FilesByShard sortFiles = priorResult.getOutputResult();
      int maxFilesPerShard = findMaxFilesPerShard(sortFiles);
      if (maxFilesPerShard <= settings.getMergeFanin()) {
        return immediate(priorResult);
      }

View Full Code Here
454
455
456
457
458
459
460
461
462
463
464
465
466
    @Override
    public Value<Void> run(MapReduceResult<FilesByShard> result) {
      Set<GcsFilename> toDelete = new HashSet<>();

      FilesByShard filesByShard = result.getOutputResult();
      for (int i = 0; i < filesByShard.getShardCount(); i++) {
        toDelete.addAll(filesByShard.getFilesForShard(i).getFiles());
      }

      CleanupPipelineJob.cleanup(new ArrayList<>(toDelete), settings.toJobSettings());
      return null;
    }
View Full Code Here
TOP

Related Classes of com.google.appengine.tools.mapreduce.impl.FilesByShard

  • com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet
  • com.google.appengine.tools.mapreduce.MapReduceJob$Cleanup
  • com.google.appengine.tools.mapreduce.MapReduceJob$MergeJob
  • com.google.appengine.tools.mapreduce.MapReduceJob$SortJob
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.