Package com.google.appengine.tools.mapreduce

Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

66
67
68
69
70
71
72
73
74
75
76
77
78
79
      out.beginSlice();
      out.write(ByteBuffer.wrap(SMALL_CONTENT));
      out.endSlice();
      out.endShard();
    }
    GoogleCloudStorageFileSet files = creator.finish(writers);
    assertEquals(NUM_SHARDS, files.getNumFiles());
    for (int i = 0; i < NUM_SHARDS; i++) {
      GcsFileMetadata metadata = gcsService.getMetadata(files.getFile(i));
      assertNotNull(metadata);
      assertEquals(SMALL_CONTENT.length, metadata.getLength());
      assertEquals(MIME_TYPE, metadata.getOptions().getMimeType());
    }
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
      out.beginSlice();
      out.write(ByteBuffer.wrap(content));
      out.endSlice();
      out.endShard();
    }
    GoogleCloudStorageFileSet files = creator.finish(writers);
    assertEquals(NUM_SHARDS, files.getNumFiles());
    ByteBuffer expectedContent = ByteBuffer.allocate(content.length * 2);
    expectedContent.put(content);
    expectedContent.put(content);
    for (int i = 0; i < NUM_SHARDS; i++) {
      expectedContent.rewind();
      ByteBuffer actualContent = ByteBuffer.allocate(content.length * 2 + 1);
      GcsFileMetadata metadata = gcsService.getMetadata(files.getFile(i));
      assertNotNull(metadata);
      assertEquals(expectedContent.capacity(), metadata.getLength());
      assertEquals(MIME_TYPE, metadata.getOptions().getMimeType());
      try (ReadableByteChannel readChannel = gcsService.openReadChannel(files.getFile(i), 0)) {
        int read = readChannel.read(actualContent);
        assertEquals(read, content.length * 2);
        actualContent.limit(actualContent.position());
        actualContent.rewind();
        assertEquals(expectedContent, actualContent);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
      w.write(ByteBuffer.wrap(new byte[9]));
      w.endSlice();
      w.endShard();
      finished.add(w);
    }
    GoogleCloudStorageFileSet filesWritten = segmenter.finish(finished);
    assertEquals(15, filesWritten.getNumFiles());
    for (int i = 0; i < filesWritten.getNumFiles(); i++) {
      GcsFileMetadata metadata = gcsService.getMetadata(filesWritten.getFile(i));
      assertNotNull(metadata);
      assertEquals(MIME_TYPE, metadata.getOptions().getMimeType());
    }
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
    int countFiles = 0;
    for (OutputWriter<ByteBuffer> w : writers) {
      writeMultipleValues(w, 3, 9);
      countFiles += 3;
    }
    GoogleCloudStorageFileSet filesWritten = segmenter.finish(writers);
    assertEquals(countFiles, filesWritten.getNumFiles());
    for (int i = 0; i < filesWritten.getNumFiles(); i++) {
      GcsFileMetadata metadata = gcsService.getMetadata(filesWritten.getFile(i));
      assertNotNull(metadata);
      assertEquals(MIME_TYPE, metadata.getOptions().getMimeType());
    }
  }
View Full Code Here
14
15
16
17
18
19
20
21
22
23
24
25
  public void testAdd() {
    FilesByShard fbs = new FilesByShard(10, "Foo");
    fbs.addFileToShard(1, "Bar");
    fbs.addFileToShard(1, "Baz");
    fbs.addFileToShard(1, "Bat");
    GoogleCloudStorageFileSet forShard = fbs.getFilesForShard(1);
    List<GcsFilename> files = forShard.getFiles();
    assertEquals(3, files.size());
    assertEquals("Foo", files.get(0).getBucketName());
    assertEquals("Bar", files.get(0).getObjectName());
    assertEquals("Baz", files.get(1).getObjectName());
    assertEquals("Bat", files.get(2).getObjectName());
View Full Code Here
26
27
28
29
30
31
32
33
34
35
36
37
  }

  public void testAddAll() {
    FilesByShard fbs = new FilesByShard(10, "Foo");
    fbs.addFilesToShard(1, Arrays.asList("Bar", "Baz", "Bat"));
    GoogleCloudStorageFileSet forShard = fbs.getFilesForShard(1);
    List<GcsFilename> files = forShard.getFiles();
    assertEquals(3, files.size());
    assertEquals("Foo", files.get(0).getBucketName());
    assertEquals("Bar", files.get(0).getObjectName());
    assertEquals("Baz", files.get(1).getObjectName());
    assertEquals("Bat", files.get(2).getObjectName());
View Full Code Here
37
38
39
40
41
42
43
44
45
    assertEquals("Bat", files.get(2).getObjectName());
  }

  public void testEmptyShards() {
    FilesByShard fbs = new FilesByShard(10, "Foo");
    GoogleCloudStorageFileSet forShard = fbs.getFilesForShard(1);
    List<GcsFilename> files = forShard.getFiles();
    assertEquals(0, files.size());
  }
View Full Code Here
58
59
60
61
62
63
64
    List<String> out = Lists.newArrayList();
    for (OutputWriter<ByteBuffer> w : writers) {
      GoogleCloudStorageFileOutputWriter writer = (GoogleCloudStorageFileOutputWriter) w;
      out.add(writer.getFile().getObjectName());
    }
    return new GoogleCloudStorageFileSet(bucket, out);
  }
View Full Code Here
28
29
30
31
32
33
34
  }

  public GoogleCloudStorageFileSet getFilesForShard(int shardNumber) {
    Preconditions.checkArgument(shardNumber < shardCount);
    List<String> fileNames = allFiles.get(shardNumber);
    return new GoogleCloudStorageFileSet(getBucket(), fileNames);
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
        ImmutableList.builder();
    for (int shard = 0; shard < filesByShard.getShardCount(); shard++) {
      List<InputReader<KeyValue<ByteBuffer, Iterator<ByteBuffer>>>> readers = new ArrayList<>();
      for (List<String> group : Lists.partition(filesByShard.getFilesForShard(shard).getFileNames(),
          mergeFanin)) {
        GoogleCloudStorageFileSet fileSet =
            new GoogleCloudStorageFileSet(filesByShard.getBucket(), group);
        readers.add(createReaderForShard(marshaller, fileSet));
      }
      result.add(new ConcatenatingInputReader<>(readers));
    }
    return result.build();
View Full Code Here
TOP

Related Classes of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

  • com.google.appengine.tools.cloudstorage.GcsFilename
  • com.google.appengine.tools.mapreduce.impl.FilesByShard
  • com.google.appengine.tools.mapreduce.impl.FilesByShardTest
  • com.google.appengine.tools.mapreduce.impl.GoogleCloudStorageMergeInput
  • com.google.appengine.tools.mapreduce.outputs.GoogleCloudStorageFileOutput
  • com.google.appengine.tools.mapreduce.outputs.GoogleCloudStorageFileOutputTest
  • com.google.appengine.tools.mapreduce.outputs.SizeSegmentedGoogleCloudStorageFileOutput
  • com.google.appengine.tools.mapreduce.outputs.SizeSegmentedGoogleCloudStorageFileOutputTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.