Examples of com.google.appengine.tools.mapreduce.inputs.RandomLongInput

Package com.google.appengine.tools.mapreduce.inputs

Examples of com.google.appengine.tools.mapreduce.inputs.RandomLongInput

com.google.appengine.tools.mapreduce.inputs.RandomLongInput
An input that produce random values. @author ohler@google.com (Christian Ohler)

    }
  }


  @Test
  public void testMapOnlyJob() throws Exception {
    RandomLongInput input = new RandomLongInput(100, 2);
    InMemoryOutput<String> output = new InMemoryOutput<>();
    MapSpecification<Long, String, List<List<String>>> specification =
        new MapSpecification.Builder<>(input, new MapOnly(), output).setJobName("mr-only").build();
    runTest(specification, new Verifier<List<List<String>>>() {
      @Override

View Full Code Here

    }
  }


  @Test
  public void testAdaptedMapOnlyJob() throws Exception {
    RandomLongInput input = new RandomLongInput(100, 2);
    InMemoryOutput<String> output = new InMemoryOutput<>();
    MapOnlyMapper<Long, String> mapper = MapOnlyMapper.forMapper(new VoidKeyMapper());
    MapSpecification<Long, String, List<List<String>>> specification =
        new MapSpecification.Builder<>(input, mapper, output).setJobName("adapted-mr-only").build();
    runTest(specification, new Verifier<List<List<String>>>() {

View Full Code Here

  public void testShardAndSliceRetriesSuccess() throws Exception {
    int shardsCount = 1;
    // {input-size, shard-failure, slice-failure}
    int[][] runs = { {10, 0, 0}, {10, 2, 0}, {10, 0, 10}, {10, 3, 5}, {10, 0, 22}, {10, 1, 50}};
    for (int[] run : runs) {
      RandomLongInput input = new RandomLongInput(run[0], shardsCount);
      runWithPipeline(new MapReduceSettings.Builder().setMillisPerSlice(0)
          .build(), new MapReduceSpecification.Builder<>(input,
          new RougeMapper(shardsCount, run[1], run[2]), NoReducer.<String, Long, String>create(),
          new NoOutput<String, String>()).setKeyMarshaller(Marshallers.getStringMarshaller())
          .setValueMarshaller(Marshallers.getLongMarshaller()).setJobName("Shard-retry test")
          .build(), new RougeMapperVerifier(shardsCount, run[0], run[1], run[2]) {
        @Override
        public void verify(MapReduceResult<String> result) throws Exception {
          super.verify(result);
          assertNull(result.getOutputResult());
        }
      });
    }


    // Disallow slice-retry
    runs = new int[][] { {10, 0, 0}, {10, 4, 0}, {10, 0, 4}, {10, 3, 1}, {10, 1, 3}};
    for (int[] run : runs) {
      RandomLongInput input = new RandomLongInput(run[0], shardsCount);
      RougeMapper mapper = new RougeMapper(shardsCount, run[1], run[2]);
      mapper.setAllowSliceRetry(false);
      runWithPipeline(new MapReduceSettings.Builder().setMillisPerSlice(0)
          .build(), new MapReduceSpecification.Builder<>(input, mapper,
          NoReducer.<String, Long, String>create(), new NoOutput<String, String>())

View Full Code Here

  public void testShardAndSliceRetriesFailure() throws Exception {
    int shardsCount = 1;
    // {shard-failure, slice-failure}
    int[][] runs = { {5, 0}, {4, 21}, {3, 50}};
    for (int[] run : runs) {
      RandomLongInput input = new RandomLongInput(10, shardsCount);
      MapReduceSettings mrSettings = new MapReduceSettings.Builder().setMillisPerSlice(0).build();
      MapReduceSpecification<Long, String, Long, String, String> mrSpec =
          new MapReduceSpecification.Builder<>(input, new RougeMapper(shardsCount, run[0], run[1]),
              NoReducer.<String, Long, String>create(), new NoOutput<String, String>())
              .setJobName("Shard-retry failed").setKeyMarshaller(Marshallers.getStringMarshaller())
              .setValueMarshaller(Marshallers.getLongMarshaller()).build();
      String jobId = pipelineService.startNewPipeline(new MapReduceJob<>(mrSpec, mrSettings));
      assertFalse(jobId.isEmpty());
      executeTasksUntilEmpty("default");
      JobInfo info = pipelineService.getJobInfo(jobId);
      assertNull(info.getOutput());
      assertEquals(JobInfo.State.STOPPED_BY_ERROR, info.getJobState());
      assertTrue(info.getException().getMessage()
          .matches("Stage map-.* was not completed successfuly \\(status=ERROR, message=.*\\)"));
    }


    // Disallow slice-retry
    runs = new int[][] { {5, 0}, {0, 5}, {4, 1}, {1, 4}};
    for (int[] run : runs) {
      RandomLongInput input = new RandomLongInput(10, shardsCount);
      RougeMapper mapper = new RougeMapper(shardsCount, run[0], run[1]);
      mapper.setAllowSliceRetry(false);
      MapReduceSettings mrSettings = new MapReduceSettings.Builder().setMillisPerSlice(0).build();
      MapReduceSpecification<Long, String, Long, String, String> mrSpec =
          new MapReduceSpecification.Builder<>(input, mapper,

View Full Code Here

    }
  }


  @Test
  public void testPassThroughToString() throws Exception {
    final RandomLongInput input = new RandomLongInput(10, 1);
    input.setSeed(0L);
    runTest(new MapReduceSpecification.Builder<>(input, new Mod37Mapper(), ValueProjectionReducer
        .<String, Long>create(), new StringOutput<Long, List<AppEngineFile>>(",",
        new BlobFileOutput("Foo-%02d", "testType")))
        .setKeyMarshaller(Marshallers.getStringMarshaller())
        .setValueMarshaller(Marshallers.getLongMarshaller()).setJobName("TestPassThroughToString")
        .build(), new Verifier<List<AppEngineFile>>() {
      @Override
      public void verify(MapReduceResult<List<AppEngineFile>> result) throws Exception {
        assertEquals(1, result.getOutputResult().size());
        assertEquals(10, result.getCounters().getCounter(CounterNames.MAPPER_CALLS).getValue());
        AppEngineFile file = result.getOutputResult().get(0);
        FileReadChannel ch = FileServiceFactory.getFileService().openReadChannel(file, false);
        BufferedReader reader =
            new BufferedReader(Channels.newReader(ch, US_ASCII.newDecoder(), -1));
        String line = reader.readLine();
        List<String> strings = Arrays.asList(line.split(","));
        assertEquals(10, strings.size());
        input.setSeed(0L);
        InputReader<Long> source = input.createReaders().get(0);
        for (int i = 0; i < 10; i++) {
          assertTrue(strings.contains(source.next().toString()));
        }
      }
    });

View Full Code Here

    });
  }


  @Test
  public void testPassByteBufferToGcs() throws Exception {
    final RandomLongInput input = new RandomLongInput(10, 1);
    input.setSeed(0L);
    MapReduceSpecification.Builder<Long, ByteBuffer, ByteBuffer, ByteBuffer,
        GoogleCloudStorageFileSet> builder = new MapReduceSpecification.Builder<>();
    builder.setJobName("TestPassThroughToByteBuffer");
    builder.setInput(input);
    builder.setMapper(new LongToBytesMapper());
    builder.setKeyMarshaller(Marshallers.getByteBufferMarshaller());
    builder.setValueMarshaller(Marshallers.getByteBufferMarshaller());
    builder.setReducer(ValueProjectionReducer.<ByteBuffer, ByteBuffer>create());
    builder.setOutput(new GoogleCloudStorageFileOutput("bucket", "fileNamePattern-%04d",
        "application/octet-stream"));
    builder.setNumReducers(2);
    runTest(builder.build(), new Verifier<GoogleCloudStorageFileSet>() {
      @Override
      public void verify(MapReduceResult<GoogleCloudStorageFileSet> result) throws Exception {
        assertEquals(2, result.getOutputResult().getNumFiles());
        assertEquals(10, result.getCounters().getCounter(CounterNames.MAPPER_CALLS).getValue());
        ArrayList<Long> results = new ArrayList<>();
        GcsService gcsService = GcsServiceFactory.createGcsService();
        ByteBuffer holder = ByteBuffer.allocate(8);
        for (GcsFilename file : result.getOutputResult().getFiles()) {
          try (GcsInputChannel channel = gcsService.openPrefetchingReadChannel(file, 0, 4096)) {
            int read = channel.read(holder);
            while (read != -1) {
              holder.rewind();
              results.add(holder.getLong());
              holder.rewind();
              read = channel.read(holder);
            }
          }
        }
        assertEquals(10, results.size());
        RandomLongInput input = new RandomLongInput(10, 1);
        input.setSeed(0L);
        InputReader<Long> source = input.createReaders().get(0);
        for (int i = 0; i < results.size(); i++) {
          Long expected = source.next();
          assertTrue(results.contains(expected));
        }
      }

View Full Code Here

    MapReduceSpecification.Builder<Long, Long, String, Long, List<List<Long>>> builder =
        new MapReduceSpecification.Builder<>();
    builder.setJobName("Test MR");
    final long sortMem = 1000;
    final long inputItems = 10 * sortMem / 100; // Forces 3 levels of merging
    builder.setInput(new RandomLongInput(inputItems, 1));
    builder.setMapper(new DummyValueMapper(100));
    builder.setKeyMarshaller(Marshallers.getLongMarshaller());
    builder.setValueMarshaller(Marshallers.getStringMarshaller());
    builder.setReducer(KeyProjectionReducer.<Long, String>create());
    builder.setOutput(new InMemoryOutput<Long>());

View Full Code Here

TOP

Related Classes of com.google.appengine.tools.mapreduce.inputs.RandomLongInput

com.google.appengine.tools.mapreduce.EndToEndTest

java.security.SecureRandom

java.util.NoSuchElementException

java.util.Random

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.