Package org.apache.crunch.impl.mr

Examples of org.apache.crunch.impl.mr.MRPipeline.run()

165
166
167
168
169
170
171
172
173
174
175
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target trevniFile = new TrevniKeyTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, trevniFile);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();

    PCollection<Person> retrievedPeople = pipeline.read(new TrevniKeySource<Person>(
        new Path(outputFile.toURI()), Avros.records(Person.class)));
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
      public String map(KeyValue input) {
        return input.toString();
      }
    }, strings());
    texts.write(To.textFile(outputPath));
    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());

    List<String> lines = FileUtils.readLines(new File(outputPath.toString(), "part-m-00000"));
    assertEquals(kvs.size(), lines.size());
    for (int i = 0; i < kvs.size(); i++) {
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    // The aggregate method groups a collection into a single PObject.
    PObject<Long> totalCount = numberOfWords.aggregate(Aggregators.SUM_LONGS()).first();

    // Execute the pipeline as a MapReduce.
    PipelineResult result = pipeline.run();

    System.out.println("Total number of words: " + totalCount.getValue());
   
    pipeline.done();
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
    PCollection<String> words = split(shakespeare, "\\s+");
    PTable<String,Long> wordCounts = words.count();
    PCollection<KeyValue> wordCountKeyValues = convertToKeyValues(wordCounts);
    pipeline.write(wordCountKeyValues, ToHBase.hfile(outputPath));

    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());

    FileSystem fs = FileSystem.get(conf);
    KeyValue kv = readFromHFiles(fs, outputPath, "and");
    assertEquals(427L, Bytes.toLong(kv.getValue()));
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
    HFileUtils.writePutsToHFilesForIncrementalLoad(
        wordCountPuts,
        testTable,
        outputPath);

    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());

    new LoadIncrementalHFiles(HBASE_TEST_UTILITY.getConfiguration())
        .doBulkLoad(outputPath, testTable);
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
    HFileUtils.writePutsToHFilesForIncrementalLoad(
        convertToPuts(longWordCounts),
        table2,
        outputPath2);

    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());
    loader.doBulkLoad(outputPath1, table1);
    loader.doBulkLoad(outputPath2, table2);

    assertEquals(396L, getWordCountFromTable(table1, "of"));
View Full Code Here
276
277
278
279
280
281
282
283
284
285
286
    HFileUtils.writePutsToHFilesForIncrementalLoad(
        wordCountPuts,
        testTable,
        outputPath);

    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());

    int hfilesCount = 0;
    FileSystem fs = outputPath.getFileSystem(conf);
    for (FileStatus e : fs.listStatus(new Path(outputPath, Bytes.toString(TEST_FAMILY)))) {
View Full Code Here
34
35
36
37
38
39
40
41
  @Test(expected=CrunchRuntimeException.class)
  public void testTextToAvro() throws Exception {
    String shakes = tmpDir.copyResourceFileName("shakes.txt");
    Pipeline pipeline = new MRPipeline(TextToAvroIT.class, tmpDir.getDefaultConfiguration());
    pipeline.read(From.textFile(shakes)).write(To.avroFile("output"));
    pipeline.run();
  }
}
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    Pipeline mrPipeline = new MRPipeline(getClass());
    PCollection<String> mrPColl = mrPipeline.readTextFile(infilename);
    Target mrTarget = new TextFileTarget(mrOutFilename);
    mrPipeline.write(mrPColl, mrTarget, WriteMode.OVERWRITE);
    mrPipeline.run();
    String actualMrText = Files.readFirstLine(new File(mrOutFilename + "/part-m-00000"), Charsets.UTF_8);

    Assert.assertEquals("MR file mismatch", expected, actualMrText);
    Assert.assertEquals("Mem file mismatch", expected, actualMemText);
  }
View Full Code Here
184
185
186
187
188
189
190
191
192
193
194
    // for map side join
    final MapsideJoinStrategy<Text, Text, Text> mapSideJoinStrategy = new MapsideJoinStrategy<Text, Text, Text>();

    final PTable<Text, Pair<Text, Text>> updatedJoinedRows = mapSideJoinStrategy.join(convertedValues1, convertedValues2, JoinType.INNER_JOIN);
    pipeline.run();

    // Join should have 2 results
    // Join should have contentBytes1 and contentBytes2
    assertEquals(4, updatedJoinedRows.materializeToMap().size());
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.