Package org.apache.crunch

Examples of org.apache.crunch.Pipeline.run()

245
246
247
248
249
250
251
252
253
254
255
    HFileUtils.writePutsToHFilesForIncrementalLoad(
        convertToPuts(longWordCounts),
        table2,
        outputPath2);

    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());
    loader.doBulkLoad(outputPath1, table1);
    loader.doBulkLoad(outputPath2, table2);

    assertEquals(396L, getWordCountFromTable(table1, "of"));
View Full Code Here
276
277
278
279
280
281
282
283
284
285
286
    HFileUtils.writePutsToHFilesForIncrementalLoad(
        wordCountPuts,
        testTable,
        outputPath);

    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());

    int hfilesCount = 0;
    FileSystem fs = outputPath.getFileSystem(conf);
    for (FileStatus e : fs.listStatus(new Path(outputPath, Bytes.toString(TEST_FAMILY)))) {
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
      public String map(KeyValue input) {
        return input.toString();
      }
    }, strings());
    texts.write(To.textFile(outputPath));
    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());

    List<String> lines = FileUtils.readLines(new File(outputPath.toString(), "part-m-00000"));
    assertEquals(kvs.size(), lines.size());
    for (int i = 0; i < kvs.size(); i++) {
View Full Code Here
34
35
36
37
38
39
40
41
  @Test(expected=CrunchRuntimeException.class)
  public void testTextToAvro() throws Exception {
    String shakes = tmpDir.copyResourceFileName("shakes.txt");
    Pipeline pipeline = new MRPipeline(TextToAvroIT.class, tmpDir.getDefaultConfiguration());
    pipeline.read(From.textFile(shakes)).write(To.avroFile("output"));
    pipeline.run();
  }
}
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    Pipeline memPipeline = MemPipeline.getInstance();
    PCollection<String> memPColl = memPipeline.readTextFile(infilename);
    Target memTarget = new TextFileTarget(memOutFilename);
    memPipeline.write(memPColl, memTarget, WriteMode.OVERWRITE);
    memPipeline.run();
    File outDir = new File(memOutFilename);
    File actualMemOut = null;
    for (File f : outDir.listFiles()) {
      String name = f.getName();
      if (name.contains("out") && name.endsWith(".txt")) {
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    Pipeline mrPipeline = new MRPipeline(getClass());
    PCollection<String> mrPColl = mrPipeline.readTextFile(infilename);
    Target mrTarget = new TextFileTarget(mrOutFilename);
    mrPipeline.write(mrPColl, mrTarget, WriteMode.OVERWRITE);
    mrPipeline.run();
    String actualMrText = Files.readFirstLine(new File(mrOutFilename + "/part-m-00000"), Charsets.UTF_8);

    Assert.assertEquals("MR file mismatch", expected, actualMrText);
    Assert.assertEquals("Mem file mismatch", expected, actualMemText);
  }
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target trevniFile = new TrevniKeyTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, trevniFile);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();

    File trvFile = new File(outputFile, "part-m-00000.trv-part-0.trv");
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
        Avros.records(Person.class)));
    File output1File = tmpDir.getFile("output1");
    File output2File = tmpDir.getFile("output2");
    pipeline.write(genericCollection, new TrevniKeyTarget(output1File.getAbsolutePath()));
    pipeline.write(genericCollection, new TrevniKeyTarget(output2File.getAbsolutePath()));
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();

    File trv1File = new File(output1File, "part-m-00000.trv-part-0.trv");
    File trv2File = new File(output2File, "part-m-00000.trv-part-0.trv");
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target trevniFile = new TrevniKeyTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, trevniFile);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();

    PCollection<Person> retrievedPeople = pipeline.read(new TrevniKeySource<Person>(
        new Path(outputFile.toURI()), Avros.records(Person.class)));
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target textFile = To.textFile(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, textFile);
    pipeline.run();
    Person person = genericCollection.materialize().iterator().next();
    String outputString = FileUtils.readFileToString(new File(outputFile, "part-m-00000"));
    assertTrue(outputString.contains(person.toString()));
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.