Package org.apache.crunch.impl.mr

Examples of org.apache.crunch.impl.mr.MRPipeline.run()

49
50
51
52
53
54
55
56
57
58
59
        emitter.emit(new Pair<String, etypes>(input, etypes.type1));
      }
    }, Writables.tableOf(Writables.strings(), PTypes.enums(etypes.class, set1.getTypeFamily())));

    Iterable<Pair<String, etypes>> materialized = data.materialize();
    pipeline.run();
    for (Pair<String, etypes> pair : materialized) {
      assertEquals(etypes.type1, pair.second());
    }
  }
}
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
        addressesTable.materialize();

        PTable<String, Pair<String, String>> orderAddresses = assignedOrders.join(addressesTable);
        orderAddresses.materialize();

        PipelineResult result = pipeline.run();
        assertTrue(result != null && result.succeeded());
    }

    private static class StringToPairMapFn extends MapFn<String, Pair<String, String>> {
        private transient Splitter splitter;
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target trevniFile = new TrevniKeyTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, trevniFile);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();

    File trvFile = new File(outputFile, "part-m-00000.trv-part-0.trv");
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
        Avros.records(Person.class)));
    File output1File = tmpDir.getFile("output1");
    File output2File = tmpDir.getFile("output2");
    pipeline.write(genericCollection, new TrevniKeyTarget(output1File.getAbsolutePath()));
    pipeline.write(genericCollection, new TrevniKeyTarget(output2File.getAbsolutePath()));
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();

    File trv1File = new File(output1File, "part-m-00000.trv-part-0.trv");
    File trv2File = new File(output2File, "part-m-00000.trv-part-0.trv");
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target trevniFile = new TrevniKeyTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, trevniFile);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();

    PCollection<Person> retrievedPeople = pipeline.read(new TrevniKeySource<Person>(
        new Path(outputFile.toURI()), Avros.records(Person.class)));
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target textFile = To.textFile(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, textFile);
    pipeline.run();
    Person person = genericCollection.materialize().iterator().next();
    String outputString = FileUtils.readFileToString(new File(outputFile, "part-m-00000"));
    assertTrue(outputString.contains(person.toString()));
  }
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
  @Test
  public void materializedColShouldBeWritten() throws Exception {
    File textFile = tmpDir.copyResourceFile("shakes.txt");
    Pipeline pipeline = new MRPipeline(MRPipelineIT.class, tmpDir.getDefaultConfiguration());
    PCollection<String> genericCollection = pipeline.readTextFile(textFile.getAbsolutePath());
    pipeline.run();
    PCollection<String> filter = genericCollection.filter("Filtering data", FilterFns.<String>ACCEPT_ALL());
    filter.materialize();
    pipeline.run();
    File file = tmpDir.getFile("output.txt");
    Target outFile = To.textFile(file.getAbsolutePath());
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
    Pipeline pipeline = new MRPipeline(MRPipelineIT.class, tmpDir.getDefaultConfiguration());
    PCollection<String> genericCollection = pipeline.readTextFile(textFile.getAbsolutePath());
    pipeline.run();
    PCollection<String> filter = genericCollection.filter("Filtering data", FilterFns.<String>ACCEPT_ALL());
    filter.materialize();
    pipeline.run();
    File file = tmpDir.getFile("output.txt");
    Target outFile = To.textFile(file.getAbsolutePath());
    PCollection<String> write = filter.write(outFile);
    write.materialize();
    pipeline.run();
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    pipeline.run();
    File file = tmpDir.getFile("output.txt");
    Target outFile = To.textFile(file.getAbsolutePath());
    PCollection<String> write = filter.write(outFile);
    write.materialize();
    pipeline.run();
  }
 
 
 
  @Test
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target parquetFileTarget = new AvroParquetFileTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, parquetFileTarget);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();

    Path parquetFile = new Path(new File(outputFile, "part-m-00000.parquet").getPath());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.