Examples of readTextFile()

Examples of org.apache.crunch.impl.mr.MRPipeline.readTextFile()

40
41
42
43
44
45
46
47
48
49
50
  @Test
  public void testCompressText() throws Exception {
    String urlsFile = tmpDir.copyResourceFileName("urls.txt");
    String out = tmpDir.getFileName("out");
    MRPipeline p = new MRPipeline(CompressIT.class, tmpDir.getDefaultConfiguration());
    PCollection<String> in = p.readTextFile(urlsFile);
    in.write(Compress.gzip(To.textFile(out)));
    p.done();
    assertTrue(checkDirContainsExt(out, ".gz"));
  }
View Full Code Here

Examples of org.apache.crunch.impl.mr.MRPipeline.readTextFile()

44
45
46
47
48
49
50
51
52
53
54
    File src2 = tmpDir.copyResourceFile(Tests.resource(this, "src2.txt"));
    Files.copy(src1, new File(srcFiles, "src1.txt"));
    Files.copy(src2, new File(srcFiles, "src2.txt"));

    MRPipeline p = new MRPipeline(CombineFileIT.class, tmpDir.getDefaultConfiguration());
    PCollection<String> in = p.readTextFile(srcFiles.getAbsolutePath());
    in.write(To.textFile(outputFiles.getAbsolutePath()));
    p.done();
    assertEquals(4, outputFiles.listFiles().length);
  }
View Full Code Here

Examples of org.apache.crunch.impl.mr.MRPipeline.readTextFile()

174
175
176
177
178
179
180
181
182
183
184
    out2.flush();
    out2.close();

    final MRPipeline pipeline = new MRPipeline(MapsideJoinStrategyIT.class, tmpDir.getDefaultConfiguration());

    final PCollection<String> values1 = pipeline.readTextFile(path1.toString());
    final PCollection<String> values2 = pipeline.readTextFile(path2.toString());

    final PTable<Text, Text> convertedValues1 = convertStringToText(values1);
    final PTable<Text, Text> convertedValues2 = convertStringToText(values2);
View Full Code Here

Examples of org.apache.crunch.impl.mr.MRPipeline.readTextFile()

175
176
177
178
179
180
181
182
183
184
185
    out2.close();

    final MRPipeline pipeline = new MRPipeline(MapsideJoinStrategyIT.class, tmpDir.getDefaultConfiguration());

    final PCollection<String> values1 = pipeline.readTextFile(path1.toString());
    final PCollection<String> values2 = pipeline.readTextFile(path2.toString());

    final PTable<Text, Text> convertedValues1 = convertStringToText(values1);
    final PTable<Text, Text> convertedValues2 = convertStringToText(values2);

    // for map side join
View Full Code Here

Examples of org.apache.crunch.impl.spark.SparkPipeline.readTextFile()

135
136
137
138
139
140
141
142
143
144
145
  @Test
  public void testAvroReflectSortPair() throws IOException {
    Pipeline pipeline = new SparkPipeline("local", "sort");
    pipeline.enableDebug();
    String rsrc = tmpDir.copyResourceFileName("set2.txt");
    PCollection<Pair<String, StringWrapper>> in = pipeline.readTextFile(rsrc)
        .parallelDo(new MapFn<String, Pair<String, StringWrapper>>() {

          @Override
          public Pair<String, StringWrapper> map(String input) {
            return Pair.of(input, wrap(input));
View Full Code Here

Examples of org.apache.crunch.impl.spark.SparkPipeline.readTextFile()

157
158
159
160
161
162
163
164
165
166
167
  }

  @Test
  public void testAvroReflectSortTable() throws IOException {
    Pipeline pipeline = new SparkPipeline("local", "sort");
    PTable<String, StringWrapper> unsorted = pipeline.readTextFile(tmpDir.copyResourceFileName("set2.txt")).parallelDo(
        new MapFn<String, Pair<String, StringWrapper>>() {

          @Override
          public Pair<String, StringWrapper> map(String input) {
            return Pair.of(input, wrap(input));
View Full Code Here

Examples of org.apache.flink.api.java.ExecutionEnvironment.readTextFile()

111
112
113
114
115
116
117
118
119
120
121
   
    // construct the plan it will be multiple flat maps, all unioned
    // and the "unioned" dataSet will be grouped
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
   
    DataSet<String> source = env.readTextFile(IN_FILE);
    DataSet<Tuple2<String, Integer>> lastUnion = source.flatMap(new DummyFlatMap());
 
    for (int i = 1; i< NUM_INPUTS; i++){
      lastUnion = lastUnion.union(source.flatMap(new DummyFlatMap()));
    }
View Full Code Here

Examples of org.apache.flink.streaming.api.environment.LocalStreamEnvironment.readTextFile()

121
122
123
124
125
126
127
128
129
130
131
      fail();
    } catch (RuntimeException e) {
    }

    try {
      env.readTextFile("random/path/that/is/not/valid");
      fail();
    } catch (IllegalArgumentException e) {
    }
  }
View Full Code Here

Examples of org.platformlayer.ops.OpsTarget.readTextFile()

25
26
27
28
29
30
31
32
33
34
35
      return;
    }

    String fstabLine = "\ncgroup\t/cgroup\tcgroup\tdefaults\t0\t0";
    File fstabFile = new File("/etc/fstab");
    String fstab = target.readTextFile(fstabFile);
    fstab += fstabLine;
    FileUpload.upload(target, fstabFile, fstab);

    target.mkdir(cgroupsFile);
View Full Code Here

Examples of org.platformlayer.ops.OpsTarget.readTextFile()

45
46
47
48
49
50
51
52
53
54
55
    @Override
    protected byte[] getContentsBytes() throws OpsException {
      if (!built) {
        OpsTarget target = OpsContext.get().getInstance(OpsTarget.class);

        String etcIssue = target.readTextFile(new File("/etc/issue"));
        if (etcIssue.startsWith("Debian")) {
          addDefaultsDebian(target);
        } else if (etcIssue.startsWith("Ubuntu")) {
          addDefaultsUbuntu(target);
        } else {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.