Package org.apache.crunch

Examples of org.apache.crunch.Pipeline.done()

133
134
135
136
137
138
139
140
141
142
          public Pair<Long, String> map(String input) {
            return Pair.of(1L, input);
          }
        }, Avros.pairs(Avros.longs(), STRING_PTYPE));
    List<Pair<Long, String>> pairs = Lists.newArrayList(stringWrapperCollection.materialize());
    pipeline.done();
    assertEquals(pairs.size(), 4);
    assertEquals(Pair.of(1L, "a"), pairs.get(2));
  }
}
View Full Code Here
116
117
118
119
120
121
122
123
124
      }
    }, Avros.tableOf(Avros.longs(), Avros.reflects(StringWrapper.class)))
        .groupByKey()
        .ungroup();
    List<Pair<Long, StringWrapper>> ret = Lists.newArrayList(pt.materialize());
    pipeline.done();
    assertEquals(1, ret.size());
  }
}
View Full Code Here
137
138
139
140
141
142
143
144
            }
          }
        });

    pipeline.writeTextFile(result, tmpDir.getFileName("unused"));
    Assert.assertTrue("Should succeed", pipeline.done().succeeded());
  }
}
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
            .build(new Path(avroFile.getAbsolutePath())));

    File outputFile = tmpDir.getFile("output");
    Target avroFile = To.avroFile(outputFile.getAbsolutePath());
    genericCollection.write(avroFile);
    pipeline.done();
   
    Pipeline pipeline2 = new MRPipeline(AvroParquetFileSourceTargetIT.class,
        tmpDir.getDefaultConfiguration());
    PCollection<Person> ageOnly = pipeline2.read(
        new AvroFileSource<Person>(new Path(outputFile.getAbsolutePath()), Avros.specifics(Person.class)));
View Full Code Here
167
168
169
170
171
172
173
174
175
176
177
    PCollection<GenericRecord> genericCollection = pipeline.read(src);

    File outputFile = tmpDir.getFile("output");
    Target avroFile = To.avroFile(outputFile.getAbsolutePath());
    genericCollection.write(avroFile);
    pipeline.done();

    Pipeline pipeline2 = new MRPipeline(AvroParquetFileSourceTargetIT.class,
        tmpDir.getDefaultConfiguration());
    PCollection<Record> ageOnly = pipeline2.read(
        new AvroFileSource<Record>(new Path(outputFile.getAbsolutePath()), Avros.generics(src.getProjectedSchema())));
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
      @Override
      public boolean accept(Person input) {
        return input != null;
      }
    }).write(avroFile);
    pipeline.done();

    Pipeline pipeline2 = new MRPipeline(AvroParquetFileSourceTargetIT.class,
        tmpDir.getDefaultConfiguration());
    PCollection<Person> ageOnly = pipeline2.read(
        new AvroFileSource<Person>(new Path(outputFile.getAbsolutePath()), Avros.specifics(Person.class)));
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
  public void testMemPipelineFileWriter() throws Exception {
    File tmpDir = baseTmpDir.getFile("mempipe");
    Pipeline p = MemPipeline.getInstance();
    PCollection<String> lines = MemPipeline.collectionOf("hello", "world");
    p.writeTextFile(lines, tmpDir.toString());
    p.done();
    assertTrue(tmpDir.exists());
    File[] files = tmpDir.listFiles();
    assertTrue(files != null && files.length > 0);
    for (File f : files) {
      if (!f.getName().startsWith(".")) {
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
  public void testWritables() throws Exception {
    Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration());
    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
    PCollection<String> shakes = pipeline.readTextFile(shakesInputPath);
    runMinMax(shakes, WritableTypeFamily.getInstance());
    pipeline.done();
  }

  @Test
  public void testAvro() throws Exception {
    Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration());
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
  public void testAvro() throws Exception {
    Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration());
    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
    PCollection<String> shakes = pipeline.readTextFile(shakesInputPath);
    runMinMax(shakes, AvroTypeFamily.getInstance());
    pipeline.done();
  }

  @Test
  public void testInMemoryAvro() throws Exception {
    PCollection<String> someText = MemPipeline.collectionOf("first line", "second line", "third line");
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
        expectedSize = 2;
      } else if ("www.C.com".equals(key) || "www.D.com".equals(key) || "www.E.com".equals(key)) {
        expectedSize = 1;
      }
      assertEquals("Checking key = " + key, expectedSize, e.second().size());
      p.done();
    }
  }

  @Test
  public void testTopN() throws Exception {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.