Examples of openForRead()

Examples of cascading.tap.hadoop.Hfs.openForRead()

116
117
118
119
120
121
122
123
124
125
126
      put("io.serializations",
          new JobConf().get("io.serializations") + "," + ProtobufSerialization.class.getName());
    }};
    new HadoopFlowConnector(properties).connect(t, sink, groupByPipe).complete();

    TupleEntryIterator tei = sink.openForRead(new HadoopFlowProcess(new JobConf()));
    Set<Tuple> tuples = new HashSet<Tuple>();
    while (tei.hasNext()) {
      tuples.add(tei.next().getTupleCopy());
    }
View Full Code Here

Examples of cascading.tap.hadoop.Hfs.openForRead()

35
36
37
38
39
40
41
42
43
44
45
    }
    tec.close();

    // read results back out
    Tap outputTap = new Hfs(new ProtobufScheme("value", Example.Person.class), "/tmp/input");
    TupleEntryIterator iter = outputTap.openForRead(new HadoopFlowProcess(), null);
    List<Tuple> tuples = new ArrayList<Tuple>();
    while (iter.hasNext()) {
      tuples.add(iter.next().getTupleCopy());
    }
View Full Code Here

Examples of cascading.tap.hadoop.Hfs.openForRead()

161
162
163
164
165
166
167
168
169
170
171
    Pipe p = new Each(inPipe, new Fields("value"), new ExpandProto(Example.Person.class), new Fields("id", "name", "email", "position"));

    Hfs sink = new Hfs(new TextLine(), "/tmp/output");
    new HadoopFlowConnector().connect(inTap, sink, p).complete();

    TupleEntryIterator iter = sink.openForRead(new HadoopFlowProcess());
    List<Tuple> results = new ArrayList<Tuple>();
    while (iter.hasNext()) {
      results.add(iter.next().getTupleCopy());
    }
    assertEquals(2, results.size());
View Full Code Here

Examples of cascading.tap.hadoop.Hfs.openForRead()

76
77
78
79
80
81
82
83
84
85
86
  private static BloomFilter mergeBloomParts(String tapPath, long numBloomBits, long splitSize, int numBloomHashes, long numElems) throws IOException {
    FixedSizeBitSet bitSet = new FixedSizeBitSet(numBloomBits);

    if (FileSystemHelper.getFS().exists(new Path(tapPath))) {
      Hfs tap = new Hfs(new SequenceFile(new Fields("split", "filter")), tapPath);
      TupleEntryIterator itr = tap.openForRead(CascadingUtil.get().getFlowProcess());
      while (itr.hasNext()) {
        TupleEntry cur = itr.next();
        long split = cur.getLong(0);
        FixedSizeBitSet curSet = new FixedSizeBitSet(splitSize, ((BytesWritable) cur.getObject(1)).getBytes());
        for (long i = 0; i < curSet.numBits(); i++) {
View Full Code Here

Examples of cascading.tap.hadoop.Hfs.openForRead()

163
164
165
166
167
168
169
170
171
172
173
      return 0;
    }

    Hfs approxCountsTap = new Hfs(new SequenceFile(new Fields("bytes")), partsDir);

    TupleEntryIterator in = approxCountsTap.openForRead(CascadingUtil.get().getFlowProcess());
    List<HyperLogLog> countParts = new LinkedList<HyperLogLog>();

    long totalSum = 0;
    while (in.hasNext()) {
      TupleEntry tuple = in.next();
View Full Code Here

Examples of cascading.tap.hadoop.Hfs.openForRead()

54
55
56
57
58
59
60
61
62
63
64
    Pipe joined = new BloomJoin(source1, new Fields("field1"), source2, new Fields("field3"));

    CascadingUtil.get().getFlowConnector().connect("Example flow", sources, sink, joined).complete();

    //  Take a look at the output tuples
    TupleEntryIterator output = sink.openForRead(CascadingUtil.get().getFlowProcess());
    System.out.println("Output tuples from flow:");
    while (output.hasNext()) {
      System.out.println(output.next().getTuple());
    }
  }
View Full Code Here

Examples of cascading.tap.hadoop.Hfs.openForRead()

54
55
56
57
58
59
60
61
62
63
64
    sources.put("source2", ExampleFixtures.SOURCE_TAP_2);

    CascadingUtil.get().getFlowConnector().connect("Example flow", sources, sink, joined).complete();

    //  Take a look at the output tuples
    TupleEntryIterator output = sink.openForRead(CascadingUtil.get().getFlowProcess());
    System.out.println("Output tuples from flow:");
    while (output.hasNext()) {
      System.out.println(output.next().getTuple());
    }
  }
View Full Code Here

Examples of cascading.tap.hadoop.Hfs.openForRead()

66
67
68
69
70
71
72
73
74
75
76
    f.setFlowStepStrategy(new BloomAssemblyStrategy());

    f.complete();

    //  Take a look at the output tuples
    TupleEntryIterator output = sink.openForRead(CascadingUtil.get().getFlowProcess());
    System.out.println("Output tuples from flow:");
    while (output.hasNext()) {
      System.out.println(output.next().getTuple());
    }
  }
View Full Code Here

Examples of cascading.tap.hadoop.Hfs.openForRead()

99
100
101
102
103
104
105
106
107
108
109
    //  use MyCascadingUtil just like CascadingUtil would be used
    MyCascadingUtil.get().getFlowConnector(flowSpecificProps).connect("Example flow", sources, sink, joined).complete();

    //  Take a look at the output tuples
    TupleEntryIterator output = sink.openForRead(CascadingUtil.get().getFlowProcess());
    System.out.println("Output tuples from flow:");
    while (output.hasNext()) {
      System.out.println(output.next().getTuple());
    }
  }
View Full Code Here

Examples of cascading.tap.hadoop.Hfs.openForRead()

90
91
92
93
94
95
96
97
98
99
100
        new CustomBuffer(new Fields("result", " result1", "result2", "result3", "result4", "result5")));
    results = new Each(results, new Fields("key"), new FilterNull());

    CascadingUtil.get().getFlowConnector().connect(sources, sink, results).complete();

    TupleEntryIterator iter = sink.openForRead(CascadingUtil.get().getFlowProcess());

    assertEquals(new Tuple(1, 108, 108, 108, 108, 108, 108), iter.next().getTuple());
    assertEquals(new Tuple(2, 5, 5, 5, 5, 5, 5), iter.next().getTuple());
    assertEquals(new Tuple(3, 6, 6, 6, 6, 6, 6), iter.next().getTuple());
    assertEquals(new Tuple(5, 7, 7, 7, 7, 7, 7), iter.next().getTuple());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.