Package cascading.tuple

Examples of cascading.tuple.TupleEntryIterator

55
56
57
58
59
60
61
        properties.remove( "mapred.input.dir" );

        jobConf = HadoopUtil.createJobConf( properties, null );

        return new TupleEntryIterator( getSourceFields(), new TapIterator( this, jobConf ) );
    }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
        return temp.getAbsoluteFile().getPath();
    }

    @Override
    public boolean commitResource(JobConf conf) throws java.io.IOException {
        TupleEntryIterator it = new HadoopFlowProcess(conf).openTapForRead(this);
        System.out.println("");
        System.out.println("");
        System.out.println("RESULTS");
        System.out.println("-----------------------");
        while (it.hasNext()) {
            System.out.println(it.next().getTuple());
        }
        System.out.println("-----------------------");
        it.close();
        return true;
    }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
        return temp.getAbsoluteFile().getPath();
    }

    @Override
    public boolean commitResource(JobConf conf) throws java.io.IOException {
        TupleEntryIterator it = new HadoopFlowProcess(conf).openTapForRead(this);

        boolean first_time = true;
        while (it.hasNext()) {
            TupleEntry tuple = it.next();
            results.add(tuple.getTupleCopy());

            if (first_time) {
                fields = tuple.getFields();
                first_time = false;
            }
        }
        it.close();
        return true;
    }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
  private static BloomFilter mergeBloomParts(String tapPath, long numBloomBits, long splitSize, int numBloomHashes, long numElems) throws IOException {
    FixedSizeBitSet bitSet = new FixedSizeBitSet(numBloomBits);

    if (FileSystemHelper.getFS().exists(new Path(tapPath))) {
      Hfs tap = new Hfs(new SequenceFile(new Fields("split", "filter")), tapPath);
      TupleEntryIterator itr = tap.openForRead(CascadingUtil.get().getFlowProcess());
      while (itr.hasNext()) {
        TupleEntry cur = itr.next();
        long split = cur.getLong(0);
        FixedSizeBitSet curSet = new FixedSizeBitSet(splitSize, ((BytesWritable) cur.getObject(1)).getBytes());
        for (long i = 0; i < curSet.numBits(); i++) {
          if (curSet.get(i)) {
            bitSet.set(split * splitSize + i);
          }
        }
      }
      itr.close();
    }

    return new BloomFilter(numBloomBits, numBloomHashes, bitSet, numElems);
  }
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
      return 0;
    }

    Hfs approxCountsTap = new Hfs(new SequenceFile(new Fields("bytes")), partsDir);

    TupleEntryIterator in = approxCountsTap.openForRead(CascadingUtil.get().getFlowProcess());
    List<HyperLogLog> countParts = new LinkedList<HyperLogLog>();

    long totalSum = 0;
    while (in.hasNext()) {
      TupleEntry tuple = in.next();
      HyperLogLog card = HyperLogLog.Builder.build(Bytes.getBytes((BytesWritable) tuple.getObject("bytes")));
      countParts.add(card);
      totalSum += card.cardinality();
    }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
    Pipe joined = new BloomJoin(source1, new Fields("field1"), source2, new Fields("field3"));

    CascadingUtil.get().getFlowConnector().connect("Example flow", sources, sink, joined).complete();

    //  Take a look at the output tuples
    TupleEntryIterator output = sink.openForRead(CascadingUtil.get().getFlowProcess());
    System.out.println("Output tuples from flow:");
    while (output.hasNext()) {
      System.out.println(output.next().getTuple());
    }
  }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
    sources.put("source2", ExampleFixtures.SOURCE_TAP_2);

    CascadingUtil.get().getFlowConnector().connect("Example flow", sources, sink, joined).complete();

    //  Take a look at the output tuples
    TupleEntryIterator output = sink.openForRead(CascadingUtil.get().getFlowProcess());
    System.out.println("Output tuples from flow:");
    while (output.hasNext()) {
      System.out.println(output.next().getTuple());
    }
  }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
    f.setFlowStepStrategy(new BloomAssemblyStrategy());

    f.complete();

    //  Take a look at the output tuples
    TupleEntryIterator output = sink.openForRead(CascadingUtil.get().getFlowProcess());
    System.out.println("Output tuples from flow:");
    while (output.hasNext()) {
      System.out.println(output.next().getTuple());
    }
  }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
    //  use MyCascadingUtil just like CascadingUtil would be used
    MyCascadingUtil.get().getFlowConnector(flowSpecificProps).connect("Example flow", sources, sink, joined).complete();

    //  Take a look at the output tuples
    TupleEntryIterator output = sink.openForRead(CascadingUtil.get().getFlowProcess());
    System.out.println("Output tuples from flow:");
    while (output.hasNext()) {
      System.out.println(output.next().getTuple());
    }
  }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
    return TEST_ROOT;
  }

  protected List<Tuple> getAllTuples(Tap sink) throws IOException {
    List<Tuple> ret = Lists.newArrayList();
    TupleEntryIterator tupleEntryIterator = sink.openForRead(CascadingUtil.get().getFlowProcess());
    while (tupleEntryIterator.hasNext()) {
      ret.add(tupleEntryIterator.next().getTuple());
    }
    return ret;
  }
View Full Code Here
TOP

Related Classes of cascading.tuple.TupleEntryIterator

  • backtype.hadoop.ElasticTap
  • bixo.examples.crawl.DemoCrawlWorkflowLRTest
  • bixo.examples.crawl.DemoStatusTool
  • bixo.examples.crawl.LatestUrlDatumBufferTest
  • bixo.examples.webmining.DemoWebMiningWorkflowTest
  • bixo.fetcher.FetcherTest
  • bixo.pipes.AbstractFetchPipeTest
  • cascading.CascadingTestCase
  • cascading.dbmigrate.tap.DBMigrateTap
  • cascading.detail.PipeAssemblyTestBase
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.