Examples of cascading.tuple.TupleEntryCollector

cascading.tuple.TupleEntryCollector
Interface TupleEntryCollector is used to allow {@link cascading.operation.BaseOperation} instances to emitone or more result {@link Tuple} values.
The general rule in Cascading is if you are handed a Tuple, you cannot change or cache it. Attempts at modifying such a Tuple will result in an Exception. Preventing caching is harder, see below.
If you create the Tuple, you can re-use or modify it.
When calling {@link #add(Tuple)} or {@link #add(TupleEntry)}, you are passing a Tuple to the down stream pipes and operations. Since no downstream operation may modify or cache the Tuple instance, it is safe to re-use the Tuple instance when {@code add()} returns.
That said, Tuple copies do get cached in order to perform specific operations in the underlying platforms. Currently only a shallow copy is made (via the {@link Tuple} copy constructor). Thus, any mutable type or collectionplaced inside a Tuple will not be copied, but will likely be cached if a copy of the Tuple passed downstream is copied.
So any subsequent changes to that nested type or collection will be reflected in the cached copy, a likely source of hard to find errors.
There is currently no way to specify that a deep copy must be performed when making a Tuple copy.


  @Test
  public void testComplexCombiner() throws IOException {


    Hfs source = new Hfs(new SequenceFile(new Fields("key", "value")), INPUT_PATH);
    TupleEntryCollector tc = source.openForWrite(CascadingUtil.get().getFlowProcess());
    tc.add(new Tuple("k0", 1));
    tc.add(new Tuple("k0", 2));
    tc.add(new Tuple("k1", 1));
    tc.add(new Tuple("k1", 4));
    tc.add(new Tuple("k1", 10));
    tc.close();


    Tap sink = new Hfs(new SequenceFile(new Fields("key", "sum", "num_values", "average")), OUTPUT_PATH);


    Pipe pipe = new Pipe("pipe");
    pipe = Combiner.assembly(pipe,

View Full Code Here

  private final String OUTPUT_PATH = getTestRoot() + "/output";


  @Test
  public void testMain() throws IOException {
    Hfs source = new Hfs(new SequenceFile(new Fields("key", "value")), INPUT_PATH);
    TupleEntryCollector tc = source.openForWrite(CascadingUtil.get().getFlowProcess());
    tc.add(new Tuple("k0", 1));
    tc.add(new Tuple("k0", 2));
    tc.add(new Tuple("k1", 10));
    tc.add(new Tuple("k1", -2));
    tc.add(new Tuple("k1", -9));
    tc.add(new Tuple("k2", -3));
    tc.add(new Tuple("k2", -6));
    tc.close();


    Tap sink = new Hfs(new SequenceFile(new Fields("key", "sum")), OUTPUT_PATH);
    
    Pipe pipe = new Pipe("pipe");
    pipe = Combiner.assembly(pipe, new SumExactAggregator(1), new Fields("key"), new Fields("value"), new Fields("sum"));

View Full Code Here


  @Test
  public void testMain() throws IOException {


    Hfs source = new Hfs(new SequenceFile(new Fields("key", "a", "b")), INPUT_PATH);
    TupleEntryCollector tc = source.openForWrite(CascadingUtil.get().getFlowProcess());
    tc.add(new Tuple("k0", 1, 1));
    tc.add(new Tuple("k0", 2, 5));
    tc.add(new Tuple("k1", 1, 7));
    tc.add(new Tuple("k1", -2, 10));
    tc.add(new Tuple("k1", -2, -9));
    tc.close();


    Tap sink = new Hfs(new SequenceFile(new Fields("key", "a_sum", "b_sum", "a_b_sum", "a_b_sum_doubled")), OUTPUT_PATH);


    Pipe pipe = new Pipe("pipe");
    pipe = Combiner.assembly(pipe,

View Full Code Here

  private final String OUTPUT_PATH = getTestRoot() + "/output";


  @Test
  public void testMain() throws IOException {
    Hfs source = new Hfs(new SequenceFile(new Fields("key")), INPUT_PATH);
    TupleEntryCollector tc = source.openForWrite(CascadingUtil.get().getFlowProcess());
    tc.add(new Tuple("k0"));
    tc.add(new Tuple("k0"));
    tc.add(new Tuple("k1"));
    tc.add(new Tuple("k1"));
    tc.add(new Tuple("k1"));
    tc.add(new Tuple("k2"));
    tc.add(new Tuple("k2"));
    tc.close();


    Tap sink = new Hfs(new SequenceFile(new Fields("key", "count")), OUTPUT_PATH);
    
    Pipe pipe = new Pipe("pipe");
    pipe = Combiner.assembly(pipe, new CountExactAggregator(), new Fields("key"), new Fields(), new Fields("count"));

View Full Code Here

  public static void writeToTap(Tap<JobConf, ?, ?> t, FlowProcess<JobConf> conf, Tuple... tuples) throws IOException {
    writeToTap(t, conf, Lists.newArrayList(tuples));
  }


  public static void writeToTap(Tap<JobConf, ?, ?> t, FlowProcess<JobConf> conf, List<Tuple> tuples) throws IOException {
    TupleEntryCollector collector = t.openForWrite(conf);
    for (Tuple tuple : tuples) {
      collector.add(tuple);
    }
    collector.close();
  }

View Full Code Here


public class TestNullTap extends BaseTestCase {


  @Test
  public void testWrite() throws IOException {
    TupleEntryCollector tc = new NullTap().openForWrite(CascadingUtil.get().getFlowProcess());
    tc.add(new Tuple("testing if it fails"));
  }

View Full Code Here

    SchemaCatalogManager catalog = platformBroker.getCatalogManager();
    String identifier = getIdentifierFor( platformBroker, head );


    TableDef tableDef = createTableFor( catalog, head, identifier );


    TupleEntryCollector collector = catalog.createTapFor( tableDef, SinkMode.KEEP ).openForWrite( platformBroker.getFlowProcess() );


    for( List<RexLiteral> values : head.tuples )
      collector.add( EnumerableUtil.createTupleFrom( values ) );


    collector.close();
    }

View Full Code Here

    {
    PlatformBroker platformBroker = getPlatformBroker();
    Optiq.writeSQLPlan( platformBroker.getProperties(), createUniqueName(), getVolcanoPlanner() );


    Branch branch = getBranch();
    TupleEntryCollector collector = getTupleEntryCollector( platformBroker, branch );


    long rowCount;


    try
      {
      rowCount = 0;


      for( List<RexLiteral> values : branch.tuples )
        {
        collector.add( createTupleFrom( values ) );


        rowCount++;
        }
      }
    finally
      {
      if( platformBroker.getCollectorCache() == null )
        collector.close();
      }


    LOG.debug( "inserted {} rows", rowCount );


    return new Linq4j().singletonEnumerable( rowCount ).enumerator();

View Full Code Here


    FlowProcess flowProcess = platformBroker.getFlowProcess();
    SchemaCatalogManager schemaCatalog = platformBroker.getCatalogManager();
    Map<String, TupleEntryCollector> cache = platformBroker.getCollectorCache();


    TupleEntryCollector collector;


    try
      {
      String identifier = tableDef.getIdentifier();

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of cascading.tuple.TupleEntryCollector

bixo.datum.ScoredUrlDatumTest

bixo.examples.crawl.CreateUrlDatumFromOutlinksFunction

bixo.examples.crawl.DemoCrawlTool

bixo.examples.crawl.LatestUrlDatumBufferTest

bixo.examples.webmining.CreateLinkDatumFromOutlinksFunction

bixo.examples.webmining.CreateResultsFunction

bixo.examples.webmining.DemoWebMiningWorkflow

bixo.operations.FilterAndScoreByUrlAndRobotsTest

bixo.operations.MakeFetchSetsBuffer

bixo.pipes.AbstractFetchPipeTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.