Examples of cascading.tuple.Tuple

cascading.tuple.Tuple
A Tuple represents a set of values. Consider a Tuple the same as a database record where every value is a column in that table.
A "tuple stream" is a set of Tuple instances passed consecutively through a Pipe assembly.
Tuples work in tandem with {@link Fields} and the {@link TupleEntry} classes. A TupleEntry holds an instance ofFields and a Tuple. It allows a tuple to be accessed by its field names, and will help maintain consistent types if any are given on the Fields instance. That is, if a field is specified at an Integer, calling {@link #set(int,Object)}with a String will force the String to be coerced into a Integer instance.
For managing custom types, see the {@link CoercibleType} interface which extends {@link Type}.
Tuple instances created by user code, by default, are mutable (or modifiable). Tuple instances created by the system are immutable (or unmodifiable, tested by calling {@link #isUnmodifiable()}).
For example tuples returned by {@link cascading.operation.FunctionCall#getArguments()}, will always be unmodifiable. Thus they must be copied if they will be changed by user code or cached in the local context. See the Tuple copy constructor, or {@code *Copy()} methodson {@link TupleEntry}.
Because a Tuple can hold any Object type, it is suitable for storing custom types. But all custom types must have a serialization support per the underlying framework.
For Hadoop, a {@link org.apache.hadoop.io.serializer.Serialization} implementationmust be registered with Hadoop. For further performance improvements, see the {@link cascading.tuple.hadoop.SerializationToken} Java annotation. @see org.apache.hadoop.io.serializer.Serialization @see cascading.tuple.hadoop.SerializationToken

        // it's ok to use NULL here so the collector does not write anything
        TupleEntry tupleEntry = sinkCall.getOutgoingEntry();
        OutputCollector outputCollector = sinkCall.getOutput();
        if( updateBy != null )
        {
            Tuple allValues = tupleEntry.selectTuple( updateValueFields );
            Tuple updateValues = tupleEntry.selectTuple( updateByFields );


            allValues = cleanTuple( allValues );


            TupleRecord key = new TupleRecord( allValues );


            if( updateValues.equals( updateIfTuple ) )
                outputCollector.collect( key, null );
            else
                outputCollector.collect( key, key );


            return;
        }


        Tuple result = tupleEntry.selectTuple( getSinkFields() );


        result = cleanTuple( result );


        outputCollector.collect( new TupleRecord( result ), null );
    }

View Full Code Here

        DirectoryTap solrSink = new DirectoryTap(new SolrScheme(testFields, SOLR_CORE_DIR), out, SinkMode.REPLACE);
        
        TupleEntryCollector writer = solrSink.openForWrite(new LocalFlowProcess());


        for (int i = 0; i < 100; i++) {
            writer.add(new Tuple(i, "product #" + i, i * 1.0f, true));
        }


        writer.close();
    }

View Full Code Here


        byte[] imageData = new byte[] {0, 1, 2, 3, 5};
        
        Tap source = makeSourceTap(testFields, in);
        TupleEntryCollector write = source.openForWrite(makeFlowProcess());
        Tuple t = new Tuple();
        t.add(1);
        t.add("TurboWriter 2.3");
        t.add(395.50f);
        t.add(new Tuple("wordprocessor", "Japanese"));
        t.add(true);
        t.add(imageData);
        write.add(t);
        
        t = new Tuple();
        t.add(2);
        t.add("Shasta 1.0");
        t.add(95.00f);
        t.add("Chinese");
        t.add(false);
        
        BytesWritable bw = new BytesWritable(imageData);
        bw.setCapacity(imageData.length + 10);
        t.add(bw);
        write.add(t);
        write.close();


        // Now read from the results, and write to a Solr index.
        Pipe writePipe = new Pipe("tuples to Solr");

View Full Code Here

            String name = (String)_sinkFields.get(i);
            Object fieldValue = value.getObject(i);
            if (fieldValue == null) {
                // Don't add null values.
            } else if (fieldValue instanceof Tuple) {
                Tuple list = (Tuple)fieldValue;
                for (int j = 0; j < list.size(); j++) {
                    safeAdd(doc, name, list.getObject(j).toString());
                }
            } else if (fieldValue instanceof byte[]) {
                safeAdd(doc, name, fieldValue);
            } else if (fieldValue instanceof BytesWritable) {
                BytesWritable bw = (BytesWritable)fieldValue;

View Full Code Here

    int token_count2 = argument.getInteger( 3 );
    int common = argument.getInteger( 4 );


    double similarity = calcSimilarity( token_count1, token_count2, common );


    Tuple result = new Tuple();
    result.add( uid1 );
    result.add( uid2 );
    result.add( similarity );
    functionCall.getOutputCollector().add( result );


    result = new Tuple();
    result.add( uid2 );
    result.add( uid1 );
    result.add( similarity );
    functionCall.getOutputCollector().add( result );
    }

View Full Code Here

      Object v = sourceCall.getContext()[1];
      boolean result = sourceCall.getInput().next(k, v);
      if (!result) { return false; }
      String relPath = ((Text) k).toString();
      Object value = deserialize((BytesWritable) v);
      sourceCall.getIncomingEntry().setTuple(new Tuple(relPath, value));
      return true;
    }

View Full Code Here

    String doc_id = argument.getString( 0 );
    String token = scrubText( argument.getString( 1 ) );


    if( token.length() > 0 )
      {
      Tuple result = new Tuple();
      result.add( doc_id );
      result.add( token );
      functionCall.getOutputCollector().add( result );
      }
    }

View Full Code Here

    Fields inputFields = new Fields("num", "lower", "upper");
    TupleEntryCollector input = mHelper.makeCollectorForWrite("input", inputFields);


    // Set up the input.
    Tuple[] expected = new Tuple[] {
      new Tuple("1", "a", "b"),
      new Tuple("2", "test", "other"),
    };


    for (Tuple t : expected) {
      input.add(t);
    }

View Full Code Here

   * @param expected - expected tuple[] to match.
   */
  public void expectResult(TupleEntryIterator results, Tuple[] expected) {
    int resultCount = 0;
    while (results.hasNext()) {
      Tuple actual = results.next().getTuple();
      Assert.assertTrue("Unexpected extra entry: " + actual,
          resultCount < expected.length);
      assertTupleEquals(expected[resultCount], actual);
      resultCount++;
    }

View Full Code Here

    expectedSet.addAll(Arrays.asList(expected));


    TupleEntryIterator output = flow.openSink();


    while (output.hasNext()) {
      Tuple actual = output.next().getTuple();
      Assert.assertTrue(actual + " doesn't exist in expected.", expectedSet.contains(actual));
      expectedSet.remove(actual);
    }
    Assert.assertEquals("Did not find expected items: " + expectedSet, 0, expectedSet.size());
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.tuple.Tuple

bixo.config.BixoPlatform

bixo.datum.FetchSetDatum

bixo.datum.ParsedDatum

bixo.examples.webmining.AnalyzedDatum

bixo.operations.FilterAndScoreByUrlAndRobotsTest

cascading.BasicPipesPlatformTest

cascading.BufferPipesPlatformTest

cascading.CoGroupFieldedPipesPlatformTest

cascading.DistanceUseCasePlatformTest

cascading.FieldedPipesPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.