Examples of cascading.tuple.Tuple

cascading.tuple.Tuple
A Tuple represents a set of values. Consider a Tuple the same as a database record where every value is a column in that table.
A "tuple stream" is a set of Tuple instances passed consecutively through a Pipe assembly.
Tuples work in tandem with {@link Fields} and the {@link TupleEntry} classes. A TupleEntry holds an instance ofFields and a Tuple. It allows a tuple to be accessed by its field names, and will help maintain consistent types if any are given on the Fields instance. That is, if a field is specified at an Integer, calling {@link #set(int,Object)}with a String will force the String to be coerced into a Integer instance.
For managing custom types, see the {@link CoercibleType} interface which extends {@link Type}.
Tuple instances created by user code, by default, are mutable (or modifiable). Tuple instances created by the system are immutable (or unmodifiable, tested by calling {@link #isUnmodifiable()}).
For example tuples returned by {@link cascading.operation.FunctionCall#getArguments()}, will always be unmodifiable. Thus they must be copied if they will be changed by user code or cached in the local context. See the Tuple copy constructor, or {@code *Copy()} methodson {@link TupleEntry}.
Because a Tuple can hold any Object type, it is suitable for storing custom types. But all custom types must have a serialization support per the underlying framework.
For Hadoop, a {@link org.apache.hadoop.io.serializer.Serialization} implementationmust be registered with Hadoop. For further performance improvements, see the {@link cascading.tuple.hadoop.SerializationToken} Java annotation. @see org.apache.hadoop.io.serializer.Serialization @see cascading.tuple.hadoop.SerializationToken

            return false;
        }


        @Override
        public void sink(FlowProcess<JobConf> flowProcess, SinkCall<Object, Object> sinkCall) throws IOException {
            Tuple tuple = sinkCall.getOutgoingEntry().getTuple();
            StringBuffer sb = new StringBuffer();
            for (Object object : tuple) {
                if (object instanceof Writable) {
                    sb.append(WritableUtils.fromWritable((Writable) object));
                }

View Full Code Here

        }


        static Tuple coerceToString(SinkCall<?, ?> sinkCall) {
            TupleEntry entry = sinkCall.getOutgoingEntry();
            Fields fields = entry.getFields();
            Tuple tuple = entry.getTuple();


            if (fields.hasTypes()) {
                Type types[] = new Type[fields.size()];
                for (int index = 0; index < fields.size(); index++) {
                    Type type = fields.getType(index);

View Full Code Here

        Assert.isTrue(from instanceof SinkCall,
                String.format("Unexpected object type, expecting [%s], given [%s]", SinkCall.class, from.getClass()));


        // handle common cases
        SinkCall sinkCall = (SinkCall) from;
        Tuple rawTuple = sinkCall.getOutgoingEntry().getTuple();


        if (rawTuple == null || rawTuple.isEmpty()) {
            to.bytes("{}");
            return;
        }
        Assert.isTrue(rawTuple.size() == 1, "When using JSON input, only one field is expected");


        // postpone the coercion
        Tuple tuple = CascadingUtils.coerceToString(sinkCall);
        super.convert(tuple.getObject(0), to);
    }

View Full Code Here

    }


    @SuppressWarnings("unchecked")
    @Override
    public boolean write(SinkCall<Object[], ?> sinkCall, Generator generator) {
        Tuple tuple = CascadingUtils.coerceToString(sinkCall);
        // consider names (in case of aliases these are already applied)
        List<String> names = (List<String>) sinkCall.getContext()[0];


        generator.writeBeginObject();
        for (int i = 0; i < tuple.size(); i++) {
            String name = (i < names.size() ? names.get(i) : "tuple" + i);
            generator.writeFieldName(name);
            Object object = tuple.getObject(i);
            if (!jdkWriter.write(object, generator)) {
                if (!(object instanceof Writable) || !writableWriter.write((Writable) object, generator)) {
                    return false;
                }
            }

View Full Code Here

  }


  @Override
  public boolean source(FlowProcess<JobConf> flowProcess,
      SourceCall<Object[], RecordReader> sourceCall) throws IOException {
    Tuple result = new Tuple();


    Object key = sourceCall.getContext()[0];
    Object value = sourceCall.getContext()[1];
    boolean hasNext = sourceCall.getInput().next(key, value);
    if (!hasNext) { return false; }


    // Skip nulls
    if (key == null || value == null) { return true; }


    ImmutableBytesWritable keyWritable = (ImmutableBytesWritable) key;
    Result row = (Result) value;
    result.add(keyWritable);


    for (int i = 0; i < this.familyNames.length; i++) {
      String familyName = this.familyNames[i];
      byte[] familyNameBytes = Bytes.toBytes(familyName);
      Fields fields = this.valueFields[i];
      for (int k = 0; k < fields.size(); k++) {
        String fieldName = (String) fields.get(k);
        byte[] fieldNameBytes = Bytes.toBytes(fieldName);
        byte[] cellValue = row.getValue(familyNameBytes, fieldNameBytes);
        if (cellValue == null) {
            cellValue = new byte[0];
        }
        result.add(new ImmutableBytesWritable(cellValue));
      }
    }


    sourceCall.getIncomingEntry().setTuple(result);

View Full Code Here

  @Override
  public void sink(FlowProcess<JobConf> flowProcess, SinkCall<Object[], OutputCollector> sinkCall)
      throws IOException {
    TupleEntry tupleEntry = sinkCall.getOutgoingEntry();
    OutputCollector outputCollector = sinkCall.getOutput();
    Tuple key = tupleEntry.selectTuple(keyField);
    ImmutableBytesWritable keyBytes = (ImmutableBytesWritable) key.getObject(0);
    Put put = new Put(keyBytes.get());


    for (int i = 0; i < valueFields.length; i++) {
      Fields fieldSelector = valueFields[i];
      TupleEntry values = tupleEntry.selectEntry(fieldSelector);


      for (int j = 0; j < values.getFields().size(); j++) {
        Fields fields = values.getFields();
        Tuple tuple = values.getTuple();


        ImmutableBytesWritable valueBytes = (ImmutableBytesWritable) tuple.getObject(j);
        put.add(Bytes.toBytes(familyNames[i]), Bytes.toBytes((String) fields.get(j)), valueBytes.get());
      }
    }


    outputCollector.collect(null, put);

View Full Code Here

        boolean result = sourceCall.getInput().next( key, value );


        if( !result )
            return false;


        Tuple newTuple = ( (TupleRecord) value ).getTuple();
        sourceCall.getIncomingEntry().setTuple( newTuple );


        return true;
    }

View Full Code Here

        // it's ok to use NULL here so the collector does not write anything
        TupleEntry tupleEntry = sinkCall.getOutgoingEntry();
        OutputCollector outputCollector = sinkCall.getOutput();
        if( updateBy != null )
        {
            Tuple allValues = tupleEntry.selectTuple( updateValueFields );
            Tuple updateValues = tupleEntry.selectTuple( updateByFields );


            allValues = cleanTuple( allValues );


            TupleRecord key = new TupleRecord( allValues );


            if( updateValues.equals( updateIfTuple ) )
                outputCollector.collect( key, null );
            else
                outputCollector.collect( key, key );


            return;
        }


        Tuple result = tupleEntry.selectTuple( getSinkFields() );


        result = cleanTuple( result );


        outputCollector.collect( new TupleRecord( result ), null );
    }

View Full Code Here

        for( int i = 0; i < tuple.size(); i++ )
            statement.setObject( i + 1, tuple.getObject( i ) );
    }


    public void readFields( ResultSet resultSet ) throws SQLException {
        tuple = new Tuple();


        for( int i = 0; i < resultSet.getMetaData().getColumnCount(); i++ )
            tuple.add( resultSet.getObject( i + 1 ) );
    }

View Full Code Here

            
            boolean isCrawlDatum = entry.getString(CRAWLDBDATUM_URL_FIELD) != null;
            boolean isStatus = entry.getString(STATUSDATUM_URL_FIELD) != null;
            boolean isAnalyzed = entry.getString(ANALYZEDDATUM_URL_FIELD) != null;
            if (isCrawlDatum) {
               Tuple crawlDbTuple = TupleEntry.select(CrawlDbDatum.FIELDS, entry);
               crawlDbDatum = new CrawlDbDatum(crawlDbTuple);
               url = crawlDbDatum.getUrl();
            }
            
            if (isStatus) {
                statusDatum = new StatusDatum(entry);
                url = statusDatum.getUrl();
            }


            if (isAnalyzed) {
                Tuple analyzedTuple = TupleEntry.select(AnalyzedDatum.FIELDS, entry);
                analyzedDatum = new AnalyzedDatum(analyzedTuple);
                url = analyzedDatum.getUrl();
            }


            // we could have either status + link or just link tuple entry

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.tuple.Tuple

bixo.config.BixoPlatform

bixo.datum.FetchSetDatum

bixo.datum.ParsedDatum

bixo.examples.webmining.AnalyzedDatum

bixo.operations.FilterAndScoreByUrlAndRobotsTest

cascading.BasicPipesPlatformTest

cascading.BufferPipesPlatformTest

cascading.CoGroupFieldedPipesPlatformTest

cascading.DistanceUseCasePlatformTest

cascading.FieldedPipesPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.