Examples of cascading.tuple.Tuple

cascading.tuple.Tuple
A Tuple represents a set of values. Consider a Tuple the same as a database record where every value is a column in that table.
A "tuple stream" is a set of Tuple instances passed consecutively through a Pipe assembly.
Tuples work in tandem with {@link Fields} and the {@link TupleEntry} classes. A TupleEntry holds an instance ofFields and a Tuple. It allows a tuple to be accessed by its field names, and will help maintain consistent types if any are given on the Fields instance. That is, if a field is specified at an Integer, calling {@link #set(int,Object)}with a String will force the String to be coerced into a Integer instance.
For managing custom types, see the {@link CoercibleType} interface which extends {@link Type}.
Tuple instances created by user code, by default, are mutable (or modifiable). Tuple instances created by the system are immutable (or unmodifiable, tested by calling {@link #isUnmodifiable()}).
For example tuples returned by {@link cascading.operation.FunctionCall#getArguments()}, will always be unmodifiable. Thus they must be copied if they will be changed by user code or cached in the local context. See the Tuple copy constructor, or {@code *Copy()} methodson {@link TupleEntry}.
Because a Tuple can hold any Object type, it is suitable for storing custom types. But all custom types must have a serialization support per the underlying framework.
For Hadoop, a {@link org.apache.hadoop.io.serializer.Serialization} implementationmust be registered with Hadoop. For further performance improvements, see the {@link cascading.tuple.hadoop.SerializationToken} Java annotation. @see org.apache.hadoop.io.serializer.Serialization @see cascading.tuple.hadoop.SerializationToken

        
        return result;
    }
    
    private Tuple convertMapToTuple(Map<String, String> map) {
        Tuple result = new Tuple();
        if (map != null) {
            for (Entry<String, String> entry : map.entrySet()) {
                result.add(entry.getKey());
                result.add(entry.getValue());
            }
        }
        
        return result;
    }

View Full Code Here

    }
    
    private void skipUrls(List<ScoredUrlDatum> urls, UrlStatus status, String traceMsg) {
        for (ScoredUrlDatum datum : urls) {
            FetchedDatum result = new FetchedDatum(datum);
            Tuple tuple = result.getTuple();
            tuple.add(status.toString());
            _collector.add(BixoPlatform.clone(tuple, _flowProcess));
        }


        _flowProcess.increment(FetchCounters.URLS_SKIPPED, urls.size());
        if (status == UrlStatus.SKIPPED_PER_SERVER_LIMIT) {

View Full Code Here

        }


        @SuppressWarnings("rawtypes")
        @Override
        public void operate(FlowProcess process, FunctionCall<NullContext> funcCall) {
            Tuple t = funcCall.getArguments().getTuple();
            
            // Get the status to decide if it's a good fetch
            Object status = t.getObject(_fieldPos);
            if ((status instanceof String) && (UrlStatus.valueOf((String)status) == UrlStatus.FETCHED)) {
                funcCall.getOutputCollector().add(BixoPlatform.clone(t.get(_fieldsToCopy), process));
            }
        }

View Full Code Here

    private static String normalize(String name) {
        return name.toLowerCase();
    }
    
    public Tuple toTuple() {
        Tuple result = new Tuple();
        for (String name : _headers.keySet()) {
            List<String> values = _headers.get(name);
            result.add(encodeString(name));
            result.add(encodeValues(values));
        }
        
        return result;
    }

View Full Code Here

                if (!results.next()) {
                    return false;
                }
                key.set(pos + split.startId);


                value.tuple = new Tuple();


                for (int i = 0; i < results.getMetaData().getColumnCount(); i++) {
                    Object o = results.getObject(i + 1);
                    if (o instanceof byte[]) {
                        o = new BytesWritable((byte[]) o);

View Full Code Here

        try {
            String doJoinString = (String)flowProcess.getProperty("joining");
            String itemIDString = rowIndex.inverse().get(String.valueOf(key));
            Vector va = ((VectorWritable)arguments.getObject(arguments.getFields().get(1))).get();
            String vaDoc = createOrderedDoc(va, itemIndex);
            Tuple tuple;
            if(doJoinString.equals("true")){
                Vector vb = ((VectorWritable)arguments.getObject(arguments.getFields().get(3))).get();
                String vbDoc = createOrderedDoc(vb, itemIndex);
                tuple = new Tuple(itemIDString, vaDoc, vbDoc);
            } else { // not joining, just converting to CSV
                tuple = new Tuple(itemIDString, vaDoc);
            }
            functionCall.getOutputCollector().add(tuple);


        } catch (Exception e) {
            e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.

View Full Code Here

        for( int i = 0; i < tuple.size(); i++ )
            statement.setObject( i + 1, tuple.get( i ) );
    }


    public void readFields( ResultSet resultSet ) throws SQLException {
        tuple = new Tuple();


        for( int i = 0; i < resultSet.getMetaData().getColumnCount(); i++ )
            tuple.add( (Comparable) resultSet.getObject( i + 1 ) );
    }

View Full Code Here

  }


  @Override
  public boolean source(FlowProcess<JobConf> flowProcess,
      SourceCall<Object[], RecordReader> sourceCall) throws IOException {
    Tuple result = new Tuple();


    Object key = sourceCall.getContext()[0];
    Object value = sourceCall.getContext()[1];
    boolean hasNext = sourceCall.getInput().next(key, value);
    if (!hasNext) { return false; }


    ImmutableBytesWritable keyWritable = (ImmutableBytesWritable) key;
    Result row = (Result) value;


    result.add(keyWritable.get());


    for (int i = 0; i < this.familyNames.length; i++) {
      String familyName = this.familyNames[i];
      byte[] familyNameBytes = Bytes.toBytes(familyName);
      Fields fields = this.valueFields[i];
      for (int k = 0; k < fields.size(); k++) {
        String fieldName = (String) fields.get(k);
        byte[] fieldNameBytes = Bytes.toBytes(fieldName);
        byte[] cellValue = row.getValue(familyNameBytes, fieldNameBytes);
        result.add(cellValue);
      }
    }


    sourceCall.getIncomingEntry().setTuple(result);

View Full Code Here

  @Override
  public void sink(FlowProcess<JobConf> flowProcess, SinkCall<Object[], OutputCollector> sinkCall)
      throws IOException {
    TupleEntry tupleEntry = sinkCall.getOutgoingEntry();
    OutputCollector outputCollector = sinkCall.getOutput();
    Tuple key = tupleEntry.selectTuple(keyField);
    byte[] keyBytes = Bytes.toBytes(key.getString(0));
    Put put = new Put(keyBytes);


    for (int i = 0; i < valueFields.length; i++) {
      Fields fieldSelector = valueFields[i];
      TupleEntry values = tupleEntry.selectEntry(fieldSelector);
      
      for (int j = 0; j < values.getFields().size(); j++) {
        Fields fields = values.getFields();
        Tuple tuple = values.getTuple();


        String value = tuple.getString(j);
        byte[] asBytes = value == null ? null : Bytes.toBytes(value);
        put.add(Bytes.toBytes(familyNames[i]), Bytes.toBytes((String) fields.get(j)), asBytes);
      }
    }

View Full Code Here

        boolean result = sourceCall.getInput().next( key, value );


        if( !result )
            return false;


        Tuple newTuple = ( (TupleRecord) value ).getTuple();
        sourceCall.getIncomingEntry().setTuple( newTuple );


        return true;
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.tuple.Tuple

bixo.config.BixoPlatform

bixo.datum.FetchSetDatum

bixo.datum.ParsedDatum

bixo.examples.webmining.AnalyzedDatum

bixo.operations.FilterAndScoreByUrlAndRobotsTest

cascading.BasicPipesPlatformTest

cascading.BufferPipesPlatformTest

cascading.CoGroupFieldedPipesPlatformTest

cascading.DistanceUseCasePlatformTest

cascading.FieldedPipesPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.