Examples of PigNullableWritable

org.apache.pig.impl.io.PigNullableWritable
A base class for all types that pig uses to move data between map and reduce. It implements WritableComparable so that compareTo etc. can be called. It also wraps a WritableComparable 'value'. This is set by each different type to be an object of its specific type. It also provides a getIndex() and setIndex() calls that are used to get and set the index. These can be used by LocalRearrange, the partitioner, and Package to determine the index. Index and the null indicator are packed into one byte to save space.

Examples of org.apache.pig.impl.io.PigNullableWritable

        tupleValIdx--;
      } else {
        partitionIndex = (Byte)tuple.get(1);
      }


            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(tupleKeyIdx), DataType.TUPLE);


      NullablePartitionWritable wrappedKey = new NullablePartitionWritable(key);
      //key.setIndex(index);
      //NullableTuple wrappedKey = new NullableTuple((Tuple)tuple);

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

                        Result redRes = leaf.getNext(DUMMYTUPLE);
                        
                        if(redRes.returnStatus==POStatus.STATUS_OK){
                            Tuple tuple = (Tuple)redRes.result;
                            Byte index = (Byte)tuple.get(0);
                            PigNullableWritable outKey =
                                HDataType.getWritableComparableTypes(tuple.get(1), this.keyType);
                            NullableTuple val =
                                new NullableTuple((Tuple)tuple.get(2));
                            // Both the key and the value need the index.  The key needs it so
                            // that it can be sorted on the index in addition to the key
                            // value.  The value needs it so that POPackage can properly
                            // assign the tuple to its slot in the projection.
                            outKey.setIndex(index);
                            val.setIndex(index);
                            oc.collect(outKey, val);
                            continue;
                        }

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

    Pair <Integer, Integer> indexes;
    Integer curIndex = -1;
    Tuple keyTuple = DefaultTupleFactory.getInstance().newTuple(1);
    
    // extract the key from nullablepartitionwritable
    PigNullableWritable key = ((NullablePartitionWritable) wrappedKey).getKey();


    try {
      keyTuple.set(0, key.getValueAsPigType());
    } catch (ExecException e) {
      return -1;
    }
    
    // if the key is not null and key 
    if (key instanceof NullableTuple && key.getValueAsPigType() != null) {
      keyTuple = (Tuple)key.getValueAsPigType();
    }


    indexes = reducerMap.get(keyTuple);
    // if the reducerMap does not contain the key, do the default hash based partitioning
    if (indexes == null) {

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

        }
        //ITIterator iti = new TestPackage.ITIterator(db.iterator());
        POPackage pop = new POPackage(new OperatorKey("", r.nextLong()));
        pop.setNumInps(2);
        pop.setInner(inner);
        PigNullableWritable k = HDataType.getWritableComparableTypes(key, (byte)0);
        pop.attachInput(k, db.iterator());
        
        // we are not doing any optimization to remove
        // parts of the "value" which are present in the "key" in this
        // unit test - so set up the "keyInfo" accordingly in

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

    private PhysicalOperator attachInputWithIndex(Tuple res) throws ExecException {
        
        // unwrap the first field of the tuple to get the wrapped value which
        // is expected by the inner plans, as well as the index of the associated
        // inner plan.
        PigNullableWritable fld = (PigNullableWritable)res.get(0);        
    
        // choose an inner plan to run based on the index set by
        // the POLocalRearrange operator and passed to this operator
        // by POMultiQueryPackage
        int index = fld.getIndex();
        index &= idxPart;                      
        
        PhysicalPlan pl = myPlans.get(index);
        if (!(pl.getRoots().get(0) instanceof PODemux)) {                             
            if (!sameMapKeyType && !inCombiner && isKeyWrapped.get(index)) {                                       
                
                // unwrap the keys
                boolean[] keys = keyPositions.get(index);
                for (int pos = 0; pos < keys.length; pos++) {
                    if (keys[pos]) {
                        Tuple tup = (pos == 0) ? 
                                (Tuple)fld.getValueAsPigType() : (Tuple)res.get(pos);
                        res.set(pos, tup.get(0));
                    } 
                    else if (pos == 0) {                 
                        res.set(0, fld.getValueAsPigType());
                    }
                }
                
            } else {
                res.set(0, fld.getValueAsPigType());
            }
        }
    
        myPlans.get(index).attachInput(res);
        return myPlans.get(index).getLeaves().get(0);

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

        Object obj = tuple.get(0);
        if (obj instanceof PigNullableWritable) {
            ((PigNullableWritable)obj).setIndex(origIndex);
        }
        else {
            PigNullableWritable myObj = null;
            if (obj == null) {
                myObj = new NullableUnknownWritable();
                myObj.setNull(true);
            }
            else {
                myObj = HDataType.getWritableComparableTypes(obj, (byte)0);
            }
            myObj.setIndex(origIndex);
            tuple.set(0, myObj);
        }
        
        return res;
    }

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

                        Result redRes = leaf.getNext(DUMMYTUPLE);
                        
                        if(redRes.returnStatus==POStatus.STATUS_OK){
                            Tuple tuple = (Tuple)redRes.result;
                            Byte index = (Byte)tuple.get(0);
                            PigNullableWritable outKey =
                                HDataType.getWritableComparableTypes(tuple.get(1), this.keyType);
                            NullableTuple val =
                                new NullableTuple((Tuple)tuple.get(2));
                            // Both the key and the value need the index.  The key needs it so
                            // that it can be sorted on the index in addition to the key
                            // value.  The value needs it so that POPackage can properly
                            // assign the tuple to its slot in the projection.
                            outKey.setIndex(index);
                            val.setIndex(index);
                            oc.collect(outKey, val);
                            continue;
                        }

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

        }


        @Override
        public int compare(WritableComparable a, WritableComparable b)
        {
            PigNullableWritable wa = (PigNullableWritable)a;
            PigNullableWritable wb = (PigNullableWritable)b;
            if ((wa.getIndex() & PigNullableWritable.mqFlag) != 0) { // this is a multi-query index
                if ((wa.getIndex() & PigNullableWritable.idxSpace) < (wb.getIndex() & PigNullableWritable.idxSpace)) return -1;
                else if ((wa.getIndex() & PigNullableWritable.idxSpace) > (wb.getIndex() & PigNullableWritable.idxSpace)) return 1;
                // If equal, we fall through
            }
            
            // wa and wb are guaranteed to be not null, POLocalRearrange will create a tuple anyway even if main key and secondary key
            // are both null; however, main key can be null, we need to check for that using the same logic we have in PigNullableWritable
            Object valuea = null;
            Object valueb = null;
            try {
                // Get the main key from compound key
                valuea = ((Tuple)wa.getValueAsPigType()).get(0);
                valueb = ((Tuple)wb.getValueAsPigType()).get(0);
            } catch (ExecException e) {
                throw new RuntimeException("Unable to access tuple field", e);
            }
            if (!wa.isNull() && !wb.isNull()) {
                
                int result = DataType.compare(valuea, valueb);
                
                // If any of the field inside tuple is null, then we do not merge keys
                // See PIG-927
                if (result == 0 && valuea instanceof Tuple && valueb instanceof Tuple)
                {
                    try {
                        for (int i=0;i<((Tuple)valuea).size();i++)
                            if (((Tuple)valueb).get(i)==null)
                                return (wa.getIndex()&PigNullableWritable.idxSpace) - (wb.getIndex()&PigNullableWritable.idxSpace);
                    } catch (ExecException e) {
                        throw new RuntimeException("Unable to access tuple field", e);
                    }
                }
                return result;
            } else if (valuea==null && valueb==null) {
                // If they're both null, compare the indicies
                if ((wa.getIndex() & PigNullableWritable.idxSpace) < (wb.getIndex() & PigNullableWritable.idxSpace)) return -1;
                else if ((wa.getIndex() & PigNullableWritable.idxSpace) > (wb.getIndex() & PigNullableWritable.idxSpace)) return 1;
                else return 0;
            }
            else if (valuea==null) return -1; 
            else return 1;
        }

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

            Mapper<Text, Tuple, PigNullableWritable, Writable> {


        @Override
        public void collect(OutputCollector<PigNullableWritable, Writable> oc, Tuple tuple) throws ExecException, IOException {
            Byte index = (Byte)tuple.get(0);
            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(1), keyType);
            NullableTuple val = new NullableTuple((Tuple)tuple.get(2));
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            key.setIndex(index);
            val.setIndex(index);
            oc.collect(key, val);
        }

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

                keyTuple = tuple.get(1);
            }
            


            Byte index = (Byte)tuple.get(0);
            PigNullableWritable key =
                HDataType.getWritableComparableTypes(keyTuple, DataType.TUPLE);
            NullableTuple val = new NullableTuple((Tuple)tuple.get(2));
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            key.setIndex(index);
            val.setIndex(index);
            oc.collect(key, val);
        }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.