Examples of org.apache.pig.data.NonSpillableDataBag

org.apache.pig.data.NonSpillableDataBag
An unordered collection of Tuples (possibly) with multiples. The tuples are stored in an ArrayList, since there is no concern for order or distinctness. The implicit assumption is that the user of this class is storing only those many tuples as will fit in memory - no spilling will be done on this bag to disk.

        if (PigMapReduce.sJobConfInternal.get() != null) {
         bagType = PigMapReduce.sJobConfInternal.get().get("pig.cachedbag.type");             
         }
                                               
      if (bagType != null && bagType.equalsIgnoreCase("default")) {
        return new NonSpillableDataBag();
      }
      return new InternalCachedBag(numBags);    
    }

View Full Code Here

                // constructor argument should be 2 * numInputs. But for one obscure
                // case we don't want to pay the penalty all the time.        
                        : new InternalCachedBag(numInputs-1);                    
            }
            // For last bag, we always use NonSpillableBag.
            dbs[lastBagIndex] = new NonSpillableDataBag((int)chunkSize);
            
            //For each Nullable tuple in the input, put it
            //into the corresponding bag based on the index,
            // except for the last input, which we will stream
            // The tuples will arrive in the order of the index,

View Full Code Here

        if(input.returnStatus!=POStatus.STATUS_OK) {
            if(input.returnStatus == POStatus.STATUS_EOP && sendEmptyBagOnEOP)  {
                // we received an EOP from the predecessor
                // since the successor in the pipeline is
                // expecting a bag, send an empty bag
                input.result = new NonSpillableDataBag();
                input.returnStatus = POStatus.STATUS_OK;
                // we should send EOP the next time we are called
                // if the foreach in which this operator is present
                // calls this.getNext(bag) with new inputs then
                // this flag will be reset in this.reset()

View Full Code Here

    }
    
    @Test
    public void testNonSpillableDataBag() throws Exception {
        String[][] tupleContents = new String[][] {{"a", "b"},{"c", "d" }, { "e", "f"} };
        NonSpillableDataBag bg = new NonSpillableDataBag();
        for (int i = 0; i < tupleContents.length; i++) {
            bg.add(Util.createTuple(tupleContents[i]));
        }
        Iterator<Tuple> it = bg.iterator();
        int j = 0;
        while(it.hasNext()) {
            Tuple t = it.next();
            assertEquals(Util.createTuple(tupleContents[j]), t);
            j++;

View Full Code Here

    public Result getNext(Tuple t) throws ExecException {
        int keyField = -1;
        //Create numInputs bags
        Object[] fields = new Object[mBags.length];
        for (int i = 0; i < mBags.length; i++) {
            if (mBags[i]) fields[i] = new NonSpillableDataBag();
        }
        
        // For each indexed tup in the inp, split them up and place their
        // fields into the proper bags.  If the given field isn't a bag, just
        // set the value as is.

View Full Code Here

        if(input.returnStatus!=POStatus.STATUS_OK) {
            if(input.returnStatus == POStatus.STATUS_EOP && sendEmptyBagOnEOP)  {
                // we received an EOP from the predecessor
                // since the successor in the pipeline is
                // expecting a bag, send an empty bag
                input.result = new NonSpillableDataBag();
                input.returnStatus = POStatus.STATUS_OK;
                // we should send EOP the next time we are called
                // if the foreach in which this operator is present
                // calls this.getNext(bag) with new inputs then
                // this flag will be reset in this.reset()

View Full Code Here

        if(input.returnStatus!=POStatus.STATUS_OK) {
            if(input.returnStatus == POStatus.STATUS_EOP && sendEmptyBagOnEOP)  {
                // we received an EOP from the predecessor
                // since the successor in the pipeline is
                // expecting a bag, send an empty bag
                input.result = new NonSpillableDataBag();
                input.returnStatus = POStatus.STATUS_OK;
                // we should send EOP the next time we are called
                // if the foreach in which this operator is present
                // calls this.getNext(bag) with new inputs then
                // this flag will be reset in this.reset()

View Full Code Here

                // constructor argument should be 2 * numInputs. But for one obscure
                // case we don't want to pay the penalty all the time.        
                        : new InternalCachedBag(numInputs-1);                    
            }
            // For last bag, we always use NonSpillableBag.
            dbs[lastBagIndex] = new NonSpillableDataBag((int)chunkSize);
            
            //For each Nullable tuple in the input, put it
            //into the corresponding bag based on the index,
            // except for the last input, which we will stream
            // The tuples will arrive in the order of the index,

View Full Code Here

        createJoinPlans(k);
        processingPlan = false;
        mTupleFactory = TupleFactory.getInstance();
        List<Tuple> tupList = new ArrayList<Tuple>();
        tupList.add(nullTuple);
        nullBag = new NonSpillableDataBag(tupList);
        this.isLeftOuterJoin = isLeftOuter;
    }

View Full Code Here

                        ce.setValue(nullBag);
                    }
                    noMatch = true;
                    break;
                }
                ce.setValue(new NonSpillableDataBag(replicate.get(key)));
            }


            // If this is not LeftOuter Join and there was no match we
            // skip the processing of this left tuple and move ahead
            if (!isLeftOuterJoin && noMatch)

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of org.apache.pig.data.NonSpillableDataBag

com.endgame.binarypig.loaders.ExecutingJsonLoader

com.endgame.binarypig.loaders.ExecutingJsonLoaderTest

com.endgame.binarypig.util.JsonUtil

com.endgame.binarypig.util.JsonUtilTest

com.twitter.elephantbird.pig.load.JsonLoader

com.twitter.elephantbird.pig.piggybank.Fixtures

com.twitter.elephantbird.pig.piggybank.TestInvoker

com.twitter.elephantbird.pig.util.ProtobufToPig

com.twitter.elephantbird.pig.util.TestPigToProtobuf

com.twitter.elephantbird.pig.util.TestPigToThrift

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.