Package org.apache.pig.data

Examples of org.apache.pig.data.NonSpillableDataBag$NonSpillableDataBagIterator

260
261
262
263
264
265
266
267
268
269
270
                for (Entry<Integer,Integer> valEnt : value.entrySet()) {
                    probVec.set(valEnt.getKey(), (float)valEnt.getValue()/total);
                }
                weightedParts.put(key, probVec);
            }
            output.put(QUANTILES_LIST, new NonSpillableDataBag(quantilesList));
            output.put(WEIGHTED_PARTS, weightedParts);
            return output;
        }catch (Exception e){
            e.printStackTrace();
            throw new RuntimeException(e);
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    @Override
    public DataBag exec(Tuple input) throws IOException {
        try {
            // The assumption is that if the bag contents fits into
            // an input tuple, it will not need to be spilled.
            DataBag bag = new NonSpillableDataBag(input.size());

            for (int i = 0; i < input.size(); ++i) {
                final Object object = input.get(i);
                if (object instanceof Tuple) {
                    bag.add( (Tuple) object);
                } else {
                    Tuple tp2 = TupleFactory.getInstance().newTuple(1);
                    tp2.set(0, object);
                    bag.add(tp2);
                }
            }

            return bag;
        } catch (Exception ee) {
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
        if(m == null) {
            return null;
        }

        Collection c = m.values();
        DataBag bag = new NonSpillableDataBag(c.size());
        Iterator<Object> iter = c.iterator();
        while(iter.hasNext()) {
            Tuple t = TUPLE_FACTORY.newTuple(iter.next());
            bag.add(t);
        }

        return bag;
    }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    }

    @Test
    public void testNonSpillableDataBag() throws Exception {
        String[][] tupleContents = new String[][] {{"a", "b"},{"c", "d" }, { "e", "f"} };
        NonSpillableDataBag bg = new NonSpillableDataBag();
        for (int i = 0; i < tupleContents.length; i++) {
            bg.add(Util.createTuple(tupleContents[i]));
        }
        Iterator<Tuple> it = bg.iterator();
        int j = 0;
        while(it.hasNext()) {
            Tuple t = it.next();
            assertEquals(Util.createTuple(tupleContents[j]), t);
            j++;
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
        createJoinPlans(k);
        processingPlan = false;
        mTupleFactory = TupleFactory.getInstance();
        List<Tuple> tupList = new ArrayList<Tuple>();
        tupList.add(nullTuple);
        nullBag = new NonSpillableDataBag(tupList);
        this.isLeftOuterJoin = isLeftOuter;
        if (inputSchemas != null) {
            this.inputSchemas = inputSchemas;
        } else {
            this.inputSchemas = new Schema[replFiles == null ? 0 : replFiles.length];
View Full Code Here
304
305
306
307
308
309
310
311
312
313
314
                        ce.setValue(nullBag);
                    }
                    noMatch = true;
                    break;
                }
                ce.setValue(new NonSpillableDataBag(replicate.get(key).getList()));
            }

            // If this is not LeftOuter Join and there was no match we
            // skip the processing of this left tuple and move ahead
            if (!isLeftOuterJoin && noMatch) {
View Full Code Here
102
103
104
105
106
107
108
                if (bagType != null && bagType.equalsIgnoreCase("default")) {
                    useDefaultBag = true;
                }
            }
        }
        return useDefaultBag ? new NonSpillableDataBag() : new InternalCachedBag(numBags);
    }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
            for (int i = 0; i < numInputs - 1; i++) {
                dbs[i] = bags[i];
            }

            // For last bag, we always use NonSpillableBag.
            dbs[lastBagIndex] = new NonSpillableDataBag((int)chunkSize);

            lastBagIter = bags[lastBagIndex].iterator();

            // If we don't have any tuple for input#n
            // we do not need any further process, return EOP
View Full Code Here
100
101
102
103
104
105
106
            Tuple t = tFact.newTuple(1);
            t.set(0, rand.nextInt(max));
            samples.add(t);
        }
        Collections.sort(samples);
        return new NonSpillableDataBag(samples);
    }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    }

    @Test
    public void testNonSpillableDataBag() throws Exception {
        String[][] tupleContents = new String[][] {{"a", "b"},{"c", "d" }, { "e", "f"} };
        NonSpillableDataBag bg = new NonSpillableDataBag();
        for (int i = 0; i < tupleContents.length; i++) {
            bg.add(Util.createTuple(tupleContents[i]));
        }
        Iterator<Tuple> it = bg.iterator();
        int j = 0;
        while(it.hasNext()) {
            Tuple t = it.next();
            assertEquals(Util.createTuple(tupleContents[j]), t);
            j++;
View Full Code Here
TOP

Related Classes of org.apache.pig.data.NonSpillableDataBag$NonSpillableDataBagIterator

  • com.endgame.binarypig.loaders.ExecutingJsonLoader
  • com.endgame.binarypig.loaders.ExecutingJsonLoaderTest
  • com.endgame.binarypig.util.JsonUtil
  • com.endgame.binarypig.util.JsonUtilTest
  • com.twitter.elephantbird.pig.load.JsonLoader
  • com.twitter.elephantbird.pig.piggybank.Fixtures
  • com.twitter.elephantbird.pig.piggybank.TestInvoker
  • com.twitter.elephantbird.pig.util.ProtobufToPig
  • com.twitter.elephantbird.pig.util.TestPigToProtobuf
  • com.twitter.elephantbird.pig.util.TestPigToThrift
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.