Package org.apache.pig.data

Examples of org.apache.pig.data.NonSpillableDataBag

79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    }
   
    @Test
    public void testNonSpillableDataBag() throws Exception {
        String[][] tupleContents = new String[][] {{"a", "b"},{"c", "d" }, { "e", "f"} };
        NonSpillableDataBag bg = new NonSpillableDataBag();
        for (int i = 0; i < tupleContents.length; i++) {
            bg.add(Util.createTuple(tupleContents[i]));
        }
        Iterator<Tuple> it = bg.iterator();
        int j = 0;
        while(it.hasNext()) {
            Tuple t = it.next();
            assertEquals(Util.createTuple(tupleContents[j]), t);
            j++;
View Full Code Here
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
            for(int i = 0; i < inputSchema.size(); i++) {
                t.set(i, null);
            }
            List<Tuple> bagContents = new ArrayList<Tuple>(1);
            bagContents.add(t);
            DataBag bg = new NonSpillableDataBag(bagContents);
            ce.setValue(bg);
            ce.setResultType(DataType.BAG);
            //this operator doesn't have any predecessors
            fePlan.add(ce);
           
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    }
   
    @Test
    public void testNonSpillableDataBag() throws Exception {
        String[][] tupleContents = new String[][] {{"a", "b"},{"c", "d" }, { "e", "f"} };
        NonSpillableDataBag bg = new NonSpillableDataBag();
        for (int i = 0; i < tupleContents.length; i++) {
            bg.add(Util.createTuple(tupleContents[i]));
        }
        Iterator<Tuple> it = bg.iterator();
        int j = 0;
        while(it.hasNext()) {
            Tuple t = it.next();
            assertEquals(Util.createTuple(tupleContents[j]), t);
            j++;
View Full Code Here
127
128
129
130
131
132
133
134
135
        if (PigMapReduce.sJobConf != null) {
         bagType = PigMapReduce.sJobConf.get("pig.cachedbag.type");            
         }
                                              
      if (bagType != null && bagType.equalsIgnoreCase("default")) {
        return new NonSpillableDataBag();
      }
      return new InternalCachedBag();   
    }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
        if(m == null) {
            return null;
        }

        Collection c = m.values();
        DataBag bag = new NonSpillableDataBag(c.size());
        Iterator<Object> iter = c.iterator();
        while(iter.hasNext()) {
            Tuple t = TUPLE_FACTORY.newTuple(iter.next());
            bag.add(t);
        }

        return bag;
    }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
            }

            // Create a new bag if "newKey" does not exist in Map
            DataBag bag = inverseMap.get(newKey);
            if (bag == null) {
                bag = new NonSpillableDataBag();
                bag.add(TUPLE_FACTORY.newTuple(entry.getKey()));
                inverseMap.put(newKey, bag);
            } else {
                bag.add(TUPLE_FACTORY.newTuple(entry.getKey()));
            }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
        m = (Map<String, Object>)(input.get(0));
        if(m == null) {
            return null;
        }

        DataBag bag = new NonSpillableDataBag(m.size());
        for (String s : m.keySet()) {
            Tuple t = TUPLE_FACTORY.newTuple(s);
            bag.add(t);
        }

        return bag;
    }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    @Override
    public DataBag exec(Tuple input) throws IOException {
        try {
            // The assumption is that if the bag contents fits into
            // an input tuple, it will not need to be spilled.
            DataBag bag = new NonSpillableDataBag(input.size());

            for (int i = 0; i < input.size(); ++i) {
                final Object object = input.get(i);
                if (object instanceof Tuple) {
                    bag.add( (Tuple) object);
                } else {
                    Tuple tp2 = TupleFactory.getInstance().newTuple(1);
                    tp2.set(0, object);
                    bag.add(tp2);
                }
            }

            return bag;
        } catch (Exception ee) {
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
            return null;
        }

        int initialSetSize = getInitialSetSize(m.values());
        Set<Object> uniqueElements = new HashSet<Object>(initialSetSize);
        DataBag bag = new NonSpillableDataBag();

        Iterator<Object> iter = m.values().iterator();

        while (iter.hasNext()) {
            Object val = iter.next();
            if (!uniqueElements.contains(val)) {
                uniqueElements.add(val);
                Tuple t = TUPLE_FACTORY.newTuple(val);
                bag.add(t);
            }
        }

        return bag;
    }
View Full Code Here
266
267
268
269
270
271
272
273
274
275
276
                for (Entry<Integer,Integer> valEnt : value.entrySet()) {
                    probVec.set(valEnt.getKey(), (float)valEnt.getValue()/total);
                }
                weightedParts.put(key, probVec);
            }
            output.put(QUANTILES_LIST, new NonSpillableDataBag(quantilesList));
            output.put(WEIGHTED_PARTS, weightedParts);
            return output;
        }catch (Exception e){
            e.printStackTrace();
            throw new RuntimeException(e);
View Full Code Here
TOP

Related Classes of org.apache.pig.data.NonSpillableDataBag

  • com.endgame.binarypig.loaders.ExecutingJsonLoader
  • com.endgame.binarypig.loaders.ExecutingJsonLoaderTest
  • com.endgame.binarypig.util.JsonUtil
  • com.endgame.binarypig.util.JsonUtilTest
  • com.twitter.elephantbird.pig.load.JsonLoader
  • com.twitter.elephantbird.pig.piggybank.Fixtures
  • com.twitter.elephantbird.pig.piggybank.TestInvoker
  • com.twitter.elephantbird.pig.util.ProtobufToPig
  • com.twitter.elephantbird.pig.util.TestPigToProtobuf
  • com.twitter.elephantbird.pig.util.TestPigToThrift
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.