Package org.apache.pig.data

Examples of org.apache.pig.data.NonSpillableDataBag

118
119
120
121
122
123
124
125
126
        if (PigMapReduce.sJobConfInternal.get() != null) {
         bagType = PigMapReduce.sJobConfInternal.get().get("pig.cachedbag.type");            
         }
                                              
      if (bagType != null && bagType.equalsIgnoreCase("default")) {
        return new NonSpillableDataBag();
      }
      return new InternalCachedBag(numBags);   
    }
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    }
   
    @Test
    public void testNonSpillableDataBag() throws Exception {
        String[][] tupleContents = new String[][] {{"a", "b"},{"c", "d" }, { "e", "f"} };
        NonSpillableDataBag bg = new NonSpillableDataBag();
        for (int i = 0; i < tupleContents.length; i++) {
            bg.add(Util.createTuple(tupleContents[i]));
        }
        Iterator<Tuple> it = bg.iterator();
        int j = 0;
        while(it.hasNext()) {
            Tuple t = it.next();
            assertEquals(Util.createTuple(tupleContents[j]), t);
            j++;
View Full Code Here
55
56
57
58
59
60
61
62
63
64
    // when adding, might want to consider doing explicit casts from Writables to Pig datatypes - does not appear to be needed at this time
    //This is the spot to do the generic JSON loading. some override function for data formatting would be here
    try{
      protoTuple.add(jsonUtil.wrap(jsonParser.parse(output)));
    } catch (Exception e){
      protoTuple.add(new NonSpillableDataBag());
    }
   
    return tupleFactory.newTuple(protoTuple);
  }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
57
58
    if (isNestedLoadEnabled && value instanceof JSONObject) {
      return walkJson((JSONObject) value);
    } else if (isNestedLoadEnabled && value instanceof JSONArray) {

      JSONArray a = (JSONArray) value;
      DataBag mapValue = new NonSpillableDataBag(a.size());
      for (int i = 0; i < a.size(); i++) {
        Tuple t = tupleFactory.newTuple(wrap(a.get(i)));
        mapValue.add(t);
      }
      return mapValue;

    } else {
      return value != null ? value.toString() : null;
View Full Code Here
55
56
57
58
59
60
61
   
    tuple = underTest.outputToTuple(new Text("mykey"), new BytesWritable("data".getBytes()), "{\"name\":\"val1\", MALFORMED}", false);
    assertEquals(3, tuple.size());
    assertEquals("mykey", tuple.get(0));
    assertEquals(false, tuple.get(1));
    assertEquals(new NonSpillableDataBag(), tuple.get(2));
  }
View Full Code Here
59
60
61
62
63
64
65
66
        TupleFactory.getInstance().newTuple((Object)"1"),
        TupleFactory.getInstance().newTuple((Object)"2"),
        TupleFactory.getInstance().newTuple((Object)"3")
    );
   
    assertEquals(map.get("list"), new NonSpillableDataBag(tuples) );
   
  }
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
        createJoinPlans(k);
        processingPlan = false;
        mTupleFactory = TupleFactory.getInstance();
        List<Tuple> tupList = new ArrayList<Tuple>();
        tupList.add(nullTuple);
        nullBag = new NonSpillableDataBag(tupList);
        this.isLeftOuterJoin = isLeftOuter;
        if (inputSchemas != null) {
            this.inputSchemas = inputSchemas;
        } else {
            this.inputSchemas = new Schema[replFiles == null ? 0 : replFiles.length];
View Full Code Here
294
295
296
297
298
299
300
301
302
303
304
                        ce.setValue(nullBag);
                    }
                    noMatch = true;
                    break;
                }
                ce.setValue(new NonSpillableDataBag(replicate.get(key).getList()));
            }

            // If this is not LeftOuter Join and there was no match we
            // skip the processing of this left tuple and move ahead
            if (!isLeftOuterJoin && noMatch)
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
                // constructor argument should be 2 * numInputs. But for one obscure
                // case we don't want to pay the penalty all the time.       
                        : new InternalCachedBag(numInputs-1);                   
            }
            // For last bag, we always use NonSpillableBag.
            dbs[lastBagIndex] = new NonSpillableDataBag((int)chunkSize);
           
            //For each Nullable tuple in the input, put it
            //into the corresponding bag based on the index,
            // except for the last input, which we will stream
            // The tuples will arrive in the order of the index,
View Full Code Here
205
206
207
208
209
210
211
        tuples.add((Tuple)pValue);
      } else {
        tuples.add(tupleFactory.newTuple(pValue));
      }
    }
    return new NonSpillableDataBag(tuples);
  }
View Full Code Here
TOP

Related Classes of org.apache.pig.data.NonSpillableDataBag

  • com.endgame.binarypig.loaders.ExecutingJsonLoader
  • com.endgame.binarypig.loaders.ExecutingJsonLoaderTest
  • com.endgame.binarypig.util.JsonUtil
  • com.endgame.binarypig.util.JsonUtilTest
  • com.twitter.elephantbird.pig.load.JsonLoader
  • com.twitter.elephantbird.pig.piggybank.Fixtures
  • com.twitter.elephantbird.pig.piggybank.TestInvoker
  • com.twitter.elephantbird.pig.util.ProtobufToPig
  • com.twitter.elephantbird.pig.util.TestPigToProtobuf
  • com.twitter.elephantbird.pig.util.TestPigToThrift
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.