Examples of org.apache.pig.data.DataBag

org.apache.pig.data.DataBag
A collection of Tuples. A DataBag may or may not fit into memory. DataBag extends spillable, which means that it registers with a memory manager. By default, it attempts to keep all of its contents in memory. If it is asked by the memory manager to spill to disk (by a call to spill()), it takes whatever it has in memory, opens a spill file, and writes the contents out. This may happen multiple times. The bag tracks all of the files it's spilled to. DataBag provides an Iterator interface, that allows callers to read through the contents. The iterators are aware of the data spilling. They have to be able to handle reading from files, as well as the fact that data they were reading from memory may have been spilled to disk underneath them. The DataBag interface assumes that all data is written before any is read. That is, a DataBag cannot be used as a queue. If data is written after data is read, the results are undefined. This condition is not checked on each add or read, for reasons of speed. Caveat emptor. Since spills are asynchronous (the memory manager requesting a spill runs in a separate thread), all operations dealing with the mContents Collection (which is the collection of tuples contained in the bag) have to be synchronized. This means that reading from a DataBag is currently serialized. This is ok for the moment because pig execution is currently single threaded. A ReadWriteLock was experimented with, but it was found to be about 10x slower than using the synchronize keyword. If pig changes its execution model to be multithreaded, we may need to return to this issue, as synchronizing reads will most likely defeat the purpose of multi-threading execution. DataBag come in several types, default, sorted, and distinct. The type must be chosen up front, there is no way to convert a bag on the fly.

    private static TupleFactory mTupleFactory = TupleFactory.getInstance();


    @Override
    public Long exec(Tuple input) throws IOException {
        try {
            DataBag bag = (DataBag)input.get(0);
            Iterator it = bag.iterator();
            long cnt = 0;
            while (it.hasNext()){
                    Tuple t = (Tuple)it.next();
                    if (t != null && t.size() > 0 && t.get(0) != null )
                            cnt++;

View Full Code Here

            }
        }
    }


    static protected Long sum(Tuple input) throws ExecException, NumberFormatException {
        DataBag values = (DataBag)input.get(0);
        long sum = 0;
        for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
            Tuple t = it.next();
            // Have faith here.  Checking each value before the cast is
            // just too much.
            sum += (Long)t.get(0);
        }

View Full Code Here

    private long intermediateCount = 0L;


    @Override
    public void accumulate(Tuple b) throws IOException {
        try {
            DataBag bag = (DataBag)b.get(0);
            Iterator it = bag.iterator();
            while (it.hasNext()){
                Tuple t = (Tuple)it.next();
                if (t != null && t.size() > 0 && t.get(0) != null) {
                    intermediateCount += 1;
                }

View Full Code Here

    m2.put("x", m3);
    m2.put("y", m4);
    tuple.set(9, m2);


    // c:collection(f13:double, f14:float, f15:bytes)
    DataBag bagColl = TypesUtils.createBag();
    Schema schColl = schema.getColumn(10).getSchema().getColumn(0).getSchema();
    Tuple tupColl1 = TypesUtils.createTuple(schColl);
    Tuple tupColl2 = TypesUtils.createTuple(schColl);
    byte[] abs1 = new byte[3];
    byte[] abs2 = new byte[4];
    tupColl1.set(0, 3.1415926);
    tupColl1.set(1, 1.6);
    abs1[0] = 11;
    abs1[1] = 12;
    abs1[2] = 13;
    tupColl1.set(2, new DataByteArray(abs1));
    bagColl.add(tupColl1);
    tupColl2.set(0, 123.456789);
    tupColl2.set(1, 100);
    abs2[0] = 21;
    abs2[1] = 22;
    abs2[2] = 23;
    abs2[3] = 24;
    tupColl2.set(2, new DataByteArray(abs2));
    bagColl.add(tupColl2);
    tuple.set(10, bagColl);


    int row = 0;
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);


    // row 2
    row++;
    TypesUtils.resetTuple(tuple);
    TypesUtils.resetTuple(tupRecord1);
    TypesUtils.resetTuple(tupRecord2);
    TypesUtils.resetTuple(tupRecord3);
    m1.clear();
    m2.clear();
    m3.clear();
    m4.clear();
    tuple.set(0, false);
    tuple.set(1, 2); // int
    tuple.set(2, 1002L); // long
    tuple.set(3, 3.1); // float
    tuple.set(4, "hello world 2"); // string
    tuple.set(5, new DataByteArray("hello byte 2")); // byte


    // r1:record(f1:int, f2:long
    tupRecord1.set(0, 2);
    tupRecord1.set(1, 1002L);
    tuple.set(6, tupRecord1);


    // r2:record(r3:record(f3:float, f4))
    tupRecord2.set(0, tupRecord3);
    tupRecord3.set(0, 2.3);
    tupRecord3.set(1, new DataByteArray("r3 row2  byte array"));
    tuple.set(7, tupRecord2);


    // m1:map(string)
    m1.put("a2", "A2");
    m1.put("b2", "B2");
    m1.put("c2", "C2");
    tuple.set(8, m1);


    // m2:map(map(int))
    m3.put("m321", 321);
    m3.put("m322", 322);
    m3.put("m323", 323);
    m2.put("z", m3);
    tuple.set(9, m2);


    // c:collection(f13:double, f14:float, f15:bytes)
    bagColl.clear();
    TypesUtils.resetTuple(tupColl1);
    TypesUtils.resetTuple(tupColl2);
    tupColl1.set(0, 7654.321);
    tupColl1.set(1, 0.0001);
    abs1[0] = 31;
    abs1[1] = 32;
    abs1[2] = 33;
    tupColl1.set(2, new DataByteArray(abs1));
    bagColl.add(tupColl1);
    tupColl2.set(0, 0.123456789);
    tupColl2.set(1, 0.3333);
    abs2[0] = 41;
    abs2[1] = 42;
    abs2[2] = 43;
    abs2[3] = 44;
    tupColl2.set(2, new DataByteArray(abs2));
    bagColl.add(tupColl2);
    tuple.set(10, bagColl);


    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);

View Full Code Here

    while (it3.hasNext()) {
      RowValue3 = it3.next();
      Assert.assertEquals(2, RowValue3.size());
      row++;
      String key = (String) RowValue3.get(0);
      DataBag bag = (DataBag) RowValue3.get(1);
      Iterator<Tuple> it = bag.iterator();
      if (key.equals("so")) {
        Assert.assertEquals(1, bag.size());
        Tuple t = it.next();
        Assert.assertEquals(-100, t.get(0));
      } else if (key.equals("something")) {
        Assert.assertEquals(1, bag.size());
        Tuple t = it.next();
        Assert.assertEquals(100, t.get(0));
      } else if (key.equals("some")) {
        Assert.assertEquals(2, bag.size());
        Tuple t1 = it.next(), t2 = it.next();
        int i1 = (Integer) t1.get(0), i2 = (Integer) t2.get(0);
        Assert.assertTrue((i1 == 1 && i2 == 3) || (i1 == 3 || i1 ==1));
      } else {
        Assert.fail("Unexprected key: " + key);

View Full Code Here

        public Tuple exec(Tuple input) throws IOException {
            // Since Initial is guaranteed to be called
            // only in the map, it will be called with an
            // input of a bag with a single tuple - the 
            // count should always be 1 if bag is non empty
            DataBag bag = (DataBag)input.get(0);
            Iterator it = bag.iterator();
            if (it.hasNext()){
                Tuple t = (Tuple)it.next();
                if (t != null && t.size() > 0 && t.get(0) != null)
                    return mTupleFactory.newTuple(Long.valueOf(1));
            }

View Full Code Here

            }
        }
    }


    static protected Long min(Tuple input) throws ExecException {
        DataBag values = (DataBag)input.get(0);


        // if we were handed an empty bag, return NULL
        // this is in compliance with SQL standard
        if(values.size() == 0) {
            return null;
        }


        long curMin = Long.MAX_VALUE;
        boolean sawNonNull = false;
        for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
            Tuple t = it.next();
            try {
                Long l = (Long)(t.get(0));
                if (l == null) continue;
                sawNonNull = true;

View Full Code Here

        @Override
        public Tuple exec(Tuple input) throws IOException {
            try {
                // input is a bag with one tuple containing
                // the column we are trying to min on
                DataBag bg = (DataBag) input.get(0);
                Long l = null;
                if(bg.iterator().hasNext()) {
                    Tuple tp = bg.iterator().next();
                    l = (Long)(tp.get(0)); 
                }
                return tfact.newTuple(l);
            } catch (ExecException ee) {
                throw ee;

View Full Code Here

        output.set(1, Long.valueOf(count));
        return output;
    }


    static protected long count(Tuple input) throws ExecException {
        DataBag values = (DataBag)input.get(0);
        long cnt = 0;
        Iterator it = values.iterator();
        while (it.hasNext()){
            Tuple t = (Tuple)it.next(); 
            if (t != null && t.size() > 0 && t.get(0) != null)
                cnt ++;
        }

View Full Code Here

                    
        return cnt;
    }


    static protected Double sum(Tuple input) throws ExecException, IOException {
        DataBag values = (DataBag)input.get(0);
        
        // if we were handed an empty bag, return NULL
        if(values.size() == 0) {
            return null;
        }


        double sum = 0;
        boolean sawNonNull = false;
        for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
            Tuple t = it.next();
            try{
                DataByteArray dba = (DataByteArray)t.get(0);
                Double d = dba != null ? Double.valueOf(dba.toString()) : null;
                if (d == null) continue;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.data.DataBag

com.inadco.hbl.piggybank.AggregationFromMeasureBag

com.mongodb.hadoop.pig.JSONPigReplaceTest

com.mongodb.hadoop.pig.MongoLoaderTest

com.mozilla.pig.eval.json.JsonMap

com.twitter.elephantbird.pig.piggybank.Fixtures

com.twitter.elephantbird.pig.util.ProtobufToPig

datafu.pig.linkanalysis.PageRank

org.apache.cassandra.pig.CqlTableTest

org.apache.cassandra.pig.ThriftColumnFamilyTest

org.apache.hadoop.zebra.io.TestColumnGroupSplits

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.