Examples of org.apache.pig.data.DefaultDataBag$DefaultDataBagIterator

org.apache.pig.data.DefaultDataBag
An iterator that handles getting the next tuple from the bag. This iterator has a couple of issues to deal with. First, data can be stored in a combination of in memory and on disk. Second, the bag may be asked to spill while the iterator is reading it. This means that it will be pointing to someplace in memory and suddenly it will need to switch to a disk file.

        String expected = "{\"bag\":[[{\"first\":\"one\",\"second\":\"two\",\"third\":\"three\"}]," +
                                     "[{\"first\":\"one\",\"second\":\"two\",\"third\":\"three\"}]," +
                                     "[{\"first\":\"one\",\"second\":\"two\",\"third\":\"three\"}]]}";


        Tuple tuple = TupleFactory.getInstance().newTuple(Arrays.asList(new String[] { "one", "two", "three" }));
        assertThat(pigTypeToJson(createTuple(new DefaultDataBag(Arrays.asList(new Tuple[] { tuple, tuple, tuple })),
                createSchema("bag: {t:(first:chararray, second:chararray, third: chararray)}"))), is(expected));
    }

View Full Code Here

                                       "[{\"val_0\":\"xxx\",\"val_1\":\"yyy\"}]," +
                                       "[{\"val_0\":\"xxx\",\"val_1\":\"yyy\"}]]}";


        Tuple tuple = TupleFactory.getInstance().newTuple(Arrays.asList(new String[] { "xxx", "yyy" }));


        assertThat(pigTypeToJson(createTuple(new DefaultDataBag(Arrays.asList(new Tuple[] { tuple, tuple, tuple })),
                createSchema("{t:(chararray, chararray)}"))), is(expected));
    }

View Full Code Here

    @Test
    public void testNamedBag() {
        String expected = "{\"bag\":[[\"one\",\"two\",\"three\"],[\"one\",\"two\",\"three\"],[\"one\",\"two\",\"three\"]]}";


        Tuple tuple = TupleFactory.getInstance().newTuple(Arrays.asList(new String[] { "one", "two", "three" }));
        assertThat(pigTypeToJson(createTuple(new DefaultDataBag(Arrays.asList(new Tuple[] { tuple, tuple, tuple })),
                createSchema("bag: {t:(first:chararray, second:chararray, third: chararray)}"))), is(expected));
    }

View Full Code Here

    @Test
    public void testBagWithAnonTuple() {
        String expected = "{\"bag\":[[\"xxx\",\"yyy\"],[\"xxx\",\"yyy\"],[\"xxx\",\"yyy\"]]}";


        Tuple tuple = TupleFactory.getInstance().newTuple(Arrays.asList(new String[] { "xxx", "yyy" }));
        assertThat((pigTypeToJson(createTuple(new DefaultDataBag(Arrays.asList(new Tuple[] { tuple, tuple, tuple })),
                createSchema("bag: {t:(chararray, chararray)}")))), is(expected));
    }

View Full Code Here

        innerTuple.append("k");


        input.append(innerTuple);
        result = rbsc.exec(input);
        assertEquals("a b c d e f g h i j k", result);
        DataBag db = new DefaultDataBag();
        Tuple dbTuple = new DefaultTuple();
        dbTuple.append("l");
        dbTuple.append("m");
        db.add(dbTuple);
        innerTuple.append(db);
        result = rbsc.exec(input);
        assertEquals("a b c d e f g h i j k l m", result);
    }

View Full Code Here


        input.append(innerTuple);
        rbsc = new RangeBasedStringConcat("0,9", " ");
        result = rbsc.exec(input);
        assertEquals("a j k", result);
        DataBag db = new DefaultDataBag();
        Tuple dbTuple = new DefaultTuple();
        dbTuple.append("l");
        dbTuple.append("m");
        db.add(dbTuple);
        innerTuple.append(db);
        rbsc = new RangeBasedStringConcat("0,9,10", " ");
        result = rbsc.exec(input);
        assertEquals("a j k l m", result);
    }

View Full Code Here

    @Test
    public void testSkewedJoinUDF() throws IOException {
        PartitionSkewedKeys udf = new PartitionSkewedKeys(new String[]{"0.1", "2", "1.txt"});
        Tuple t = TupleFactory.getInstance().newTuple();
        t.append(3);    // use 3 reducers
        DataBag db = new DefaultDataBag();
        Tuple sample;
        for (int i=0;i<=3;i++) {
            sample = TupleFactory.getInstance().newTuple();
            if (i!=3)
                sample.append("1");
            else
                sample.append("2");
            sample.append((long)200);
            if (i!=3)
                sample.append((long)0);
            else
                sample.append((long)30);
            db.add(sample);
        }
        t.append(db);
        Map<String, Object> output = udf.exec(t);
        DataBag parList = (DataBag)output.get(PartitionSkewedKeys.PARTITION_LIST);
        for (Tuple par : parList) {

View Full Code Here

    }


    @Test
    public void testCompareDataBag() throws IOException {
        list = new ArrayList<Object>(list);
        list.add(new DefaultDataBag(Arrays.asList(tf.newTuple(Arrays.asList(0)))));
        NullableTuple t1 = new NullableTuple(tf.newTuple(list));
        list.set(list.size() - 1, new DefaultDataBag(Arrays.asList(tf.newTuple(Arrays.asList(1)))));
        NullableTuple t2 = new NullableTuple(tf.newTuple(list));
        int res = compareHelper(t1, t2, comparator);
        assertEquals(Math.signum(t1.compareTo(t2)), Math.signum(res), 0);
        assertTrue(res < 0);

View Full Code Here

        Tuple t = tf.newTuple(Arrays.asList(0));
        ArrayList<Tuple> tuplist = new ArrayList<Tuple>(size); 
        for(int i=0; i<size; i++){
            tuplist.add(t);
        }
        return new DefaultDataBag(tuplist);
    }

View Full Code Here


    @Test
    public void testCompareDifferentSizes() throws IOException {
        list = new ArrayList<Object>(list);
        // this object should be never get into the comparison loop
        list.add(new DefaultDataBag());
        NullableTuple t = new NullableTuple(tf.newTuple(list));
        int res = compareHelper(prototype, t, comparator);
        assertEquals(Math.signum(prototype.compareTo(t)), Math.signum(res), 0);
        assertTrue(res < 0);
    }

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of org.apache.pig.data.DefaultDataBag$DefaultDataBagIterator

com.inadco.hbl.piggybank.Dimensions2CuboidKey

com.packetloop.packetpig.udf.util.Explode

org.apache.cassandra.hadoop.pig.CassandraStorage

org.apache.hadoop.zebra.types.TypesUtils

org.apache.hcatalog.data.HCatArrayBag

org.apache.hcatalog.pig.PigHCatUtil

org.apache.hive.hcatalog.pig.PigHCatUtil

org.apache.howl.data.HowlArrayBag

org.apache.howl.pig.PigHowlUtil

org.apache.pig.backend.hadoop.accumulo.TestAbstractAccumuloStorage

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.