Examples of org.apache.pig.data.DataByteArray

org.apache.pig.data.DataByteArray
An implementation of byte array. This is done as an object because we need to be able to implement compareTo, toString, hashCode, and some other methods.

    public Tuple createTuple(byte[] content) throws Exception {
        if (mProtoTuple == null) {
            mProtoTuple = new ArrayList<Object>();
        }
        if (content.length > 0) {
            mProtoTuple.add(new DataByteArray(content));
        }
        Tuple t = TupleFactory.getInstance().newTupleNoCopy(mProtoTuple);
        mProtoTuple = null;


        return t;

View Full Code Here

        
        // DataByteArray concat
        byte[] a = {1,2,3};
        byte[] b = {4,5,6};
        byte[] expected = {1,2,3,4,5,6};
        DataByteArray dbaExpected = new DataByteArray(expected);
        
        DataByteArray dbaA = new DataByteArray(a);
        DataByteArray dbaB = new DataByteArray(b);
        EvalFunc<DataByteArray> concat = new CONCAT();
        
        Tuple t = TupleFactory.getInstance().newTuple(2);
        t.set(0, dbaA);
        t.set(1, dbaB);
        DataByteArray result = concat.exec(t);
        String msg = "[Testing CONCAT on input type: bytearray]";
        assertTrue(msg, result.equals(dbaExpected));
        
        // String concat
        String s1 = "unit ";
        String s2 = "test";
        String exp = "unit test";

View Full Code Here

        byte[] b = {4,5,6};
        byte[] c = {7,8,9};
        byte[] d = {10,11,12};
        byte[] e = {13,14,15};
        byte[] expected = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
        DataByteArray dbaExpected = new DataByteArray(expected);


        DataByteArray dbaA = new DataByteArray(a);
        DataByteArray dbaB = new DataByteArray(b);
        DataByteArray dbaC = new DataByteArray(c);
        DataByteArray dbaD = new DataByteArray(d);
        DataByteArray dbaE = new DataByteArray(e);


        EvalFunc<DataByteArray> concat = new CONCAT();


        Tuple t = TupleFactory.getInstance().newTuple(5);
        t.set(0, dbaA);
        t.set(1, dbaB);
        t.set(2, dbaC);
        t.set(3, dbaD);
        t.set(4, dbaE);


        DataByteArray result = concat.exec(t);
        String msg = "[Testing CONCAT on >2 tuples for input type: bytearray]";
        assertTrue(msg, result.equals(dbaExpected));


        // String concat
        String s1 = "high ";
        String s2 = "fives ";
        String s3 = "kick ";

View Full Code Here

    @Test
    public void testSIZE() throws Exception {
        
        // DataByteArray size
        byte[] a = {1,2,3};
        DataByteArray dba = new DataByteArray(a);
        Long expected = new Long(3);
        Tuple t = TupleFactory.getInstance().newTuple(1);
        t.set(0, dba);
        EvalFunc<Long> size = new SIZE();        
        String msg = "[Testing SIZE on input type: bytearray]";

View Full Code Here

    @Test
    public void testSFPig() throws Exception {
        String inputStr = "amy\tbob\tcharlene\tdavid\terin\tfrank";
        Util.createInputFile(cluster, "testSFPig-input.txt", new String[]
                                                                    {inputStr});
        DataByteArray[] input = { new DataByteArray("amy"),
            new DataByteArray("bob"), new DataByteArray("charlene"),
            new DataByteArray("david"), new DataByteArray("erin"),
            new DataByteArray("frank") };
        Tuple f1 = Util.loadTuple(TupleFactory.getInstance().
                newTuple(input.length), input);
        
        String query = "a = load 'testSFPig-input.txt';" +
            "store a into 'testSFPig-output.txt';";

View Full Code Here

    tupColl1.set(0, 3.1415926);
    tupColl1.set(1, 1.6);
    abs1[0] = 11;
    abs1[1] = 12;
    abs1[2] = 13;
    tupColl1.set(2, new DataByteArray(abs1));
    bagColl.add(tupColl1);
    tupColl2.set(0, 123.456789);
    tupColl2.set(1, 100);
    abs2[0] = 21;
    abs2[1] = 22;
    abs2[2] = 23;
    abs2[3] = 24;
    tupColl2.set(2, new DataByteArray(abs2));
    bagColl.add(tupColl2);
    tuple.set(3, bagColl);


    int row = 0;
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);


    // row 2
    row++;
    TypesUtils.resetTuple(tuple);
    TypesUtils.resetTuple(tupRecord);
    map.clear();
    tuple.set(0, false);
    tupRecord.set(0, 2);
    tupRecord.set(1, 1002L);
    tuple.set(1, tupRecord);
    map.put("boy", "girl");
    map.put("adam", "amy");
    map.put("bob", "becky");
    map.put("carl", "cathy");
    tuple.set(2, map);
    bagColl.clear();
    TypesUtils.resetTuple(tupColl1);
    TypesUtils.resetTuple(tupColl2);
    tupColl1.set(0, 7654.321);
    tupColl1.set(1, 0.0001);
    abs1[0] = 31;
    abs1[1] = 32;
    abs1[2] = 33;
    tupColl1.set(2, new DataByteArray(abs1));
    bagColl.add(tupColl1);
    tupColl2.set(0, 0.123456789);
    tupColl2.set(1, 0.3333);
    abs2[0] = 41;
    abs2[1] = 42;
    abs2[2] = 43;
    abs2[3] = 44;
    tupColl2.set(2, new DataByteArray(abs2));
    bagColl.add(tupColl2);
    tuple.set(3, bagColl);
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);

View Full Code Here

    m1 = new HashMap<String, String>();
    m1.put("a","m1-a");
    m1.put("b","m1-b");
    
    table1 = new Object[][]{
      {5,    -3.25f,  1001L,  51e+2,  "Zebra",  new DataByteArray("Zebra"),    m1},
      {-1,  3.25f,  1000L,  50e+2,  "zebra",  new DataByteArray("zebra"),    m1},
      {1001,  100.0f,  1003L,  50e+2,  "Apple",  new DataByteArray("Apple"),    m1},
      {1001,  101.0f,  1001L,  50e+2,  "apple",  new DataByteArray("apple"),    m1},
      {1001,  50.0f,  1000L,  50e+2,  "Pig",    new DataByteArray("Pig"),    m1},
      {1001,  52.0f,  1001L,  50e+2,  "pig",    new DataByteArray("pig"),    m1},
      {1002,  28.0f,  1000L,  50e+2,  "Hadoop",  new DataByteArray("Hadoop"),  m1},
      {1000,  0.0f,  1002L,  52e+2,  "hadoop",  new DataByteArray("hadoop"),  m1} };
    
    // Create table1
    createTable(pathTable1, TABLE1_SCHEMA, TABLE1_STORAGE, table1);
    
    // Create table2 data
    m2 = new HashMap<String, String>();
    m2.put("a","m2-a");
    m2.put("b","m2-b");
    
    table2 = new Object[][] {
      {15,  56.0f,  1004L,  50e+2,  "green",  new DataByteArray("green"),    m2},
      {-1,  -99.0f,  1002L,  51e+2,  "orange",  new DataByteArray("orange"),  m2},
      {1001,  100.0f,  1003L,  55e+2,  "white",  new DataByteArray("white"),    m2},
      {1001,  102.0f,  1001L,  52e+2,  "purple",  new DataByteArray("purple"),  m2},
      {1001,  50.0f,  1008L,  52e+2,  "gray",    new DataByteArray("gray"),    m2},
      {1001,  53.0f,  1001L,  52e+2,  "brown",  new DataByteArray("brown"),    m2},
      {2000,  33.0f,  1006L,  52e+2,  "beige",  new DataByteArray("beige"),    m2} };
    
    // Create table2
    createTable(pathTable2, TABLE2_SCHEMA, TABLE2_STORAGE, table2);
    
    // Create table3 data
    m3 = new HashMap<String, String>();
    m3.put("a","m3-a");
    m3.put("b","m3-b");
    m3.put("c","m3-b");
    
    table3 = new Object[][] {
      {"a 8",  "Cupertino",  "California",  1,  m3},
      {"a 7",  "San Jose",    "California",  2,  m3},
      {"a 6",  "Santa Cruz",  "California",  3,  m3},
      {"a 5",  "Las Vegas",  "Nevada",    4,  m3},
      {"a 4",  "New York",    "New York",    5,  m3},
      {"a 3",  "Phoenix",    "Arizona",    6,  m3},
      {"a 2",  "Dallas",    "Texas",    7,  m3},
      {"a 1",  "Reno",      "Nevada",    8,  m3} };
    
    // Create table3
    createTable(pathTable3, TABLE3_SCHEMA, TABLE3_STORAGE, table3);
    
    // Create table4 data
    table4 = new Object[][] {
      {1,  "Cupertino",  1001L},
      {2,  "San Jose",    1008L},
      {3,  "Santa Cruz",  1008L},
      {4,  "Las Vegas",  1008L},
      {5,  "Dallas",    1010L},
      {6,  "Reno",      1000L} };
    
    // Create table4
    createTable(pathTable4, TABLE4_SCHEMA, TABLE4_STORAGE, table4);
    
    // Create table5 data
    table5 = new Object[][] {
      {3.25f,    51e+2,  1001L,  "string1",  new DataByteArray("green"),  100},
      {3.25f,    50e+2,  1000L,  "string1",  new DataByteArray("blue"),  100},
      {3.26f,    51e+2,  1001L,  "string0",  new DataByteArray("purple"),100},
      {3.24f,    53e+2,  1001L,  "string1",  new DataByteArray("yellow"),100},
      {3.28f,    51e+2,  1001L,  "string2",  new DataByteArray("white"),  100},
      {3.25f,    53e+2,  1000L,  "string0",  new DataByteArray("orange"),100} };
    
    // Create table5
    createTable(pathTable5, TABLE5_SCHEMA, TABLE5_STORAGE, table5);
    
    // Create table6 data

View Full Code Here

    testOrderPreserveUnion(inputTables, "a,b,c,d,e,f,m1");
    
    // Verify union table
    ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
    
    addResultRow(resultTable, -1,  -99.0f,  1002L,  51e+2,  "orange",  new DataByteArray("orange"),m2);
    addResultRow(resultTable, -1,  3.25f,  1000L,  50e+2,  "zebra",  new DataByteArray("zebra"),  m1);
    addResultRow(resultTable, 5,  -3.25f,  1001L,  51e+2,  "Zebra",  new DataByteArray("Zebra"),  m1);
    addResultRow(resultTable, 15,  56.0f,  1004L,  50e+2,  "green",  new DataByteArray("green"),  m2);
    
    addResultRow(resultTable, 1000,  0.0f,  1002L,  52e+2,  "hadoop",  new DataByteArray("hadoop"),m1);
    addResultRow(resultTable, 1001,  50.0f,  1000L,  50e+2,  "Pig",    new DataByteArray("Pig"),  m1);
    addResultRow(resultTable, 1001,  50.0f,  1008L,  52e+2,  "gray",    new DataByteArray("gray"),  m2);
    addResultRow(resultTable, 1001,  52.0f,  1001L,  50e+2,  "pig",    new DataByteArray("pig"),  m1);
    
    addResultRow(resultTable, 1001,  53.0f,  1001L,  52e+2,  "brown",  new DataByteArray("brown"),  m2);
    addResultRow(resultTable, 1001,  100.0f,  1003L,  50e+2,  "Apple",  new DataByteArray("Apple"),  m1);
    addResultRow(resultTable, 1001,  100.0f,  1003L,  55e+2,  "white",  new DataByteArray("white"),  m2);
    addResultRow(resultTable, 1001,  101.0f,  1001L,  50e+2,  "apple",  new DataByteArray("apple"),  m1);
    
    addResultRow(resultTable, 1001,  102.0f,  1001L,  52e+2,  "purple",  new DataByteArray("purple"),m2);
    addResultRow(resultTable, 1002,  28.0f,  1000L,  50e+2,  "Hadoop",  new DataByteArray("Hadoop"),m1);
    addResultRow(resultTable, 2000,  33.0f,  1006L,  52e+2,  "beige",  new DataByteArray("beige"),  m2);
    
    // Verify union table
    Iterator<Tuple> it = pigServer.openIterator("records1");
    int numbRows = verifyTable(resultTable, 0, it);

View Full Code Here

    testOrderPreserveUnion(inputTables, "a,b,c,d,e,f,m1,source_table");
    
    // Verify union table
    ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
    
    addResultRow(resultTable, -1,  -99.0f,  1002L,  51e+2,  "orange",  new DataByteArray("orange"),m2,  1);
    addResultRow(resultTable, -1,  3.25f,  1000L,  50e+2,  "zebra",  new DataByteArray("zebra"),  m1,  0);
    addResultRow(resultTable, 5,  -3.25f,  1001L,  51e+2,  "Zebra",  new DataByteArray("Zebra"),  m1,  0);
    addResultRow(resultTable, 15,  56.0f,  1004L,  50e+2,  "green",  new DataByteArray("green"),  m2,  1);
    
    addResultRow(resultTable, 1000,  0.0f,  1002L,  52e+2,  "hadoop",  new DataByteArray("hadoop"),m1,  0);
    addResultRow(resultTable, 1001,  50.0f,  1000L,  50e+2,  "Pig",    new DataByteArray("Pig"),  m1,  0);
    addResultRow(resultTable, 1001,  50.0f,  1008L,  52e+2,  "gray",    new DataByteArray("gray"),  m2,  1);
    addResultRow(resultTable, 1001,  52.0f,  1001L,  50e+2,  "pig",    new DataByteArray("pig"),  m1,  0);
    
    addResultRow(resultTable, 1001,  53.0f,  1001L,  52e+2,  "brown",  new DataByteArray("brown"),  m2,  1);
    addResultRow(resultTable, 1001,  100.0f,  1003L,  50e+2,  "Apple",  new DataByteArray("Apple"),  m1,  0);
    addResultRow(resultTable, 1001,  100.0f,  1003L,  55e+2,  "white",  new DataByteArray("white"),  m2,  1);
    addResultRow(resultTable, 1001,  101.0f,  1001L,  50e+2,  "apple",  new DataByteArray("apple"),  m1,  0);
    
    addResultRow(resultTable, 1001,  102.0f,  1001L,  52e+2,  "purple",  new DataByteArray("purple"),m2,  1);
    addResultRow(resultTable, 1002,  28.0f,  1000L,  50e+2,  "Hadoop",  new DataByteArray("Hadoop"),m1,  0);
    addResultRow(resultTable, 2000,  33.0f,  1006L,  52e+2,  "beige",  new DataByteArray("beige"),  m2,  1);
    
    // Verify union table
    Iterator<Tuple> it = pigServer.openIterator("records1");
    int numbRows = verifyTable(resultTable, 0, it);

View Full Code Here

    testOrderPreserveUnion(inputTables, "a,b,c,d,e,f,m1,source_table");
    
    // Verify union table
    ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
    
    addResultRow(resultTable, -1,  -99.0f,  1002L,  51e+2,  "orange",  new DataByteArray("orange"),m2,  1);
    addResultRow(resultTable, -1,  3.25f,  1000L,  50e+2,  "zebra",  new DataByteArray("zebra"),  m1,  0);
    addResultRow(resultTable, 5,  -3.25f,  1001L,  51e+2,  "Zebra",  new DataByteArray("Zebra"),  m1,  0);
    addResultRow(resultTable, 15,  56.0f,  1004L,  50e+2,  "green",  new DataByteArray("green"),  m2,  1);
    
    addResultRow(resultTable, 1000,  0.0f,  1002L,  52e+2,  "hadoop",  new DataByteArray("hadoop"),m1,  0);
    addResultRow(resultTable, 1001,  50.0f,  1008L,  52e+2,  "gray",    new DataByteArray("gray"),  m2,  1);
    addResultRow(resultTable, 1001,  50.0f,  1000L,  50e+2,  "Pig",    new DataByteArray("Pig"),  m1,  0);
    addResultRow(resultTable, 1001,  52.0f,  1001L,  50e+2,  "pig",    new DataByteArray("pig"),  m1,  0);
    
    addResultRow(resultTable, 1001,  53.0f,  1001L,  52e+2,  "brown",  new DataByteArray("brown"),  m2,  1);
    addResultRow(resultTable, 1001,  100.0f,  1003L,  55e+2,  "white",  new DataByteArray("white"),  m2,  1);
    addResultRow(resultTable, 1001,  100.0f,  1003L,  50e+2,  "Apple",  new DataByteArray("Apple"),  m1,  0);
    addResultRow(resultTable, 1001,  101.0f,  1001L,  50e+2,  "apple",  new DataByteArray("apple"),  m1,  0);
    
    addResultRow(resultTable, 1001,  102.0f,  1001L,  52e+2,  "purple",  new DataByteArray("purple"),m2,  1);
    addResultRow(resultTable, 1002,  28.0f,  1000L,  50e+2,  "Hadoop",  new DataByteArray("Hadoop"),m1,  0);
    addResultRow(resultTable, 2000,  33.0f,  1006L,  52e+2,  "beige",  new DataByteArray("beige"),  m2,  1);
    
    // Verify union table
    Iterator<Tuple> it = pigServer.openIterator("records1");
    int numbRows = verifyTable(resultTable, 0, it);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.data.DataByteArray

com.twitter.elephantbird.pig.load.LzoRawBytesLoader

com.twitter.elephantbird.pig.load.TestRCFileProtobufStorage$B64ToTuple

com.twitter.elephantbird.pig.load.TestRCFileThriftStorage$B64ToTuple

com.twitter.elephantbird.pig.piggybank.TestProtoToPig

com.twitter.elephantbird.pig.store.LzoRawBytesStorage

com.twitter.elephantbird.pig.store.RCFilePigStorage

com.twitter.elephantbird.pig.util.BytesWritableConverter

com.twitter.elephantbird.pig.util.ProtobufToPig

com.twitter.elephantbird.pig.util.TestThriftToPig

com.twitter.elephantbird.pig.util.ThriftToPig

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.