Examples of org.apache.pig.builtin.mock.Storage.Data

Package org.apache.pig.builtin.mock.Storage

Examples of org.apache.pig.builtin.mock.Storage.Data

org.apache.pig.builtin.mock.Storage.Data

                "   return bi"
        };
        Util.createLocalInputFile( "pyfile_bi.py", pythonScript);


        
        Data data = resetData(pigServerLocal);
        Tuple t0 = tf.newTuple(new BigInteger("123456789012345678901234567890"));
        Tuple t1 = tf.newTuple(new BigInteger("9123456789012345678901234567890"));
        data.set("testBiTuples", "bi:biginteger", t0, t1);


        pigServerLocal.registerQuery("REGISTER 'pyfile_bi.py' USING streaming_python AS pf;");
        pigServerLocal.registerQuery("A = LOAD 'testBiTuples' USING mock.Storage();");
        pigServerLocal.registerQuery("B = FOREACH A generate pf.py_func(bi);");
        pigServerLocal.registerQuery("STORE B INTO 'bi_out' USING mock.Storage();");
        
        List<Tuple> out = data.get("bi_out");
        assertEquals(t0, out.get(0));
        assertEquals(t1, out.get(1));
    }

View Full Code Here

                "def py_func(bd):",
                "   return bd"
        };
        Util.createLocalInputFile( "pyfile_bd.py", pythonScript);


        Data data = resetData(pigServerLocal);
        Tuple t0 = tf.newTuple(new BigDecimal("123456789012345678901234567890.12345"));
        Tuple t1 = tf.newTuple(new BigDecimal("9123456789012345678901234567890.12345"));
        data.set("testBdTuples", "bd:bigdecimal", t0, t1);


        pigServerLocal.registerQuery("REGISTER 'pyfile_bd.py' USING streaming_python AS pf;");
        pigServerLocal.registerQuery("A = LOAD 'testBdTuples' USING mock.Storage();");
        pigServerLocal.registerQuery("B = FOREACH A generate pf.py_func(bd);");
        pigServerLocal.registerQuery("STORE B INTO 'bd_out' USING mock.Storage();");
        
        //We lose precision when we go to python.
        List<Tuple> out = data.get("bd_out");
        Float e0 = ((BigDecimal)t0.get(0)).floatValue();
        Float e1 = ((BigDecimal)t1.get(0)).floatValue();
        assertEquals(e0, ((BigDecimal) out.get(0).get(0)).floatValue(), 0.1);
        assertEquals(e1, ((BigDecimal) out.get(1).get(0)).floatValue(), 0.1);
    }

View Full Code Here

                "   return dt"
        };
        Util.createLocalInputFile( "pyfile_dt.py", pythonScript);


        
        Data data = resetData(pigServerLocal);
        Tuple t0 = tf.newTuple(new DateTime());
        Tuple t1 = tf.newTuple(new DateTime());
        data.set("testDateTuples", "d:datetime", t0, t1);


        pigServerLocal.registerQuery("REGISTER 'pyfile_dt.py' USING streaming_python AS pf;");
        pigServerLocal.registerQuery("A = LOAD 'testDateTuples' USING mock.Storage();");
        pigServerLocal.registerQuery("B = FOREACH A generate pf.py_func(d);");
        pigServerLocal.registerQuery("STORE B INTO 'date_out' USING mock.Storage();");
        
        List<Tuple> out = data.get("date_out");
        assertEquals(t0, out.get(0));
        assertEquals(t1, out.get(1));
    }

View Full Code Here

            "    return bag"
        };
        Util.createLocalInputFile("allfile.py", pythonScript);


        
        Data data = resetData(pigServerLocal);
        Tuple t0 = tf.newTuple(2);
        t0.set(0, "user1");
        t0.set(1, 10);
        Tuple t1 = tf.newTuple(2);
        t1.set(0, "user2");
        t1.set(1, 11);
        data.set("userTuples", "user_id:chararray,age:int", t0, t1);
        
        
        pigServerLocal.registerQuery("REGISTER 'allfile.py' USING streaming_python AS pf;");
        pigServerLocal.registerQuery("users = LOAD 'userTuples' USING mock.Storage();");
        pigServerLocal.registerQuery("crazy_tuple = FOREACH users GENERATE user_id, age, pf.get_tuple_output();");
        pigServerLocal.registerQuery("crazy_tuple_fun = FOREACH crazy_tuple GENERATE user_id, age, pf.crazy_tuple_identity(tuple_output);");
        pigServerLocal.registerQuery("crazy_tuple_with_map = FOREACH crazy_tuple_fun GENERATE user_id, age, tuple_output, pf.add_map(), pf.silly('\u2026');");
        pigServerLocal.registerQuery("crazy_group = GROUP crazy_tuple_with_map BY (age);");
        pigServerLocal.registerQuery("out = FOREACH crazy_group GENERATE group, pf.bag_identity(crazy_tuple_with_map);");


        pigServerLocal.registerQuery("STORE out INTO 'all_out' USING mock.Storage();");
        
        List<Tuple> out = data.get("all_out");


        /*
         * Expected output for first tuple.
         * (10,
         * {(user1,10,(,32,1000000099990000,32.0,3200.12346785,Some String,Hello\u2026Hello,Hello\u2026Hello,Some Byte Array),

View Full Code Here

    }


    @Test
    public void testLoadStoreFunc() throws Exception {
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        Data data = Storage.resetData(pigServer.getPigContext());
        data.set("foo",
                tuple("a"),
                tuple("b"),
                tuple("c")
                );


        pigServer.registerQuery(
                "A = LOAD 'foo' USING " + TestLoadStoreFuncLifeCycle.class.getName() + "$Loader();\n" +
                        "STORE A INTO 'bar' USING " + TestLoadStoreFuncLifeCycle.class.getName() + "$Storer();");


        List<Tuple> out = data.get("bar");


        assertEquals("a", out.get(0).get(0));
        assertEquals("b", out.get(1).get(0));
        assertEquals("c", out.get(2).get(0));

View Full Code Here

    public void testJoinType(String joinType, boolean preSort) throws Exception {
        Properties props = PropertiesUtil.loadDefaultProperties();
        props.setProperty("pig.schematuple", "true");
        PigServer pigServer = new PigServer(ExecType.LOCAL, props);


        Data data = resetData(pigServer);


        data.set("foo1",
            tuple(0),
            tuple(1),
            tuple(2),
            tuple(3),
            tuple(4),
            tuple(5),
            tuple(6),
            tuple(7),
            tuple(8),
            tuple(9)
            );


        data.set("foo2",
            tuple(0),
            tuple(1),
            tuple(2),
            tuple(3),
            tuple(4),
            tuple(5),
            tuple(6),
            tuple(7),
            tuple(8),
            tuple(9)
            );


        pigServer.registerQuery("A = LOAD 'foo1' USING mock.Storage() as (x:int);");
        pigServer.registerQuery("B = LOAD 'foo2' USING mock.Storage() as (x:int);");
        if (preSort) {
            pigServer.registerQuery("A = ORDER A BY x ASC;");
            pigServer.registerQuery("B = ORDER B BY x ASC;");
        }
        pigServer.registerQuery("C = JOIN A by x, B by x using '"+joinType+"';");
        pigServer.registerQuery("D = ORDER C BY $0 ASC;");


        Iterator<Tuple> out = pigServer.openIterator("D");
        for (int i = 0; i < 10; i++) {
            if (!out.hasNext()) {
                throw new Exception("Output should have had more elements! Failed on element: " + i);
            }
            assertEquals(tuple(i, i), out.next());
        }
        assertFalse(out.hasNext());


        pigServer.registerQuery("STORE D INTO 'bar' USING mock.Storage();");


        List<Tuple> tuples = data.get("bar");


        if (tuples.size() != 10) {
            throw new Exception("Output does not have enough elements! List: " + tuples);
        }

View Full Code Here

    }


    @Test
    public void testRetrieveDataFromMap() throws Exception {
        pigServerLocal = new PigServer(ExecType.LOCAL);
        Data data = resetData(pigServerLocal);
        Map<String, String> mapv1 = new HashMap<String, String>();
        mapv1.put("key1", "v11");
        mapv1.put("key2", "v12");
        Map<String, String> mapv2 = new HashMap<String, String>();
        mapv2.put("key1", "v21");
        mapv2.put("key2", "v22");
        data.set("testMap", "maps:map[chararray]", tuple(mapv1), tuple(mapv2));
        String schemaDescription = new String(
                "{" +
                      "\"type\": \"record\"," +
                      "\"name\": \"record\"," +
                      "\"fields\" : [" +
                      "{\"name\" : \"maps\", \"type\" :{\"type\" : \"map\", \"values\" : \"string\"}}" +
                      "]" +
                      "}");
        pigServerLocal.registerQuery("A = LOAD 'testMap' USING mock.Storage();");
        pigServerLocal.registerQuery("STORE A INTO '" + createOutputName() + "' USING AvroStorage('"+ schemaDescription +"');");
        pigServerLocal.registerQuery("B = LOAD '" + createOutputName() + "' USING AvroStorage();");
        pigServerLocal.registerQuery("C = FOREACH B generate maps#'key1';");
        pigServerLocal.registerQuery("STORE C INTO 'out' USING mock.Storage();");


        List<Tuple> out = data.get("out");
        assertEquals(tuple("v11"), out.get(0));
        assertEquals(tuple("v21"), out.get(1));
    }

View Full Code Here

        r = new Random(42L);
    }


    @Test
    public void testConcat() throws Exception {
        Data data = resetData(pigServer);


        Set<Tuple> inputs = ImmutableSet.of(tuple("a"), tuple("b"), tuple("c"));
        Set<Tuple> expected = Sets.newHashSet();


        for (Tuple t : inputs) {
            String str = (String)t.get(0);
            expected.add(tuple(str.concat(str)));
        }


        data.set("foo", Utils.getSchemaFromString("x:chararray"), inputs);


        pigServer.registerQuery("define concat InvokerGenerator('java.lang.String','concat','String');");
        pigServer.registerQuery("a = load 'foo' using mock.Storage();");
        pigServer.registerQuery("b = foreach @ generate concat($0, $0);");
        pigServer.registerQuery("store b into 'bar' using mock.Storage();");


        List<Tuple> results = data.get("bar");
        assertEquals(expected.size(), results.size());
        for (Tuple t : results) {
            assertTrue(expected.remove(t));
        }
        assertEquals(0, expected.size());

View Full Code Here

        assertEquals(0, expected.size());
    }


    @Test
    public void testValueOf() throws Exception {
        Data data = resetData(pigServer);


        Set<Tuple> inputs = Sets.newHashSet();
        while (inputs.size() < 1000) {
            inputs.add(tuple(Integer.toString(r.nextInt())));
        }
        Set<Tuple> expected = Sets.newHashSet();


        for (Tuple t : inputs) {
            String str = (String)t.get(0);
            expected.add(tuple(Integer.valueOf(str)));
        }


        data.set("foo", Utils.getSchemaFromString("x:chararray"), inputs);


        pigServer.registerQuery("define valueOf InvokerGenerator('java.lang.Integer','valueOf','String');");
        pigServer.registerQuery("a = load 'foo' using mock.Storage();");
        pigServer.registerQuery("b = foreach @ generate valueOf($0);");
        pigServer.registerQuery("store b into 'bar' using mock.Storage();");


        List<Tuple> results = data.get("bar");
        assertEquals(expected.size(), results.size());
        for (Tuple t : results) {
            assertTrue(expected.remove(t));
        }
        assertEquals(0, expected.size());

View Full Code Here

    }
    
    // See: PIG-2937
    @Test
    public void testRelationAliasInNestedForeachWhereUnspecified() throws Exception {
        Data data = resetData(pigServer);
        List<Tuple> values = Lists.newArrayList();
        for (int i = 0; i < 10; i++) {
          values.add(tuple(i % 3 == 0 ? null : "a", "b"));
        }
        data.set("foo", values);
        pigServer.registerQuery("raw_data = load 'foo' using mock.Storage() as (field_a:chararray, field_b:chararray);");
        pigServer.registerQuery("records = foreach raw_data {" +
            "  generated_field = (field_a is null ? '-' : field_b);"+
            "  GENERATE" +
            "    field_a," +

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.builtin.mock.Storage.Data

org.apache.phoenix.pig.PhoenixHBaseLoaderIT

org.apache.phoenix.pig.PhoenixHBaseStorerIT

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.TestJobControlSleep

org.apache.pig.builtin.mock.TestMockStorage

org.apache.pig.builtin.TestAvroStorage

org.apache.pig.builtin.TestBigTypeSort

org.apache.pig.builtin.TestCurrentTime

org.apache.pig.builtin.TestInvokerGenerator

org.apache.pig.builtin.TestPluckTuple

org.apache.pig.data.TestSchemaTuple

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.