Package org.apache.pig.builtin.mock.Storage

Examples of org.apache.pig.builtin.mock.Storage.Data

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
        pigServer = new PigServer(ExecType.LOCAL);
    }

    @Test
    public void testAllCurrentTimeSame() throws Exception {
        Data data = resetData(pigServer);

        List<Tuple> justSomeRows = Lists.newArrayList();
        for (int i = 0; i < 1000; i++) {
            justSomeRows.add(tuple(i));
        }

        data.set("justSomeRows", justSomeRows);

        pigServer.registerQuery("A = load 'justSomeRows' using mock.Storage();");
        pigServer.registerQuery("B = foreach A generate CurrentTime();");
        pigServer.registerQuery("C = foreach (group B by $0) generate COUNT($1);");
        Iterator<Tuple> it = pigServer.openIterator("C");
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
        assertTrue(Schema.equals(pigServer.dumpSchema("a"), pigServer.dumpSchema("d"), false, true));
    }

    @Test
    public void testOutput() throws Exception {
        Data data = resetData(pigServer);

        Tuple exp1 = tuple(1, "hey", 2L);
        Tuple exp2 = tuple(2, "woah", 3L);

        data.set("a",
            Utils.getSchemaFromString("x:int,y:chararray,z:long"),
            exp1,
            exp2,
            tuple(3, "c", 4L)
            );
        data.set("b",
            Utils.getSchemaFromString("x:int,y:chararray,z:long"),
            tuple(1, "sasf", 5L),
            tuple(2, "woah", 6L),
            tuple(4, "c", 7L)
            );
View Full Code Here
276
277
278
279
280
281
282
283
284
285
286
287
288
289
        grunt.exec();
    }

    @Test
    public void testDumpWithPreviousRelation() throws Exception {
        Data data = resetData(pigServer);
        Set<Tuple> expected = Sets.newHashSet(tuple("a"), tuple("b"), tuple("c"));

        data.set("foo", Utils.getSchemaFromString("x:chararray"), expected);
        pigServer.registerQuery("=> load 'foo' using mock.Storage();");
        Iterator<Tuple> op = pigServer.openIterator("@");
        while (op.hasNext()) {
            assertTrue(expected.remove(op.next()));
        }
View Full Code Here
291
292
293
294
295
296
297
298
299
300
301
302
303
304
        assertTrue(expected.isEmpty());
    }
   
    @Test
    public void testDescribeWithPreviousRelation() throws Exception {
        Data data = resetData(pigServer);
        Set<Tuple> expected = Sets.newHashSet(tuple("a"), tuple("b"), tuple("c"));

        Schema s = Utils.getSchemaFromString("x:chararray");
        data.set("foo", s, expected);
        pigServer.registerQuery("=> load 'foo' using mock.Storage();");
        Schema s2 = pigServer.dumpSchema("@");
        assertEquals(s,s2);
    }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
public class TestMockStorage {

  @Test
  public void testMockStoreAndLoad() throws Exception {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Data data = resetData(pigServer);

    data.set("foo",
        tuple("a"),
        tuple("b"),
        tuple("c")
        );

    pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
    pigServer.registerQuery("STORE A INTO 'bar' USING mock.Storage();");

    List<Tuple> out = data.get("bar");
    assertEquals(tuple("a"), out.get(0));
    assertEquals(tuple("b"), out.get(1));
    assertEquals(tuple("c"), out.get(2));
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
  }
 
  @Test
  public void testMockSchema() throws Exception {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Data data = resetData(pigServer);

    data.set("foo", "blah:chararray",
        tuple("a"),
        tuple("b"),
        tuple("c")
        );

    pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
    pigServer.registerQuery("B = FOREACH A GENERATE blah as a, blah as b;");
    pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");

    assertEquals(schema("a:chararray,b:chararray"), data.getSchema("bar"));

    List<Tuple> out = data.get("bar");
    assertEquals(tuple("a", "a"), out.get(0));
    assertEquals(tuple("b", "b"), out.get(1));
    assertEquals(tuple("c", "c"), out.get(2));
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
  }

  @Test
  public void testMockStoreUnion() throws Exception {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Data data = resetData(pigServer);

    data.set("input1",
        tuple("a"),
        tuple("b"),
        tuple("c")
        );

    data.set("input2",
            tuple("d"),
            tuple("e"),
            tuple("f")
            );

    pigServer.registerQuery("A = LOAD 'input1' USING mock.Storage();");
    pigServer.registerQuery("B = LOAD 'input2' USING mock.Storage();");
    pigServer.registerQuery("C = UNION A, B;");
    pigServer.registerQuery("STORE C INTO 'output' USING mock.Storage();");

    List<Tuple> out = data.get("output");
    assertEquals(out + " size", 6, out.size());
    Set<String> set = new HashSet<String>();
    for (Tuple tuple : out) {
        if (!set.add((String)tuple.get(0))) {
            fail(tuple.get(0) + " is present twice in " + out);
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
  }
 
  @Test
  public void testBadUsage1() throws Exception {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Data data = resetData(pigServer);

    data.set("input1",
            tuple("a"),
            tuple("b"),
            tuple("c")
            );

    try {
        data.set("input1",
                tuple("d"),
                tuple("e"),
                tuple("f")
                );
        fail("should have thrown an exception for setting twice the same input");
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
  }
 
  @Test
  public void testBadUsage2() throws Exception {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Data data = resetData(pigServer);

    data.set("input",
        tuple("a"),
        tuple("b"),
        tuple("c")
        );
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
public class TestImplicitSplitOnTuple {

    @Test
    public void testImplicitSplitterOnTuple() throws IOException {
        PigServer pigServer = new PigServer(LOCAL);
        Data data = Storage.resetData(pigServer);
        data.set("input",
                tuple("1", "1001", "101"),
                tuple("1", "1002", "103"),
                tuple("1", "1003", "102"),
                tuple("1", "1004", "102"),
                tuple("2", "1005", "101"),
                tuple("2", "1003", "101"),
                tuple("2", "1002", "123"),
                tuple("3", "1042", "101"),
                tuple("3", "1005", "101"),
                tuple("3", "1002", "133"));

        pigServer.registerQuery(
                "inp = LOAD 'input' USING mock.Storage() AS (memberId:long, shopId:long, score:int);"+
                "tuplified = FOREACH inp GENERATE (memberId, shopId) AS tuplify, score;"+
                "D1 = FOREACH tuplified GENERATE tuplify.memberId as memberId, tuplify.shopId as shopId, score AS score;"+
                "D2 = FOREACH tuplified GENERATE tuplify.memberId as memberId, tuplify.shopId as shopId, score AS score;"+
                "J = JOIN D1 By shopId, D2 by shopId;"+
                "K = FOREACH J GENERATE D1::memberId AS member_id1, D2::memberId AS member_id2, D1::shopId as shop;"+
                "L = ORDER K by shop;"+
                "STORE L into 'output' using mock.Storage;");
        List<Tuple> list = data.get("output");
        assertEquals("list: "+list, 20, list.size());
        assertEquals("(1,1,1001)", list.get(0).toString());
        assertEquals("(1,1,1002)", list.get(1).toString());
        assertEquals("(1,2,1002)", list.get(2).toString());
        assertEquals("(1,3,1002)", list.get(3).toString());
View Full Code Here
TOP

Related Classes of org.apache.pig.builtin.mock.Storage.Data

  • org.apache.phoenix.pig.PhoenixHBaseLoaderIT
  • org.apache.phoenix.pig.PhoenixHBaseStorerIT
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.TestJobControlSleep
  • org.apache.pig.builtin.mock.TestMockStorage
  • org.apache.pig.builtin.TestAvroStorage
  • org.apache.pig.builtin.TestBigTypeSort
  • org.apache.pig.builtin.TestCurrentTime
  • org.apache.pig.builtin.TestInvokerGenerator
  • org.apache.pig.builtin.TestPluckTuple
  • org.apache.pig.data.TestSchemaTuple
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.