Package org.apache.pig.builtin.mock.Storage

Examples of org.apache.pig.builtin.mock.Storage.Data

68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
    }

    @Test
    public void testSchemaResetterExec() throws IOException {
        PigServer pigServer = new PigServer(LOCAL);
        Data data = Storage.resetData(pigServer);
        data.set("input",
                tuple(tuple("1", "2")),
                tuple(tuple("2", "3")),
                tuple(tuple("2", "4")));
        pigServer.registerQuery(
                "A = LOAD 'input' USING mock.Storage() AS (group:tuple(uid, dst_id));" +
                "edges_both = FOREACH A GENERATE" +
                "    group.uid AS src_id," +
                "    group.dst_id AS dst_id;" +
                "both_counts = GROUP edges_both BY src_id;" +
                "both_counts = FOREACH both_counts GENERATE" +
                "    group AS src_id, SIZE(edges_both) AS size_both;" +
                "edges_bq = FOREACH A GENERATE" +
                "    group.uid AS src_id," +
                "    group.dst_id AS dst_id;" +
                "bq_counts = GROUP edges_bq BY src_id;" +
                "bq_counts = FOREACH bq_counts GENERATE" +
                "    group AS src_id, SIZE(edges_bq) AS size_bq;" +
                "per_user_set_sizes = JOIN bq_counts BY src_id LEFT OUTER, both_counts BY src_id;" +
                "store per_user_set_sizes into 'output' USING mock.Storage();");
        List<Tuple> list = data.get("output");
        Collections.sort(list);
        assertEquals("list: "+list, 2, list.size());
        assertEquals("(1,1,1,1)", list.get(0).toString());
        assertEquals("(2,2,2,2)", list.get(1).toString());
    }
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
   * @see HadoopShims#newJobControl(String, org.apache.hadoop.conf.Configuration, org.apache.pig.impl.PigContext)
   */
  @Test
  public void testLocalModeTakesLessThan5secs() throws Exception {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Data data = resetData(pigServer);
    data.set("in", tuple("a"));
    long t0 = System.currentTimeMillis();
    pigServer.registerQuery(
        "A = LOAD 'in' using mock.Storage();\n"
        + "STORE A INTO 'out' USING mock.Storage();");
    long t1 = System.currentTimeMillis();
    List<Tuple> list = data.get("out");
    assertEquals(1, list.size());
    assertEquals("a", list.get(0).get(0));
    assertTrue("must take less than 5 seconds", (t1 - t0) < 5000);
  }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
        pigServer = new PigServer(ExecType.LOCAL);
    }
   
    @Test
    public void testDuplicate() throws Exception {
        Data data = resetData(pigServer);

        Map<String,String> m1 = ImmutableMap.of("name", "jon", "age", "26");
       
        data.set("foo", Utils.getSchemaFromString("a:[chararray]"), tuple(m1));
       
        pigServer.registerQuery("a = load 'foo' using mock.Storage() as (a:map[chararray]);");
        pigServer.registerQuery("b = foreach @ generate a#'name' as name, a#'age' as age;");
        pigServer.registerQuery("c = foreach @ generate *;");
        Schema s = pigServer.dumpSchema("c");
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
     * @throws Exception
     */
    @Test
    public void testNoElseBranch() throws Exception {
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        Data data = resetData(pigServer);

        data.set("foo",
                tuple(1),
                tuple(2),
                tuple(3),
                tuple(4),
                tuple(5),
                tuple(6),
                tuple(7)
                );

        pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (i:int);");
        pigServer.registerQuery("B = FOREACH A GENERATE i, (" +
                "  CASE i % 5" +
                "    WHEN 0 THEN '5n'" +
                "    WHEN 1 THEN '5n+1'" +
                "    WHEN 2 THEN '5n+2'" +
                "    WHEN 3 THEN '5n+3'" +
                "  END" +
                ") AS s;");
        pigServer.registerQuery("C = FILTER B BY s IS NOT NULL;");
        pigServer.registerQuery("STORE C INTO 'bar' USING mock.Storage();");

        List<Tuple> out = data.get("bar");
        assertEquals(6, out.size());
        assertEquals(tuple(1,"5n+1"), out.get(0));
        assertEquals(tuple(2,"5n+2"), out.get(1));
        assertEquals(tuple(3,"5n+3"), out.get(2));
        assertEquals(tuple(5,"5n"),   out.get(3));
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
     * @throws Exception
     */
    @Test
    public void testWithElseBranch() throws Exception {
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        Data data = resetData(pigServer);

        data.set("foo",
                tuple(1),
                tuple(2),
                tuple(3),
                tuple(4),
                tuple(5),
                tuple(6),
                tuple(7)
                );

        pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (i:int);");
        pigServer.registerQuery("B = FOREACH A GENERATE i, (" +
                "  CASE i % 4" +
                "    WHEN 0 THEN '4n'" +
                "    WHEN 1 THEN '4n+1'" +
                "    WHEN 2 THEN '4n+2'" +
                "    ELSE        '4n+3'" +
                "  END" +
                ");");
        pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");

        List<Tuple> out = data.get("bar");
        assertEquals(7, out.size());
        assertEquals(tuple(1,"4n+1"), out.get(0));
        assertEquals(tuple(2,"4n+2"), out.get(1));
        assertEquals(tuple(3,"4n+3"), out.get(2));
        assertEquals(tuple(4,"4n"),   out.get(3));
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
     * @throws Exception
     */
    @Test
    public void testConditionalWithNoElse() throws Exception {
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        Data data = resetData(pigServer);

        data.set("foo",
                tuple(1),
                tuple(2),
                tuple(3),
                tuple(4),
                tuple(5),
                tuple(6),
                tuple(7)
                );

        pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (i:int);");
        pigServer.registerQuery("B = FOREACH A GENERATE i, (" +
                "  CASE " +
                "    WHEN i % 5 == 0 THEN '5n'" + // Conditional expression in when branch
                "    WHEN i % 5 == 1 THEN '5n+1'" +
                "    WHEN i % 5 == 2 THEN '5n+2'" +
                "    WHEN i % 5 == 3 THEN '5n+3'" +
                "  END" +
                ") AS s;");
        pigServer.registerQuery("C = FILTER B BY s IS NOT NULL;");
        pigServer.registerQuery("STORE C INTO 'bar' USING mock.Storage();");

        List<Tuple> out = data.get("bar");
        assertEquals(6, out.size());
        assertEquals(tuple(1,"5n+1"), out.get(0));
        assertEquals(tuple(2,"5n+2"), out.get(1));
        assertEquals(tuple(3,"5n+3"), out.get(2));
        assertEquals(tuple(5,"5n"),   out.get(3));
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
     * @throws Exception
     */
    @Test
    public void testConditionalWithElse() throws Exception {
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        Data data = resetData(pigServer);

        data.set("foo",
                tuple(1),
                tuple(2),
                tuple(3),
                tuple(4),
                tuple(5),
                tuple(6),
                tuple(7)
                );

        pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (i:int);");
        pigServer.registerQuery("B = FOREACH A GENERATE i, (" +
                "  CASE " +
                "    WHEN i % 4 == 0 THEN '4n'" + // Conditional expression in when branch
                "    WHEN i % 4 == 1 THEN '4n+1'" +
                "    WHEN i % 4 == 2 THEN '4n+2'" +
                "    ELSE                 '4n+3'" +
                "  END" +
                ");");
        pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");

        List<Tuple> out = data.get("bar");
        assertEquals(7, out.size());
        assertEquals(tuple(1,"4n+1"), out.get(0));
        assertEquals(tuple(2,"4n+2"), out.get(1));
        assertEquals(tuple(3,"4n+3"), out.get(2));
        assertEquals(tuple(4,"4n"),   out.get(3));
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
     * @throws Exception
     */
    @Test
    public void testOrderOfConditions() throws Exception {
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        Data data = resetData(pigServer);

        data.set("foo",
                tuple(1),
                tuple(5),
                tuple(10),
                tuple(15),
                tuple(20),
                tuple(25),
                tuple(30)
                );

        pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (i:int);");
        pigServer.registerQuery("B = FOREACH A GENERATE i, (" +
                "  CASE " +
                "    WHEN i > 20 THEN '> 20'" + // Conditions are not mutually exclusive
                "    WHEN i > 10 THEN '> 10'" +
                "    ELSE             '> 0'" +
                "  END" +
                ");");
        pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");

        List<Tuple> out = data.get("bar");
        assertEquals(7, out.size());
        assertEquals(tuple(1,"> 0"),   out.get(0));
        assertEquals(tuple(5,"> 0"),   out.get(1));
        assertEquals(tuple(10,"> 0"),  out.get(2));
        assertEquals(tuple(15,"> 10"), out.get(3));
View Full Code Here
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
     * @throws Exception
     */
    @Test
    public void testWithDereferenceOperator() throws Exception {
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        Data data = resetData(pigServer);

        data.set("foo",
                tuple("a","x",1),
                tuple("a","y",1),
                tuple("b","x",2),
                tuple("b","y",2),
                tuple("c","x",3),
                tuple("c","y",3)
                );

        pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (c1:chararray, c2:chararray, i:int);");
        pigServer.registerQuery("B = GROUP A BY (c1, i);");
        pigServer.registerQuery("C = FOREACH B GENERATE group.i, (" +
                "  CASE group.i % 3" +
                "    WHEN 0 THEN '3n'" +
                "    WHEN 1 THEN '3n+1'" +
                "    ELSE        '3n+2'" +
                "  END" +
                "), A.(c1, c2);");
        pigServer.registerQuery("STORE C INTO 'bar' USING mock.Storage();");

        List<Tuple> out = data.get("bar");
        assertEquals(3, out.size());
        assertEquals(tuple(1, "3n+1", bag(tuple("a","x"), tuple("a","y"))), out.get(0));
        assertEquals(tuple(2, "3n+2", bag(tuple("b","x"), tuple("b","y"))), out.get(1));
        assertEquals(tuple(3, "3n",   bag(tuple("c","x"), tuple("c","y"))), out.get(2));
    }
View Full Code Here
280
281
282
283
284
285
286
287
288
289
290
291
292
     * @throws Exception
     */
    @Test(expected = FrontendException.class)
    public void testMissingCaseExpression() throws Exception {
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        Data data = resetData(pigServer);

        data.set("foo",
                tuple(1),
                tuple(2),
                tuple(3),
                tuple(4),
                tuple(5)
View Full Code Here
TOP

Related Classes of org.apache.pig.builtin.mock.Storage.Data

  • org.apache.phoenix.pig.PhoenixHBaseLoaderIT
  • org.apache.phoenix.pig.PhoenixHBaseStorerIT
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.TestJobControlSleep
  • org.apache.pig.builtin.mock.TestMockStorage
  • org.apache.pig.builtin.TestAvroStorage
  • org.apache.pig.builtin.TestBigTypeSort
  • org.apache.pig.builtin.TestCurrentTime
  • org.apache.pig.builtin.TestInvokerGenerator
  • org.apache.pig.builtin.TestPluckTuple
  • org.apache.pig.data.TestSchemaTuple
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.