Package org.apache.pig.builtin.mock.Storage

Examples of org.apache.pig.builtin.mock.Storage.Data

236
237
238
239
240
241
242
243
244
245
246
247
248
     */
    @Test(expected = FrontendException.class)
    public void testWhitelistWithPigServer() throws Exception {
        ctx.getProperties().setProperty(PigConfiguration.PIG_WHITELIST, "load");
        PigServer pigServer = new PigServer(ctx);
        Data data = resetData(pigServer);

        data.set("foo", tuple("a", 1, "b"), tuple("b", 2, "c"),
                tuple("c", 3, "d"));

        pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
        pigServer.registerQuery("B = order A by f1,f2,f3 DESC;");
        pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");
View Full Code Here
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
     * Tests whether 'pig.job.submitted.timestamp' has been set by FetchLauncher
     * @throws Exception
     */
    @Test
    public void test7() throws Exception {
        Data data = resetData(pigServer);

        List<Tuple> justSomeRows = Lists.newArrayListWithCapacity(1);
        justSomeRows.add(tuple(1));
        data.set("justSomeRows", justSomeRows);

        pigServer.registerQuery("A = load 'justSomeRows' using mock.Storage();");
        pigServer.registerQuery("B = foreach A generate CurrentTime();");
        Iterator<Tuple> it = pigServer.openIterator("B");
        DateTime received = (DateTime) it.next().get(0);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
        log.info("Total no. of iterations to run for test data = " + total);

        Random rand = new Random();

        pig = new PigServer(ExecType.LOCAL);
        Data data = resetData(pig);
        List<Tuple> theInput = Lists.newArrayList();

        for(int i = 0; i < total; i++) {
            Double x = new Double(rand.nextDouble());
            Double y = new Double(rand.nextDouble());
            double x1 = x.doubleValue() - 0.5;
            double y1 = y.doubleValue() - 0.5;
            double sq_dist = (x1*x1) + (y1*y1);
            if(sq_dist <= 0.25) {
                inCircle ++;

            }

            theInput.add(tuple(x, y));

            totalLength += String.valueOf(sq_dist).length();
        }
        data.set("foo", theInput);
    }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
    // tests that when a filter inside a foreach filters away all tuples
    // for a group, an empty bag is still provided to udfs whose
    // input is the filter
    @Test
    public void testFilterCount1() throws IOException, ParserException {
        Data data = resetData(pigServer);
        data.set("foo", tuple(1,1,3), tuple(1,2,3), tuple(2,1,3), tuple(2,1,3));
        String script = "test   = load 'foo' using mock.Storage() as (col1: int, col2: int, col3: int);" +
                "test2 = group test by col1;" +
                "test3 = foreach test2 {" +
                "        filter_one    = filter test by (col2==1);" +
                "        filter_notone = filter test by (col2!=1);" +
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
    // tests that when a filter inside a foreach filters away all tuples
    // for a group, an empty bag is still provided to udfs whose
    // input is the filter
    @Test
    public void testFilterCount2() throws IOException, ParserException {
        Data data = resetData(pigServer);
        data.set("foo",
                tuple("a", "hello"),
                tuple("a", "goodbye"),
                tuple("b", "goodbye"),
                tuple("c", "hello"),
                tuple("c", "hello"),
View Full Code Here
191
192
193
194
195
196
197
198
199
200
201
202
    // test case where RelationToExprProject is present in the
    // single inner plan of foreach - this will test that it does
    // send an EOP eventually for each input of the foreach
    @Test
    public void testFilter1() throws IOException, ParserException {
        Data data = resetData(pigServer);
        data.set("foo", tuple(1,1,3), tuple(1,2,3), tuple(2,1,3), tuple(2,1,3), tuple(3,4,4));

        String script = "test   = load 'foo' using mock.Storage() as (col1: int, col2: int, col3: int);" +
                "test2 = group test by col1;" +
                "test3 = foreach test2 {" +
                "        filter_one    = filter test by (col2==1);" +
View Full Code Here
231
232
233
234
235
236
237
238
239
240
241
242
    // in foreach - this will test that reset() correctly resets
    // the state that empty bags need to be sent on EOP if no non-EOP
    // input has been seen on a fresh input from foreach.
    @Test
    public void testFilter2() throws IOException, ParserException {
        Data data = resetData(pigServer);
        data.set("foo", tuple(1,1,3), tuple(1,2,3), tuple(2,1,3), tuple(2,1,3), tuple(3,4,4));

        String script = "test   = load 'foo' using mock.Storage() as (col1: int, col2: int, col3: int);" +
                "test2 = group test by col1;" +
                "test3 = foreach test2 {" +
                "        filter_one    = filter test by (col2==1);" +
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
    public void testRemoteServerList() throws ExecException, IOException {
        Properties pigProperties = pigServer.getPigContext().getProperties();
        pigProperties.setProperty("fs.default.name", "hdfs://a.com:8020");
        Configuration conf;
       
        Data data = Storage.resetData(pigServer.getPigContext());
        data.set("/user/pig/1.txt");// no data

        pigServer.registerQuery("a = load '/user/pig/1.txt' using mock.Storage;");
        conf = ConfigurationUtil.toConfiguration(pigProperties);
        assertTrue(conf.get("mapreduce.job.hdfs-servers") == null ||
                conf.get("mapreduce.job.hdfs-servers").equals(pigProperties.get("fs.default.name"))||
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
149
        pigProperties.setProperty("fs.default.name", "hdfs://a.com:8020");
        Configuration conf;

        pigServer.setBatchOn();

        Data data = Storage.resetData(pigServer.getPigContext());
        data.set("/user/pig/1.txt");// no data

        pigServer.registerQuery("a = load '/user/pig/1.txt' using mock.Storage;");
        pigServer.registerQuery("store a into '/user/pig/1.txt';");

        System.out.println("hdfs-servers: "
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
    @Test
    public void testRestrictedColumnNamesWhitelist() throws Exception {
        pigServer = new PigServer(LOCAL);

        Data data = resetData(pigServer);

        Set<Tuple> tuples = Sets.newHashSet(tuple(1),tuple(2),tuple(3));
        data.set("foo",
            "x:int",
            tuples
            );

        pigServer.registerQuery("a = load 'foo' using mock.Storage();");
        pigServer.registerQuery("a = foreach a generate x as rank;");
        pigServer.registerQuery("a = foreach a generate rank as cube;");
        pigServer.registerQuery("a = foreach a generate cube as y;");
        pigServer.registerQuery("rank = a;");
        pigServer.registerQuery("cube = rank;");
        pigServer.registerQuery("rank = cube;");
        pigServer.registerQuery("cube = foreach rank generate y as cube;");
        pigServer.registerQuery("store cube into 'baz' using mock.Storage();");
        List<Tuple> tuples2 = data.get("baz");
        assertEquals(tuples.size(), tuples2.size());
        for (Tuple t : tuples2) {
            tuples.remove(t);
        }
        assertTrue(tuples.isEmpty());
View Full Code Here
TOP

Related Classes of org.apache.pig.builtin.mock.Storage.Data

  • org.apache.phoenix.pig.PhoenixHBaseLoaderIT
  • org.apache.phoenix.pig.PhoenixHBaseStorerIT
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.TestJobControlSleep
  • org.apache.pig.builtin.mock.TestMockStorage
  • org.apache.pig.builtin.TestAvroStorage
  • org.apache.pig.builtin.TestBigTypeSort
  • org.apache.pig.builtin.TestCurrentTime
  • org.apache.pig.builtin.TestInvokerGenerator
  • org.apache.pig.builtin.TestPluckTuple
  • org.apache.pig.data.TestSchemaTuple
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.