Package org.apache.pig

Examples of org.apache.pig.PigServer

180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
        for(int i = 0; i < 10; i++) {
            if(nos[i] > 0) count ++;
        }

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group;");

        ExecJob job = pigServer.store("c", "output");
        PigStats pigStats = job.getStatistics();
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());

        long filesize = 0;
        while(is.read() != -1) filesize++;
       
        is.close();
View Full Code Here
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
        for(int i = 0; i < 10; i++) {
            if(nos[i] > 0) count ++;
        }

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group;");
        ExecJob job = pigServer.store("c", "output", "BinStorage");
        PigStats pigStats = job.getStatistics();

        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
        is.close();
       
View Full Code Here
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
        for(int i = 0; i < 10; i++) {
            if(nos[i] > 0) count ++;
        }

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, SUM(a.$1);");
        ExecJob job = pigServer.store("c", "output");
        PigStats pigStats = job.getStatistics();

        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
        is.close();
View Full Code Here
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
        for(int i = 0; i < 10; i++) {
            if(nos[i] > 0) count ++;
        }

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, SUM(a.$1);");

        ExecJob job = pigServer.store("c", "output", "BinStorage");
        PigStats pigStats = job.getStatistics();
       
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());

        long filesize = 0;
        while(is.read() != -1) filesize++;
       
        is.close();
View Full Code Here
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
        for(int i = 0; i < 10; i++) {
            if(nos[i] > 0) count ++;
        }

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = order a by $0;");
        pigServer.registerQuery("c = group b by $0;");
        pigServer.registerQuery("d = foreach c generate group, SUM(b.$1);");
        ExecJob job = pigServer.store("d", "output");
        PigStats pigStats = job.getStatistics();
       
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
        is.close();
       
View Full Code Here
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
            int t = r.nextInt(100);
            pw.println(t);
        }
        pw.close();
       
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE,
                cluster.getProperties());
        pigServer.setBatchOn();
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 > 50;");
        pigServer.registerQuery("c = filter a by $0 <= 50;");
        pigServer.registerQuery("store b into '/tmp/outout1';");
        pigServer.registerQuery("store c into '/tmp/outout2';");
        List<ExecJob> jobs = pigServer.executeBatch();
        PigStats stats = jobs.get(0).getStatistics();
        assertTrue(stats.getOutputLocations().size() == 2);
       
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("/tmp/outout1"), true);
View Full Code Here
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
        int groups = 0;
        for (int i : nums) {
            if (i > 0) groups++;
        }
       
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE,
                cluster.getProperties());
        pigServer.setBatchOn();
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 >= 50;");
        pigServer.registerQuery("c = group b by $0;");
        pigServer.registerQuery("d = foreach c generate group;");
        pigServer.registerQuery("e = filter a by $0 < 50;");
        pigServer.registerQuery("f = group e by $0;");
        pigServer.registerQuery("g = foreach f generate group;");
        pigServer.registerQuery("store d into '/tmp/outout1';");
        pigServer.registerQuery("store g into '/tmp/outout2';");
        List<ExecJob> jobs = pigServer.executeBatch();
        PigStats stats = jobs.get(0).getStatistics();
       
        assertTrue(stats.getOutputLocations().size() == 2);
              
        cluster.getFileSystem().delete(new Path(file), true);
View Full Code Here
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
            }
            pw2.close();
            multiInputCreated = true;
        }
       
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE,
                cluster.getProperties());
        pigServer.setBatchOn();
        pigServer.registerQuery("a = load '" + file1 + "';");
        pigServer.registerQuery("b = load '" + file2 + "';");
        if (keyword.equals("join") || keyword.endsWith("cogroup")) {
            pigServer.registerQuery("c = " + keyword + " a by $0, b by $0;");
        } else if (keyword.equals("skewed")) {
            pigServer.registerQuery("c = join a by $0, b by $0 using 'skewed';");
        }
        ExecJob job = pigServer.store("c", output + "_output");
       
        PigStats stats = job.getStatistics();
        assertTrue(stats.isSuccessful());
        List<InputStats> inputs = stats.getInputStats();
        if (keyword.equals("join") || keyword.endsWith("cogroup")) {
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
                // backslash - hence 4. In a pig script in a file, this would be
                // www\\.xyz\\.com
                "define minelogs org.apache.pig.test.RegexGroupCount('www\\\\.xyz\\\\.com/sports');" ,
            "A = load 'file://" + f.getAbsolutePath() + "'  using PigStorage() as (source : chararray);" ,
            "B = foreach A generate minelogs(source) as sportslogs;" };
        PigServer ps = new PigServer(ExecType.LOCAL);
        for (String line : queryLines) {
            ps.registerQuery(line);
        }
        Iterator<Tuple> it = ps.openIterator("B");
        int[] expectedResults = new int[] {2,0,1,0,0};
        int i = 0;
        while(it.hasNext()) {
            Tuple t = it.next();
            assertEquals(expectedResults[i++], t.get(0));
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
            int t = r.nextInt(100);
            pw.println(t);
            if(t > 50) count ++;
        }
        pw.close();
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 > 50;");
        pigServer.registerQuery("c = foreach b generate $0 - 50;");
        ExecJob job = pigServer.store("c", "output_map_only");
        PigStats pigStats = job.getStatistics();
       
        //counting the no. of bytes in the output file
        //long filesize = cluster.getFileSystem().getFileStatus(new Path("output_map_only")).getLen();
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath(
                "output_map_only", pigServer.getPigContext()), pigServer
                .getPigContext());

        long filesize = 0;
        while(is.read() != -1) filesize++;
       
View Full Code Here
TOP

Related Classes of org.apache.pig.PigServer

  • org.apache.howl.mapreduce.TestHowlHiveCompatibility
  • org.apache.howl.pig.TestHowlLoader
  • org.apache.howl.pig.TestHowlStorer
  • org.apache.howl.pig.TestHowlStorerMulti
  • org.apache.howl.pig.TestPigStorageDriver
  • org.apache.pig.backend.datastorage.ContainerDescriptor
  • org.apache.pig.backend.datastorage.DataStorage
  • org.apache.pig.backend.datastorage.ElementDescriptor
  • org.apache.pig.backend.executionengine.ExecJob
  • org.apache.pig.backend.executionengine.ExecPhysicalPlan
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.