Examples of org.apache.pig.PigServer$SortInfoSetter

org.apache.pig.PigServer
A class for Java programs to connect to Pig. Typically a program will create a PigServer instance. The programmer then registers queries using registerQuery() and retrieves results using openIterator() or store(). After doing so, the shutdown() method should be called to free any resources used by the current PigServer instance. Not doing so could result in a memory leak.


        for(int i = 0; i < 10; i++) {
            if(nos[i] > 0) count ++;
        }


        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group;");


        ExecJob job = pigServer.store("c", "output");
        PigStats pigStats = job.getStatistics();
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());


        long filesize = 0;
        while(is.read() != -1) filesize++;
        
        is.close();

View Full Code Here


        for(int i = 0; i < 10; i++) {
            if(nos[i] > 0) count ++;
        }


        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group;");
        ExecJob job = pigServer.store("c", "output", "BinStorage");
        PigStats pigStats = job.getStatistics();


        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());
        long filesize = 0;
        while(is.read() != -1) filesize++;
        
        is.close();

View Full Code Here


        for(int i = 0; i < 10; i++) {
            if(nos[i] > 0) count ++;
        }


        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, SUM(a.$1);");
        ExecJob job = pigServer.store("c", "output");
        PigStats pigStats = job.getStatistics();


        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());
        long filesize = 0;
        while(is.read() != -1) filesize++;
        
        is.close();

View Full Code Here


        for(int i = 0; i < 10; i++) {
            if(nos[i] > 0) count ++;
        }


        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, SUM(a.$1);");


        ExecJob job = pigServer.store("c", "output", "BinStorage");
        PigStats pigStats = job.getStatistics();
        
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());


        long filesize = 0;
        while(is.read() != -1) filesize++;
        
        is.close();

View Full Code Here


        for(int i = 0; i < 10; i++) { 
            if(nos[i] > 0) count ++;
        }


        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = order a by $0;");
        pigServer.registerQuery("c = group b by $0;");
        pigServer.registerQuery("d = foreach c generate group, SUM(b.$1);");
        ExecJob job = pigServer.store("d", "output");
        PigStats pigStats = job.getStatistics();
        
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());
        long filesize = 0;
        while(is.read() != -1) filesize++;
        
        is.close();

View Full Code Here

            int t = r.nextInt(100);
            pw.println(t);
        }
        pw.close();
        
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, 
                cluster.getProperties());
        pigServer.setBatchOn();
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 > 50;");
        pigServer.registerQuery("c = filter a by $0 <= 50;");
        pigServer.registerQuery("store b into '/tmp/outout1';");
        pigServer.registerQuery("store c into '/tmp/outout2';");
        List<ExecJob> jobs = pigServer.executeBatch();
        PigStats stats = jobs.get(0).getStatistics();
        assertTrue(stats.getOutputLocations().size() == 2);
        
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("/tmp/outout1"), true);

View Full Code Here

        int groups = 0;
        for (int i : nums) {
            if (i > 0) groups++;
        }
        
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, 
                cluster.getProperties());
        pigServer.setBatchOn();
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 >= 50;");
        pigServer.registerQuery("c = group b by $0;");
        pigServer.registerQuery("d = foreach c generate group;");
        pigServer.registerQuery("e = filter a by $0 < 50;");
        pigServer.registerQuery("f = group e by $0;");
        pigServer.registerQuery("g = foreach f generate group;");
        pigServer.registerQuery("store d into '/tmp/outout1';");
        pigServer.registerQuery("store g into '/tmp/outout2';");
        List<ExecJob> jobs = pigServer.executeBatch();
        PigStats stats = jobs.get(0).getStatistics();
        
        assertTrue(stats.getOutputLocations().size() == 2);
               
        cluster.getFileSystem().delete(new Path(file), true);

View Full Code Here

            }
            pw2.close();
            multiInputCreated = true;
        }
        
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, 
                cluster.getProperties());
        pigServer.setBatchOn();
        pigServer.registerQuery("a = load '" + file1 + "';");
        pigServer.registerQuery("b = load '" + file2 + "';");
        if (keyword.equals("join") || keyword.endsWith("cogroup")) {
            pigServer.registerQuery("c = " + keyword + " a by $0, b by $0;");
        } else if (keyword.equals("skewed")) {
            pigServer.registerQuery("c = join a by $0, b by $0 using 'skewed';");
        }
        ExecJob job = pigServer.store("c", output + "_output");
        
        PigStats stats = job.getStatistics();
        assertTrue(stats.isSuccessful());
        List<InputStats> inputs = stats.getInputStats();
        if (keyword.equals("join") || keyword.endsWith("cogroup")) {

View Full Code Here

                // backslash - hence 4. In a pig script in a file, this would be
                // www\\.xyz\\.com
                "define minelogs org.apache.pig.test.RegexGroupCount('www\\\\.xyz\\\\.com/sports');" ,
            "A = load 'file://" + f.getAbsolutePath() + "'  using PigStorage() as (source : chararray);" ,
            "B = foreach A generate minelogs(source) as sportslogs;" };
        PigServer ps = new PigServer(ExecType.LOCAL);
        for (String line : queryLines) {
            ps.registerQuery(line);
        }
        Iterator<Tuple> it = ps.openIterator("B");
        int[] expectedResults = new int[] {2,0,1,0,0};
        int i = 0;
        while(it.hasNext()) {
            Tuple t = it.next();
            assertEquals(expectedResults[i++], t.get(0));

View Full Code Here

            int t = r.nextInt(100);
            pw.println(t);
            if(t > 50) count ++;
        }
        pw.close();
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 > 50;");
        pigServer.registerQuery("c = foreach b generate $0 - 50;");
        ExecJob job = pigServer.store("c", "output_map_only");
        PigStats pigStats = job.getStatistics();
        
        //counting the no. of bytes in the output file
        //long filesize = cluster.getFileSystem().getFileStatus(new Path("output_map_only")).getLen();
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath(
                "output_map_only", pigServer.getPigContext()), pigServer
                .getPigContext());


        long filesize = 0;
        while(is.read() != -1) filesize++;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.PigServer$SortInfoSetter

org.apache.howl.mapreduce.TestHowlHiveCompatibility

org.apache.howl.pig.TestHowlLoader

org.apache.howl.pig.TestHowlStorer

org.apache.howl.pig.TestHowlStorerMulti

org.apache.howl.pig.TestPigStorageDriver

org.apache.pig.backend.datastorage.ContainerDescriptor

org.apache.pig.backend.datastorage.DataStorage

org.apache.pig.backend.datastorage.ElementDescriptor

org.apache.pig.backend.executionengine.ExecJob

org.apache.pig.backend.executionengine.ExecPhysicalPlan

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.