Package org.apache.pig

Examples of org.apache.pig.PigServer.store()


            pigServer.registerQuery("a = load 'test/org/apache/pig/test/data/passwd';");
            pigServer.registerQuery("b = group a by $0;");
            pigServer.registerQuery("c = foreach b generate group, COUNT(a) as cnt;");
            pigServer.registerQuery("d = group c by cnt;");
            pigServer.registerQuery("e = foreach d generate group;");
            ExecJob job = pigServer.store("e", filePath);
            JobGraph jobGraph = job.getStatistics().getJobGraph();
            assertEquals(2, jobGraph.getJobList().size());

        } catch (IOException e) {
            LOG.error("IOException while creating file ", e);
View Full Code Here


            cos.write((-i + "\n").getBytes());
        }
        cos.close();
        pig.registerQuery("AA=load '" + Util.generateURI(in.getAbsolutePath()) + "';");
        pig.registerQuery("A=foreach (group (filter AA by $0 > 0) all) generate flatten($1);");
        pig.store("A", Util.generateURI(out.getAbsolutePath()));
        CBZip2InputStream cis = new CBZip2InputStream(
                new LocalSeekableInputStream(new File(out, "part-00000.bz")));
        // Just a sanity check, to make sure it was a bzip file; we
        // will do the value verification later
        assertEquals(100, cis.read(new byte[100]));
View Full Code Here

        fos.close();
        System.out.println(in.getAbsolutePath());
        pig.registerQuery("AA=load '" + Util.generateURI(in.getAbsolutePath()) + "';");
        pig
                .registerQuery("A=foreach (group (filter AA by $0 < '0') all) generate flatten($1);");
        pig.store("A", Util.generateURI(out.getAbsolutePath()));
        CBZip2InputStream cis = new CBZip2InputStream(
                new LocalSeekableInputStream(new File(out, "part-00000.bz")));
        assertEquals(-1, cis.read(new byte[100]));
        cis.close();
        pig.registerQuery("B=load '" + Util.generateURI(out.getAbsolutePath()) + "';");
View Full Code Here

        disabledOptimizerRules.add("PredicatePushdownOptimizer");
        pigServer_disabledRule.getPigContext().getProperties().setProperty(PigImplConstants.PIG_OPTIMIZER_RULES_KEY,
                ObjectSerializer.serialize(disabledOptimizerRules));
        pigServer_disabledRule.registerQuery("B = load '" + inputFile + "' using OrcStorage();");
        pigServer_disabledRule.registerQuery("C = filter B by " + filterStmt + ";");
        ExecJob job = pigServer_disabledRule.store("C", OUTPUT3);
        //Util.copyFromClusterToLocal(cluster, OUTPUT3 + "/part-m-00000", OUTPUT3);
        JobStats stats = (JobStats) job.getStatistics().getJobGraph().getSources().get(0);
        assertEquals(expectedRows, stats.getRecordWrittern());
        long bytesWithoutPushdown = stats.getHdfsBytesRead();
View Full Code Here

        pw.close();
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 > 50;");
        pigServer.registerQuery("c = foreach b generate $0 - 50;");
        PigStats pigStats = pigServer.store("c", "output_map_only").getStatistics();

        //PigStats pigStats = pigServer.getPigStats();
        Map<String, Map<String, String>> stats = pigStats.getPigStats();
       
        //counting the no. of bytes in the output file
View Full Code Here

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 > 50;");
        pigServer.registerQuery("c = foreach b generate $0 - 50;");
        //pigServer.store("c", "output_map_only");
        PigStats pigStats = pigServer.store("c", "output_map_only", "BinStorage").getStatistics();
       
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output_map_only", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
View Full Code Here

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group;");
        PigStats pigStats = pigServer.store("c", "output").getStatistics();
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
        is.close();
View Full Code Here

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group;");
        PigStats pigStats = pigServer.store("c", "output", "BinStorage").getStatistics();

        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
View Full Code Here

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, SUM(a.$1);");
        PigStats pigStats = pigServer.store("c", "output").getStatistics();

        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
View Full Code Here

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, SUM(a.$1);");
        PigStats pigStats = pigServer.store("c", "output", "BinStorage").getStatistics();

        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
View Full Code Here

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.