Examples of PigContext


Examples of org.apache.pig.impl.PigContext

            lpt.buildPlan("B = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
            lpt.buildPlan("D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
            LogicalPlan lp = lpt.buildPlan("C = cogroup A by id, B by id, D by id using 'merge';");
            assertEquals(LOCogroup.GROUPTYPE.MERGE, ((LOCogroup)lp.getLeaves().get(0)).getGroupType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
            PhysicalPlan phyP = Util.buildPhysicalPlan(lp, pc);
            PhysicalOperator phyOp = phyP.getLeaves().get(0);
            assertTrue(phyOp instanceof POMergeCogroup);

            lp = lpt.buildPlan("store C into 'out';");
View Full Code Here

Examples of org.apache.pig.impl.PigContext

        lpt.buildPlan("B = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
        lpt.buildPlan("D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
        LogicalPlan lp = lpt.buildPlan("C = cogroup E by A.id, B by id, D by id using 'merge';");
        assertEquals(LOCogroup.GROUPTYPE.MERGE, ((LOCogroup)lp.getLeaves().get(0)).getGroupType());

        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        boolean exceptionCaught = false;
        try{
            Util.buildPhysicalPlan(lp, pc);  
        }catch (LogicalToPhysicalTranslatorException e){
            assertEquals(1103,e.getErrorCode());
View Full Code Here

Examples of org.apache.pig.impl.PigContext

        lpt.buildPlan("B = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
        lpt.buildPlan("D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
        LogicalPlan lp = lpt.buildPlan("C = cogroup A by id inner, B by id, D by id inner using 'merge';");
        assertEquals(LOCogroup.GROUPTYPE.MERGE, ((LOCogroup)lp.getLeaves().get(0)).getGroupType());

        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        boolean exceptionCaught = false;
        try{
            Util.buildPhysicalPlan(lp, pc);  
        }catch (LogicalToPhysicalTranslatorException e){
            exceptionCaught = true;
View Full Code Here

Examples of org.apache.pig.impl.PigContext

        return buildPlan(query, LogicalPlanBuilder.class.getClassLoader());
    }

    public LogicalPlan buildPlan(String query, ClassLoader cldr) {
        LogicalPlanBuilder.classloader = cldr;
        PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
        try {

            pigContext.connect();
           
            LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //


            LogicalPlan lp = builder.parse("Test-Plan-Builder",
View Full Code Here

Examples of org.apache.pig.impl.PigContext

                throw new AssertionError("Unhandled option " + cc.toString());
                     }
            }
        }
        // create the context with the parameter
        PigContext pigContext = new PigContext(execType, properties);
       
        // create the static script state object
        String commandLine = LoadFunc.join((AbstractList<String>)Arrays.asList(args), " ");
        ScriptState scriptState = ScriptState.start(commandLine);
        if (listener != null) {
            scriptState.registerListener(listener);
        }
       

        if(logFileName == null && !userSpecifiedLog) {
            logFileName = validateLogFile(properties.getProperty("pig.logfile"), null);
        }
       
        pigContext.getProperties().setProperty("pig.logfile", (logFileName == null? "": logFileName));
    
        // configure logging
        configureLog4J(properties, pigContext);
       
        if(logFileName != null) {
            log.info("Logging error messages to: " + logFileName);
        }
       
        if(optimizerRules.size() > 0) {
          pigContext.getProperties().setProperty("pig.optimizer.rules", ObjectSerializer.serialize(optimizerRules));
        }
       
        if (properties.get("udf.import.list")!=null)
            PigContext.initializeImportList((String)properties.get("udf.import.list"));

        LogicalPlanBuilder.classloader = pigContext.createCl(null);

        // construct the parameter substitution preprocessor
        Grunt grunt = null;
        BufferedReader in;
        String substFile = null;
        switch (mode) {
        case FILE: {
            FileLocalizer.FetchFileRet localFileRet = FileLocalizer.fetchFile(properties, file);
            if (localFileRet.didFetch) {
                properties.setProperty("pig.jars.relative.to.dfs", "true");
            }
            in = new BufferedReader(new FileReader(localFileRet.file));

            // run parameter substitution preprocessor first
            substFile = file + ".substituted";
            pin = runParamPreprocessor(properties, in, params, paramFiles, substFile, debug || dryrun || checkScriptOnly);
            if (dryrun) {
                log.info("Dry run completed. Substituted pig script is at " + substFile);
                return ReturnCode.SUCCESS;
            }

            logFileName = validateLogFile(logFileName, file);
            pigContext.getProperties().setProperty("pig.logfile", logFileName);

            // Set job name based on name of the script
            pigContext.getProperties().setProperty(PigContext.JOB_NAME,
                                                   "PigLatin:" +new File(file).getName()
            );
           
            if (!debug) {
                new File(substFile).deleteOnExit();
            }
           
            scriptState.setScript(new File(file));
           
            grunt = new Grunt(pin, pigContext);
            gruntCalled = true;
           
            if(checkScriptOnly) {
                grunt.checkScript(substFile);
                System.err.println(file + " syntax OK");
                rc = ReturnCode.SUCCESS;
            } else {
                int results[] = grunt.exec();
                rc = getReturnCodeForStats(results);
            }
           
            return rc;
        }

        case STRING: {
            if(checkScriptOnly) {
                System.err.println("ERROR:" +
                        "-c (-check) option is only valid " +
                        "when executing pig with a pig script file)");
                return ReturnCode.ILLEGAL_ARGS;
            }
            // Gather up all the remaining arguments into a string and pass them into
            // grunt.
            StringBuffer sb = new StringBuffer();
            String remainders[] = opts.getRemainingArgs();
            for (int i = 0; i < remainders.length; i++) {
                if (i != 0) sb.append(' ');
                sb.append(remainders[i]);
            }
           
            scriptState.setScript(sb.toString());
           
            in = new BufferedReader(new StringReader(sb.toString()));
            grunt = new Grunt(in, pigContext);
            gruntCalled = true;
            int results[] = grunt.exec();
            return getReturnCodeForStats(results);
        }

        default:
            break;
        }

        // If we're here, we don't know yet what they want.  They may have just
        // given us a jar to execute, they might have given us a pig script to
        // execute, or they might have given us a dash (or nothing) which means to
        // run grunt interactive.
        String remainders[] = opts.getRemainingArgs();
        if (remainders == null) {
            if(checkScriptOnly) {
                System.err.println("ERROR:" +
                        "-c (-check) option is only valid " +
                        "when executing pig with a pig script file)");
                return ReturnCode.ILLEGAL_ARGS;
            }
            // Interactive
            mode = ExecMode.SHELL;
            ConsoleReader reader = new ConsoleReader(System.in, new OutputStreamWriter(System.out));
            reader.setDefaultPrompt("grunt> ");
            final String HISTORYFILE = ".pig_history";
            String historyFile = System.getProperty("user.home") + File.separator  + HISTORYFILE;
            reader.setHistory(new History(new File(historyFile)));
            ConsoleReaderInputStream inputStream = new ConsoleReaderInputStream(reader);
            grunt = new Grunt(new BufferedReader(new InputStreamReader(inputStream)), pigContext);
            grunt.setConsoleReader(reader);
            gruntCalled = true;
            grunt.run();
            return ReturnCode.SUCCESS;
        } else {
            // They have a pig script they want us to run.
            if (remainders.length > 1) {
                   throw new RuntimeException("Encountered unexpected arguments on command line - please check the command line.");
            }
            mode = ExecMode.FILE;
           
            FileLocalizer.FetchFileRet localFileRet = FileLocalizer.fetchFile(properties, remainders[0]);
            if (localFileRet.didFetch) {
                properties.setProperty("pig.jars.relative.to.dfs", "true");
            }
            in = new BufferedReader(new FileReader(localFileRet.file));

            // run parameter substitution preprocessor first
            substFile = remainders[0] + ".substituted";
            pin = runParamPreprocessor(properties, in, params, paramFiles, substFile, debug || dryrun || checkScriptOnly);
            if (dryrun){
                log.info("Dry run completed. Substituted pig script is at " + substFile);
                return ReturnCode.SUCCESS;
            }
           
            logFileName = validateLogFile(logFileName, remainders[0]);
            pigContext.getProperties().setProperty("pig.logfile", logFileName);

            if (!debug) {
                new File(substFile).deleteOnExit();
            }

            // Set job name based on name of the script
            pigContext.getProperties().setProperty(PigContext.JOB_NAME,
                                                   "PigLatin:" +new File(remainders[0]).getName()
            );

            scriptState.setScript(localFileRet.file);
           
View Full Code Here

Examples of org.apache.pig.impl.PigContext

        Properties props = new Properties();
        for (Entry<Object, Object> entry : cluster.getProperties().entrySet()) {
            props.put(entry.getKey(), entry.getValue());
        }
        props.setProperty("mapred.max.split.size", Integer.toString(splitSize));
        PigContext pigContext = new PigContext(ExecType.MAPREDUCE, props);
        PigServer pig = new PigServer(pigContext);
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(props));
        fs.delete(new Path(outputFile), true);
        Util.registerMultiLineQuery(pig, scriptToTestSplitting);
       
View Full Code Here

Examples of org.apache.pig.impl.PigContext

        String inputFileName = "input2.txt";
        Util.createInputFile(cluster, inputFileName, inputData);
       
        PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster
                .getProperties());
        PigContext pigContext = pig.getPigContext();
        pigContext.getProperties().setProperty( "output.compression.enabled", "true" );
        pigContext.getProperties().setProperty( "output.compression.codec", "org.apache.hadoop.io.compress.BZip2Codec" );
       
        pig.setBatchOn();
        pig.registerQuery("a = load '" +  inputFileName + "';");
        pig.registerQuery("store a into 'output2.bz2';");
        pig.registerQuery("store a into 'output2';");
View Full Code Here

Examples of org.apache.pig.impl.PigContext

    public PigServer(ExecType execType) throws ExecException {
        this(execType, PropertiesUtil.loadDefaultProperties());
    }

    public PigServer(ExecType execType, Properties properties) throws ExecException {
        this(new PigContext(execType, properties));
    }
View Full Code Here

Examples of org.apache.pig.impl.PigContext

*/
   
    @Test
    public void testDefine() throws Throwable {
        PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        PigContext context = server.getPigContext();
       
        String strCmd = "define myudf org.apache.pig.builtin.AVG();\n";
       
        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
        InputStreamReader reader = new InputStreamReader(cmd);
       
        Grunt grunt = new Grunt(new BufferedReader(reader), context);
   
        try {
            grunt.exec();
        } catch (Exception e) {
            assertTrue(e.getMessage().contains("Encountered \"define\""));
        }
        assertTrue(null != context.getFuncSpecFromAlias("myudf"));
    }
View Full Code Here

Examples of org.apache.pig.impl.PigContext

    }

    @Test
    public void testBagSchema() throws Throwable {
        PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        PigContext context = server.getPigContext();
       
        String strCmd = "a = load 'input1' as (b: bag{t(i: int, c:chararray, f: float)});\n";
       
        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
        InputStreamReader reader = new InputStreamReader(cmd);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.