Examples of org.apache.pig.PigServer.executeBatch()

org.apache.pig.PigServer.executeBatch()
Submits a batch of Pig commands for execution. @throws FrontendException @throws ExecException

                Util.deleteFile(ps.getPigContext(), cleanupSuccessFile2);
                ps.setBatchOn();
                Util.createInputFile(ps.getPigContext(), 
                        inputFileName, inputData);
                Util.registerMultiLineQuery(ps, script);
                ps.executeBatch();
                assertEquals(
                        "Checking if file indicating that cleanupOnFailure failed " +
                        " does not exists in " + execType + " mode", false, 
                        Util.exists(ps.getPigContext(), cleanupFailFile1));
                assertEquals(

View Full Code Here

                        cleanupFiles(ps, files);
                        ps.setBatchOn();
                        Util.createInputFile(ps.getPigContext(), 
                                inputFileName, inputData);
                        Util.registerMultiLineQuery(ps, script);
                        ps.executeBatch();
                        for(int i = 1; i <= (isMultiStore ? 3 : 1); i++) {
                            String sucFile = outputFileName + "_" + i + "/" + 
                                               MapReduceLauncher.SUCCEEDED_FILE_NAME;
                            assertEquals("Checking if _SUCCESS file exists in " + 
                                    execType + " mode", isPropertySet,

View Full Code Here

                        ps.setBatchOn();
                        Util.createInputFile(ps.getPigContext(), 
                                inputFileName, inputData);
                        Util.registerMultiLineQuery(ps, script);
                        try {
                            ps.executeBatch();
                        } catch(IOException ioe) {
                            if(!ioe.getMessage().equals("FailUDFException")) {
                                // an unexpected exception
                                throw ioe;
                            }

View Full Code Here

        
        pig.setBatchOn();
        pig.registerQuery("a = load '" +  inputFileName + "';");
        pig.registerQuery("store a into 'output.bz2';");
        pig.registerQuery("store a into 'output';");
        pig.executeBatch();
        
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FileStatus stat = fs.getFileStatus(new Path("output/part-m-00000"));        
        assertTrue(stat.getLen() > 0);

View Full Code Here

        
        pig.setBatchOn();
        pig.registerQuery("a = load '" +  inputFileName + "';");
        pig.registerQuery("store a into 'output2.bz2';");
        pig.registerQuery("store a into 'output2';");
        pig.executeBatch();
        
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
                pig.getPigContext().getProperties()));
        FileStatus stat = fs.getFileStatus(new Path("output2/part-m-00000.bz2"));        
        assertTrue(stat.getLen() > 0);

View Full Code Here

    UDFContext.getUDFContext().setClientSystemProps();
    server.setBatchOn();
    server.registerQuery("A = load '"+basicFileFullName+"' as (a:int, b:chararray);");
    server.registerQuery("store A into '"+BASIC_TABLE+"' using org.apache.howl.pig.HowlStorer();");


    server.executeBatch();


    driver.run("select * from "+BASIC_TABLE);
    ArrayList<String> unpartitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(unpartitionedTableValuesReadFromHiveDriver);
    assertEquals(basicInputData.size(),unpartitionedTableValuesReadFromHiveDriver.size());

View Full Code Here

    server.registerQuery("B2 = filter A by a < 2;");
    server.registerQuery("store B2 into '"+PARTITIONED_TABLE+"' using org.apache.howl.pig.HowlStorer('bkt=0');");
    server.registerQuery("C2 = filter A by a >= 2;");
    server.registerQuery("store C2 into '"+PARTITIONED_TABLE+"' using org.apache.howl.pig.HowlStorer('bkt=1');");


    server.executeBatch();


    driver.run("select * from "+PARTITIONED_TABLE);
    ArrayList<String> partitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(partitionedTableValuesReadFromHiveDriver);
    assertEquals(basicInputData.size(),partitionedTableValuesReadFromHiveDriver.size());

View Full Code Here

    server.registerQuery("B2 = filter A by a < 2;");
    server.registerQuery("store B2 into '"+PARTITIONED_TABLE+"' using org.apache.howl.pig.HowlStorer('bkt=0');");
    server.registerQuery("C2 = filter A by a >= 2;");
    server.registerQuery("store C2 into '"+PARTITIONED_TABLE+"' using org.apache.howl.pig.HowlStorer('bkt=1');");


    server.executeBatch();


    driver.run("select * from "+BASIC_TABLE);
    ArrayList<String> unpartitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(unpartitionedTableValuesReadFromHiveDriver);
    driver.run("select * from "+PARTITIONED_TABLE);

View Full Code Here

    server.registerQuery("C2 = filter C by a >= 2;");
    server.registerQuery("store C2 into '"+PARTITIONED_TABLE+"' using org.apache.howl.pig.HowlStorer('bkt=1');");


    server.registerQuery("D = load '"+fullFileNameComplex+"' as (name:chararray, studentid:int, contact:tuple(phno:chararray,email:chararray), currently_registered_courses:bag{innertup:tuple(course:chararray)}, current_grades:map[ ] , phnos :bag{innertup:tuple(phno:chararray,type:chararray)});");
    server.registerQuery("store D into '"+COMPLEX_TABLE+"' using org.apache.howl.pig.HowlStorer();");
    server.executeBatch();


  }
  private void cleanup() throws IOException {
    MiniCluster.deleteFile(cluster, basicFile);
    MiniCluster.deleteFile(cluster, complexFile);

View Full Code Here

    pig.registerQuery("AP = FILTER A BY emp_state == 'AP';");
    pig.registerQuery("STORE TN INTO 'employee' USING org.apache.howl.pig.HowlStorer('emp_country=IN,emp_state=TN');");
    pig.registerQuery("STORE KA INTO 'employee' USING org.apache.howl.pig.HowlStorer('emp_country=IN,emp_state=KA');");
    pig.registerQuery("STORE KL INTO 'employee' USING org.apache.howl.pig.HowlStorer('emp_country=IN,emp_state=KL');");
    pig.registerQuery("STORE AP INTO 'employee' USING org.apache.howl.pig.HowlStorer('emp_country=IN,emp_state=AP');");
    pig.executeBatch();
    driver.run("select * from employee");
    ArrayList<String> results = new ArrayList<String>();
    driver.getResults(results);
    assertEquals(4, results.size());
    Collections.sort(results);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.