Package org.apache.pig.backend.executionengine

Examples of org.apache.pig.backend.executionengine.ExecJob

161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
        String dbStore = "org.apache.pig.piggybank.storage.DBStorage('" + driver
                + "', '" + Util.encodeEscape(url) + "', '" + insertQuery + "');";
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE
                + "' as (id:int, fruit:chararray, ratio:double, dt : datetime);");
        pigServer.registerQuery("STORE A INTO 'dummy' USING " + dbStore);
      ExecJob job = pigServer.executeBatch().get(0);
        try {
            while(!job.hasCompleted()) Thread.sleep(1000);
        } catch(InterruptedException ie) {// ignore
        }

        assertNotSame("Failed: " + job.getException(), job.getStatus(),
                        ExecJob.JOB_STATUS.FAILED);

        Connection con = null;
        String selectQuery = "select id, name, ratio, dt from ttt order by name";
        try {
View Full Code Here
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
                int errCode = 1003;
                String msg = "Unable to find an operator for alias " + id;
                throw new FrontendException(msg, errCode, PigException.INPUT);
            }
//            ExecJob job = execute(getPlanFromAlias(id, op.getClass().getName()));
            ExecJob job = store(id, FileLocalizer.getTemporaryPath(null, pigContext).toString(), BinStorage.class.getName() + "()");
            // invocation of "execute" is synchronous!

            if (job.getStatus() == JOB_STATUS.COMPLETED) {
                    return job.getResults();
            } else {
                throw new IOException("Job terminated with anomalous status "
                    + job.getStatus().toString());
            }
        } catch (Exception e) {
            int errCode = 1066;
            String msg = "Unable to open iterator for alias " + id;
            throw new FrontendException(msg, errCode, PigException.INPUT, e);
View Full Code Here
677
678
679
680
681
682
683
684
685
686
687
        ExampleGenerator exgen = new ExampleGenerator(plan, pigContext);
        return exgen.getExamples();
    }
   
    private ExecJob execute(String alias) throws FrontendException, ExecException {
        ExecJob job = null;
//        lp.explain(System.out, System.err);
        LogicalPlan typeCheckedLp = compileLp(alias);
       
        return executeCompiledLogicalPlan(typeCheckedLp);
//        typeCheckedLp.explain(System.out, System.err);
View Full Code Here
690
691
692
693
694
695
696
697
698
699
   
    private ExecJob executeCompiledLogicalPlan(LogicalPlan compiledLp) throws ExecException {
        PhysicalPlan pp = compilePp(compiledLp);
        // execute using appropriate engine
        FileLocalizer.clearDeleteOnFail();
        ExecJob execJob = pigContext.getExecutionEngine().execute(pp, "execute");
        if (execJob.getStatus()==ExecJob.JOB_STATUS.FAILED)
            FileLocalizer.triggerDeleteOnFail();
        return execJob;
    }
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
        String dbStore = "org.apache.pig.piggybank.storage.DBStorage('" + driver
                + "', '" + Util.encodeEscape(url) + "', '" + insertQuery + "');";
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE
                + "' as (id:int, fruit:chararray, ratio:double, dt : datetime);");
        pigServer.registerQuery("STORE A INTO 'dummy' USING " + dbStore);
      ExecJob job = pigServer.executeBatch().get(0);
        try {
            while(!job.hasCompleted()) Thread.sleep(1000);
        } catch(InterruptedException ie) {// ignore
        }

        assertNotSame("Failed: " + job.getException(), job.getStatus(),
                        ExecJob.JOB_STATUS.FAILED);

        Connection con = null;
        String selectQuery = "select id, name, ratio, dt from ttt order by name";
        try {
View Full Code Here
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
        try {
            pigContext.getProperties().setProperty( PigContext.JOB_NAME, jobName );
            if( jobPriority != null ) {
                pigContext.getProperties().setProperty( PigContext.JOB_PRIORITY, jobPriority );
            }
            ExecJob job = store(id, FileLocalizer.getTemporaryPath(pigContext)
                    .toString(), Utils.getTmpFileCompressorName(pigContext)
                    + "()");

            // invocation of "execute" is synchronous!

            if (job.getStatus() == JOB_STATUS.COMPLETED) {
                return job.getResults();
            } else if (job.getStatus() == JOB_STATUS.FAILED
                    && job.getException() != null) {
                // throw the backend exception in the failed case
                Exception e = job.getException();
                int errCode = 1066;
                String msg = "Unable to open iterator for alias " + id
                        + ". Backend error : " + e.getMessage();
                throw new FrontendException(msg, errCode, PigException.INPUT, e);
            } else {
                throw new IOException("Job terminated with anomalous status "
                        + job.getStatus().toString());
            }
        } catch (FrontendException e) {
            throw e;
        } catch (Exception e) {
            int errCode = 1066;
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
        assertNotNull(plans.lp);

        List<ExecJob> jobs = pig.runPlan(plans.lp, "testToolsPigServer");

        assertEquals(1, jobs.size());
        ExecJob job = jobs.get(0);
        assertEquals(JOB_STATUS.COMPLETED, job.getStatus());
        Iterator<Tuple> results = job.getResults();
        assertTrue(results.hasNext()); // should have one record
        Tuple t = results.next();
        assertEquals("abc", t.get(0).toString());
        assertTrue(results.hasNext());
    }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
        assertNotNull(plans.lp);

        List<ExecJob> jobs = pig.runPlan(plans.lp, "testToolsPigServer");

        assertEquals(1, jobs.size());
        ExecJob job = jobs.get(0);
        assertEquals(JOB_STATUS.COMPLETED, job.getStatus());
        Iterator<Tuple> results = job.getResults();
        assertTrue(results.hasNext()); // should have one record
        Tuple t = results.next();
        assertEquals("abc", t.get(0).toString());
        assertTrue(results.hasNext());
    }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
        pigServer.registerQuery(
                "A = LOAD '" + Util.encodeEscape(input.getAbsolutePath()) + "' using PigStorage();\n"
            +    "B = GROUP A BY $0;\n"
            +   "A = FOREACH B GENERATE COUNT(A);\n"
            +  "STORE A INTO '" + Util.encodeEscape(output.getAbsolutePath()) + "';");
        ExecJob job = pigServer.executeBatch().get(0);
        List<OriginalLocation> originalLocations = job.getPOStore().getOriginalLocations();
        Assert.assertEquals(1, originalLocations.size());
        OriginalLocation originalLocation = originalLocations.get(0);
        Assert.assertEquals(4, originalLocation.getLine());
        Assert.assertEquals(0, originalLocation.getOffset());
        Assert.assertEquals("A", originalLocation.getAlias());
        JobStats jStats = (JobStats)job.getStatistics().getJobGraph().getSinks().get(0);
        Assert.assertEquals("M: A[1,4],A[3,4],B[2,4] C: A[3,4],B[2,4] R: A[3,4]", jStats.getAliasLocation());
    }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
        pw.close();
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 > 50;");
        pigServer.registerQuery("c = foreach b generate $0 - 50;");
        ExecJob job = pigServer.store("c", "output_map_only");
        PigStats pigStats = job.getStatistics();
       
        //counting the no. of bytes in the output file
        //long filesize = cluster.getFileSystem().getFileStatus(new Path("output_map_only")).getLen();
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath(
                "output_map_only", pigServer.getPigContext()), pigServer
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.executionengine.ExecJob

  • org.apache.hadoop.zebra.pig.TestCogroup
  • org.apache.hadoop.zebra.pig.TestCollectionTableStorer
  • org.apache.hadoop.zebra.pig.TestMapSideGroupBy
  • org.apache.hadoop.zebra.pig.TestMapTableStorer
  • org.apache.hadoop.zebra.pig.TestMergeJoin
  • org.apache.hadoop.zebra.pig.TestMergeJoinPrune
  • org.apache.hadoop.zebra.pig.TestMultipleOutputs1
  • org.apache.hadoop.zebra.pig.TestMultipleOutputs2
  • org.apache.hadoop.zebra.pig.TestSimpleType
  • org.apache.hadoop.zebra.pig.TestTableStorer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.