Package org.apache.pig.backend.executionengine

Examples of org.apache.pig.backend.executionengine.ExecJob

82
83
84
85
86
87
88
89
90
91
92
93
94
        assertNotNull(plans.lp);

        List<ExecJob> jobs = pig.runPlan(plans.lp, "testToolsPigServer");

        assertEquals(1, jobs.size());
        ExecJob job = jobs.get(0);
        assertEquals(JOB_STATUS.COMPLETED, job.getStatus());
        Iterator<Tuple> results = job.getResults();
        assertTrue(results.hasNext()); // should have one record
        Tuple t = results.next();
        assertEquals("abc", t.get(0).toString());
        assertTrue(results.hasNext());
    }
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
130
        assertNotNull(plans.lp);

        List<ExecJob> jobs = pig.runPlan(plans.lp, "testToolsPigServer");

        assertEquals(1, jobs.size());
        ExecJob job = jobs.get(0);
        assertEquals(JOB_STATUS.COMPLETED, job.getStatus());
        Iterator<Tuple> results = job.getResults();
        assertTrue(results.hasNext()); // should have one record
        Tuple t = results.next();
        assertEquals("abc", t.get(0).toString());
        assertTrue(results.hasNext());
    }
View Full Code Here
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
    public void testColumnPruning() throws Exception {
        Util.resetStateForExecModeSwitch();
        pigServer = new PigServer(cluster.getExecType(), cluster.getProperties());

        pigServer.registerQuery("A = load '" + basedir + "orc-file-11-format.orc' using OrcStorage();");
        ExecJob job = pigServer.store("A", OUTPUT1);
        JobStats stats = (JobStats) job.getStatistics().getJobGraph().getSources().get(0);
        long bytesWithoutPushdown = stats.getHdfsBytesRead();

        pigServer.registerQuery("PRUNE = load '" + basedir + "orc-file-11-format.orc' using OrcStorage();");
        pigServer.registerQuery("PRUNE = foreach PRUNE generate boolean1;");
        job = pigServer.store("PRUNE", OUTPUT2);
        Util.checkLogFileMessage(logFile, new String[]{"Columns pruned for PRUNE: $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13"}, true);
        stats = (JobStats) job.getStatistics().getJobGraph().getSources().get(0);
        long bytesWithPushdown = stats.getHdfsBytesRead();

        assertTrue((bytesWithoutPushdown - bytesWithPushdown) > 300000);
    }
View Full Code Here
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
        disabledOptimizerRules.add("PredicatePushdownOptimizer");
        pigServer_disabledRule.getPigContext().getProperties().setProperty(PigImplConstants.PIG_OPTIMIZER_RULES_KEY,
                ObjectSerializer.serialize(disabledOptimizerRules));
        pigServer_disabledRule.registerQuery("B = load '" + inputFile + "' using OrcStorage();");
        pigServer_disabledRule.registerQuery("C = filter B by " + filterStmt + ";");
        ExecJob job = pigServer_disabledRule.store("C", OUTPUT3);
        //Util.copyFromClusterToLocal(cluster, OUTPUT3 + "/part-m-00000", OUTPUT3);
        JobStats stats = (JobStats) job.getStatistics().getJobGraph().getSources().get(0);
        assertEquals(expectedRows, stats.getRecordWrittern());
        long bytesWithoutPushdown = stats.getHdfsBytesRead();

        // Test with PredicatePushdownOptimizer enabled. Only 2 blocks should be read
        pigServer.registerQuery("D = load '" + inputFile + "' using OrcStorage();");
        pigServer.registerQuery("E = filter D by " + filterStmt + ";");
        job = pigServer.store("E", OUTPUT4);
        //Util.copyFromClusterToLocal(cluster, OUTPUT4 + "/part-m-00000", OUTPUT4);
        stats = (JobStats) job.getStatistics().getJobGraph().getSources().get(0);
        assertEquals(expectedRows, stats.getRecordWrittern());
        long bytesWithPushdown = stats.getHdfsBytesRead();

        System.out.println("bytesWithoutPushdown was " + bytesWithoutPushdown +
                " and bytesWithPushdown was " + bytesWithPushdown);
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    String dbStore = "org.apache.pig.piggybank.storage.DBStorage('" + driver                                                                                                                      
              + "', '" + Util.encodeEscape(url) + "', '" + insertQuery + "');";
    pigServer.registerQuery("A = LOAD '" + INPUT_FILE
        + "' as (id:int, fruit:chararray, ratio:double, dt : datetime);");
    pigServer.registerQuery("STORE A INTO 'dummy' USING " + dbStore);
    ExecJob job = pigServer.executeBatch().get(0);
    try {
      while(!job.hasCompleted()) Thread.sleep(1000);
    } catch(InterruptedException ie) {// ignore
    }
   
    assertNotSame("Failed: " + job.getException(), job.getStatus(),
            ExecJob.JOB_STATUS.FAILED);
   
    Connection con = null;
    String selectQuery = "select id, name, ratio, dt from ttt order by name";
    try {
View Full Code Here
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
        try {
            pigContext.getProperties().setProperty( PigContext.JOB_NAME, jobName );
            if( jobPriority != null ) {
                pigContext.getProperties().setProperty( PigContext.JOB_PRIORITY, jobPriority );
            }
            ExecJob job = store(id, FileLocalizer.getTemporaryPath(pigContext)
                    .toString(), Utils.getTmpFileCompressorName(pigContext)
                    + "()");

            // invocation of "execute" is synchronous!

            if (job.getStatus() == JOB_STATUS.COMPLETED) {
                return job.getResults();
            } else if (job.getStatus() == JOB_STATUS.FAILED
                    && job.getException() != null) {
                // throw the backend exception in the failed case
                Exception e = job.getException();
                int errCode = 1066;
                String msg = "Unable to open iterator for alias " + id
                        + ". Backend error : " + e.getMessage();
                throw new FrontendException(msg, errCode, PigException.INPUT, e);
            } else {
                throw new IOException("Job terminated with anomalous status "
                        + job.getStatus().toString());
            }
        } catch (FrontendException e) {
            throw e;
        } catch (Exception e) {
            int errCode = 1066;
View Full Code Here
232
233
234
235
236
237
238
239
240
241
242
243
244
      Schema dumpedASchema = server.dumpSchema("A");
      server.registerQuery("STORE A into '" + tablename + "' using org.apache.hive.hcatalog.pig.HCatStorer("
        + (provideSchemaToStorer ? "'', '" + pigSchema + "'" : "")
        + ");");

      ExecJob execJob = server.executeBatch().get(0);
      if (!execJob.getStatistics().isSuccessful()) {
        throw new RuntimeException("Import failed", execJob.getException());
      }
      // test that schema was loaded correctly
      server.registerQuery("X = load '" + tablename + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
      server.dumpSchema("X");
      Iterator<Tuple> it = server.openIterator("X");
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
    String dbStore = "org.apache.pig.piggybank.storage.DBStorage('" + driver                                                                                                                      
              + "', '" + url + "', '" + insertQuery + "');";
    pigServer.registerQuery("A = LOAD '" + INPUT_FILE
        + "' as (id:int, fruit:chararray, ratio:double);");
    pigServer.registerQuery("STORE A INTO 'dummy' USING " + dbStore);
    ExecJob job = pigServer.executeBatch().get(0);
    try {
      while(!job.hasCompleted()) Thread.sleep(1000);
    } catch(InterruptedException ie) {// ignore
    }
   
    assertNotSame("Failed: " + job.getException(), job.getStatus(),
            ExecJob.JOB_STATUS.FAILED);
   
    Connection con = null;
    String selectQuery = "select id, name, ratio from ttt order by name";
    try {
View Full Code Here
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
     * @return iterator of tuples returned from the script
     * @throws IOException
     */
    public Iterator<Tuple> openIterator(String id) throws IOException {
        try {
            ExecJob job = store(id, FileLocalizer.getTemporaryPath(pigContext)
                    .toString(), Utils.getTmpFileCompressorName(pigContext)
                    + "()");

            // invocation of "execute" is synchronous!

            if (job.getStatus() == JOB_STATUS.COMPLETED) {
                return job.getResults();
            } else if (job.getStatus() == JOB_STATUS.FAILED
                    && job.getException() != null) {
                // throw the backend exception in the failed case
                Exception e = job.getException();
                int errCode = 1066;
                String msg = "Unable to open iterator for alias " + id
                        + ". Backend error : " + e.getMessage();
                throw new FrontendException(msg, errCode, PigException.INPUT, e);
            } else {
                throw new IOException("Job terminated with anomalous status "
                        + job.getStatus().toString());
            }
        } catch (FrontendException e) {
            throw e;
        } catch (Exception e) {
            int errCode = 1066;
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
137
            outputFile = File.createTempFile("JIAR_1027", ".out");
            String filePath = outputFile.getAbsolutePath();
            outputFile.delete();
            PigServer pig = new PigServer(ExecType.LOCAL);
            pig.registerQuery("A = load 'test/org/apache/pig/test/data/passwd';");
            ExecJob job = pig.store("A", filePath);
            PigStats stats = job.getStatistics();
            File dataFile = new File( outputFile.getAbsoluteFile() + File.separator + "part-00000" );
            assertEquals(dataFile.length(), stats.getBytesWritten());
        } catch (IOException e) {
            e.printStackTrace();
            System.err.println( e.getMessage() );
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.executionengine.ExecJob

  • org.apache.hadoop.zebra.pig.TestCogroup
  • org.apache.hadoop.zebra.pig.TestCollectionTableStorer
  • org.apache.hadoop.zebra.pig.TestMapSideGroupBy
  • org.apache.hadoop.zebra.pig.TestMapTableStorer
  • org.apache.hadoop.zebra.pig.TestMergeJoin
  • org.apache.hadoop.zebra.pig.TestMergeJoinPrune
  • org.apache.hadoop.zebra.pig.TestMultipleOutputs1
  • org.apache.hadoop.zebra.pig.TestMultipleOutputs2
  • org.apache.hadoop.zebra.pig.TestSimpleType
  • org.apache.hadoop.zebra.pig.TestTableStorer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.