Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange

1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
        POProject prj = (POProject)cogroupPlan.getLeaves().get(0);
        assertEquals( 0, prj.getColumn() );
        assertEquals( DataType.CHARARRAY, prj.getResultType() );
       
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
       
        assertEquals( POLoad.class, phyPlan.getRoots().get(1).getClass() );
        POLoad load2 = (POLoad)phyPlan.getRoots().get(0);
       
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load2).get(0).getClass() );
View Full Code Here
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 1, prj2.getColumn() );
        assertEquals( DataType.INTEGER, prj2.getResultType() );
       
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
       
        assertEquals( POLoad.class, phyPlan.getRoots().get(1).getClass() );
        POLoad load2 = (POLoad)phyPlan.getRoots().get(0);
       
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load2).get(0).getClass() );
View Full Code Here
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
    @Override
    public void visit(LOCross cross) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        List<Operator> inputs = cross.getPlan().getPredecessors(cross);
       
        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                scope, nodeGen.getNextNodeId(scope)), cross
                .getRequestedParallelisam());
        poGlobal.setAlias(cross.getAlias());
        POPackage poPackage = new POPackage(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), cross.getRequestedParallelisam());
        poGlobal.setAlias(cross.getAlias());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);
       
        int count = 0;
       
View Full Code Here
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
    }
   
    private POPackage compileToLR_GR_PackTrio(LogicalRelationalOperator relationalOp, String customPartitioner,
            boolean[] innerFlags, MultiMap<Integer, LogicalExpressionPlan> innerPlans) throws FrontendException {

        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelisam());
        poGlobal.setAlias(relationalOp.getAlias());
        poGlobal.setCustomPartitioner(customPartitioner);
        POPackage poPackage = new POPackage(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelisam());
        poPackage.setAlias(relationalOp.getAlias());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);
View Full Code Here
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
      rearrangeOutputs[1] = curMROp;                    
      compiledInputs = rearrangeOutputs;
            
     
      // create POGlobalRearrange
      POGlobalRearrange gr = new POGlobalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      // Skewed join has its own special partitioner
      gr.setResultType(DataType.TUPLE);
      gr.visit(this);
      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
View Full Code Here
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
      rearrangeOutputs[1] = curMROp;                    
      compiledInputs = rearrangeOutputs;
            
     
      // create POGlobalRearrange
      POGlobalRearrange gr = new POGlobalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      // Skewed join has its own special partitioner
      gr.setResultType(DataType.TUPLE);
      gr.visit(this);
      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
        POLoad lC = GenPhyOp.topLoadOp();
        POFilter fC = GenPhyOp.topFilterOp();
        fC.setRequestedParallelism(20);
        POLocalRearrange lrC = GenPhyOp.topLocalRearrangeOp();
        lrC.setRequestedParallelism(10);
        POGlobalRearrange grC = GenPhyOp.topGlobalRearrangeOp();
        POPackage pkC = GenPhyOp.topPackageOp();
        part1.add(lC);
        part1.add(fC);
        part1.connect(lC, fC);
        part1.add(lrC);
        part1.connect(fC, lrC);
        part1.add(grC);
        part1.connect(lrC, grC);
        part1.add(pkC);
        part1.connect(grC, pkC);

        POPackage pkD = GenPhyOp.topPackageOp();
        pkD.setRequestedParallelism(20);
        POLocalRearrange lrD = GenPhyOp.topLocalRearrangeOp();
        lrD.setRequestedParallelism(30);
        POGlobalRearrange grD = GenPhyOp.topGlobalRearrangeOp();
        POLoad lD = GenPhyOp.topLoadOp();
        part1.add(lD);
        part1.add(lrD);
        part1.connect(lD, lrD);

        part1.add(grD);
        part1.connect(lrD, grD);
        part1.add(pkD);
        part1.connect(grD, pkD);

        POPackage pkCD = GenPhyOp.topPackageOp();
        POLocalRearrange lrCD1 = GenPhyOp.topLocalRearrangeOp();
        POLocalRearrange lrCD2 = GenPhyOp.topLocalRearrangeOp();
        POGlobalRearrange grCD = GenPhyOp.topGlobalRearrangeOp();
        part1.add(lrCD1);
        part1.add(lrCD2);
        part1.connect(pkC, lrCD1);
        part1.connect(pkD, lrCD2);
        part1.add(grCD);
        part1.connect(lrCD1, grCD);
        part1.connect(lrCD2, grCD);
        part1.add(pkCD);
        part1.connect(grCD, pkCD);

        POLoad lA = GenPhyOp.topLoadOp();
        POLoad lB = GenPhyOp.topLoadOp();

        // POLoad lC = lA;
        POFilter fA = GenPhyOp.topFilterOp();

        POLocalRearrange lrA = GenPhyOp.topLocalRearrangeOp();
        POLocalRearrange lrB = GenPhyOp.topLocalRearrangeOp();

        POGlobalRearrange grAB = GenPhyOp.topGlobalRearrangeOp();

        POPackage pkAB = GenPhyOp.topPackageOp();

        POFilter fAB = GenPhyOp.topFilterOp();
        POUnion unABC = GenPhyOp.topUnionOp();
View Full Code Here
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
        PhysicalPlan part1 = new PhysicalPlan();
        POLoad lC = GenPhyOp.topLoadOp();
        POFilter fC = GenPhyOp.topFilterOp();
        POLocalRearrange lrC = GenPhyOp.topLocalRearrangeOp();
        POGlobalRearrange grC = GenPhyOp.topGlobalRearrangeOp();
        POPackage pkC = GenPhyOp.topPackageOp();
        part1.add(lC);
        part1.add(fC);
        part1.connect(lC, fC);
        part1.add(lrC);
        part1.connect(fC, lrC);
        part1.add(grC);
        part1.connect(lrC, grC);
        part1.add(pkC);
        part1.connect(grC, pkC);

        POPackage pkD = GenPhyOp.topPackageOp();
        POLocalRearrange lrD = GenPhyOp.topLocalRearrangeOp();
        POGlobalRearrange grD = GenPhyOp.topGlobalRearrangeOp();
        POLoad lD = GenPhyOp.topLoadOp();
        part1.add(lD);
        part1.add(lrD);
        part1.connect(lD, lrD);

        part1.add(grD);
        part1.connect(lrD, grD);
        part1.add(pkD);
        part1.connect(grD, pkD);
        part1.connect(pkD, grC);

        POLoad lA = GenPhyOp.topLoadOp();
        POLoad lB = GenPhyOp.topLoadOp();

        // POLoad lC = lA;
        POFilter fA = GenPhyOp.topFilterOp();

        POLocalRearrange lrA = GenPhyOp.topLocalRearrangeOp();
        POLocalRearrange lrB = GenPhyOp.topLocalRearrangeOp();

        POGlobalRearrange grAB = GenPhyOp.topGlobalRearrangeOp();

        POPackage pkAB = GenPhyOp.topPackageOp();

        POFilter fAB = GenPhyOp.topFilterOp();
        POUnion unABC = GenPhyOp.topUnionOp();
View Full Code Here
386
387
388
389
390
391
392
393
394
395
396
        php.add(lr1);
        php.add(lr2);
        php.connect(fl1, lr1);
        php.connect(fl2, lr2);

        POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
        php.add(gr);
        php.connect(lr1, gr);
        php.connect(lr2, gr);

        POPackage pk = GenPhyOp.topPackageOp();
View Full Code Here
426
427
428
429
430
431
432
433
434
435
436
        php.add(lr1);
        php.add(lr2);
        php.connect(fl1, lr1);
        php.connect(fl2, lr2);

        POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
        php.addAsLeaf(gr);

        POPackage pk = GenPhyOp.topPackageOp();
        php.addAsLeaf(pk);
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange

  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler
  • org.apache.pig.experimental.logical.relational.LogToPhyTranslationVisitor
  • org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor
  • org.apache.pig.test.TestExperimentalLogToPhyTranslationVisitor
  • org.apache.pig.test.TestLocalJobSubmission
  • org.apache.pig.test.TestMRCompiler
  • org.apache.pig.test.TestNewPlanLogToPhyTranslationVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.