Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange

1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 1, prj2.getColumn() );
        assertEquals( DataType.INTEGER, prj2.getResultType() );
       
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
       
        assertEquals( POLoad.class, phyPlan.getRoots().get(1).getClass() );
        POLoad load2 = (POLoad)phyPlan.getRoots().get(0);
       
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load2).get(0).getClass() );
View Full Code Here
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
      rearrangeOutputs[1] = curMROp;                    
      compiledInputs = rearrangeOutputs;
            
     
      // create POGlobalRearrange
      POGlobalRearrange gr = new POGlobalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);                         
      gr.setResultType(DataType.TUPLE);
      gr.visit(this);
      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
View Full Code Here
435
436
437
438
439
440
441
442
443
444
445
446
447
    }
   
    private void translateRegularCogroup(LOCogroup cg) throws IOException {
        List<Operator> preds = plan.getPredecessors(cg);
       
        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), cg.getRequestedParallelisam() );
        poGlobal.setAlias(cg.getAlias());
        POPackage poPackage = new POPackage(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)), cg.getRequestedParallelisam());
        poPackage.setAlias(cg.getAlias());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);
View Full Code Here
728
729
730
731
732
733
734
735
736
737
738
739
740
741
            }
            logToPhyMap.put(loj, smj);
            return;
        }
        else if (loj.getJoinType() == LOJoin.JOINTYPE.HASH){
            POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                    scope, nodeGen.getNextNodeId(scope)), loj
                    .getRequestedParallelisam());
            poGlobal.setAlias(loj.getAlias());
            POPackage poPackage = new POPackage(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), loj.getRequestedParallelisam());
            poPackage.setAlias(loj.getAlias());
            currentPlan.add(poGlobal);
            currentPlan.add(poPackage);
View Full Code Here
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
    @Override
    public void visit(LOCross cross) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        List<Operator> inputs = cross.getPlan().getPredecessors(cross);
       
        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                scope, nodeGen.getNextNodeId(scope)), cross
                .getRequestedParallelisam());
        poGlobal.setAlias(cross.getAlias());
        POPackage poPackage = new POPackage(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), cross.getRequestedParallelisam());
        poGlobal.setAlias(cross.getAlias());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);
       
        int count = 0;
       
View Full Code Here
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
    }
   
    private POPackage compileToLR_GR_PackTrio(LogicalRelationalOperator relationalOp, String customPartitioner,
            boolean[] innerFlags, MultiMap<Integer, LogicalExpressionPlan> innerPlans) throws FrontendException {

        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelisam());
        poGlobal.setAlias(relationalOp.getAlias());
        poGlobal.setCustomPartitioner(customPartitioner);
        POPackage poPackage = new POPackage(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelisam());
        poPackage.setAlias(relationalOp.getAlias());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);
View Full Code Here
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
      rearrangeOutputs[1] = curMROp;                    
      compiledInputs = rearrangeOutputs;
            
     
      // create POGlobalRearrange
      POGlobalRearrange gr = new POGlobalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      // Skewed join has its own special partitioner
      gr.setResultType(DataType.TUPLE);
      gr.visit(this);
      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
View Full Code Here
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
      rearrangeOutputs[1] = curMROp;                    
      compiledInputs = rearrangeOutputs;
            
     
      // create POGlobalRearrange
      POGlobalRearrange gr = new POGlobalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      // Skewed join has its own special partitioner
      gr.setResultType(DataType.TUPLE);
      gr.visit(this);
      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
View Full Code Here
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
      rearrangeOutputs[1] = curMROp;                    
      compiledInputs = rearrangeOutputs;
            
     
      // create POGlobalRearrange
      POGlobalRearrange gr = new POGlobalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);                         
      gr.setResultType(DataType.TUPLE);
      gr.visit(this);
      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
View Full Code Here
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
      rearrangeOutputs[1] = curMROp;                    
      compiledInputs = rearrangeOutputs;
            
     
      // create POGlobalRearrange
      POGlobalRearrange gr = new POGlobalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);                         
      gr.setResultType(DataType.TUPLE);
      gr.visit(this);
      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange

  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler
  • org.apache.pig.experimental.logical.relational.LogToPhyTranslationVisitor
  • org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor
  • org.apache.pig.test.TestExperimentalLogToPhyTranslationVisitor
  • org.apache.pig.test.TestLocalJobSubmission
  • org.apache.pig.test.TestMRCompiler
  • org.apache.pig.test.TestNewPlanLogToPhyTranslationVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.