Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange

1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
            rearrangeOutputs[1] = joinJobs[1];

            compiledInputs = rearrangeOutputs;

            // Create POGlobalRearrange
            POGlobalRearrange gr =
                    new POGlobalRearrange(OperatorKey.genOpKey(scope), rp);
            // Skewed join has its own special partitioner
            gr.setResultType(DataType.TUPLE);
            gr.visit(this);
            joinJobs[2] = curTezOp;
            joinJobs[2].setRequestedParallelism(rp);

            compiledInputs = new TezOperator[] {joinJobs[2]};
View Full Code Here
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
                    String msg = "Invalid physical operators in the physical plan" ;
                    throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
                }
            }
        } else {
            POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                    scope, nodeGen.getNextNodeId(scope)), cross
                    .getRequestedParallelism());
            poGlobal.addOriginalLocation(cross.getAlias(), cross.getLocation());
            POPackage poPackage = new POPackage(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), cross.getRequestedParallelism());
            poGlobal.addOriginalLocation(cross.getAlias(), cross.getLocation());
            currentPlan.add(poGlobal);
            currentPlan.add(poPackage);

            int count = 0;
View Full Code Here
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
    }

    private POPackage compileToLR_GR_PackTrio(LogicalRelationalOperator relationalOp, String customPartitioner,
            boolean[] innerFlags, MultiMap<Integer, LogicalExpressionPlan> innerPlans) throws FrontendException {

        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelism());
        poGlobal.addOriginalLocation(relationalOp.getAlias(), relationalOp.getLocation());
        poGlobal.setCustomPartitioner(customPartitioner);
        POPackage poPackage = new POPackage(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelism());
        poPackage.addOriginalLocation(relationalOp.getAlias(), relationalOp.getLocation());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);
View Full Code Here
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
      rearrangeOutputs[1] = curMROp;                    
      compiledInputs = rearrangeOutputs;
            
     
      // create POGlobalRearrange
      POGlobalRearrange gr = new POGlobalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      // Skewed join has its own special partitioner
      gr.setResultType(DataType.TUPLE);
      gr.visit(this);
      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
View Full Code Here
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
      rearrangeOutputs[1] = curMROp;                    
      compiledInputs = rearrangeOutputs;
            
     
      // create POGlobalRearrange
      POGlobalRearrange gr = new POGlobalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);                         
      gr.setResultType(DataType.TUPLE);
      gr.visit(this);
      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
                    String msg = "Invalid physical operators in the physical plan" ;
                    throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
                }
            }
        } else {
            POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                    scope, nodeGen.getNextNodeId(scope)), cross
                    .getRequestedParallelisam());
            poGlobal.setAlias(cross.getAlias());
            POPackage poPackage = new POPackage(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), cross.getRequestedParallelisam());
            poGlobal.setAlias(cross.getAlias());
            currentPlan.add(poGlobal);
            currentPlan.add(poPackage);
           
            int count = 0;
           
View Full Code Here
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
    }
   
    private POPackage compileToLR_GR_PackTrio(LogicalRelationalOperator relationalOp, String customPartitioner,
            boolean[] innerFlags, MultiMap<Integer, LogicalExpressionPlan> innerPlans) throws FrontendException {

        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelisam());
        poGlobal.setAlias(relationalOp.getAlias());
        poGlobal.setCustomPartitioner(customPartitioner);
        POPackage poPackage = new POPackage(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelisam());
        poPackage.setAlias(relationalOp.getAlias());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);
View Full Code Here
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
      rearrangeOutputs[1] = curMROp;                    
      compiledInputs = rearrangeOutputs;
            
     
      // create POGlobalRearrange
      POGlobalRearrange gr = new POGlobalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      // Skewed join has its own special partitioner
      gr.setResultType(DataType.TUPLE);
      gr.visit(this);
      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
View Full Code Here
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
        POProject prj = (POProject)cogroupPlan.getLeaves().get(0);
        assertEquals( 0, prj.getColumn() );
        assertEquals( DataType.CHARARRAY, prj.getResultType() );
       
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
       
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );
        POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
        assertEquals( DataType.TUPLE, pack.getResultType() );
    }
View Full Code Here
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 1, prj2.getColumn() );
        assertEquals( DataType.INTEGER, prj2.getResultType() );
       
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
       
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );
        POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
        assertEquals( DataType.TUPLE, pack.getResultType() );
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange

  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler
  • org.apache.pig.experimental.logical.relational.LogToPhyTranslationVisitor
  • org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor
  • org.apache.pig.test.TestExperimentalLogToPhyTranslationVisitor
  • org.apache.pig.test.TestLocalJobSubmission
  • org.apache.pig.test.TestMRCompiler
  • org.apache.pig.test.TestNewPlanLogToPhyTranslationVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.