Examples of PhysicalOperator


Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

            assertEquals(LOCogroup.GROUPTYPE.MERGE, ((LOCogroup)lp.getLeaves().get(0)).getGroupType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
            PhysicalPlan phyP = Util.buildPhysicalPlan(lp, pc);
            PhysicalOperator phyOp = phyP.getLeaves().get(0);
            assertTrue(phyOp instanceof POMergeCogroup);

            lp = lpt.buildPlan("store C into 'out';");
            MROperPlan mrPlan = Util.buildMRPlan(Util.buildPhysicalPlan(lp, pc),pc);           
            assertEquals(2,mrPlan.size());
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator

      pp.addAsLeaf(store);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
     
      for(MapReduceOper mro: mrPlan.getLeaves()) {
        if(mro.reducePlan != null) {
          PhysicalOperator po = mro.reducePlan.getRoots().get(0);
          if(po instanceof POPackage) {
            ((POPackage)po).setKeyType(DataType.BAG);
            mro.setGlobalSort(true);
          }
        }
View Full Code Here

Examples of org.apache.pig.impl.physicalLayer.PhysicalOperator

    }
   
    static DataBag EvaluateOperator(LogicalOperator logOp, LineageTracer lineageTracer, Map<LogicalOperator, DataBag> derivedData, Map<OperatorKey, OperatorKey> logicalToPhysicalKeys, Map<OperatorKey, ExecPhysicalOperator> physicalOpTable) throws IOException {
      //We have the compiled physical plan. We just replace the inputs to the physical operator corresponding to the supplied logical
      //operator by POReads reading from databags from the derivedData
      PhysicalOperator pOp = (PhysicalOperator)physicalOpTable.get(logicalToPhysicalKeys.get(logOp.getOperatorKey()));
      if(lineageTracer != null) {
        pOp.setLineageTracer(lineageTracer);
      }
      OperatorKey[] origInputs = new OperatorKey[pOp.inputs.length];
      for(int i = 0; i < origInputs.length; ++i) {
        origInputs[i] = pOp.inputs[i];
      }
     
      for(int i = 0; i < pOp.inputs.length; ++i) {
        PORead read = new PORead(logOp.getScope(),
          NodeIdGenerator.getGenerator().getNextNodeId(logOp.getScope()),
          physicalOpTable,
          LogicalOperator.FIXED,
          derivedData.get(logOp.getOpTable().get(logOp.getInputs().get(i))));
       
        pOp.inputs[i] = read.getOperatorKey();
      }
     
      //get the output data
      DataBag outputData = BagFactory.getInstance().newDefaultBag();
        pOp.open();
        Tuple t = pOp.getNext();
        while(t != null) {
          outputData.add(t);
          t = pOp.getNext();
        }
       
        pOp.close();
       
        //restore the physical operator to its previous form
        pOp.setLineageTracer(null);
        for(int i = 0; i < origInputs.length; ++i) {
          physicalOpTable.remove(pOp.inputs[i]);
          pOp.inputs[i] = origInputs[i];
        }
       
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.