Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage$POPackageTupleBuffer


            throw new MRCompilerException(msg, errCode, PigException.BUG, e);
        }
       
        mro.mapPlan.addAsLeaf(lr);
       
        POPackage spkg = (POPackage)sortMROp.reducePlan.getRoots().get(0);

        POPackage pkg = null;
        try {
            pkg = spkg.clone();
        } catch (Exception e) {
            int errCode = 2148;
            String msg = "Error cloning POPackageLite for limit after sort";
View Full Code Here


        lr.setPlans(eps);
        lr.setResultType(DataType.TUPLE);
       
        mro.mapPlan.addAsLeaf(lr);
       
        POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pkg.setKeyType(DataType.TUPLE);
        pkg.setNumInps(1);
        boolean[] inner = {false};
        pkg.setInner(inner);
        mro.reducePlan.add(pkg);
       
        mro.reducePlan.addAsLeaf(getPlainForEachOP());
    }
View Full Code Here

            // On the reduce side of this indexing job, there will be a global rearrange followed by POSort.
            // Output of POSort will be index file dumped on the DFS.

            // First add POPackage.
            POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)));
            pkg.setKeyType(DataType.CHARARRAY);
            pkg.setNumInps(1);
            pkg.setInner(new boolean[]{false});
            rightMROpr.reducePlan.add(pkg);

            // Next project tuples from the bag created by POPackage.
            POProject topPrj = new POProject(new OperatorKey(scope,nig.getNextNodeId(scope)));
            topPrj.setColumn(1);
View Full Code Here

                mro.reducePlan.addAsLeaf(lr);
            }
           
            blocking(op);
           
            POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)));
            pkg.setKeyType(DataType.TUPLE);
            pkg.setDistinct(true);
            pkg.setNumInps(1);
            boolean[] inner = {false};
            pkg.setInner(inner);
            curMROp.reducePlan.add(pkg);
           
            List<PhysicalPlan> eps1 = new ArrayList<PhysicalPlan>();
            List<Boolean> flat1 = new ArrayList<Boolean>();
            PhysicalPlan ep1 = new PhysicalPlan();
View Full Code Here

      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
      pkg.setResultType(DataType.TUPLE);
      pkg.setNumInps(2);
      boolean[] inner = {true, true};
      pkg.setInner(inner);           
      pkg.visit(this);      
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POForEach
      List<PhysicalPlan> eps = new ArrayList<PhysicalPlan>();
      List<Boolean> flat = new ArrayList<Boolean>();
View Full Code Here

        mro.mapPlan.add(lr);
        mro.mapPlan.connect(nfe1, lr);
       
        mro.setMapDone(true);
       
        POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pkg.setKeyType(DataType.CHARARRAY);
        pkg.setNumInps(1);
        boolean[] inner = {false};
        pkg.setInner(inner);
        mro.reducePlan.add(pkg);
       
        // Lets start building the plan which will have the sort
        // for the foreach
        PhysicalPlan fe2Plan = new PhysicalPlan();
View Full Code Here

        return index;
    }
   
    private void mergeOneReducePlanWithIndex(PhysicalPlan from,
            PhysicalPlan to, int initial, int current, byte mapKeyType) throws VisitorException {                   
        POPackage pk = (POPackage)from.getRoots().get(0);
        from.remove(pk);
        if(!(pk instanceof POMultiQueryPackage)){
            // XXX the index of the original keyInfo map is always 0,
            // we need to shift the index so that the lookups works
            // with the new indexed key
            addShiftedKeyInfoIndex(initial, pk);
        }
       
        if (pk instanceof POMultiQueryPackage) {
            POMultiQueryPackage mpkg = (POMultiQueryPackage)pk;
            setBaseIndexOnPackage(initial, mpkg);
            // we should update the keyinfo map of the
            // POPackage objects in the POMultiQueryPackage to
            // have the shifted index - The index now will be
            // starting from "initial" going up to "current"
            // ORed with the multi query bit mask
            int retIndex = addShiftedKeyInfoIndex(initial, current, mpkg);
            if(retIndex != current) {
                int errCode = 2146;
                String msg = "Internal Error. Inconsistency in key index found during optimization.";
                throw new OptimizerException(msg, errCode, PigException.BUG);
            }
        }
                               
        PhysicalOperator root = from.getRoots().get(0);
        if (root instanceof PODemux) {
            PODemux demux = (PODemux)root;
            setBaseIndexOnDemux(initial, demux);
        }
                   
        POMultiQueryPackage pkg = (POMultiQueryPackage)to.getRoots().get(0);       
        for (int i=initial; i<current; i++) {
            pkg.addPackage(pk);
        }
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
        for (int i=initial; i<current; i++) {
            demux.addPlan(from, mapKeyType);
        }
              
        if (demux.isSameMapKeyType()) {
            pkg.setKeyType(pk.getKeyType());
        } else {
            pkg.setKeyType(DataType.TUPLE);
        }               
    }
View Full Code Here

            throw new OptimizerException(msg, errCode, PigException.BUG);
        }
        int i = 0;
        int curIndex = initialIndex;
        while (i < end) {
            POPackage pkg = pkgs.get(i);
            if(pkg instanceof POMultiQueryPackage) {
                curIndex = addShiftedKeyInfoIndex(curIndex, onePastEndIndex, (POMultiQueryPackage)pkg);
            } else {
                addShiftedKeyInfoIndex(curIndex, pkg);
                curIndex++;
View Full Code Here

       
    }

    private void mergeOneCombinePlanWithIndex(PhysicalPlan from,
            PhysicalPlan to, int initial, int current, byte mapKeyType) throws VisitorException {
        POPackage cpk = (POPackage)from.getRoots().get(0);
        from.remove(cpk);
      
        if (cpk instanceof POMultiQueryPackage) {
            POMultiQueryPackage mpkg = (POMultiQueryPackage)cpk;
            setBaseIndexOnPackage(initial, mpkg);
        }
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
       
        boolean isSameKeyType = demux.isSameMapKeyType();
       
        PhysicalOperator leaf = from.getLeaves().get(0);
        if (leaf instanceof POLocalRearrange) {
            POLocalRearrange clr = (POLocalRearrange)leaf;
            try {
                clr.setMultiQueryIndex(initial);           
            } catch (ExecException e) {                                       
                int errCode = 2136;
                String msg = "Internal Error. Unable to set multi-query index for optimization.";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
            }
           
            // change the map key type to tuple when
            // multiple splittees have different map key types
            if (!isSameKeyType) {
                clr.setKeyType(DataType.TUPLE);
            }
        } else if (leaf instanceof PODemux) {
            PODemux locDemux = (PODemux)leaf;
            setBaseIndexOnDemux(initial, locDemux);
        }
      
        POMultiQueryPackage pkg = (POMultiQueryPackage)to.getRoots().get(0);
       
        // if current > initial + 1, it means we had
        // a split in the map of the MROper we are trying to
        // merge. In that case we would have changed the indices
        // of the POLocalRearranges in the split to be in the
        // range initial to current. To handle key, value pairs
        // coming out of those POLocalRearranges, we replicate
        // the Package as many times (in this case, the package
        // would have to be a POMultiQueryPackage since we had
        // a POSplit in the map). That Package would have a baseindex
        // correctly set (in the beginning of this method) and would
        // be able to handle the outputs from the different
        // POLocalRearranges.
        for (int i=initial; i<current; i++) {
            pkg.addPackage(cpk);
        }
       
        // all packages should have the same key type
        if (!isSameKeyType) {
            cpk.setKeyType(DataType.TUPLE);         
        }
       
        pkg.setKeyType(cpk.getKeyType());
       
        // See comment above for why we replicated the Package
        // in the from plan - for the same reason, we replicate
        // the Demux operators now.
        for (int i=initial; i<current; i++) {
View Full Code Here

            NullableTuple it = new NullableTuple((Tuple) db2Iter.next());
            it.setIndex((byte)1);
            db.add(it);
        }
        //ITIterator iti = new TestPackage.ITIterator(db.iterator());
        POPackage pop = new POPackage(new OperatorKey("", r.nextLong()));
        pop.setNumInps(2);
        pop.setInner(inner);
        PigNullableWritable k = HDataType.getWritableComparableTypes(key, (byte)0);
        pop.attachInput(k, db.iterator());
       
        // we are not doing any optimization to remove
        // parts of the "value" which are present in the "key" in this
        // unit test - so set up the "keyInfo" accordingly in
        // the POPackage
        Map<Integer, Pair<Boolean, Map<Integer, Integer>>> keyInfo =
            new HashMap<Integer, Pair<Boolean, Map<Integer,Integer>>>();
        Pair<Boolean, Map<Integer, Integer>> p =
            new Pair<Boolean, Map<Integer, Integer>>(false, new HashMap<Integer, Integer>());
        keyInfo.put(0, p);
        keyInfo.put(1, p);
        pop.setKeyInfo(keyInfo);
        Tuple t = null;
        Result res = null;
        res = (Result) pop.getNext(t);
        if(res.returnStatus==POStatus.STATUS_NULL && inner[0]) return;
        assertEquals(POStatus.STATUS_OK, res.returnStatus);

        t = (Tuple) res.result;
        Object outKey = t.get(0);
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage$POPackageTupleBuffer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.