Examples of POSort

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort
This implementation is applicable for both the physical plan and for the local backend, as the conversion of physical to mapreduce would see the SORT operator and take necessary steps to convert it to a quantile and a sort job. This is a blocking operator. The sortedDataBag accumulates Tuples and sorts them only when there an iterator is started. So all the tuples from the input operator should be accumulated and filled into the dataBag. The attachInput method is not applicable here.

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

        List<Boolean> mAscCols = new LinkedList<Boolean>();
        mAscCols.add(true);
        PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
        List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
        inputs.add(read);
        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
                sortPlans, mAscCols, null);
        Tuple t = null;
        Result res1 = sort.getNextTuple();
        // System.out.println(res1.result);
        Result res2 = sort.getNextTuple();
        while (res2.returnStatus != POStatus.STATUS_EOP) {
            Object i1 = ((Tuple) res1.result).get(1);
            Object i2 = ((Tuple) res2.result).get(1);
            int i = DataType.compare(i1, i2);
            assertEquals(true, (i <= 0));
            // System.out.println(res2.result);
            res1 = res2;
            res2 = sort.getNextTuple();
        }
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

        List<Boolean> mAscCols = new LinkedList<Boolean>();
        mAscCols.add(false);
        PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
        List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
        inputs.add(read);
        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
                sortPlans, mAscCols, null);
        Tuple t = null;
        Result res1 = sort.getNextTuple();
        // System.out.println(res1.result);
        Result res2 = sort.getNextTuple();
        while (res2.returnStatus != POStatus.STATUS_EOP) {
            Object i1 = ((Tuple) res1.result).get(1);
            Object i2 = ((Tuple) res2.result).get(1);
            int i = DataType.compare(i1, i2);
            assertEquals(true, (i >= 0));
            // System.out.println(res2.result);
            res1 = res2;
            res2 = sort.getNextTuple();
        }
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort


        PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
        List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
        inputs.add(read);


        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
                                 sortPlans, mAscCols, null);


        Tuple t = null;
        Result res ;
        // output line 1
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 1) ;
        assertEquals(((Tuple) res.result).get(1), 10) ;
        // output line 2
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 2) ;
        assertEquals(((Tuple) res.result).get(1), 8) ;
        // output line 3
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 3) ;
        assertEquals(((Tuple) res.result).get(1), 8) ;


    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort


        PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
        List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
        inputs.add(read);


        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
                                 sortPlans, mAscCols, null);


        Tuple t = null;
        Result res ;
        // output line 1
        res = sort.getNextTuple();
        assertNull(((Tuple) res.result).get(0)) ;
        assertEquals(((Tuple) res.result).get(1), 10) ;
         // output line 2
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 1) ;
        assertEquals(((Tuple) res.result).get(1), 8) ;
        // output line 3
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 3) ;
        assertEquals(((Tuple) res.result).get(1), 8) ;
        // output line 4
        res = sort.getNextTuple();
        assertNull(((Tuple) res.result).get(0)) ;
        assertNull(((Tuple) res.result).get(1)) ;
        // output line 5
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 1 );
        assertNull(((Tuple) res.result).get(1)) ;




    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort


        PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
        List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
        inputs.add(read);


        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
                                 sortPlans, mAscCols, null);


        Tuple t = null;
        Result res ;
        // output line 1
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 3) ;
        assertEquals(((Tuple) res.result).get(1), 5) ;
        // output line 2
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 3) ;
        assertEquals(((Tuple) res.result).get(1), 8) ;
        // output line 3
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 1) ;
        assertEquals(((Tuple) res.result).get(1), 2) ;


    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort


        PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
        List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
        inputs.add(read);


        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
                                 sortPlans, mAscCols, null);


        Tuple t = null;
        Result res ;
        // output line 1
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 3) ;
        assertEquals(((Tuple) res.result).get(1), 8) ;
        // output line 2
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 1) ;
        assertNull(((Tuple) res.result).get(1)) ;
        // output line 3
        res = sort.getNextTuple();
        assertEquals(((Tuple) res.result).get(0), 1) ;
        assertEquals(((Tuple) res.result).get(1), 8) ;
        // output line 4
        res = sort.getNextTuple();
        assertNull(((Tuple) res.result).get(0)) ;
        assertNull(((Tuple) res.result).get(1)) ;
        // output line 5
        res = sort.getNextTuple();
        assertNull(((Tuple) res.result).get(0)) ;
        assertEquals(((Tuple) res.result).get(1), 10) ;




    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

        String funcName = WeirdComparator.class.getName() + "()";
        /*POUserFunc comparator = new POUserFunc(
                new OperatorKey("", r.nextLong()), -1, inputs, funcName);*/
        POUserComparisonFunc comparator = new POUserComparisonFunc(
                new OperatorKey("", r.nextLong()), -1, null, new FuncSpec(funcName));
        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
                null, null, comparator);
        Tuple t = null;
        Result res1 = sort.getNextTuple();
        // System.out.println(res1.result);
        Result res2 = sort.getNextTuple();
        while (res2.returnStatus != POStatus.STATUS_EOP) {
            int i1 = ((Integer) ((Tuple) res1.result).get(1) == null ? 0 : (Integer) ((Tuple) res1.result).get(1));
            int i2 = ((Integer) ((Tuple) res2.result).get(1) == null ? 0 : (Integer) ((Tuple) res2.result).get(1));
            int i = (i1 - 50) * (i1 - 50) - (i2 - 50) * (i2 - 50);
            assertEquals(true, (i <= 0));
            System.out.println(i + " : " + res2.result);
            res1 = res2;
            res2 = sort.getNextTuple();
        }


    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

            MapReduceOper prevJob,
            FileSpec lFile,
            FileSpec quantFile,
            int rp) throws PlanException, VisitorException {
        
        POSort sort = new POSort(inpSort.getOperatorKey(), inpSort
                .getRequestedParallelism(), null, inpSort.getSortPlans(),
                inpSort.getMAscCols(), inpSort.getMSortFunc());
        sort.addOriginalLocation(inpSort.getAlias(), inpSort.getOriginalLocations());
      
      // Turn the asc/desc array into an array of strings so that we can pass it
        // to the FindQuantiles function.
        List<Boolean> ascCols = inpSort.getMAscCols();
        String[] ascs = new String[ascCols.size()];
        for (int i = 0; i < ascCols.size(); i++) ascs[i] = ascCols.get(i).toString();
        // check if user defined comparator is used in the sort, if so
        // prepend the name of the comparator as the first fields in the
        // constructor args array to the FindQuantiles udf
        String[] ctorArgs = ascs;
        if(sort.isUDFComparatorUsed) {
            String userComparatorFuncSpec = sort.getMSortFunc().getFuncSpec().toString();
            ctorArgs = new String[ascs.length + 1];
            ctorArgs[0] = USER_COMPARATOR_MARKER + userComparatorFuncSpec;
            for(int j = 0; j < ascs.length; j++) {
                ctorArgs[j+1] = ascs[j];
            }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

      List<Boolean> ascCol = new ArrayList<Boolean>();
      for(int i=0; i<groups.size(); i++) {                
        ascCol.add(false);
      }
      
      POSort sort = new POSort(op.getOperatorKey(), op.getRequestedParallelism(), null, groups, ascCol, null);
      
      // set up transform plan to get keys and memory size of input tuples
      // it first adds all the plans to get key columns,
      List<PhysicalPlan> transformPlans = new ArrayList<PhysicalPlan>(); 
      transformPlans.addAll(groups);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

        LogicalFieldSchema sortPlanFS = ((LogicalExpression)sortPlanLeaf).getFieldSchema();
        assertNull(sortPlanFS);


        PhysicalPlan pp = Util.buildPhysicalPlanFromNewLP(lp, pc);
        POStore poStore = (POStore)pp.getLeaves().get(0);
        POSort poSort = (POSort)pp.getPredecessors(poStore).get(0);
        POProject poProject = (POProject)poSort.getSortPlans().get(0).getLeaves().get(0);
        assertEquals(DataType.TUPLE, poProject.getResultType());
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.