Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LOSort


            {
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    LOSort sort = (LOSort)predecessor;
                    if (sort.getLimit()==-1)
                        sort.setLimit(limit.getLimit());
                    else
                        sort.setLimit(sort.getLimit()<limit.getLimit()?sort.getLimit():limit.getLimit());
                    try {
                        mPlan.removeAndReconnect(limit);
                    } catch (Exception e) {
                        int errCode = 2012;
                        String msg = "Can not remove LOLimit after LOSort";
                        throw new OptimizerException(msg, errCode, PigException.BUG, e);
                    }
                }
            }
            // Limit is merged into another LOLimit
            else if (predecessor instanceof LOLimit)
            {
              LOLimit beforeLimit = (LOLimit)predecessor;
              beforeLimit.setLimit(beforeLimit.getLimit()<limit.getLimit()?beforeLimit.getLimit():limit.getLimit());
              try {
                mPlan.removeAndReconnect(limit);
              } catch (Exception e) {
                  int errCode = 2012;
                  String msg = "Can not remove LOLimit after LOLimit";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
              }
            }
            // Limit and OrderBy (LOSort) can be separated by split
            else if (predecessor instanceof LOSplitOutput) {              
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    List<LogicalOperator> grandparants = mPlan
                            .getPredecessors(predecessor);
                    // After insertion of splitters, any node in the plan can
                    // have at most one predecessor
                    if (grandparants != null && grandparants.size() != 0
                            && grandparants.get(0) instanceof LOSplit) {                       
                        List<LogicalOperator> greatGrandparants = mPlan
                                .getPredecessors(grandparants.get(0));
                        if (greatGrandparants != null
                                && greatGrandparants.size() != 0
                                && greatGrandparants.get(0) instanceof LOSort) {                          
                            LOSort sort = (LOSort)greatGrandparants.get(0);
                            LOSort newSort = new LOSort(
                                    sort.getPlan(),
                                    new OperatorKey(
                                            sort.getOperatorKey().scope,
                                            NodeIdGenerator
                                                    .getGenerator()
                                                    .getNextNodeId(
                                                            sort.getOperatorKey().scope)),
                                    sort.getSortColPlans(),
                                    sort.getAscendingCols(),
                                    sort.getUserFunc());
                                                 
                            newSort.setLimit(limit.getLimit());
                            try {
                                mPlan.replace(limit, newSort);
                            } catch (PlanException e) {
                                int errCode = 2012;
                                String msg = "Can not replace LOLimit with LOSort after splitter";
View Full Code Here


            // distinct(flatten({(1), (1)})) is (1). However,
            // flatten(distinct({(1), (1)})) is (1), (1)
           
            // in both cases correctness is not affected
            if(successor instanceof LOSort) {
                LOSort sort = (LOSort) successor;
                RequiredFields sortRequiredField = sort.getRequiredFields().get(0);
               
                if(sortRequiredField.getNeedAllFields()) {
                    return false;
                }
               
View Full Code Here

    @Test
    public void testQueryOrderByNoSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a';");
        LogicalPlan lp = planTester.buildPlan("b = order a by $1;");
       
        LOSort sort = (LOSort)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields sortRelevantFields0 = sort.getRelevantInputs(0, 0).get(0);
        assertTrue(sortRelevantFields0.getNeedAllFields() == false);
        assertTrue(sortRelevantFields0.getNeedNoFields() == false);
        assertTrue(sortRelevantFields0.getFields().size() == 1);
        assertTrue(sortRelevantFields0.getFields().contains(new Pair<Integer, Integer>(0, 0)));
       
        RequiredFields sortRelevantFields1 = sort.getRelevantInputs(0, 1).get(0);
        assertTrue(sortRelevantFields1.getNeedAllFields() == false);
        assertTrue(sortRelevantFields1.getNeedNoFields() == false);
        assertTrue(sortRelevantFields1.getFields().size() == 1);
        assertTrue(sortRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 1)));
    }
View Full Code Here

    @Test
    public void testQueryOrderByWithSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a' as (url,hitCount);");
        LogicalPlan lp = planTester.buildPlan("b = order a by $1;");
       
        LOSort sort = (LOSort)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields sortRelevantFields0 = sort.getRelevantInputs(0, 0).get(0);
        assertTrue(sortRelevantFields0.getNeedAllFields() == false);
        assertTrue(sortRelevantFields0.getNeedNoFields() == false);
        assertTrue(sortRelevantFields0.getFields().size() == 1);
        assertTrue(sortRelevantFields0.getFields().contains(new Pair<Integer, Integer>(0, 0)));
       
        RequiredFields sortRelevantFields1 = sort.getRelevantInputs(0, 1).get(0);
        assertTrue(sortRelevantFields1.getNeedAllFields() == false);
        assertTrue(sortRelevantFields1.getNeedNoFields() == false);
        assertTrue(sortRelevantFields1.getFields().size() == 1);
        assertTrue(sortRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 1)));
    }
View Full Code Here

    @Test
    public void testQueryOrderByStarNoSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a';");
        LogicalPlan lp = planTester.buildPlan("b = order a by *;");
       
        LOSort sort = (LOSort)lp.getLeaves().get(0);
        RequiredFields sortRelevantFields0 = sort.getRelevantInputs(0, 0).get(0);
        assertTrue(sortRelevantFields0.getNeedAllFields() == false);
        assertTrue(sortRelevantFields0.getNeedNoFields() == false);
        assertTrue(sortRelevantFields0.getFields().size() == 1);
        assertTrue(sortRelevantFields0.getFields().get(0).first == 0);
        assertTrue(sortRelevantFields0.getFields().get(0).second == 0);
       
        RequiredFields sortRelevantFields1 = sort.getRelevantInputs(0, 1).get(0);
        assertTrue(sortRelevantFields1.getNeedAllFields() == false);
        assertTrue(sortRelevantFields1.getNeedNoFields() == false);
        assertTrue(sortRelevantFields1.getFields().size() == 1);
        assertTrue(sortRelevantFields1.getFields().get(0).first == 0);
        assertTrue(sortRelevantFields1.getFields().get(0).second == 1);
View Full Code Here

    @Test
    public void testQueryOrderByStarWithSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a' as (url,hitCount);");
        LogicalPlan lp = planTester.buildPlan("b = order a by *;");
       
        LOSort sort = (LOSort)lp.getLeaves().get(0);
        RequiredFields sortRelevantFields0 = sort.getRelevantInputs(0, 0).get(0);
        assertTrue(sortRelevantFields0.getNeedAllFields() == false);
        assertTrue(sortRelevantFields0.getNeedNoFields() == false);
        assertTrue(sortRelevantFields0.getFields().size() == 1);
        assertTrue(sortRelevantFields0.getFields().contains(new Pair<Integer, Integer>(0, 0)));
       
        RequiredFields sortRelevantFields1 = sort.getRelevantInputs(0, 1).get(0);
        assertTrue(sortRelevantFields1.getNeedAllFields() == false);
        assertTrue(sortRelevantFields1.getNeedNoFields() == false);
        assertTrue(sortRelevantFields1.getFields().size() == 1);
        assertTrue(sortRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 1)));
    }
View Full Code Here

            {
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    LOSort sort = (LOSort)predecessor;
                    if (sort.getLimit()==-1)
                        sort.setLimit(limit.getLimit());
                    else
                        sort.setLimit(sort.getLimit()<limit.getLimit()?sort.getLimit():limit.getLimit());
                    try {
                        mPlan.removeAndReconnect(limit);
                    } catch (Exception e) {
                        int errCode = 2012;
                        String msg = "Can not remove LOLimit after LOSort";
                        throw new OptimizerException(msg, errCode, PigException.BUG, e);
                    }
                }
            }
            // Limit is merged into another LOLimit
            else if (predecessor instanceof LOLimit)
            {
              LOLimit beforeLimit = (LOLimit)predecessor;
              beforeLimit.setLimit(beforeLimit.getLimit()<limit.getLimit()?beforeLimit.getLimit():limit.getLimit());
              try {
                mPlan.removeAndReconnect(limit);
              } catch (Exception e) {
                  int errCode = 2012;
                  String msg = "Can not remove LOLimit after LOLimit";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
              }
            }
            // Limit and OrderBy (LOSort) can be separated by split
            else if (predecessor instanceof LOSplitOutput) {              
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    List<LogicalOperator> grandparants = mPlan
                            .getPredecessors(predecessor);
                    // After insertion of splitters, any node in the plan can
                    // have at most one predecessor
                    if (grandparants != null && grandparants.size() != 0
                            && grandparants.get(0) instanceof LOSplit) {                       
                        List<LogicalOperator> greatGrandparants = mPlan
                                .getPredecessors(grandparants.get(0));
                        if (greatGrandparants != null
                                && greatGrandparants.size() != 0
                                && greatGrandparants.get(0) instanceof LOSort) {                          
                            LOSort sort = (LOSort)greatGrandparants.get(0);
                            LOSort newSort = new LOSort(
                                    sort.getPlan(),
                                    new OperatorKey(
                                            sort.getOperatorKey().scope,
                                            NodeIdGenerator
                                                    .getGenerator()
                                                    .getNextNodeId(
                                                            sort.getOperatorKey().scope)),
                                    sort.getSortColPlans(),
                                    sort.getAscendingCols(),
                                    sort.getUserFunc());
                                                 
                            newSort.setLimit(limit.getLimit());
                            try {
                                mPlan.replace(limit, newSort);
                            } catch (PlanException e) {
                                int errCode = 2012;
                                String msg = "Can not replace LOLimit with LOSort after splitter";
View Full Code Here

                                              lo.getRequestedParallelism());
            connectInputs(compiledInputs, pom, execEngine.getPhysicalOpTable());
            return pom.getOperatorKey();
        }
        else if (lo instanceof LOSort) {
            LOSort loSort = (LOSort) lo;
            //must break up into 2 map reduce jobs, one for gathering quantiles, another for sorting
            POMapreduce quantileJob = getQuantileJob(lo.getScope(),
                                                     nodeIdGenerator.getNextNodeId(logicalKey.getScope()),
                                                     execEngine.getPhysicalOpTable(),
                                                     logicalKey,
View Full Code Here

            // distinct(flatten({(1), (1)})) is (1). However,
            // flatten(distinct({(1), (1)})) is (1), (1)
           
            // in both cases correctness is not affected
            if(successor instanceof LOSort) {
                LOSort sort = (LOSort) successor;
                RequiredFields sortRequiredField = sort.getRequiredFields().get(0);
               
                if(sortRequiredField.getNeedAllFields()) {
                    return false;
                }
               
View Full Code Here

            {
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    LOSort sort = (LOSort)predecessor;
                    if (sort.getLimit()==-1)
                        sort.setLimit(limit.getLimit());
                    else
                        sort.setLimit(sort.getLimit()<limit.getLimit()?sort.getLimit():limit.getLimit());
                    try {
                        mPlan.removeAndReconnect(limit);
                    } catch (Exception e) {
                        int errCode = 2012;
                        String msg = "Can not remove LOLimit after LOSort";
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.LOSort

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.