Package org.apache.pig.newplan.logical.expression

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan


            LOInnerLoad innerLoad = new LOInnerLoad(newForEachInnerPlan,
                    newForEach, project.isStar()?-1:project.getCol());
            newForEachInnerPlan.add(innerLoad);
            newForEachInnerPlan.connect(innerLoad, generate);
            LogicalExpressionPlan expPlan = new LogicalExpressionPlan();
            expPlans.add(expPlan);
            ProjectExpression pe = new ProjectExpression(expPlan, i, -1, generate);
            expPlan.add(pe);
        }
       
        newInnerPlan.add(newForEach);
        innerOpsMap.put(foreach, newForEach);
        try {
View Full Code Here


                return false;
            }

            LOFilter filter = (LOFilter)current;
            List<Operator> preds = currentPlan.getPredecessors( pred );
            LogicalExpressionPlan filterPlan = filter.getFilterPlan();

            if (OptimizerUtils.planHasNonDeterministicUdf(filterPlan)) {
                return false;
            }
View Full Code Here

                insertFilter( prev, predecessor, filter );
                return;
            }

            // Find the predecessor of join that contains all required uids.
            LogicalExpressionPlan filterPlan = filter.getFilterPlan();
            List<Operator> preds = currentPlan.getPredecessors( predecessor );
            Map<Integer, Operator> inputs = findInputsToAddFilter( filterPlan, predecessor, preds );

            LOFilter newFilter = null;
            for( Entry<Integer, Operator> entry : inputs.entrySet() ) {
                int inputIndex = entry.getKey();
                Operator pred = entry.getValue();

                // Find projection field offset
                int columnOffset = 0;
                if( predecessor instanceof LOJoin || predecessor instanceof LOCross ) {
                    for( int i = 0; i < inputIndex; i++ ) {
                        columnOffset += ( (LogicalRelationalOperator)preds.get( i ) ).getSchema().size();
                    }
                }

                // Reuse the filter for the first match. For others, need to make a copy of the filter
                // and add it between input and predecessor.
                newFilter = newFilter == null ? filter : new LOFilter( (LogicalPlan)currentPlan );

                currentPlan.add( newFilter );
                subPlan.add( newFilter );
                subPlan.add( pred );
                LogicalExpressionPlan fPlan = filterPlan.deepCopy();
                List<Operator> sinks = fPlan.getSinks();
                List<ProjectExpression> projExprs = new ArrayList<ProjectExpression>();
                for( Operator sink : sinks ) {
                    if( sink instanceof ProjectExpression )
                        projExprs.add( (ProjectExpression)sink );
                }

                if( predecessor instanceof LOCogroup ) {
                    for( ProjectExpression projExpr : projExprs ) {
                        // Need to merge filter condition and cogroup by expression;
                        LogicalExpressionPlan plan = ((LOCogroup) predecessor).getExpressionPlans().get( inputIndex ).iterator().next();
                        LogicalExpressionPlan copy = plan.deepCopy();
                        LogicalExpression root = (LogicalExpression)copy.getSinks().get( 0 );
                        List<Operator> predecessors = fPlan.getPredecessors( projExpr );
                        if( predecessors == null || predecessors.size() == 0 ) {
                            fPlan.remove( projExpr );
                            fPlan.add( root );
                        } else {
View Full Code Here

        return buildOp( loc, op, alias, inputAlias, null );
    }
   
    String buildSampleOp(SourceLocation loc, String alias, String inputAlias, double value,
            SourceLocation valLoc) {
        LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
        //  Generate a filter condition.
        LogicalExpression konst = new ConstantExpression( filterPlan, value);
        konst.setLocation( valLoc );
        UserFuncExpression udf = new UserFuncExpression( filterPlan, new FuncSpec( RANDOM.class.getName() ) );
        new LessThanEqualExpression( filterPlan, udf, konst );
View Full Code Here

                ProjectExpression projExpr = (ProjectExpression)sink;
                String colAlias = projExpr.getColAlias();
                if( projExpr.isRangeProject()){
                
                    LOInnerLoad innerLoad = new LOInnerLoad( lp, foreach,
                            new ProjectExpression(projExpr, new LogicalExpressionPlan())
                    );
                    setupInnerLoadAndProj(innerLoad, projExpr, lp, inputs);
                }
                else if( colAlias != null ) {
                    // the project is using a column alias
View Full Code Here

            Map<String, LogicalExpressionPlan> exprPlans, String colAlias, int col)
    throws RecognitionException {
        ProjectExpression result = null;
       
        if( colAlias != null ) {
            LogicalExpressionPlan exprPlan = exprPlans.get( colAlias );
            if( exprPlan != null ) {
                LogicalExpressionPlan planCopy = null;
                try {
                    planCopy = exprPlan.deepCopy();
                    plan.merge( planCopy );
                } catch (FrontendException ex) {
                    throw new PlanGenerationFailureException( intStream, loc, ex );
                }
                // The projected alias is actually expression alias, so the projections in the represented
                // expression doesn't have any operator associated with it. We need to set it when we
                // substitute the expression alias with the its expression.
                if( op != null ) {
                    Iterator<Operator> it = plan.getOperators();
                    while( it.hasNext() ) {
                        Operator o = it.next();
                        if( o instanceof ProjectExpression ) {
                            ProjectExpression projExpr = (ProjectExpression)o;
                            projExpr.setAttachedRelationalOp( op );
                        }
                    }
                }
                return (LogicalExpression)planCopy.getSources().get( 0 );// get the root of the plan
            } else {
                result = new ProjectExpression( plan, 0, colAlias, op );
                result.setLocation( loc );
                return result;
            }
View Full Code Here

    String buildSampleOp(SourceLocation loc, String alias, String inputAlias, double value,
            SourceLocation valLoc)
                    throws ParserValidationException {

        LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
        //  Generate a filter condition.
        LogicalExpression konst = new ConstantExpression( filterPlan, value);
        konst.setLocation( valLoc );
        UserFuncExpression udf = new UserFuncExpression( filterPlan, new FuncSpec( RANDOM.class.getName() ) );
        new LessThanExpression( filterPlan, udf, konst );
View Full Code Here

        return buildOp ( loc, op, alias, inputAlias, null );
    }

    String buildSplitOtherwiseOp(SourceLocation loc, LOSplitOutput op, String alias, String inputAlias)
            throws ParserValidationException, PlanGenerationFailureException {
        LogicalExpressionPlan splitPlan = new LogicalExpressionPlan();
        Operator losplit = lookupOperator(inputAlias);
        LogicalExpression currentExpr = null;
        for (Operator losplitoutput : plan.getSuccessors(losplit)) {
            // take all the LOSplitOutput and negate their filter plans
            LogicalExpressionPlan fragment = ((LOSplitOutput) losplitoutput)
                    .getFilterPlan();
            try {
                if (OptimizerUtils.planHasNonDeterministicUdf(fragment))
                    throw new ParserValidationException(
                            intStream, loc, new FrontendException(op,
                                    "Can not use Otherwise in Split with an expression containing a @Nondeterministic UDF", 1131));
            } catch (FrontendException e) {
                e.printStackTrace();
                throw new PlanGenerationFailureException(intStream, loc, e);
            }
            LogicalExpression root = null;
            try {
                // get the root expression of the filter plan in LOSplitOutput and copy it
                root = ((LogicalExpression) fragment.getSources().get(0))
                        .deepCopy(splitPlan);
            } catch (FrontendException e) {
                e.printStackTrace();
                throw new PlanGenerationFailureException(intStream, loc, e);
            }
View Full Code Here

      if (schema != null) {
    ArrayList<LogicalFieldSchema> fields = (ArrayList<LogicalFieldSchema>) schema
            .getFields();
    for (int i = 0; i < fields.size(); i++) {
        LogicalExpressionPlan lEplan = new LogicalExpressionPlan();
        new ProjectExpression(lEplan, i, fields.get(i).alias, null, gen);
        allExprPlan.add(lEplan);
    }
      }
  }

  // iterate over all operations and generate corresponding UDFs
  for (int operIdx = 0; operIdx < operations.size(); operIdx++) {
      List<LogicalExpressionPlan> lexpPlanList = new ArrayList<LogicalExpressionPlan>();
      List<LogicalExpression> lexpList = new ArrayList<LogicalExpression>();

      lexpPlanList.addAll(expressionPlans.get(operIdx));

      // If duplicates exists in the dimension list then exception is
      // thrown
      checkDuplicateProject(lexpPlanList);

      // Construct ProjectExpression from the LogicalExpressionPlans
      lexpList = getProjectExpList(lexpPlanList, gen);

      for (int i = 0; i < lexpList.size(); i++) {
    // Retain the columns that needs to be pushed down.
    // Remove the dimension columns from the input column list
    // as it will be attached to CubeDimension UDF
    for (int j = 0; j < allExprPlan.size(); j++) {
        LogicalExpression lexp = (LogicalExpression) allExprPlan.get(j).getSources()
          .get(0);
        String colAlias = ((ProjectExpression) lexpList.get(i)).getColAlias();
        if (colAlias == null) {
      colAlias = ((ProjectExpression) lexpList.get(i)).getFieldSchema().alias;
        }

        String projExpAlias = null;
        try {
      projExpAlias = ((ProjectExpression) lexp).getColAlias();
        } catch (ClassCastException e) {
      // if it is not projection then it should be
      // UserFuncExpr.
      // ignore and continue till next ProjExpr is encountered
      continue;
        }
        if (colAlias.equals(projExpAlias) == true) {
      allExprPlan.remove(j);
        } else {
      // if projected exp alias is a namespaced alias
      if (projExpAlias.lastIndexOf(":") != -1) {
          projExpAlias = projExpAlias.substring(
            projExpAlias.lastIndexOf(":") + 1, projExpAlias.length());
          if (colAlias.equals(projExpAlias) == true) {
        allExprPlan.remove(j);
          }
      }
        }
    }
      }

      // Create UDF with user specified dimensions
      LogicalExpressionPlan uexpPlan = new LogicalExpressionPlan();
      if (operations.get(operIdx).equals("CUBE")) {
    new UserFuncExpression(uexpPlan, new FuncSpec(CubeDimensions.class.getName()),
            lexpList);
      } else {
    new UserFuncExpression(uexpPlan, new FuncSpec(RollupDimensions.class.getName()),
            lexpList);
      }

      for (LogicalExpressionPlan lexp : lexpPlanList) {
    Iterator<Operator> it = lexp.getOperators();
    while (it.hasNext()) {
        uexpPlan.add(it.next());
    }
      }
      // Add the UDF to logical expression plan that contains dependent
      // attributes (pushed down from input columns)
      allExprPlan.add(operIdx, uexpPlan);
  }

  // If the operator is a UserFuncExpression then set the flatten flags.
  List<Boolean> flattenFlags = new ArrayList<Boolean>();
  for (int idx = 0; idx < allExprPlan.size(); idx++) {
      List<Operator> opers = allExprPlan.get(idx).getSources();
      for (Operator oper : opers) {
    if (oper instanceof ProjectExpression) {
        flattenFlags.add(false);
    } else if (oper instanceof UserFuncExpression) {
        flattenFlags.add(true);
    }
      }
  }

  // Generate and Foreach operator creation
  String falias = null;
  try {
      buildGenerateOp(loc, (LOForEach) foreach, (LOGenerate) gen, allExprPlan,
        flattenFlags, getUserDefinedSchema(allExprPlan));
      falias = buildForeachOp(loc, (LOForEach) foreach, "cube", inputAlias, innerPlan);
  } catch (ParserValidationException pve) {
      throw new FrontendException(pve);
  }

  List<Boolean> innerFlags = new ArrayList<Boolean>();
  List<String> inpAliases = new ArrayList<String>();
  inpAliases.add(falias);
  innerFlags.add(false);

  // Get the output schema of foreach operator and reconstruct the
  // LogicalExpressionPlan for each dimensional attributes
  MultiMap<Integer, LogicalExpressionPlan> exprPlansCopy = new MultiMap<Integer, LogicalExpressionPlan>();

  for (LogicalExpressionPlan exp : expressionPlans.values()) {
      LogicalExpression lexp = (LogicalExpression) exp.getSources().get(0);
      LogicalExpressionPlan epGrp = new LogicalExpressionPlan();
      new ProjectExpression(epGrp, 0, lexp.getFieldSchema().alias, null, groupby);
      exprPlansCopy.put(0, epGrp);
  }

  // build group by operator
View Full Code Here

    private List<LogicalExpression> getProjectExpList(List<LogicalExpressionPlan> lexpPlanList,
      LogicalRelationalOperator lro) throws FrontendException {

  List<LogicalExpression> leList = new ArrayList<LogicalExpression>();
  for (int i = 0; i < lexpPlanList.size(); i++) {
      LogicalExpressionPlan lexp = lexpPlanList.get(i);
      LogicalExpression lex = (LogicalExpression) lexp.getSources().get(0);
      Iterator<Operator> opers = lexp.getOperators();

      // ProjExpr are initially attached to CubeOp. So re-attach it to
      // specified operator
      while (opers.hasNext()) {
    Operator oper = opers.next();
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.