Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin

Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin
The PhysicalOperator that represents a skewed join. It must have two inputs. This operator does not do any actually work, it is only a place holder. When it is translated into MR plan, a POSkewedJoin is translated into a sampling job and a join job.

                tupleKeyMemberTypes.add(exprPlan.getLeaves().get(0).getResultType());
            keyTypes.add(tupleKeyMemberTypes);
        }


        if (loj.getJoinType() == LOJoin.JOINTYPE.SKEWED) {
            POSkewedJoin skj;
            try {
                skj = new POSkewedJoin(new OperatorKey(scope,nodeGen.getNextNodeId(scope)),
                                        parallel,inp, innerFlags);
                skj.setAlias(alias);
                skj.setJoinPlans(joinPlans);
            }
            catch (Exception e) {
                int errCode = 2015;
                String msg = "Skewed Join creation failed";
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
            }
            skj.setResultType(DataType.TUPLE);
            
            for (int i=0; i < inputs.size(); i++) {
                Operator op = inputs.get(i);
                if (!innerFlags[i]) {
                    try {
                        LogicalSchema s = ((LogicalRelationalOperator)op).getSchema();
                        // if the schema cannot be determined
                        if (s == null) {
                            throw new FrontendException("Cannot determine skewed join schema", 2247);
                        }
                        skj.addSchema(Util.translateSchema(s));
                    } catch (FrontendException e) {
                        int errCode = 2015;
                        String msg = "Couldn't set the schema for outer join" ;
                        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
                    }
                } else {
                    // This will never be retrieved. It just guarantees that the index will be valid when
                    // MRCompiler is trying to read the schema
                    skj.addSchema(null);
                }
            }
            
            currentPlan.add(skj);

View Full Code Here

                tupleKeyMemberTypes.add(exprPlan.getLeaves().get(0).getResultType());
            keyTypes.add(tupleKeyMemberTypes);
        }


        if (loj.getJoinType() == LOJoin.JOINTYPE.SKEWED) {
            POSkewedJoin skj;
            try {
                skj = new POSkewedJoin(new OperatorKey(scope,nodeGen.getNextNodeId(scope)),loj.getRequestedParallelisam(),
                                            inp, loj.getInnerFlags());
                skj.setAlias(loj.getAlias());
                skj.setJoinPlans(joinPlans);
            }
            catch (Exception e) {
                int errCode = 2015;
                String msg = "Skewed Join creation failed";
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
            }
            skj.setResultType(DataType.TUPLE);
            
            boolean[] innerFlags = loj.getInnerFlags();
            for (int i=0; i < inputs.size(); i++) {
                LogicalRelationalOperator op = (LogicalRelationalOperator) inputs.get(i);
                if (!innerFlags[i]) {
                    try {
                        LogicalSchema s = op.getSchema();
                        // if the schema cannot be determined
                        if (s == null) {
                            throw new FrontendException();
                        }
                        skj.addSchema(translateSchema(s));
                    } catch (FrontendException e) {
                        int errCode = 2015;
                        String msg = "Couldn't set the schema for outer join" ;
                        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
                    }
                } else {
                    // This will never be retrieved. It just guarantees that the index will be valid when
                    // MRCompiler is trying to read the schema
                    skj.addSchema(null);
                }
            }
            
            currentPlan.add(skj);

View Full Code Here

                tupleKeyMemberTypes.add(exprPlan.getLeaves().get(0).getResultType());
            keyTypes.add(tupleKeyMemberTypes);
        }


        if (loj.getJoinType() == LOJoin.JOINTYPE.SKEWED) {
            POSkewedJoin skj;
            try {
                skj = new POSkewedJoin(new OperatorKey(scope,nodeGen.getNextNodeId(scope)),
                                        parallel,inp, innerFlags);
                skj.setAlias(alias);
                skj.setJoinPlans(joinPlans);
            }
            catch (Exception e) {
                int errCode = 2015;
                String msg = "Skewed Join creation failed";
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
            }
            skj.setResultType(DataType.TUPLE);
            
            for (int i=0; i < inputs.size(); i++) {
                Operator op = inputs.get(i);
                if (!innerFlags[i]) {
                    try {
                        LogicalSchema s = ((LogicalRelationalOperator)op).getSchema();
                        // if the schema cannot be determined
                        if (s == null) {
                            throw new FrontendException("Cannot determine skewed join schema", 2247);
                        }
                        skj.addSchema(Util.translateSchema(s));
                    } catch (FrontendException e) {
                        int errCode = 2015;
                        String msg = "Couldn't set the schema for outer join" ;
                        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
                    }
                } else {
                    // This will never be retrieved. It just guarantees that the index will be valid when
                    // MRCompiler is trying to read the schema
                    skj.addSchema(null);
                }
            }
            
            currentPlan.add(skj);

View Full Code Here

                tupleKeyMemberTypes.add(exprPlan.getLeaves().get(0).getResultType());
            keyTypes.add(tupleKeyMemberTypes);
        }


        if (loj.getJoinType() == LOJoin.JOINTYPE.SKEWED) {
            POSkewedJoin skj;
            try {
                skj = new POSkewedJoin(new OperatorKey(scope,nodeGen.getNextNodeId(scope)),
                                        parallel,inp, innerFlags);
                skj.addOriginalLocation(alias, location);
                skj.setJoinPlans(joinPlans);
            }
            catch (Exception e) {
                int errCode = 2015;
                String msg = "Skewed Join creation failed";
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
            }
            skj.setResultType(DataType.TUPLE);


            for (int i=0; i < inputs.size(); i++) {
                Operator op = inputs.get(i);
                if (!innerFlags[i]) {
                    try {
                        LogicalSchema s = ((LogicalRelationalOperator)op).getSchema();
                        // if the schema cannot be determined
                        if (s == null) {
                            throw new FrontendException(loj, "Cannot determine skewed join schema", 2247);
                        }
                        skj.addSchema(Util.translateSchema(s));
                    } catch (FrontendException e) {
                        int errCode = 2015;
                        String msg = "Couldn't set the schema for outer join" ;
                        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
                    }
                } else {
                    // This will never be retrieved. It just guarantees that the index will be valid when
                    // MRCompiler is trying to read the schema
                    skj.addSchema(null);
                }
            }


            currentPlan.add(skj);

View Full Code Here

                    visitPlan(list, childNode);
                }
            }
        } else if(node instanceof POSkewedJoin){
            childNode = createPONode(node);
            POSkewedJoin skewed = (POSkewedJoin)node;
            MultiMap<PhysicalOperator, PhysicalPlan> joinPlans = skewed.getJoinPlans();
            if(joinPlans!=null) {
                List<PhysicalPlan> inner_plans = new ArrayList<PhysicalPlan>();
                inner_plans.addAll(joinPlans.values());
                visitPlan(inner_plans, childNode);
            }

View Full Code Here

              for (List<PhysicalPlan> list : joinPlans) {
                sb.append(planString(list));
              }
          }
          else if(node instanceof POSkewedJoin){
              POSkewedJoin skewed = (POSkewedJoin)node;
              MultiMap<PhysicalOperator, PhysicalPlan> joinPlans = skewed.getJoinPlans();
              if(joinPlans!=null) {
                List<PhysicalPlan> inner_plans = new ArrayList<PhysicalPlan>();
                inner_plans.addAll(joinPlans.values());                
                  sb.append(planString(inner_plans));                
              }

View Full Code Here

                tupleKeyMemberTypes.add(exprPlan.getLeaves().get(0).getResultType());
            keyTypes.add(tupleKeyMemberTypes);
        }


        if (loj.getJoinType() == LOJoin.JOINTYPE.SKEWED) {
            POSkewedJoin skj;
            try {
                skj = new POSkewedJoin(new OperatorKey(scope,nodeGen.getNextNodeId(scope)),
                                        parallel,inp, innerFlags);
                skj.addOriginalLocation(alias, location);
                skj.setJoinPlans(joinPlans);
            }
            catch (Exception e) {
                int errCode = 2015;
                String msg = "Skewed Join creation failed";
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
            }
            skj.setResultType(DataType.TUPLE);


            for (int i=0; i < inputs.size(); i++) {
                Operator op = inputs.get(i);
                if (!innerFlags[i]) {
                    try {
                        LogicalSchema s = ((LogicalRelationalOperator)op).getSchema();
                        // if the schema cannot be determined
                        if (s == null) {
                            throw new FrontendException(loj, "Cannot determine skewed join schema", 2247);
                        }
                        skj.addSchema(Util.translateSchema(s));
                    } catch (FrontendException e) {
                        int errCode = 2015;
                        String msg = "Couldn't set the schema for outer join" ;
                        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
                    }
                } else {
                    // This will never be retrieved. It just guarantees that the index will be valid when
                    // MRCompiler is trying to read the schema
                    skj.addSchema(null);
                }
            }


            currentPlan.add(skj);

View Full Code Here

                tupleKeyMemberTypes.add(exprPlan.getLeaves().get(0).getResultType());
            keyTypes.add(tupleKeyMemberTypes);
        }


        if (loj.getJoinType() == LOJoin.JOINTYPE.SKEWED) {
            POSkewedJoin skj;
            try {
                skj = new POSkewedJoin(new OperatorKey(scope,nodeGen.getNextNodeId(scope)),
                                        parallel,inp, innerFlags);
                skj.addOriginalLocation(alias, location);
                skj.setJoinPlans(joinPlans);
            }
            catch (Exception e) {
                int errCode = 2015;
                String msg = "Skewed Join creation failed";
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
            }
            skj.setResultType(DataType.TUPLE);


            for (int i=0; i < inputs.size(); i++) {
                Operator op = inputs.get(i);
                if (!innerFlags[i]) {
                    try {
                        LogicalSchema s = ((LogicalRelationalOperator)op).getSchema();
                        // if the schema cannot be determined
                        if (s == null) {
                            throw new FrontendException(loj, "Cannot determine skewed join schema", 2247);
                        }
                        skj.addSchema(Util.translateSchema(s));
                    } catch (FrontendException e) {
                        int errCode = 2015;
                        String msg = "Couldn't set the schema for outer join" ;
                        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
                    }
                } else {
                    // This will never be retrieved. It just guarantees that the index will be valid when
                    // MRCompiler is trying to read the schema
                    skj.addSchema(null);
                }
            }


            currentPlan.add(skj);

View Full Code Here

                    visitPlan(list, childNode);
                }
            }
        } else if(node instanceof POSkewedJoin){
            childNode = createPONode(node);
            POSkewedJoin skewed = (POSkewedJoin)node;
            MultiMap<PhysicalOperator, PhysicalPlan> joinPlans = skewed.getJoinPlans();
            if(joinPlans!=null) {
                List<PhysicalPlan> inner_plans = new ArrayList<PhysicalPlan>();
                inner_plans.addAll(joinPlans.values());
                visitPlan(inner_plans, childNode);
            }

View Full Code Here

              for (List<PhysicalPlan> list : joinPlans) {
                sb.append(planString(list));
              }
          }
          else if(node instanceof POSkewedJoin){
              POSkewedJoin skewed = (POSkewedJoin)node;
              MultiMap<PhysicalOperator, PhysicalPlan> joinPlans = skewed.getJoinPlans();
              if(joinPlans!=null) {
                  List<PhysicalPlan> inner_plans = new ArrayList<PhysicalPlan>();
                  inner_plans.addAll(joinPlans.values());
                  sb.append(planString(inner_plans));
              }

View Full Code Here

0 1

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin

org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PlanPrinter

org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.XMLPhysicalPlanPrinter

org.apache.pig.experimental.logical.relational.LogToPhyTranslationVisitor

org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.