Package org.apache.pig.impl.plan

Examples of org.apache.pig.impl.plan.OperatorKey

1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
    }
   
    private POPackage compileToLR_GR_PackTrio(LogicalRelationalOperator relationalOp, String customPartitioner,
            boolean[] innerFlags, MultiMap<Integer, LogicalExpressionPlan> innerPlans) throws FrontendException {

        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelisam());
        poGlobal.setAlias(relationalOp.getAlias());
        poGlobal.setCustomPartitioner(customPartitioner);
        POPackage poPackage = new POPackage(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelisam());
        poPackage.setAlias(relationalOp.getAlias());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);

        try {
            currentPlan.connect(poGlobal, poPackage);
        } catch (PlanException e1) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);
        }

        int count = 0;
        Byte type = null;
        List<Operator> inputs = relationalOp.getPlan().getPredecessors(relationalOp);
        for (int i=0;i<inputs.size();i++) {
            Operator op = inputs.get(i);
            List<LogicalExpressionPlan> plans = (List<LogicalExpressionPlan>)innerPlans.get(i);
            POLocalRearrange physOp = new POLocalRearrange(new OperatorKey(
                    DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelisam());
            physOp.setAlias(relationalOp.getAlias());
            List<PhysicalPlan> exprPlans = translateExpressionPlans(relationalOp, plans);
            try {
                physOp.setPlans(exprPlans);
View Full Code Here
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
        List<Boolean> flattenLst = new ArrayList<Boolean>();
        POForEach fe;
        try{
            for(int i=0;i< inputs.size();i++){
                PhysicalPlan fep1 = new PhysicalPlan();
                POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
                        parallel, i+1); //i+1 since the first column is the "group" field
                feproj1.setAlias(alias);
                feproj1.setResultType(DataType.BAG);
                feproj1.setOverloaded(false);
                fep1.add(feproj1);
                fePlans.add(fep1);
                // the parser would have marked the side
                // where we need to keep empty bags on
                // non matched as outer (innerFlags[i] would be
                // false)
                if(!(innerFlags[i])) {
                    Operator joinInput = inputs.get(i);
                    // for outer join add a bincond
                    // which will project nulls when bag is
                    // empty
                    updateWithEmptyBagCheck(fep1, joinInput);
                }
                flattenLst.add(true);
            }
           
            fe = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
                    parallel, fePlans, flattenLst );
            fe.setAlias(alias);

        }catch (PlanException e1) {
            int errCode = 2015;
View Full Code Here
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
    }

    @Override
    public void visit(LOUnion loUnion) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        POUnion physOp = new POUnion(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loUnion.getRequestedParallelisam());
        physOp.setAlias(loUnion.getAlias());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loUnion, physOp);
        List<Operator> ops = loUnion.getPlan().getPredecessors(loUnion);
View Full Code Here
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
    }
   
    @Override
    public void visit(LODistinct loDistinct) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        PODistinct physOp = new PODistinct(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loDistinct.getRequestedParallelisam());
        physOp.setAlias(loDistinct.getAlias());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loDistinct, physOp);
        Operator op = loDistinct.getPlan().getPredecessors(loDistinct).get(0);
View Full Code Here
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
    }
   
    @Override
    public void visit(LOLimit loLimit) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        POLimit physOp = new POLimit(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loLimit.getRequestedParallelisam());
        physOp.setLimit(loLimit.getLimit());
        physOp.setAlias(loLimit.getAlias());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loLimit, physOp);
View Full Code Here
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
    }
   
    @Override
    public void visit(LOSplit loSplit) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        POSplit physOp = new POSplit(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), loSplit.getRequestedParallelisam());
        physOp.setAlias(loSplit.getAlias());
        FileSpec splStrFile;
        try {
            splStrFile = new FileSpec(FileLocalizer.getTemporaryPath(pc).toString(),new FuncSpec(Utils.getTmpFileCompressorName(pc)));
View Full Code Here
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
   
    @Override
    public void visit(LOSplitOutput loSplitOutput) throws FrontendException {
        String scope = DEFAULT_SCOPE;
//        System.err.println("Entering Filter");
        POFilter poFilter = new POFilter(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), loSplitOutput.getRequestedParallelisam());
        poFilter.setAlias(loSplitOutput.getAlias());
        poFilter.setResultType(DataType.BAG);
        currentPlan.add(poFilter);
        logToPhyMap.put(loSplitOutput, poFilter);
View Full Code Here
496
497
498
499
500
501
502
        mTupleFactory = TupleFactory.getInstance();
    }


    private OperatorKey genKey(){
        return new OperatorKey(opKey.scope,NodeIdGenerator.getGenerator().getNextNodeId(opKey.scope));
    }
View Full Code Here
136
137
138
139
140
141
142
143
    public List<List<PhysicalPlan>> getJoinPlans() {
        return phyPlanLists;
    }

    private OperatorKey genKey(OperatorKey old) {
        return new OperatorKey(old.scope, NodeIdGenerator.getGenerator()
                .getNextNodeId(old.scope));
    }
View Full Code Here
315
316
317
318
319
320
321
322
323
324
325
            if (i == fragment) {
                replicates[i] = null;
                continue;
            }

            POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L),
                    replFile);
           
            Properties props = ConfigurationUtil.getLocalFSProperties();
            PigContext pc = new PigContext(ExecType.LOCAL, props);  
            ld.setPc(pc);
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.plan.OperatorKey

  • com.twitter.elephantbird.pig.store.TestSequenceFileStorage
  • com.twitter.elephantbird.pig.util.AbstractTestWritableConverter
  • org.apache.pig.backend.executionengine.util.ExecTools
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer$DistinctPatcher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LimitAdjuster
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$LastInputStreamingOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$LimitAdjuster
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.