Package org.apache.pig.backend.hadoop.executionengine.tez.plan.operator

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueInputTez

535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
                    break;
                }
            }
            identityInOut.setInputKey(identityInOutPred.getOperatorKey().toString());
        } else if (roots.size() == 1 && roots.get(0) instanceof POValueInputTez) {
            POValueInputTez valueInput = (POValueInputTez) roots.get(0);

            LinkedList<String> scalarInputs = new LinkedList<String>();
            for (POUserFunc userFunc : PlanHelper.getPhysicalOperators(tezOp.plan, POUserFunc.class) ) {
                if (userFunc.getFunc() instanceof ReadScalarsTez) {
                    scalarInputs.add(((ReadScalarsTez)userFunc.getFunc()).getTezInputs()[0]);
                }
            }
            // Make sure we don't find the scalar
            for (TezOperator pred : mPlan.getPredecessors(tezOp)) {
                if (!scalarInputs.contains(pred.getOperatorKey().toString())) {
                    valueInput.setInputKey(pred.getOperatorKey().toString());
                    break;
                }
            }
        }
        setOutputFormat(job);
View Full Code Here
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
                    oper.setSplitter(true);

                    // Create a splittee of store only
                    TezOperator storeOnlyTezOperator = getTezOp();
                    PhysicalPlan storeOnlyPhyPlan = new PhysicalPlan();
                    POValueInputTez valueInput = new POValueInputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
                    valueInput.setInputKey(oper.getOperatorKey().toString());
                    storeOnlyPhyPlan.addAsLeaf(valueInput);
                    storeOnlyPhyPlan.addAsLeaf(store);
                    storeOnlyTezOperator.plan = storeOnlyPhyPlan;
                    tezPlan.add(storeOnlyTezOperator);
                    phyToTezOpMap.put(store, storeOnlyTezOperator);

                    // Create new operator as second splittee
                    curTezOp = getTezOp();
                    POValueInputTez valueInput2 = new POValueInputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
                    valueInput2.setInputKey(oper.getOperatorKey().toString());
                    curTezOp.plan.add(valueInput2);
                    tezPlan.add(curTezOp);

                    // Connect splitter to splittee
                    TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, oper, storeOnlyTezOperator);
View Full Code Here
857
858
859
860
861
862
863
864
865
866
867
868
869
            } else {
                curTezOp.markLimit();
            }

            // Then add a POValueInputTez to the start of the new tezOp.
            POValueInputTez input = new POValueInputTez(OperatorKey.genOpKey(scope));
            input.copyAliasFrom(op);
            input.setInputKey(prevOp.getOperatorKey().toString());
            curTezOp.plan.addAsLeaf(input);

            if (!pigContext.inIllustrator) {
                POLimit limitCopy = new POLimit(OperatorKey.genOpKey(scope));
                limitCopy.copyAliasFrom(op);
View Full Code Here
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
                edge.inputClassName = OrderedGroupedKVInput.class.getName();
                edge.setIntermediateOutputKeyClass(TezCompilerUtil.TUPLE_CLASS);
                edge.setIntermediateOutputKeyComparatorClass(PigTupleWritableComparator.class.getName());

                // Then add a POValueInputTez to the start of the new tezOp followed by a LIMIT
                POValueInputTez input = new POValueInputTez(OperatorKey.genOpKey(scope));
                input.copyAliasFrom(op);
                input.setInputKey(sortOpers[1].getOperatorKey().toString());
                curTezOp.plan.addAsLeaf(input);

                POLimit limit = new POLimit(OperatorKey.genOpKey(scope));
                limit.setLimit(op.getLimit());
                curTezOp.plan.addAsLeaf(limit);
View Full Code Here
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
            output.addOutputKey(curTezOp.getOperatorKey().toString());
            TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, splitOp, curTezOp);
            //TODO shared edge once support is available in Tez
            TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.ONE_TO_ONE);
            curTezOp.setRequestedParallelismByReference(splitOp);
            POValueInputTez input = new POValueInputTez(OperatorKey.genOpKey(scope));
            input.copyAliasFrom(op);
            input.setInputKey(splitOp.getOperatorKey().toString());
            curTezOp.plan.addAsLeaf(input);
        } catch (Exception e) {
            int errCode = 2034;
            String msg = "Error compiling operator "
                    + op.getClass().getSimpleName();
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueInputTez

  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.pig.data.Tuple
  • org.apache.tez.runtime.api.LogicalInput
  • org.apache.tez.runtime.api.Reader
  • org.apache.pig.backend.executionengine.ExecException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.