Package org.apache.pig.impl.plan

Examples of org.apache.pig.impl.plan.OperatorKey

136
137
138
139
140
141
142
143
144
145
146
           
        }
       
        private void setUpHashTable() throws IOException {
            FileSpec replFile = new FileSpec(repl,new FuncSpec(PigStorage.class.getName()+"()"));
            POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L), replFile);
            PigContext pc = new PigContext(ExecType.MAPREDUCE,ConfigurationUtil.toProperties(PigMapReduce.sJobConfInternal.get()));
            try {
                pc.connect();
           
                ld.setPc(pc);
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
        physOp.setLineageTracer(lineage);

        // replace the original inputs by POReads
        for (int i = 0; i < inputs.size(); i++) {
            DataBag bag = derivedData.get(cg.getInputs().get(i));
            PORead por = new PORead(new OperatorKey("", r.nextLong()), bag);
            phy.add(por);
            try {
                phy.connect(por, physOp.getInputs().get(i));
            } catch (PlanException e) {
                // TODO Auto-generated catch block
View Full Code Here
335
336
337
338
339
340
341
342
343
344
345
        phy.add(physOp);

        // replace the original inputs by POReads
        for (LogicalOperator l : op.getPlan().getPredecessors(op)) {
            DataBag bag = derivedData.get(l);
            PORead por = new PORead(new OperatorKey("", r.nextLong()), bag);
            phy.add(por);
            try {
                phy.connect(por, physOp);
            } catch (PlanException e) {
                // TODO Auto-generated catch block
View Full Code Here
429
430
431
432
433
434
435
436
437
438
439
        physOp.setLineageTracer(null);

        // replace the original inputs by POReads
        for (int i = 0; i < inputs.size(); i++) {
            DataBag bag = derivedData.get(op.getInputs().get(i));
            PORead por = new PORead(new OperatorKey("", r.nextLong()), bag);
            phy.add(por);
            try {
                phy.connect(por, physOp.getInputs().get(i));
            } catch (PlanException e) {
                // TODO Auto-generated catch block
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
   
    public POJoinPackage(OperatorKey k, int rp, POPackage p, POForEach f) {
        super(k, rp);
        String scope = getOperatorKey().getScope();
        NodeIdGenerator nig = NodeIdGenerator.getGenerator();
        forEach = new POOptimizedForEach(new OperatorKey(scope,nig.getNextNodeId(scope)));
        if (p!=null)
        {
            setKeyType(p.getKeyType());
            setNumInps(p.getNumInps());
            lastBagIndex = numInputs - 1;
View Full Code Here
387
388
389
390
391
392
393
394
395
        POUserComparisonFunc cloneFunc = null;
        if (mSortFunc != null) {
            cloneFunc = mSortFunc.clone();
        }
        // Don't set inputs as PhysicalPlan.clone will take care of that
        return new POSort(new OperatorKey(mKey.scope,
            NodeIdGenerator.getGenerator().getNextNodeId(mKey.scope)),
            requestedParallelism, null, clonePlans, cloneAsc, cloneFunc);
    }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
        List<PhysicalPlan> clonePlans = new
            ArrayList<PhysicalPlan>(plans.size());
        for (PhysicalPlan plan : plans) {
            clonePlans.add(plan.clone());
        }
        POLocalRearrangeForIllustrate clone = new POLocalRearrangeForIllustrate(new OperatorKey(
            mKey.scope,
            NodeIdGenerator.getGenerator().getNextNodeId(mKey.scope)),
            requestedParallelism);
        try {
            clone.setPlans(clonePlans);
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
     * or a simple field (false).
     * @param keyPos for each field in the output tuple of the foreach operator,
     * indicates whether it's the group key.
     */
    public POCombinerPackage(POPackage pkg, boolean[] bags, boolean[] keyPos) {
        super(new OperatorKey(pkg.getOperatorKey().scope,
            NodeIdGenerator.getGenerator().getNextNodeId(pkg.getOperatorKey().scope)),
            pkg.getRequestedParallelism(), pkg.getInputs());
        resultType = pkg.getResultType();
        keyType = pkg.keyType;
        numInputs = 1;
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
    public void visit(LOLoad loLoad) throws FrontendException {
        String scope = DEFAULT_SCOPE;
//        System.err.println("Entering Load");
        // The last parameter here is set to true as we assume all files are
        // splittable due to LoadStore Refactor
        POLoad load = new POLoad(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), loLoad.getLoadFunc());
        load.setAlias(loLoad.getAlias());
        load.setLFile(loLoad.getFileSpec());
        load.setPc(pc);
        load.setResultType(DataType.BAG);
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
   
    @Override
    public void visit(LONative loNative) throws FrontendException{    
        String scope = DEFAULT_SCOPE;
       
        PONative poNative = new PONative(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)));
        poNative.setAlias(loNative.getAlias());
        poNative.setNativeMRjar(loNative.getNativeMRJar());
        poNative.setParams(loNative.getParams());
        poNative.setResultType(DataType.BAG);
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.plan.OperatorKey

  • com.twitter.elephantbird.pig.store.TestSequenceFileStorage
  • com.twitter.elephantbird.pig.util.AbstractTestWritableConverter
  • org.apache.pig.backend.executionengine.util.ExecTools
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer$DistinctPatcher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LimitAdjuster
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$LastInputStreamingOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$LimitAdjuster
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.