Package org.apache.pig.pen.util

Examples of org.apache.pig.pen.util.LineageTracer

398
399
400
401
402
403
404
405
406
407
408
409
        for (LogicalOperator in : op.getPlan().getPredecessors(op)) {
            if (derivedData.get(in) == null)
                return null;
        }

        LineageTracer oldLineage = this.lineage;
        this.lineage = new LineageTracer();

        PhysicalOperator physOp = LogToPhyMap.get(op);
        Random r = new Random();
        // get the list of original inputs
        // List<PhysicalOperator> inputs = physOp.getInputs();
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
            POLoad poLoad = (POLoad) LogToPhyMap.get(op);
//            PigContext oldPC = poLoad.getPc();
//            poLoad.setPc(pigContext);

            poLoad.setLineageTracer(new LineageTracer());

            Tuple t = null;
            int count = 0;
            for (Result res = poLoad.getNext(t); res.returnStatus != POStatus.STATUS_EOP
                    && count < MAX_RECORDS; res = poLoad.getNext(t)) {
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
            visitor.visit();
        } catch (VisitorException e) {
            log.error(e.getMessage());
        }

        LineageTracer lineage = visitor.lineage;
        Lineage.put(plan.getLeaves().get(0), lineage);
        Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = visitor.OpToEqClasses;
        Collection<IdentityHashSet<Tuple>> EqClasses = visitor.EqClasses;
        Map<IdentityHashSet<Tuple>, Integer> affinityGroup = new HashMap<IdentityHashSet<Tuple>, Integer>();
        for (IdentityHashSet<Tuple> set : EqClasses) {
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
            Map<IdentityHashSet<Tuple>, Integer> affinityGroups = null;

            continueTrimming = checkCompleteness(cg);
           
            DerivedDataVisitor visitor = null;
            LineageTracer lineage = null;
            // create affinity groups
            if (cg.getInputs().size() == 1) {
                affinityGroups = new HashMap<IdentityHashSet<Tuple>, Integer>();
                LogicalOperator childOp = cg.getInputs().get(0);
                visitor = new DerivedDataVisitor(childOp, null, baseData,
View Full Code Here
450
451
452
453
454
455
456
457
458
459
460
        Lineage.put(childOp, visitor.lineage);

    }

    private boolean checkCompleteness(LogicalOperator op) {
        LineageTracer lineage = Lineage.get(op);
        Lineage.remove(op);

        Map<IdentityHashSet<Tuple>, Integer> affinityGroups = AffinityGroups
                .get(op);
        AffinityGroups.remove(op);
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
            POLoad poLoad = (POLoad) LogToPhyMap.get(op);
//            PigContext oldPC = poLoad.getPc();
//            poLoad.setPc(pigContext);

            poLoad.setLineageTracer(new LineageTracer());

            Tuple t = null;
            int count = 0;
            for (Result res = poLoad.getNext(t); res.returnStatus != POStatus.STATUS_EOP
                    && count < MAX_RECORDS; res = poLoad.getNext(t)) {
View Full Code Here
397
398
399
400
401
402
403
404
405
406
407
408
        for (LogicalOperator in : op.getPlan().getPredecessors(op)) {
            if (derivedData.get(in) == null)
                return null;
        }

        LineageTracer oldLineage = this.lineage;
        this.lineage = new LineageTracer();

        PhysicalOperator physOp = LogToPhyMap.get(op);
        Random r = new Random();
        // get the list of original inputs
        // List<PhysicalOperator> inputs = physOp.getInputs();
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
            visitor.visit();
        } catch (VisitorException e) {
            log.error(e.getMessage());
        }

        LineageTracer lineage = visitor.lineage;
        Lineage.put(plan.getLeaves().get(0), lineage);
        Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = visitor.OpToEqClasses;
        Collection<IdentityHashSet<Tuple>> EqClasses = visitor.EqClasses;
        Map<IdentityHashSet<Tuple>, Integer> affinityGroup = new HashMap<IdentityHashSet<Tuple>, Integer>();
        for (IdentityHashSet<Tuple> set : EqClasses) {
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
            Map<IdentityHashSet<Tuple>, Integer> affinityGroups = null;

            continueTrimming = checkCompleteness(cg);
           
            DerivedDataVisitor visitor = null;
            LineageTracer lineage = null;
            // create affinity groups
            if (cg.getInputs().size() == 1) {
                affinityGroups = new HashMap<IdentityHashSet<Tuple>, Integer>();
                LogicalOperator childOp = cg.getInputs().get(0);
                visitor = new DerivedDataVisitor(childOp, null, baseData,
View Full Code Here
454
455
456
457
458
459
460
461
462
463
464
        Lineage.put(childOp, visitor.lineage);

    }

    private boolean checkCompleteness(LogicalOperator op) {
        LineageTracer lineage = Lineage.get(op);
        Lineage.remove(op);

        Map<IdentityHashSet<Tuple>, Integer> affinityGroups = AffinityGroups
                .get(op);
        AffinityGroups.remove(op);
View Full Code Here
TOP

Related Classes of org.apache.pig.pen.util.LineageTracer

  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.LitePackager
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.Packager
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCross
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackageLite
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORank
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore
  • org.apache.pig.data.Tuple
  • org.apache.pig.pen.DerivedDataVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.