Examples of LineageTracer

Examples of org.apache.pig.impl.util.LineageTracer

82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
        // (and keep track of which tuples are synthetic)
        IdentityHashSet<Tuple> syntheticTuples = new IdentityHashSet<Tuple>(); 
        Map<LOLoad, DataBag> modifiedBaseData = AugmentData.AugmentBaseData(plan.getRootOperator(), GlobalBaseData, syntheticTuples, derivedData, pigContext);
       
        {
          LineageTracer lineage = new LineageTracer();
          derivedData = DerivedData.CreateDerivedData(plan.getRootOperator(), modifiedBaseData, lineage, equivalenceClasses, OperatorToEqClasses, logicalToPhysicalKeys, physicalOpTable);
          modifiedBaseData = ShapeLineage.PruneBaseData(modifiedBaseData, derivedData.get(plan.getRoot()), syntheticTuples, lineage, equivalenceClasses);

        }

        {
          LineageTracer lineage = new LineageTracer();
          derivedData = DerivedData.CreateDerivedData(plan.getRootOperator(), modifiedBaseData, lineage, equivalenceClasses, OperatorToEqClasses, logicalToPhysicalKeys, physicalOpTable);
          modifiedBaseData = ShapeLineage.TrimLineages(plan.getRootOperator(), modifiedBaseData, derivedData, lineage, OperatorToEqClasses, logicalToPhysicalKeys, physicalOpTable);
        }
       
        {
          LineageTracer lineage = new LineageTracer();
          derivedData = DerivedData.CreateDerivedData(plan.getRootOperator(), modifiedBaseData, lineage, null, null, logicalToPhysicalKeys, physicalOpTable);
       
        }
       
        // Push finalized base data through query plan, to generate final answer
View Full Code Here

Examples of org.apache.pig.impl.util.LineageTracer

61
62
63
64
65
66
67
68
69
70
71
                        new HashMap<OperatorKey, ExecPhysicalOperator> (),
                        pigContext,
                        fileSpec,
                        LogicalOperator.FIXED
                        );
          poLoad.setLineageTracer(new LineageTracer());
            poLoad.open();
            for (int i = 0; i < sampleSize; i++) {
                Tuple t = poLoad.getNext();

                if (t == null) break;
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

398
399
400
401
402
403
404
405
406
407
408
409
        for (LogicalOperator in : op.getPlan().getPredecessors(op)) {
            if (derivedData.get(in) == null)
                return null;
        }

        LineageTracer oldLineage = this.lineage;
        this.lineage = new LineageTracer();

        PhysicalOperator physOp = LogToPhyMap.get(op);
        Random r = new Random();
        // get the list of original inputs
        // List<PhysicalOperator> inputs = physOp.getInputs();
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

158
159
160
161
162
163
164
165
166
167
168
            POLoad poLoad = (POLoad) LogToPhyMap.get(op);
//            PigContext oldPC = poLoad.getPc();
//            poLoad.setPc(pigContext);

            poLoad.setLineageTracer(new LineageTracer());

            Tuple t = null;
            int count = 0;
            for (Result res = poLoad.getNext(t); res.returnStatus != POStatus.STATUS_EOP
                    && count < MAX_RECORDS; res = poLoad.getNext(t)) {
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

94
95
96
97
98
99
100
101
102
103
104
            visitor.visit();
        } catch (VisitorException e) {
            log.error(e.getMessage());
        }

        LineageTracer lineage = visitor.lineage;
        Lineage.put(plan.getLeaves().get(0), lineage);
        Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = visitor.OpToEqClasses;
        Collection<IdentityHashSet<Tuple>> EqClasses = visitor.EqClasses;
        Map<IdentityHashSet<Tuple>, Integer> affinityGroup = new HashMap<IdentityHashSet<Tuple>, Integer>();
        for (IdentityHashSet<Tuple> set : EqClasses) {
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

118
119
120
121
122
123
124
125
126
127
128
            Map<IdentityHashSet<Tuple>, Integer> affinityGroups = null;

            continueTrimming = checkCompleteness(cg);
           
            DerivedDataVisitor visitor = null;
            LineageTracer lineage = null;
            // create affinity groups
            if (cg.getInputs().size() == 1) {
                affinityGroups = new HashMap<IdentityHashSet<Tuple>, Integer>();
                LogicalOperator childOp = cg.getInputs().get(0);
                visitor = new DerivedDataVisitor(childOp, null, baseData,
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

450
451
452
453
454
455
456
457
458
459
460
        Lineage.put(childOp, visitor.lineage);

    }

    private boolean checkCompleteness(LogicalOperator op) {
        LineageTracer lineage = Lineage.get(op);
        Lineage.remove(op);

        Map<IdentityHashSet<Tuple>, Integer> affinityGroups = AffinityGroups
                .get(op);
        AffinityGroups.remove(op);
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

157
158
159
160
161
162
163
164
165
166
167
            POLoad poLoad = (POLoad) LogToPhyMap.get(op);
//            PigContext oldPC = poLoad.getPc();
//            poLoad.setPc(pigContext);

            poLoad.setLineageTracer(new LineageTracer());

            Tuple t = null;
            int count = 0;
            for (Result res = poLoad.getNext(t); res.returnStatus != POStatus.STATUS_EOP
                    && count < MAX_RECORDS; res = poLoad.getNext(t)) {
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

397
398
399
400
401
402
403
404
405
406
407
408
        for (LogicalOperator in : op.getPlan().getPredecessors(op)) {
            if (derivedData.get(in) == null)
                return null;
        }

        LineageTracer oldLineage = this.lineage;
        this.lineage = new LineageTracer();

        PhysicalOperator physOp = LogToPhyMap.get(op);
        Random r = new Random();
        // get the list of original inputs
        // List<PhysicalOperator> inputs = physOp.getInputs();
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

98
99
100
101
102
103
104
105
106
107
108
            visitor.visit();
        } catch (VisitorException e) {
            log.error(e.getMessage());
        }

        LineageTracer lineage = visitor.lineage;
        Lineage.put(plan.getLeaves().get(0), lineage);
        Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = visitor.OpToEqClasses;
        Collection<IdentityHashSet<Tuple>> EqClasses = visitor.EqClasses;
        Map<IdentityHashSet<Tuple>, Integer> affinityGroup = new HashMap<IdentityHashSet<Tuple>, Integer>();
        for (IdentityHashSet<Tuple> set : EqClasses) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.