Examples of LogicalSchema

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
    public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
        if (fieldSchema!=null)
            return fieldSchema;
        LogicalRelationalOperator referent = findReferent();
       
        LogicalSchema schema = referent.getSchema();
       
        if (attachedRelationalOp instanceof LOGenerate && plan.getSuccessors(this)==null) {
            if (!(findReferent() instanceof LOInnerLoad)||
                    ((LOInnerLoad)findReferent()).sourceIsBag()) {
                String alias = findReferent().getAlias();
                List<LOInnerLoad> innerLoads = LOForEach.findReacheableInnerLoadFromBoundaryProject(this);
               
                // pull tuple information from innerload
                if (innerLoads.get(0).getProjection().getFieldSchema().schema!=null &&
                        innerLoads.get(0).getProjection().getFieldSchema().schema.isTwoLevelAccessRequired()) {
                    LogicalFieldSchema originalTupleFieldSchema = innerLoads.get(0).getProjection().getFieldSchema().schema.getField(0);
                    LogicalFieldSchema newTupleFieldSchema = new LogicalFieldSchema(originalTupleFieldSchema.alias,
                            schema, DataType.TUPLE);
                    newTupleFieldSchema.uid = originalTupleFieldSchema.uid;
                    LogicalSchema newTupleSchema = new LogicalSchema();
                    newTupleSchema.setTwoLevelAccessRequired(true);
                    newTupleSchema.addField(newTupleFieldSchema);
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(alias, newTupleSchema, DataType.BAG);
                }
                else {
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(alias, schema, DataType.BAG);
                }
                fieldSchema.uid = innerLoads.get(0).getProjection().getFieldSchema().uid;
            }
            else {
                if (findReferent().getSchema()!=null)
                    fieldSchema = findReferent().getSchema().getField(0);
            }
            if (fieldSchema!=null)
                uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
        }
        else {
            if (schema == null) {
                fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BYTEARRAY);
                uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
            }
            else {
                int index = -1;
                if (!isProjectStar() && uidOnlyFieldSchema!=null) {
                    long uid = uidOnlyFieldSchema.uid;
                    for (int i=0;i<schema.size();i++) {
                        LogicalFieldSchema fs = schema.getField(i);
                        if (fs.uid==uid) {
                            index = i;
                        }
                    }
                }
                if (index==-1)
                    index = col;
               
                if (!isProjectStar()) {
                    if (schema!=null && schema.size()>index)
                        fieldSchema = schema.getField(index);
                    else
                        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BYTEARRAY);
                    uidOnlyFieldSchema = fieldSchema.cloneUid();
                }
                else {
                    LogicalSchema newTupleSchema = null;
                    if (schema!=null)
                        newTupleSchema = schema.deepCopy();
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(null, newTupleSchema, DataType.TUPLE);
                    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
                }
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

318
319
320
321
322
323
324
325
326
327
328
        public void visit(LOUnion union) throws FrontendException {
            super.visit(union);
            List<Operator> preds = plan.getPredecessors(union);
            if (preds!=null) {
                for (Operator pred : preds) {
                    LogicalSchema schema = ((LogicalRelationalOperator)pred).getSchema();
                    Set<Long> uids = getMapUids(schema);
                    fullMapUids.addAll(uids);
                }
            }
        }
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

332
333
334
335
336
337
338
339
340
341
342
        public void visit(LOCogroup cogroup) throws FrontendException {
            super.visit(cogroup);
            List<Operator> preds = plan.getPredecessors(cogroup);
            if (preds!=null) {
                for (Operator pred : preds) {
                    LogicalSchema schema = ((LogicalRelationalOperator)pred).getSchema();
                    Set<Long> uids = getMapUids(schema);
                    fullMapUids.addAll(uids);
                }
            }
        }
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
            return fieldSchema;
        LogicalExpression successor = (LogicalExpression)plan.getSuccessors(this).get(0);
        LogicalFieldSchema predFS = successor.getFieldSchema();
        if (predFS!=null) {
            if (columns.size()>1 || predFS.type==DataType.BAG) {
                LogicalSchema innerSchema = null;
                if (predFS.schema!=null) {
                    innerSchema = new LogicalSchema();
                    LogicalSchema realSchema;
                    if (predFS.schema.isTwoLevelAccessRequired()) {
                        realSchema = predFS.schema.getField(0).schema;
                    }
                    else {
                        realSchema = predFS.schema;
                    }
                    if (realSchema!=null) {
                        for (int column:columns) {
                            innerSchema.addField(realSchema.getField(column));
                        }
                    }
                }
                fieldSchema = new LogicalSchema.LogicalFieldSchema(null, innerSchema, predFS.type,
                        LogicalExpression.getNextUid());
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
    public static LogicalSchema translateSchema(Schema schema) {      
        if (schema == null) {
            return null;
        }
       
        LogicalSchema s2 = new LogicalSchema();
        List<Schema.FieldSchema> ll = schema.getFields();
        for (Schema.FieldSchema f: ll) {
            LogicalSchema.LogicalFieldSchema f2 =
                new LogicalSchema.LogicalFieldSchema(f.alias, translateSchema(f.schema), f.type);
                      
            s2.addField(f2);
        }
       
        s2.setTwoLevelAccessRequired(schema.isTwoLevelAccessRequired());
       
        return s2;
    }
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

53
54
55
56
57
58
59
60
61
62
63
       
        return s2;
    }
   
    public static LogicalSchema.LogicalFieldSchema translateFieldSchema(Schema.FieldSchema fs) {     
        LogicalSchema newSchema = null;
        if (fs.schema!=null) {
            newSchema = translateSchema(fs.schema);
        }
       
        LogicalSchema.LogicalFieldSchema newFs = new LogicalSchema.LogicalFieldSchema(null, newSchema, fs.type);
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
        }
       
        LogicalPlan innerPlan = new LogicalPlan();
        foreach.setInnerPlan(innerPlan);
       
        LogicalSchema schema = op.getSchema();
       
        // build foreach inner plan
        List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
        LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[schema.size()-columnsToDrop.size()]);
        innerPlan.add(gen);
       
        for (int i=0, j=0; i<schema.size(); i++) {
            if (columnsToDrop.contains(i)) {
                continue;
            }
           
            LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

280
281
282
283
284
285
286
287
288
289
290
        translateConnection(limit, newLimit);
    }
   
    public void visit(LOStream stream) throws VisitorException {
       
        LogicalSchema s;
        try {
            s = Util.translateSchema(stream.getSchema());
        }catch(Exception e) {
            throw new VisitorException("Failed to translate schema.", e);
        }
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

333
334
335
336
337
338
339
340
341
342
343
    }
   
    public void visit(LOLoad load) throws VisitorException{     
        FileSpec fs = load.getInputFile();
       
        LogicalSchema s = null;
        try {
            s = Util.translateSchema(load.getSchema());
        }catch(Exception e) {
            throw new VisitorException("Failed to translate schema.", e);
        }
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema

73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
        // D = filter C by y > 0;
        // The plan is built with the filter having been pushed above the join
        // but the listners not yet having been called.
        // A = load
        lp = new LogicalPlan();
        LogicalSchema aschema = new LogicalSchema();
        aschema.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        aschema.getField(0).uid = 1;
        LOLoad A = new LOLoad(null, null, lp, null);
        A.neverUseForRealSetSchema(aschema);
        lp.add(A);
       
        // B = load
        LogicalSchema bschema = new LogicalSchema();
        bschema.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        bschema.getField(0).uid = 2;
        LOLoad B = new LOLoad(null, null, lp, null);
        B.neverUseForRealSetSchema(bschema);
        lp.add(B);
       
        // C = join
        LogicalSchema cschema = new LogicalSchema();
        cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        cschema.getField(0).uid = 1;
        cschema.getField(1).uid = 2;

        MultiMap<Integer, LogicalExpressionPlan> mm =
            new MultiMap<Integer, LogicalExpressionPlan>();
        LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});
       
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.