Package com.netflix.lipstick.model.operators

Examples of com.netflix.lipstick.model.operators.P2jLogicalRelationalOperator$Join

129
130
131
132
133
134
135
136
137
138
139
140
141
142
     */
    protected Boolean schemaEqualsPredecessor(P2jLogicalRelationalOperator oper) {
        if (oper.getSchemaString() != null) {
            String operString = oper.getSchemaString().substring(1, oper.getSchemaString().length() - 1);
            for (String predName : oper.getPredecessors()) {
                P2jLogicalRelationalOperator pred = p2jMap.get(predName);
                try {
                    if (pred.getSchemaString() != null) {
                        String predString = pred.getSchemaString().substring(1, pred.getSchemaString().length() - 1);
                        if (!Schema.equals(Utils.getSchemaFromString(predString),
                                           Utils.getSchemaFromString(operString),
                                           true,
                                           false)) {
                            return false;
View Full Code Here
411
412
413
414
415
416
417
418
419
420
421
            attributeGraphNode(node, e.getKey());
            if (!appendToSubgraph(subgraphs, node, e.getKey())) {
                gv.node(node);
            }
            for (String i : e.getKey().getSuccessors()) {
                P2jLogicalRelationalOperator dst = p2jMap.get(i);
                Edge edge = new Edge(node, graphMap.get(dst));
                gv.edge(edge);
            }
        }
        for (Entry<String, Graph> sg : subgraphs.entrySet()) {
View Full Code Here
62
63
64
65
66
67
68
     * @param node the LogicalRelationalOperator to convert
     * @param lp the LogicalPlan containing the LogicalRelationalOperator
     * @throws FrontendException
     */
    public LOJsonAdaptor(LogicalRelationalOperator node, LogicalPlan lp) throws FrontendException {
        this(node, new P2jLogicalRelationalOperator(), lp);
    }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
            verifyLimitAdaptor((LOLimitJsonAdaptor) adaptor, (LOLimit) lro);
        }
    }

    private void verifyGenericAdaptor(LOJsonAdaptor adaptor, LogicalRelationalOperator lro) throws FrontendException {
        P2jLogicalRelationalOperator p2j = adaptor.getToP2jOperator();
        if (lro.getSchema() != null) {
            Assert.assertEquals(p2j.getSchemaString(), Util.translateSchema(lro.getSchema()).toString());
        }
        Assert.assertEquals(p2j.getOperator(), lro.getClass().getSimpleName());
        Assert.assertEquals(p2j.getAlias(), lro.getAlias());
        Assert.assertEquals(p2j.getLocation().getLine(), (Integer) lro.getLocation().line());
        Assert.assertEquals(p2j.getLocation().getFilename(), lro.getLocation().file());
    }
View Full Code Here
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
        }

        String stepTypeString = stepType.toString();

        if (pop instanceof POStore) {
            P2jLogicalRelationalOperator node = getOpForStore((POStore) pop);
            if (node != null) {
                node.setMapReduce(jid, stepTypeString);
                return;
            }
        } else if (phy2LogMap.containsKey(pop) && reverseMap.containsKey(phy2LogMap.get(pop))) {
            String nodeId = reverseMap.get(phy2LogMap.get(pop));
            P2jLogicalRelationalOperator node = p2jMap.get(nodeId);
            node.setMapReduce(jid, stepTypeString);
            LOG.debug("Found key for: " + pop.toString());
            return;
        } else {
            LOG.debug("No node for pop: " + pop + pop.getClass() + " ... Searching locationMap.");
            boolean didAssign = false;
            for (OriginalLocation loc : pop.getOriginalLocations()) {
                LOG.debug("Checking location: " + loc);
                if (locationMap.containsKey(loc.toString())) {
                    P2jLogicalRelationalOperator node = p2jMap.get(reverseMap.get(locationMap.get(loc.toString())));
                    LOG.debug("Found location... " + node);
                    if (node.getMapReduce() == null) {
                        if (node.getOperator().equalsIgnoreCase("LOJoin")
                            || node.getOperator().equalsIgnoreCase("LOGroup")) {
                            stepTypeString = MRStepType.REDUCER.toString();
                        }
                        node.setMapReduce(jid, stepTypeString);
                        didAssign = true;
                        LOG.debug("Assign location... " + node);
                    }
                }
            }
View Full Code Here
331
332
333
334
335
336
337
338
339
340
341
342
343
     * @return a set of map/reduce job scopes
     */
    protected Set<String> generateScopesForNode(P2jLogicalRelationalOperator node, ScopeGetter scopeGetter) {
        Set<String> scopes = Sets.newHashSet();
        for (String id : scopeGetter.getScopes(node)) {
            P2jLogicalRelationalOperator job = p2jMap.get(id);
            if (job.getMapReduce() != null && job.getMapReduce().getJobId() != null) {
                scopes.add(job.getMapReduce().getJobId());
            } else {
                scopes.addAll(generateScopesForNode(job, scopeGetter));
            }
        }
        return scopes;
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
     *
     */
    protected P2jLogicalRelationalOperator convertNodeToP2j(LogicalRelationalOperator node,
                                                            LogicalPlan lp,
                                                            Map<Operator, String> reverseMap) throws FrontendException {
        P2jLogicalRelationalOperator p2jNode = convertNodeToAdaptor(node, lp).getToP2jOperator();
        p2jNode.setPredecessors(generateP2jIdList(lp.getPredecessors(node), reverseMap));
        p2jNode.setSuccessors(generateP2jIdList(lp.getSuccessors(node), reverseMap));
        p2jNode.setUid(reverseMap.get(node));
        return p2jNode;
    }
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
        expectedOps.add(limit1);

        // For each op, ensure the aliases and all predecessors/successors match
        for (String scope : plan.getPlan().keySet()) {
            P2jLogicalRelationalOperator actualOp = plan.getPlan().get(scope);

            String actualId = getIdentifier(actualOp);

            P2jLogicalRelationalOperator matchedOp = null;
            for (P2jLogicalRelationalOperator expectedOp : expectedOps) {
                String expectedId = getIdentifier(expectedOp);
                if (actualId.equals(expectedId)) {
                    matchedOp = expectedOp;
                    // Compare classes
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
        expectedIdToStepTypeMap.put("colors_filtered", MRStepType.UNKNOWN);
        expectedIdToStepTypeMap.put("tiny_colors_cogrp", MRStepType.MAPPER);
        expectedIdToStepTypeMap.put("out", MRStepType.REDUCER);

        for (String scope : opPlan.getPlan().keySet()) {
            P2jLogicalRelationalOperator actualOp = opPlan.getPlan().get(scope);

            String actualId = getIdentifier(actualOp);

            String actualStepType = actualOp.getMapReduce().getStepType();
            String expectedStepType = expectedIdToStepTypeMap.get(actualId).toString();

            Assert.assertEquals(actualStepType, expectedStepType);
        }
    }
View Full Code Here
TOP

Related Classes of com.netflix.lipstick.model.operators.P2jLogicalRelationalOperator$Join

  • com.asakusafw.vocabulary.flow.graph.FlowElementResolver
  • com.asakusafw.vocabulary.flow.graph.ShuffleKey
  • com.netflix.lipstick.adaptors.LOJsonAdaptor
  • com.netflix.lipstick.adaptors.LOJsonAdaptorTest
  • com.netflix.lipstick.model.operators.elements.JoinExpression
  • com.netflix.lipstick.MRPlanCalculator
  • com.netflix.lipstick.MRPlanCalculatorTest
  • com.netflix.lipstick.P2jPlanGenerator
  • com.netflix.lipstick.P2jPlanGeneratorTest
  • com.netflix.lipstick.Pig2DotGenerator
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.