Examples of DualInputPlanNode


Examples of eu.stratosphere.compiler.plan.DualInputPlanNode

      for (Channel c: sn.getBroadcastInputs()) {
        traverseChannel(c);
      }
    }
    else if (node instanceof DualInputPlanNode) {
      DualInputPlanNode dn = (DualInputPlanNode) node;
     
      if (!(dn.getOptimizerNode().getPactContract() instanceof DualInputOperator)) {
        throw new RuntimeException("Wrong operator type found in post pass.");
      }
     
      DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getPactContract();
     
      // parameterize the node's driver strategy
      if (dn.getDriverStrategy().requiresComparator()) {
        dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(),
          getSortOrders(dn.getKeysForInput1(), dn.getSortOrders())));
        dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(),
            getSortOrders(dn.getKeysForInput2(), dn.getSortOrders())));

        dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(),
            dualInputOperator.getOperatorInfo().getSecondInputType()));
       
      }
           
      traverseChannel(dn.getInput1());
      traverseChannel(dn.getInput2());
     
      // don't forget the broadcast inputs
      for (Channel c: dn.getBroadcastInputs()) {
        traverseChannel(c);
      }
     
    }
    // catch the sources of the iterative step functions
View Full Code Here

Examples of eu.stratosphere.compiler.plan.DualInputPlanNode

        continue;
      }
     
      placePipelineBreakersIfNecessary(operator.getStrategy(), in1, in2);
     
      DualInputPlanNode node = operator.instantiate(in1, in2, this);
      node.setBroadcastInputs(broadcastChannelsCombination);
     
      GlobalProperties gp1 = in1.getGlobalProperties().clone().filterByNodesConstantSet(this, 0);
      GlobalProperties gp2 = in2.getGlobalProperties().clone().filterByNodesConstantSet(this, 1);
      GlobalProperties combined = operator.computeGlobalProperties(gp1, gp2);

      LocalProperties lp1 = in1.getLocalProperties().clone().filterByNodesConstantSet(this, 0);
      LocalProperties lp2 = in2.getLocalProperties().clone().filterByNodesConstantSet(this, 1);
      LocalProperties locals = operator.computeLocalProperties(lp1, lp2);
     
      node.initProperties(combined, locals);
      node.updatePropertiesWithUniqueSets(getUniqueFields());
      target.add(node);
    }
  }
View Full Code Here

Examples of eu.stratosphere.compiler.plan.DualInputPlanNode

      boolean[] tmp = new boolean[this.keys1.size()];
      System.arraycopy(inputOrders, 0, tmp, 0, tmp.length);
      inputOrders = tmp;
    }
   
    return new DualInputPlanNode(node, "CoGroup ("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
  }
View Full Code Here

Examples of eu.stratosphere.compiler.plan.DualInputPlanNode

    return true;
  }

  @Override
  public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
    return new DualInputPlanNode(node, "Join("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.HYBRIDHASH_BUILD_FIRST, this.keys1, this.keys2);
  }
View Full Code Here

Examples of eu.stratosphere.compiler.plan.DualInputPlanNode

            optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
        }
      }
    }
    else if (node instanceof DualInputPlanNode) {
      DualInputPlanNode dn = (DualInputPlanNode) node;
     
      // get the nodes current schema
      T schema1;
      T schema2;
      if (dn.postPassHelper1 == null) {
        schema1 = createEmptySchema();
        schema2 = createEmptySchema();
        dn.postPassHelper1 = schema1;
        dn.postPassHelper2 = schema2;
      } else {
        schema1 = (T) dn.postPassHelper1;
        schema2 = (T) dn.postPassHelper2;
      }

      schema1.increaseNumConnectionsThatContributed();
      schema2.increaseNumConnectionsThatContributed();
      TwoInputNode optNode = dn.getTwoInputNode();
     
      // add the parent schema to the schema
      if (propagateParentSchemaDown) {
        addSchemaToSchema(parentSchema, schema1, optNode, 0);
        addSchemaToSchema(parentSchema, schema2, optNode, 1);
      }
     
      // check whether all outgoing channels have not yet contributed. come back later if not.
      if (schema1.getNumConnectionsThatContributed() < dn.getOutgoingChannels().size()) {
        return;
      }
     
      // add the nodes local information
      try {
        getDualInputNodeSchema(dn, schema1, schema2);
      } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getPactContract().getName()));
      }
     
      // parameterize the node's driver strategy
      if (createUtilities) {
        if (dn.getDriverStrategy().requiresComparator()) {
          // set the individual comparators
          try {
            dn.setComparator1(createComparator(dn.getKeysForInput1(), dn.getSortOrders(), schema1));
            dn.setComparator2(createComparator(dn.getKeysForInput2(), dn.getSortOrders(), schema2));
          } catch (MissingFieldTypeInfoException e) {
            throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
                optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
          }
         
          // set the pair comparator
          try {
            dn.setPairComparator(createPairComparator(dn.getKeysForInput1(), dn.getKeysForInput2(),
              dn.getSortOrders(), schema1, schema2));
          } catch (MissingFieldTypeInfoException e) {
            throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
                optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
          }
         
        }
      }
     
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema1, dn.getInput1(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for the first input channel to node '"
          + optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
      }
      try {
        propagateToChannel(schema2, dn.getInput2(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for the second input channel to node '"
          + optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
      }
     
      // don't forget the broadcast inputs
      for (Channel c: dn.getBroadcastInputs()) {
        try {
          propagateToChannel(createEmptySchema(), c, createUtilities);
        } catch (MissingFieldTypeInfoException e) {
          throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
            optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

      Plan plan = getTestPlanRightStatic(PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
     
      OptimizedPlan oPlan = compileNoStats(plan);
 
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
      DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
     
      // verify correct join strategy
      assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, innerJoin.getDriverStrategy());
      assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
      assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
   
      new NepheleJobGraphGenerator().compileJobGraph(oPlan);
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

      Plan plan = getTestPlanRightStatic(PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
     
      OptimizedPlan oPlan = compileNoStats(plan);
 
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
      DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
     
      // verify correct join strategy
      assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, innerJoin.getDriverStrategy());
      assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
      assertEquals(TempMode.CACHED, innerJoin.getInput2().getTempMode());
   
      new NepheleJobGraphGenerator().compileJobGraph(oPlan);
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

      Plan plan = getTestPlanLeftStatic(PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
     
      OptimizedPlan oPlan = compileNoStats(plan);
 
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
      DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
     
      // verify correct join strategy
      assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED, innerJoin.getDriverStrategy());
      assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
      assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
   
      new NepheleJobGraphGenerator().compileJobGraph(oPlan);
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

      Plan plan = getTestPlanLeftStatic(PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
     
      OptimizedPlan oPlan = compileNoStats(plan);
 
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
      DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
     
      // verify correct join strategy
      assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, innerJoin.getDriverStrategy());
      assertEquals(TempMode.CACHED, innerJoin.getInput1().getTempMode());
      assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
   
      new NepheleJobGraphGenerator().compileJobGraph(oPlan);
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

     
     
      OptimizedPlan oPlan = compileNoStats(plan);
 
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
      DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
     
      // verify correct join strategy
      assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, innerJoin.getDriverStrategy());
      assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
      assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
   
      new NepheleJobGraphGenerator().compileJobGraph(oPlan);
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.