Package org.apache.flink.api.common.operators.util

Examples of org.apache.flink.api.common.operators.util.FieldList

155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
      assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
     
      DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
      assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
      assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
     
      // check the workset set join
      DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
      assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
      assertTrue(edgeJoin.getInput1().getTempMode().isCached());
     
      assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
     
      // check that the initial partitioning is pushed out of the loop
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
      assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
      assertEquals(new FieldList(0), iteration.getInput2().getShipStrategyKeys());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here
37
38
39
40
41
42
43
44
    if (set instanceof FieldList) {
      return (FieldList) set;
    } else {
      final int[] cols = set.toArray();
      Arrays.sort(cols);
      return new FieldList(cols);
    }
  }
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
   * @return True, if the resulting properties are non trivial.
   */
  public LocalProperties filterByNodesConstantSet(OptimizerNode node, int input) {
    // check, whether the local order is preserved
    Ordering no = this.ordering;
    FieldList ngf = this.groupedFields;
    Set<FieldSet> nuf = this.uniqueFields;
   
    if (this.ordering != null) {
      final FieldList involvedIndexes = this.ordering.getInvolvedIndexes();
      for (int i = 0; i < involvedIndexes.size(); i++) {
        if (!node.isFieldConstant(input, involvedIndexes.get(i))) {
          if (i == 0) {
            no = null;
            ngf = null;
          } else {
            no = this.ordering.createNewOrderingUpToIndex(i);
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
   *
   * @return True, if the resulting properties are non trivial.
   */
  public RequestedLocalProperties filterByNodesConstantSet(OptimizerNode node, int input) {
    if (this.ordering != null) {
      final FieldList involvedIndexes = this.ordering.getInvolvedIndexes();
      for (int i = 0; i < involvedIndexes.size(); i++) {
        if (!node.isFieldConstant(input, involvedIndexes.get(i))) {
          return null;
        }
      }
    } else if (this.groupedFields != null) {
      // check, whether the local key grouping is preserved
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
      SingleInputPlanNode agg2Reducer = (SingleInputPlanNode) sink.getInput().getSource();
      SingleInputPlanNode agg2Combiner = (SingleInputPlanNode) agg2Reducer.getInput().getSource();
      SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Combiner.getInput().getSource();
     
      assertEquals(ShipStrategyType.PARTITION_HASH, agg2Reducer.getInput().getShipStrategy());
      assertEquals(new FieldList(0), agg2Reducer.getInput().getShipStrategyKeys());
     
      assertEquals(ShipStrategyType.FORWARD, agg2Combiner.getInput().getShipStrategy());
     
      assertEquals(ShipStrategyType.PARTITION_HASH, agg1Reducer.getInput().getShipStrategy());
      assertEquals(new FieldList(0, 1), agg1Reducer.getInput().getShipStrategyKeys());
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
      SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Reducer.getInput().getSource();
     
      assertEquals(ShipStrategyType.FORWARD, agg2Reducer.getInput().getShipStrategy());
     
      assertEquals(ShipStrategyType.PARTITION_HASH, agg1Reducer.getInput().getShipStrategy());
      assertEquals(new FieldList(0), agg1Reducer.getInput().getShipStrategyKeys());
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
      // the in-loop partitioning is before the final reducer
     
      // verify joinWithInvariant
      assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
     
      // verify joinWithSolutionSet
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
     
     
      // verify reducer
      assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
      assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
     
      // currently, the system may partition before or after the mapper
      ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
      ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
     
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
      // the in-loop partitioning is before the final reducer
     
      // verify joinWithInvariant
      assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
     
      // verify joinWithSolutionSet
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
     
      // verify reducer
      assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
      assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
     
      // verify solution delta
      assertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
      // the in-loop partitioning is before the final reducer
     
      // verify joinWithInvariant
      assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
     
      // verify joinWithSolutionSet
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
     
      // verify reducer
      assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
      assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
     
     
      // verify solution delta
      assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
184
185
      // check that both reduce and combiner have the same strategy
      assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
      assertEquals(DriverStrategy.SORTED_PARTIAL_REDUCE, combineNode.getDriverStrategy());
     
      // check the keys
      assertEquals(new FieldList(1), reduceNode.getKeys(0));
      assertEquals(new FieldList(1), combineNode.getKeys(0));
      assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
     
      // check DOP
      assertEquals(6, sourceNode.getDegreeOfParallelism());
      assertEquals(6, combineNode.getDegreeOfParallelism());
      assertEquals(8, reduceNode.getDegreeOfParallelism());
View Full Code Here
TOP

Related Classes of org.apache.flink.api.common.operators.util.FieldList

  • org.apache.flink.api.common.operators.util.FieldListTest
  • org.apache.flink.api.common.operators.util.FieldSetTest
  • org.apache.flink.compiler.dag.TwoInputNode
  • org.apache.flink.compiler.dag.WorksetIterationNode
  • org.apache.flink.compiler.dataproperties.LocalProperties
  • org.apache.flink.compiler.dataproperties.RequestedLocalProperties
  • org.apache.flink.compiler.FeedbackPropertiesMatchTest
  • org.apache.flink.compiler.GroupOrderTest
  • org.apache.flink.compiler.java.GroupReduceCompilationTest
  • org.apache.flink.compiler.java.ReduceCompilationTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.