Package eu.stratosphere.api.common.operators.util

Examples of eu.stratosphere.api.common.operators.util.FieldList

82
83
84
85
86
87
88
89
90
91
92
93
    super(pactContract);

    int[] k1 = pactContract.getKeyColumns(0);
    int[] k2 = pactContract.getKeyColumns(1);
   
    this.keys1 = k1 == null || k1.length == 0 ? null : new FieldList(k1);
    this.keys2 = k2 == null || k2.length == 0 ? null : new FieldList(k2);
   
    if (this.keys1 != null) {
      if (this.keys2 != null) {
        if (this.keys1.size() != this.keys2.size()) {
          throw new CompilerException("Unequal number of key fields on the two inputs.");
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
   */
  public LocalProperties filterByNodesConstantSet(OptimizerNode node, int input)
  {
    // check, whether the local order is preserved
    Ordering no = this.ordering;
    FieldList ngf = this.groupedFields;
    Set<FieldSet> nuf = this.uniqueFields;
   
    if (this.ordering != null) {
      final FieldList involvedIndexes = this.ordering.getInvolvedIndexes();
      for (int i = 0; i < involvedIndexes.size(); i++) {
        if (!node.isFieldConstant(input, involvedIndexes.get(i))) {
          if (i == 0) {
            no = null;
            ngf = null;
          } else {
            no = this.ordering.createNewOrderingUpToIndex(i);
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
149
150
   *
   * @return True, if the resulting properties are non trivial.
   */
  public RequestedLocalProperties filterByNodesConstantSet(OptimizerNode node, int input) {
    if (this.ordering != null) {
      final FieldList involvedIndexes = this.ordering.getInvolvedIndexes();
      for (int i = 0; i < involvedIndexes.size(); i++) {
        if (!node.isFieldConstant(input, involvedIndexes.get(i))) {
          return null;
        }
      }
    } else if (this.groupedFields != null) {
      // check, whether the local key grouping is preserved
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
   
    final int[] ssKeys = iteration.getSolutionSetKeyFields();
    if (ssKeys == null || ssKeys.length == 0) {
      throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
    }
    this.solutionSetKeyFields = new FieldList(ssKeys);
    this.partitionedProperties = new GlobalProperties();
    this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
   
    int weight = iteration.getMaximumNumberOfIterations() > 0 ?
      iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
View Full Code Here
31
32
33
34
35
36
37
38
    if (set instanceof FieldList) {
      return (FieldList) set;
    } else {
      final int[] cols = set.toArray();
      Arrays.sort(cols);
      return new FieldList(cols);
    }
  }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
      Assert.assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
      Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
     
      Channel c = reducer.getInput();
      Assert.assertEquals(LocalStrategy.COMBININGSORT, c.getLocalStrategy());
      FieldList l = new FieldList(0);
      Assert.assertEquals(l, c.getShipStrategyKeys());
      Assert.assertEquals(l, c.getLocalStrategyKeys());
      Assert.assertTrue(Arrays.equals(c.getLocalStrategySortOrder(), reducer.getSortOrders()));
     
      // check the combiner
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
      Assert.assertEquals(ShipStrategyType.PARTITION_RANGE, reducer.getInput().getShipStrategy());
      Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
     
      Channel c = reducer.getInput();
      Assert.assertEquals(LocalStrategy.COMBININGSORT, c.getLocalStrategy());
      FieldList l = new FieldList(0);
      Assert.assertEquals(l, c.getShipStrategyKeys());
      Assert.assertEquals(l, c.getLocalStrategyKeys());
     
      // check that the sort orders are descending
      Assert.assertFalse(c.getShipStrategySortOrder()[0]);
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
180
181
      // check that both reduce and combiner have the same strategy
      assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
      assertEquals(DriverStrategy.SORTED_PARTIAL_REDUCE, combineNode.getDriverStrategy());
     
      // check the keys
      assertEquals(new FieldList(1), reduceNode.getKeys());
      assertEquals(new FieldList(1), combineNode.getKeys());
      assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
     
      // check DOP
      assertEquals(6, sourceNode.getDegreeOfParallelism());
      assertEquals(6, combineNode.getDegreeOfParallelism());
      assertEquals(8, reduceNode.getDegreeOfParallelism());
View Full Code Here
233
234
235
236
237
238
239
240
241
242
243
244
245
      // check that both reduce and combiner have the same strategy
      assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
      assertEquals(DriverStrategy.SORTED_PARTIAL_REDUCE, combineNode.getDriverStrategy());
     
      // check the keys
      assertEquals(new FieldList(0), reduceNode.getKeys());
      assertEquals(new FieldList(0), combineNode.getKeys());
      assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
     
      // check DOP
      assertEquals(6, sourceNode.getDegreeOfParallelism());
      assertEquals(6, keyExtractor.getDegreeOfParallelism());
      assertEquals(6, combineNode.getDegreeOfParallelism());
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
175
     
      // check that both reduce and combiner have the same strategy
      assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
     
      // check the keys
      assertEquals(new FieldList(1), reduceNode.getKeys());
      assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
     
      // check DOP
      assertEquals(6, sourceNode.getDegreeOfParallelism());
      assertEquals(8, reduceNode.getDegreeOfParallelism());
      assertEquals(8, sinkNode.getDegreeOfParallelism());
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.common.operators.util.FieldList

  • eu.stratosphere.compiler.dag.TwoInputNode
  • eu.stratosphere.compiler.dag.WorksetIterationNode
  • eu.stratosphere.compiler.dataproperties.LocalProperties
  • eu.stratosphere.compiler.dataproperties.RequestedLocalProperties
  • eu.stratosphere.compiler.util.Utils
  • eu.stratosphere.pact.compiler.GroupOrderTest
  • eu.stratosphere.pact.compiler.GroupReduceCompilationTest
  • eu.stratosphere.pact.compiler.ReduceCompilationTest
  • eu.stratosphere.pact.compiler.WorksetIterationsJavaApiCompilerTest
  • eu.stratosphere.spargel.java.SpargelCompilerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.