Package eu.stratosphere.api.common.operators

Examples of eu.stratosphere.api.common.operators.Ordering

44
45
46
47
48
49
50
51
52
53
54
  public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys) {
    super(groupKeys);
   
    // if we have an additional ordering, construct the ordering to have primarily the grouping fields
    if (additionalOrderKeys != null) {
      this.ordering = new Ordering();
      for (Integer key : this.keyList) {
        this.ordering.appendOrdering(key, null, Order.ANY);
      }
   
      // and next the additional order fields
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
  }
 
  @Override
  protected void getSinkSchema(SinkPlanNode sinkPlanNode, SparseKeySchema schema) throws CompilerPostPassException {
    GenericDataSinkBase<?> sink = sinkPlanNode.getSinkNode().getPactContract();
    Ordering partitioning = sink.getPartitionOrdering();
    Ordering sorting = sink.getLocalOrder();
   
    try {
      if (partitioning != null) {
        addOrderingToSchema(partitioning, schema);
      }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
      schema.addType(localPositions[i], types[i]);
    }
   
    // this is a temporary fix, we should solve this more generic
    if (contract instanceof GroupReduceOperatorBase) {
      Ordering groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) contract).getGroupOrder();
      if (groupOrder != null) {
        addOrderingToSchema(groupOrder, schema);
      }
    }
  }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
    }
   
   
    // this is a temporary fix, we should solve this more generic
    if (contract instanceof CoGroupOperatorBase) {
      Ordering groupOrder1 = ((CoGroupOperatorBase<?, ?, ?, ?>) contract).getGroupOrderForInputOne();
      Ordering groupOrder2 = ((CoGroupOperatorBase<?, ?, ?, ?>) contract).getGroupOrderForInputTwo();
     
      if (groupOrder1 != null) {
        addOrderingToSchema(groupOrder1, input1Schema);
      }
      if (groupOrder2 != null) {
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
162
        SortedGrouping<IN> sortedGrouper = (SortedGrouping<IN>) grouper;
               
        int[] sortKeyPositions = sortedGrouper.getGroupSortKeyPositions();
        Order[] sortOrders = sortedGrouper.getGroupSortOrders();
       
        Ordering o = new Ordering();
        for(int i=0; i < sortKeyPositions.length; i++) {
          o.appendOrdering(sortKeyPositions[i], null, sortOrders[i]);
        }
        po.setGroupOrder(o);
      }
     
      return po;
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
        .fieldDelimiter(' ')
        .lenient(true)
        .field(StringValue.class, 0)
        .field(IntValue.class, 1);
     
      Ordering ordering = new Ordering(0, StringValue.class, Order.DESCENDING);
      out.setGlobalOrder(ordering, new SimpleDistribution(new StringValue[] {new StringValue("N")}));
     
      Plan p = new Plan(out, "WordCount Example");
      p.setDefaultParallelism(DEFAULT_PARALLELISM);
 
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
    ReduceOperator reducer = ReduceOperator.builder(CheckingReducer.class)
      .keyField(IntValue.class, 0)
      .input(source)
      .name("Ordered Reducer")
      .build();
    reducer.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING));
   
    FileDataSink sink = new FileDataSink(CsvOutputFormat.class, this.resultPath, reducer, "Sink");
    CsvOutputFormat.configureRecordFormat(sink)
      .recordDelimiter('\n')
      .fieldDelimiter(',')
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
    ReduceOperator sorter = ReduceOperator.builder(new IdentityReducer(), IntValue.class, 0)
      .input(input)
      .name("Reducer")
      .build();
    // sets the group sorting to the second field
    sorter.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING));

    // create and configure the output format
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, sorter, "Sorted Output");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
    // This task writes the sorted data back to disk
    final FileDataSink sink =
        new FileDataSink(new TeraOutputFormat(), output, "Data Sink");
    sink.setDegreeOfParallelism(numSubTasks);
    sink.setGlobalOrder(new Ordering(0, TeraKey.class, Order.ASCENDING), new TeraDistribution());

    sink.setInput(source);

    return new Plan(sink, "TeraSort");
  }
View Full Code Here
112
113
114
115
116
117
118
119
120
121
        .recordDelimiter('\n')
        .fieldDelimiter('|')
        .lenient(true)
        .field(IntValue.class, 0);
     
      sink.setGlobalOrder(new Ordering(0, IntValue.class, Order.ASCENDING), new UniformIntegerDistribution(Integer.MIN_VALUE, Integer.MAX_VALUE));
      sink.setInput(source);
     
      return new Plan(sink);
    }
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.common.operators.Ordering

  • eu.stratosphere.api.java.operators.ReduceGroupOperator
  • eu.stratosphere.compiler.dag.CoGroupNode
  • eu.stratosphere.compiler.dag.DataSinkNode
  • eu.stratosphere.compiler.dag.GroupReduceNode
  • eu.stratosphere.compiler.dataproperties.LocalProperties
  • eu.stratosphere.compiler.operators.CoGroupDescriptor
  • eu.stratosphere.compiler.operators.GroupReduceProperties
  • eu.stratosphere.compiler.operators.GroupReduceWithCombineProperties
  • eu.stratosphere.compiler.operators.SortMergeJoinDescriptor
  • eu.stratosphere.compiler.postpass.RecordModelPostPass
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.