Package org.apache.flink.api.common.operators

Examples of org.apache.flink.api.common.operators.Ordering

496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
      }
     
      // some required local properties at step one and some more at step 2
      {
        GlobalProperties gp = new GlobalProperties();
        LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
       
        RequestedLocalProperties reqLp1 = new RequestedLocalProperties();
        reqLp1.setGroupedFields(new FieldList(3, 1));
       
        RequestedLocalProperties reqLp2 = new RequestedLocalProperties();
        reqLp2.setOrdering(new Ordering(3, null, Order.ANY).appendOrdering(1, null, Order.ANY));
       
        toMap1.setRequiredGlobalProps(null);
        toMap1.setRequiredLocalProps(reqLp1);
       
        toMap2.setRequiredGlobalProps(null);
View Full Code Here
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
     
      // set local strategy in first channel, so later non matching local properties do not matter
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(1, 2));
        LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
       
        RequestedLocalProperties reqLp = new RequestedLocalProperties();
        reqLp.setGroupedFields(new FieldList(4, 1));
       
        toMap1.setShipStrategy(ShipStrategyType.FORWARD);
        toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false});
       
        toMap2.setShipStrategy(ShipStrategyType.FORWARD);
        toMap2.setLocalStrategy(LocalStrategy.NONE);
       
        toMap1.setRequiredGlobalProps(null);
        toMap1.setRequiredLocalProps(null);
       
        toMap2.setRequiredGlobalProps(null);
        toMap2.setRequiredLocalProps(reqLp);
       
        FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
     
      // set local strategy in second channel, so previous non matching local properties void the match
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(1, 2));
        LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
       
        RequestedLocalProperties reqLp = new RequestedLocalProperties();
        reqLp.setGroupedFields(new FieldList(4, 1));
       
        toMap1.setShipStrategy(ShipStrategyType.FORWARD);
        toMap1.setLocalStrategy(LocalStrategy.NONE);
       
        toMap2.setShipStrategy(ShipStrategyType.FORWARD);
        toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false});
       
       
        toMap1.setRequiredGlobalProps(null);
        toMap1.setRequiredLocalProps(reqLp);
       
        toMap2.setRequiredGlobalProps(null);
        toMap2.setRequiredLocalProps(null);
       
        FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
     
      // create the properties on the same node as the requirement
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(1, 2));
        LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
       
        RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
        reqGp.setAnyPartitioning(new FieldSet(5, 7));
       
        RequestedLocalProperties reqLp = new RequestedLocalProperties();
View Full Code Here
817
818
819
820
821
822
823
824
825
826
827
      SingleInputPlanNode map3 = new SingleInputPlanNode(getMapNode(), "Mapper 3", toMap3, DriverStrategy.MAP);
     
      // set local strategy in first channel, so later non matching local properties do not matter
      {
        GlobalProperties gp = new GlobalProperties();
        LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
       
        RequestedLocalProperties reqLp = new RequestedLocalProperties();
        reqLp.setGroupedFields(new FieldList(4, 1));
       
        toMap1.setShipStrategy(ShipStrategyType.FORWARD);
View Full Code Here
953
954
955
956
957
958
959
960
961
962
963
       
        RequestedGlobalProperties joinGp = new RequestedGlobalProperties();
        joinGp.setFullyReplicated();
       
        RequestedLocalProperties joinLp = new RequestedLocalProperties();
        joinLp.setOrdering(new Ordering(2, null, Order.ASCENDING).appendOrdering(7, null, Order.ASCENDING));
       
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin2.setLocalStrategy(LocalStrategy.NONE);
        toJoin2.setRequiredGlobalProps(joinGp);
        toJoin2.setRequiredLocalProps(joinLp);
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
  public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys) {
    super(groupKeys);
   
    // if we have an additional ordering, construct the ordering to have primarily the grouping fields
    if (additionalOrderKeys != null) {
      this.ordering = new Ordering();
      for (Integer key : this.keyList) {
        this.ordering.appendOrdering(key, null, Order.ANY);
      }
   
      // and next the additional order fields
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
    ReduceOperator sorter = ReduceOperator.builder(new IdentityReducer(), IntValue.class, 0)
      .input(input)
      .name("Reducer")
      .build();
    // sets the group sorting to the second field
    sorter.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING));

    // create and configure the output format
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, sorter, "Sorted Output");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
  public CoGroupDescriptor(FieldList keys1, FieldList keys2, Ordering additionalOrdering1, Ordering additionalOrdering2) {
    super(keys1, keys2);
   
    // if we have an additional ordering, construct the ordering to have primarily the grouping fields
    if (additionalOrdering1 != null) {
      this.ordering1 = new Ordering();
      for (Integer key : this.keys1) {
        this.ordering1.appendOrdering(key, null, Order.ANY);
      }
   
      // and next the additional order fields
      for (int i = 0; i < additionalOrdering1.getNumberOfFields(); i++) {
        Integer field = additionalOrdering1.getFieldNumber(i);
        Order order = additionalOrdering1.getOrder(i);
        this.ordering1.appendOrdering(field, additionalOrdering1.getType(i), order);
      }
    } else {
      this.ordering1 = Utils.createOrdering(this.keys1);
    }
   
    // if we have an additional ordering, construct the ordering to have primarily the grouping fields
    if (additionalOrdering2 != null) {
      this.ordering2 = new Ordering();
      for (Integer key : this.keys2) {
        this.ordering2.appendOrdering(key, null, Order.ANY);
      }
   
      // and next the additional order fields
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
  public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
      LocalProperties produced1, LocalProperties produced2)
  {
    int numRelevantFields = this.keys1.size();
   
    Ordering prod1 = produced1.getOrdering();
    Ordering prod2 = produced2.getOrdering();
   
    if (prod1 == null || prod2 == null || prod1.getNumberOfFields() < numRelevantFields ||
        prod2.getNumberOfFields() < prod2.getNumberOfFields())
    {
      throw new CompilerException("The given properties do not meet this operators requirements.");
    }
     
    for (int i = 0; i < numRelevantFields; i++) {
      if (prod1.getOrder(i) != prod2.getOrder(i)) {
        return false;
      }
    }
    return true;
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
    // This task writes the sorted data back to disk
    final FileDataSink sink =
        new FileDataSink(new TeraOutputFormat(), output, "Data Sink");
    sink.setDegreeOfParallelism(numSubTasks);
    sink.setGlobalOrder(new Ordering(0, TeraKey.class, Order.ASCENDING), new TeraDistribution());

    sink.setInput(source);

    return new Plan(sink, "TeraSort");
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
  public GroupReduceProperties(FieldSet groupKeys, Ordering additionalOrderKeys) {
    super(groupKeys);
   
    // if we have an additional ordering, construct the ordering to have primarily the grouping fields
    if (additionalOrderKeys != null) {
      this.ordering = new Ordering();
      for (Integer key : this.keyList) {
        this.ordering.appendOrdering(key, null, Order.ANY);
      }
   
      // and next the additional order fields
View Full Code Here
TOP

Related Classes of org.apache.flink.api.common.operators.Ordering

  • org.apache.flink.api.java.operators.GroupReduceOperator
  • org.apache.flink.compiler.dag.CoGroupNode
  • org.apache.flink.compiler.dag.DataSinkNode
  • org.apache.flink.compiler.dag.GroupReduceNode
  • org.apache.flink.compiler.dataproperties.LocalProperties
  • org.apache.flink.compiler.FeedbackPropertiesMatchTest
  • org.apache.flink.compiler.GroupOrderTest
  • org.apache.flink.compiler.operators.CoGroupDescriptor
  • org.apache.flink.compiler.operators.GroupReduceProperties
  • org.apache.flink.compiler.operators.GroupReduceWithCombineProperties
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.