Package org.apache.flink.api.common.functions.util

Examples of org.apache.flink.api.common.functions.util.RuntimeUDFContext

108
109
110
111
112
113
114
115
116
117
118
119
    final List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
    final List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));


    try {
      List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskName, 1, 0, null), true);
      List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskName, 1, 0, null), false);

      assertEquals(expected, resultSafe);
      assertEquals(expected, resultRegular);
    }
    catch (Exception e) {
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
106
     
      MapOperatorBase<String, Integer, MapFunction<String, Integer>> op = new MapOperatorBase<String, Integer, MapFunction<String,Integer>>(
          parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
     
      List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));
      List<Integer> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskName, 1, 0, null), true);
      List<Integer> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskName, 1, 0, null), false);
     
      assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
      assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
     
      assertTrue(opened.get());
View Full Code Here
63
64
65
66
67
68
69
70
71
72
  }

  private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception {
    // run on collections
    final List<String> result = getTestFlatMapOperator(udf)
        .executeOnCollections(input, new RuntimeUDFContext("Test UDF", 4, 0, null), mutableSafe);

    Assert.assertEquals(input.size(), result.size());
    Assert.assertEquals(input, result);
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
          new MapPartitionOperatorBase<String, Integer, MapPartitionFunction<String,Integer>>(
          parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
     
      List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));
     
      List<Integer> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskName, 1, 0, null), true);
      List<Integer> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskName, 1, 0, null), false);
     
      assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
      assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
     
      assertTrue(opened.get());
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
        new Tuple2<Double, String>(3.0, "3"),
        new Tuple2<Double, String>(3.0, "4")
    ));

    try {
      List<Tuple2<Double, String>> resultSafe = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext("op", 1, 0, null), true);
      List<Tuple2<Double, String>> resultRegular = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext("op", 1, 0, null), false);

      assertEquals(expected, new HashSet<Tuple2<Double, String>>(resultSafe));
      assertEquals(expected, new HashSet<Tuple2<Double, String>>(resultRegular));
    }
    catch (Exception e) {
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
141
      List<Tuple2<String, Integer>> input = new ArrayList<Tuple2<String,
          Integer>>(asList(new Tuple2<String, Integer>("foo", 1), new Tuple2<String,
          Integer>("foo", 3), new Tuple2<String, Integer>("bar", 2), new Tuple2<String,
          Integer>("bar", 4)));

      List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskName, 1, 0, null), true);
      List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskName, 1, 0, null), false);

      Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<Tuple2<String, Integer>>(resultMutableSafe);
      Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<Tuple2<String, Integer>>(resultRegular);

      Set<Tuple2<String, Integer>> expectedResult = new HashSet<Tuple2<String,
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
      List<Tuple2<String, Integer>> input = new ArrayList<Tuple2<String,
          Integer>>(asList(new Tuple2<String, Integer>("foo", 1), new Tuple2<String,
          Integer>("foo", 3), new Tuple2<String, Integer>("bar", 2), new Tuple2<String,
          Integer>("bar", 4)));

      List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskName, 1, 0, null), true);
      List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskName, 1, 0, null), false);
     
     
      Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<Tuple2<String, Integer>>(resultMutableSafe);
      Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<Tuple2<String, Integer>>(resultRegular);
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
              .add("barfoo", 1)
              .add("foo", 1)
              .build()
      );

      final RuntimeContext ctx = new RuntimeUDFContext("Test UDF", 4, 0, null);

      {
        SumCoGroup udf1 = new SumCoGroup();
        SumCoGroup udf2 = new SumCoGroup();
       
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
   
    @SuppressWarnings("unchecked")
    SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator;
   
    // build the runtime context and compute broadcast variables, if necessary
    RuntimeUDFContext ctx;
    if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
      ctx = superStep == 0 ? new RuntimeUDFContext(operator.getName(), 1, 0, getClass().getClassLoader()) :
          new IterationRuntimeUDFContext(operator.getName(), 1, 0, superStep, classLoader);
     
      for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
        List<?> bcData = execute(bcInputs.getValue());
        ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
      }
    } else {
      ctx = null;
    }
   
    List<OUT> result = typedOp.executeOnCollections(inputData, ctx, mutableObjectSafeMode);
   
    if (ctx != null) {
      AccumulatorHelper.mergeInto(this.accumulators, ctx.getAllAccumulators());
    }
    return result;
  }
View Full Code Here
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
   
    @SuppressWarnings("unchecked")
    DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
   
    // build the runtime context and compute broadcast variables, if necessary
    RuntimeUDFContext ctx;
    if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
      ctx = superStep == 0 ? new RuntimeUDFContext(operator.getName(), 1, 0, classLoader) :
        new IterationRuntimeUDFContext(operator.getName(), 1, 0, superStep, classLoader);
     
      for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
        List<?> bcData = execute(bcInputs.getValue());
        ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
      }
    } else {
      ctx = null;
    }
   
    List<OUT> result = typedOp.executeOnCollections(inputData1, inputData2, ctx, mutableObjectSafeMode);
   
    if (ctx != null) {
      AccumulatorHelper.mergeInto(this.accumulators, ctx.getAllAccumulators());
    }
    return result;
  }
View Full Code Here
TOP

Related Classes of org.apache.flink.api.common.functions.util.RuntimeUDFContext

  • org.apache.flink.api.common.operators.base.CoGroupOperatorCollectionTest
  • org.apache.flink.api.common.operators.base.FlatMapOperatorCollectionTest
  • org.apache.flink.api.common.operators.base.GroupReduceOperatorTest
  • org.apache.flink.api.common.operators.base.JoinOperatorBaseTest
  • org.apache.flink.api.common.operators.base.MapOperatorTest
  • org.apache.flink.api.common.operators.base.PartitionMapOperatorTest
  • org.apache.flink.api.common.operators.base.ReduceOperatorTest
  • org.apache.flink.api.common.operators.CollectionExecutor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.