Package org.apache.flink.core.io

Examples of org.apache.flink.core.io.GenericInputSplit

57
58
59
60
61
62
63
64
65
    }

    numSplits = (this instanceof NonParallelInput) ? 1 : numSplits;
    GenericInputSplit[] splits = new GenericInputSplit[numSplits];
    for (int i = 0; i < splits.length; i++) {
      splits[i] = new GenericInputSplit(i, numSplits);
    }
    return splits;
  }
View Full Code Here
281
282
283
284
285
286
287
288
289
  }

  @Override
  public InputSplit[] createInputSplits(int minNumSplits) throws IOException {
    GenericInputSplit[] split = {
      new GenericInputSplit(0, 1)
    };
    return split;
  }
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
    try {
      final int NUM_SPLITS = 50;
     
      Set<InputSplit> splits = new HashSet<InputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new GenericInputSplit(i, NUM_SPLITS));
      }
     
      DefaultInputSplitAssigner ia = new DefaultInputSplitAssigner(splits);
      InputSplit is = null;
      while ((is = ia.getNextInputSplit("")) != null) {
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
      final int NUM_SPLITS = 500;
      final int SUM_OF_IDS = (NUM_SPLITS-1) * (NUM_SPLITS) / 2;
     
      Set<InputSplit> splits = new HashSet<InputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new GenericInputSplit(i, NUM_SPLITS));
      }
     
      final DefaultInputSplitAssigner ia = new DefaultInputSplitAssigner(splits);
     
      final AtomicInteger splitsRetrieved = new AtomicInteger(0);
      final AtomicInteger sumOfIds = new AtomicInteger(0);
     
      Runnable retriever = new Runnable() {
       
        @Override
        public void run() {
          String host = "";
          GenericInputSplit split;
          while ((split = (GenericInputSplit) ia.getNextInputSplit(host)) != null) {
            splitsRetrieved.incrementAndGet();
            sumOfIds.addAndGet(split.getSplitNumber());
          }
        }
      };
     
      // create the threads
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
      assertTrue(serializationResult instanceof CollectionInputFormat<?>);

      @SuppressWarnings("unchecked")
      CollectionInputFormat<ElementType> result = (CollectionInputFormat<ElementType>) serializationResult;

      GenericInputSplit inputSplit = new GenericInputSplit();
      inputFormat.open(inputSplit);
      result.open(inputSplit);

      while(!inputFormat.reachedEnd() && !result.reachedEnd()){
        ElementType expectedElement = inputFormat.nextRecord(null);
View Full Code Here
175
176
177
178
179
180
181
182
183
184
185
      assertTrue(result instanceof CollectionInputFormat);
     
      int i = 0;
      @SuppressWarnings("unchecked")
      CollectionInputFormat<String> in = (CollectionInputFormat<String>) result;
      in.open(new GenericInputSplit());
     
      while (!in.reachedEnd()) {
        assertEquals(data[i++], in.nextRecord(""));
      }
     
View Full Code Here
TOP

Related Classes of org.apache.flink.core.io.GenericInputSplit

  • org.apache.flink.api.common.io.GenericInputFormat
  • org.apache.flink.api.java.io.CollectionInputFormatTest
  • org.apache.flink.api.java.io.jdbc.JDBCInputFormat
  • org.apache.flink.core.io.DefaultSplitAssignerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.