Package org.apache.flink.api.common.io

Examples of org.apache.flink.api.common.io.DefaultInputSplitAssigner


    return split;
  }

  @Override
  public InputSplitAssigner getInputSplitAssigner(InputSplit[] inputSplits) {
    return new DefaultInputSplitAssigner(inputSplits);
  }
View Full Code Here


    return hiSplit;
  }
 
  @Override
  public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
    return new DefaultInputSplitAssigner(inputSplits);
  }
View Full Code Here

    return hiSplit;
  }

  @Override
  public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
    return new DefaultInputSplitAssigner(inputSplits);
  }
View Full Code Here

    return hadoopInputSplits;
  }
 
  @Override
  public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
    return new DefaultInputSplitAssigner(inputSplits);
  }
View Full Code Here

      return null;
    }

    @Override
    public DefaultInputSplitAssigner getInputSplitAssigner(GenericInputSplit[] splits) {
      return new DefaultInputSplitAssigner(splits);
    }
View Full Code Here

      return null;
    }

    @Override
    public DefaultInputSplitAssigner getInputSplitAssigner(GenericInputSplit[] splits) {
      return new DefaultInputSplitAssigner(splits);
    }
View Full Code Here

      Set<InputSplit> splits = new HashSet<InputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new GenericInputSplit(i, NUM_SPLITS));
      }
     
      DefaultInputSplitAssigner ia = new DefaultInputSplitAssigner(splits);
      InputSplit is = null;
      while ((is = ia.getNextInputSplit("")) != null) {
        assertTrue(splits.remove(is));
      }
     
      assertTrue(splits.isEmpty());
      assertNull(ia.getNextInputSplit(""));
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
View Full Code Here

      Set<InputSplit> splits = new HashSet<InputSplit>();
      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new GenericInputSplit(i, NUM_SPLITS));
      }
     
      final DefaultInputSplitAssigner ia = new DefaultInputSplitAssigner(splits);
     
      final AtomicInteger splitsRetrieved = new AtomicInteger(0);
      final AtomicInteger sumOfIds = new AtomicInteger(0);
     
      Runnable retriever = new Runnable() {
       
        @Override
        public void run() {
          String host = "";
          GenericInputSplit split;
          while ((split = (GenericInputSplit) ia.getNextInputSplit(host)) != null) {
            splitsRetrieved.incrementAndGet();
            sumOfIds.addAndGet(split.getSplitNumber());
          }
        }
      };
     
      // create the threads
      Thread[] threads = new Thread[NUM_THREADS];
      for (int i = 0; i < NUM_THREADS; i++) {
        threads[i] = new Thread(retriever);
        threads[i].setDaemon(true);
      }
     
      // launch concurrently
      for (int i = 0; i < NUM_THREADS; i++) {
        threads[i].start();
      }
     
      // sync
      for (int i = 0; i < NUM_THREADS; i++) {
        threads[i].join(5000);
      }
     
      // verify
      for (int i = 0; i < NUM_THREADS; i++) {
        if (threads[i].isAlive()) {
          fail("The concurrency test case is erroneous, the thread did not respond in time.");
        }
      }
     
      assertEquals(NUM_SPLITS, splitsRetrieved.get());
      assertEquals(SUM_OF_IDS, sumOfIds.get());
     
      // nothing left
      assertNull(ia.getNextInputSplit(""));
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
View Full Code Here

TOP

Related Classes of org.apache.flink.api.common.io.DefaultInputSplitAssigner

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.