Package org.apache.flink.core.io

Examples of org.apache.flink.core.io.InputSplit


        splits.add(new LocatableInputSplit(i, hosts[i%3]));
      }
     
      // get all available splits
      LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
      InputSplit is = null;
      while ((is = ia.getNextInputSplit(null)) != null) {
        assertTrue(splits.remove(is));
      }
     
      // check we had all
View Full Code Here


        splits.add(new LocatableInputSplit(i, "testhost"));
      }
     
      // get all available splits
      LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
      InputSplit is = null;
      while ((is = ia.getNextInputSplit("testhost")) != null) {
        assertTrue(splits.remove(is));
      }
     
      // check we had all
View Full Code Here

        splits.add(new LocatableInputSplit(i, hosts[i % hosts.length]));
      }
     
      // get all available splits
      LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
      InputSplit is = null;
      while ((is = ia.getNextInputSplit("testhost")) != null) {
        assertTrue(splits.remove(is));
      }
     
      // check we had all
View Full Code Here

        splits.add(new LocatableInputSplit(i, hosts[i % hosts.length]));
      }
     
      // get all available splits
      LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
      InputSplit is = null;
      int i = 0;
      while ((is = ia.getNextInputSplit(hosts[i++ % hosts.length])) != null) {
        assertTrue(splits.remove(is));
      }
     
View Full Code Here

      for (int i = 0; i < NUM_SPLITS; i++) {
        splits.add(new GenericInputSplit(i, NUM_SPLITS));
      }
     
      DefaultInputSplitAssigner ia = new DefaultInputSplitAssigner(splits);
      InputSplit is = null;
      while ((is = ia.getNextInputSplit("")) != null) {
        assertTrue(splits.remove(is));
      }
     
      assertTrue(splits.isEmpty());
View Full Code Here

     
      // for each assigned input split
      while (!this.taskCanceled && splitIterator.hasNext())
      {
        // get start and end
        final InputSplit split = splitIterator.next();
       
        OT record = serializer.createInstance();
 
        if (LOG.isDebugEnabled()) {
          LOG.debug(getLogString("Opening input split " + split.toString()));
        }
       
        final InputFormat<OT, InputSplit> format = this.format;
     
        // open input format
        format.open(split);
 
        if (LOG.isDebugEnabled()) {
          LOG.debug(getLogString("Starting to read input from split " + split.toString()));
        }
       
        try {
          // ======= special-case the Record, to help the JIT and avoid some casts ======
          if (record.getClass() == Record.class) {
            Record typedRecord = (Record) record;
            @SuppressWarnings("unchecked")
            final InputFormat<Record, InputSplit> inFormat = (InputFormat<Record, InputSplit>) format;
           
            if (this.output instanceof RecordOutputCollector) {
              // Record going directly into network channels
              final RecordOutputCollector output = (RecordOutputCollector) this.output;
              while (!this.taskCanceled && !inFormat.reachedEnd()) {
                // build next pair and ship pair if it is valid
                typedRecord.clear();
                Record returnedRecord = null;
                if ((returnedRecord = inFormat.nextRecord(typedRecord)) != null) {
                  output.collect(returnedRecord);
                }
              }
            } else if (this.output instanceof ChainedCollectorMapDriver) {
              // Record going to a chained map task
              @SuppressWarnings("unchecked")
              final ChainedCollectorMapDriver<Record, ?> output = (ChainedCollectorMapDriver<Record, ?>) this.output;
             
              // as long as there is data to read
              while (!this.taskCanceled && !inFormat.reachedEnd()) {
                // build next pair and ship pair if it is valid
                typedRecord.clear();
                if ((typedRecord = inFormat.nextRecord(typedRecord)) != null) {
                  // This is where map of UDF gets called
                  output.collect(typedRecord);
                }
              }
            } else {
              // Record going to some other chained task
              @SuppressWarnings("unchecked")
              final Collector<Record> output = (Collector<Record>) this.output;
              // as long as there is data to read
              while (!this.taskCanceled && !inFormat.reachedEnd()) {
                // build next pair and ship pair if it is valid
                typedRecord.clear();
                if ((typedRecord = inFormat.nextRecord(typedRecord)) != null){
                  output.collect(typedRecord);
                }
              }
            }
          } else {
            // general types. we make a case distinction here for the common cases, in order to help
            // JIT method inlining
            if (this.output instanceof OutputCollector) {
              final OutputCollector<OT> output = (OutputCollector<OT>) this.output;
              // as long as there is data to read
              while (!this.taskCanceled && !format.reachedEnd()) {
                // build next pair and ship pair if it is valid
                if ((record = format.nextRecord(record)) != null) {
                  output.collect(record);
                }
              }
            } else if (this.output instanceof ChainedCollectorMapDriver) {
              @SuppressWarnings("unchecked")
              final ChainedCollectorMapDriver<OT, ?> output = (ChainedCollectorMapDriver<OT, ?>) this.output;
              // as long as there is data to read
              while (!this.taskCanceled && !format.reachedEnd()) {
                // build next pair and ship pair if it is valid
                if ((record = format.nextRecord(record)) != null) {
                  output.collect(record);
                }
              }
            } else {
              final Collector<OT> output = this.output;
              // as long as there is data to read
              while (!this.taskCanceled && !format.reachedEnd()) {
                // build next pair and ship pair if it is valid
                if ((record = format.nextRecord(record)) != null) {
                  output.collect(record);
                }
              }
            }
          }
         
          if (LOG.isDebugEnabled() && !this.taskCanceled) {
            LOG.debug(getLogString("Closing input split " + split.toString()));
          }
        } finally {
          // close. We close here such that a regular close throwing an exception marks a task as failed.
          format.close();
        }
View Full Code Here

       
        if (nextSplit != null) {
          return true;
        }
       
        InputSplit split = provider.getNextInputSplit();
       
        if (split != null) {
          this.nextSplit = split;
          return true;
        }
        else {
          exhausted = true;
          return false;
        }
      }

      @Override
      public InputSplit next() {
        if (this.nextSplit == null && !hasNext()) {
          throw new NoSuchElementException();
        }

        final InputSplit tmp = this.nextSplit;
        this.nextSplit = null;
        return tmp;
      }

      @Override
View Full Code Here

  }
 
 
  @Override
  public InputSplit getNextInputSplit(String host) {
    InputSplit next = null;
   
    // keep the synchronized part short
    synchronized (this.splits) {
      if (this.splits.size() > 0) {
        next = this.splits.remove(this.splits.size() - 1);
View Full Code Here

TOP

Related Classes of org.apache.flink.core.io.InputSplit

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.