Examples of FileInputSplit

eu.stratosphere.core.fs.FileInputSplit
A file input split provides information on a particular part of a file, possibly hosted on a distributed file system and replicated among several hosts.
org.apache.flink.core.fs.FileInputSplit
A file input split provides information on a particular part of a file, possibly hosted on a distributed file system and replicated among several hosts.

Examples of org.apache.flink.core.fs.FileInputSplit

    try {
      final String fileContent = "abc|dfgsdf|777|444\n" +  // good line
                  "kkz|777|foobar|hhg\n" +  // wrong data type in field
                  "kkz|777foobarhhg  \n" +  // too short, a skipped field never ends
                  "xyx|ignored|42|\n";      // another good line
      final FileInputSplit split = createTempFile(fileContent);  
    
      final Configuration parameters = new Configuration();


      format.setFieldDelimiter('|');
      format.setFieldTypesGeneric(StringValue.class, null, IntValue.class);

View Full Code Here

Examples of org.apache.flink.core.fs.FileInputSplit

  
  @Test
  public void readWithEmptyField() {
    try {
      final String fileContent = "abc|def|ghijk\nabc||hhg\n|||";
      final FileInputSplit split = createTempFile(fileContent);  
    
      final Configuration parameters = new Configuration();


      format.setFieldDelimiter('|');
      format.setFieldTypesGeneric(StringValue.class, StringValue.class, StringValue.class);

View Full Code Here

Examples of org.apache.flink.core.fs.FileInputSplit

    try {
      final String fileContent = "colname-1|colname-2|some name 3|column four|\n" +
                  "123|abc|456|def|\n"+
                  "987|xyz|654|pqr|\n";
      
      final FileInputSplit split = createTempFile(fileContent);
    
      final Configuration parameters = new Configuration();


      format.setFieldDelimiter('|');
      format.setFieldTypesGeneric(IntValue.class, StringValue.class, IntValue.class, StringValue.class);

View Full Code Here

Examples of org.apache.flink.core.fs.FileInputSplit

      final String fileContent = "colname-1|colname-2|some name 3|column four|\n" +
                  "123|abc|456|def|\n"+
                  "colname-1|colname-2|some name 3|column four|\n" +  // repeated header in the middle
                  "987|xyz|654|pqr|\n";
      
      final FileInputSplit split = createTempFile(fileContent);
    
      final Configuration parameters = new Configuration();


      format.setFieldDelimiter('|');
      format.setFieldTypesGeneric(IntValue.class, StringValue.class, IntValue.class, StringValue.class);

View Full Code Here

Examples of org.apache.flink.core.fs.FileInputSplit

    
    DataOutputStream dos = new DataOutputStream(new FileOutputStream(tempFile));
    dos.writeBytes(content);
    dos.close();
      
    return new FileInputSplit(0, new Path(this.tempFile.toURI().toString()), 0, this.tempFile.length(), new String[] {"localhost"});
  }

View Full Code Here

Examples of org.apache.flink.core.fs.FileInputSplit

  }


  @Test
  public void testOpen() throws IOException {
    final String myString = "my mocked line 1\nmy mocked line 2\n";
    final FileInputSplit split = createTempFile(myString);  
    
    int bufferSize = 5;
    format.setBufferSize(bufferSize);
    format.open(split);
    assertEquals(0, format.splitStart);

View Full Code Here

Examples of org.apache.flink.core.fs.FileInputSplit

  }


  @Test
  public void testRead() throws IOException {
    final String myString = "my key|my val$$$my key2\n$$ctd.$$|my value2";
    final FileInputSplit split = createTempFile(myString);
    
    final Configuration parameters = new Configuration();
    
    format.setDelimiter("$$$");
    format.configure(parameters);

View Full Code Here

Examples of org.apache.flink.core.fs.FileInputSplit

  
  @Test
  public void testRead2() throws IOException {
    // 2. test case
    final String myString = "my key|my val$$$my key2\n$$ctd.$$|my value2";
    final FileInputSplit split = createTempFile(myString);
    
    final Configuration parameters = new Configuration();
    // default delimiter = '\n'
    
    format.configure(parameters);

View Full Code Here

Examples of org.apache.flink.core.fs.FileInputSplit

    
    OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(this.tempFile));
    wrt.write(contents);
    wrt.close();
    
    return new FileInputSplit(0, new Path(this.tempFile.toURI().toString()), 0, this.tempFile.length(), new String[] {"localhost"});
  }

View Full Code Here

Examples of org.apache.flink.core.fs.FileInputSplit


    final long splitLength = inFile.length() / noSplits;
    long pos = 0;


    for (int i = 0; i < noSplits - 1; i++) {
      tmp[i] = new FileInputSplit(i, new Path(path), pos, splitLength, hosts);
      pos += splitLength;
    }


    tmp[noSplits - 1] = new FileInputSplit(noSplits - 1, new Path(path), pos, inFile.length() - pos, hosts);


    this.inputSplits = tmp;
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.