Package eu.stratosphere.core.fs

Examples of eu.stratosphere.core.fs.FileInputSplit

65
66
67
68
69
70
71
72
73
74
75
76
77
    final long splitLength = inFile.length() / noSplits;
    long pos = 0;

    for (int i = 0; i < noSplits - 1; i++) {
      tmp[i] = new FileInputSplit(i, new Path(path), pos, splitLength, hosts);
      pos += splitLength;
    }

    tmp[noSplits - 1] = new FileInputSplit(noSplits - 1, new Path(path), pos, inFile.length() - pos, hosts);

    this.inputSplits = tmp;
  }
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
        while (bytesUnassigned > maxBytesForLastSplit) {
          // get the block containing the majority of the data
          blockIndex = getBlockIndexForPosition(blocks, position, halfSplit, blockIndex);
          // create a new split
          final FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), position, splitSize,
            blocks[blockIndex]
              .getHosts());
          inputSplits.add(fis);

          // adjust the positions
          position += splitSize;
          bytesUnassigned -= splitSize;
        }

        // assign the last split
        if (bytesUnassigned > 0) {
          blockIndex = getBlockIndexForPosition(blocks, position, halfSplit, blockIndex);
          final FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), position,
            bytesUnassigned,
            blocks[blockIndex].getHosts());
          inputSplits.add(fis);
        }
      } else {
        // special case with a file of zero bytes size
        final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, 0);
        String[] hosts;
        if (blocks.length > 0) {
          hosts = blocks[0].getHosts();
        } else {
          hosts = new String[0];
        }
        final FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), 0, 0, hosts);
        inputSplits.add(fis);
      }
    }

    return inputSplits.toArray(new FileInputSplit[inputSplits.size()]);
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
      fail();
      e.printStackTrace();
    }
    writer.invoke();

    final FileInputSplit split = new FileInputSplit(0, new Path(this.file.toURI().toString()), 0,
      this.file.length(), null);
    when(this.environment.getInputSplitProvider()).thenReturn(this.inputSplitProvider);
    when(this.inputSplitProvider.getNextInputSplit()).thenReturn(split, (FileInputSplit) null);

    FileLineReader reader = new FileLineReader();
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
183
      // Create and populate instance specific split list
      instanceSplitList = new PriorityQueue<FileInputSplitList.QueueElem>();
      final Iterator<FileInputSplit> it = this.masterSet.iterator();
      while (it.hasNext()) {

        final FileInputSplit split = it.next();
        final String[] hostNames = split.getHostNames();
        if (hostNames == null) {
          instanceSplitList.add(new QueueElem(split, Integer.MAX_VALUE));

        } else {
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
    final Iterator<FileInputSplit> splitIterator = getFileInputSplits();

    while (splitIterator.hasNext()) {

      final FileInputSplit split = splitIterator.next();

      long start = split.getStart();
      long length = split.getLength();

      final FileSystem fs = FileSystem.get(split.getPath().toUri());

      final FSDataInputStream fdis = fs.open(split.getPath());

      final LineReader lineReader = new LineReader(fdis, start, length, (1024 * 1024));

      byte[] line = lineReader.readLine();

View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
    final Iterator<FileInputSplit> splitIterator = getFileInputSplits();

    while (splitIterator.hasNext()) {

      final FileInputSplit split = splitIterator.next();

      final long start = split.getStart();
      final long length = split.getLength();

      final FileSystem fs = FileSystem.get(split.getPath().toUri());

      final FSDataInputStream fdis = fs.open(split.getPath());

      final LineReader lineReader = new LineReader(fdis, start, length, (1024 * 1024));

      byte[] line = lineReader.readLine();

View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
  }

  @Test
  public void testOpen() throws IOException {
    final int[] fileContent = {1,2,3,4,5,6,7,8};
    final FileInputSplit split = createTempFile(fileContent); 
 
    final Configuration parameters = new Configuration();
    parameters.setInteger(FixedLengthInputFormat.RECORDLENGTH_PARAMETER_KEY, 8);
   
    format.configure(parameters);
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
  }
 
  @Test
  public void testRead() throws IOException {
    final int[] fileContent = {1,2,3,4,5,6,7,8};
    final FileInputSplit split = createTempFile(fileContent);
   
    final Configuration parameters = new Configuration();
   
    parameters.setInteger(FixedLengthInputFormat.RECORDLENGTH_PARAMETER_KEY, 8);
   
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
 
 
  @Test
  public void testReadFail() throws IOException {
    final int[] fileContent = {1,2,3,4,5,6,7,8,9};
    final FileInputSplit split = createTempFile(fileContent);
   
    final Configuration parameters = new Configuration();
    parameters.setInteger(FixedLengthInputFormat.RECORDLENGTH_PARAMETER_KEY, 8);
   
    format.configure(parameters);
View Full Code Here
178
179
180
181
182
183
184
      dos.writeInt(i);
    }
   
    dos.close();
     
    return new FileInputSplit(0, new Path(this.tempFile.toURI().toString()), 0, this.tempFile.length(), new String[] {"localhost"});
  }
View Full Code Here
TOP

Related Classes of eu.stratosphere.core.fs.FileInputSplit

  • eu.stratosphere.api.common.io.BinaryInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormatTest
  • eu.stratosphere.api.common.io.FileInputFormat
  • eu.stratosphere.api.common.io.FormatUtil
  • eu.stratosphere.api.common.io.GenericCsvInputFormatTest
  • eu.stratosphere.api.java.io.CsvInputFormatTest
  • eu.stratosphere.api.java.record.io.CsvInputFormatTest
  • eu.stratosphere.api.java.record.io.FixedLenghtInputFormatTest
  • eu.stratosphere.nephele.jobmanager.DoubleSourceTask
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.