Package org.springframework.data.hadoop.store.input

Examples of org.springframework.data.hadoop.store.input.TextFileReader

76
77
78
79
80
81
82
83
84
  public void testWriteReadManyLinesWithGzip() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath,
        Codecs.GZIP.getCodecInfo());
    TestUtils.writeData(writer, DATA09ARRAY);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath,
        Codecs.GZIP.getCodecInfo());
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
  }
View Full Code Here
87
88
89
90
91
92
93
94
95
  public void testWriteReadManyLinesWithBzip2() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath,
        Codecs.BZIP2.getCodecInfo());
    TestUtils.writeData(writer, DATA09ARRAY);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath,
        Codecs.BZIP2.getCodecInfo());
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
    fileNamingStrategy.register(new CodecFileNamingStrategy());
    fileNamingStrategy.register(new StaticFileNamingStrategy());
    writer.setFileNamingStrategy(fileNamingStrategy);
    TestUtils.writeData(writer, DATA09ARRAY);

    TextFileReader reader = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "data.gzip"),
        Codecs.GZIP.getCodecInfo());
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
  }
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    writer.setRolloverStrategy(new SizeRolloverStrategy(40));
    writer.setIdleTimeout(10000);

    TestUtils.writeData(writer, DATA09ARRAY);

    TextFileReader reader1 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "0"), null);
    List<String> splitData1 = TestUtils.readData(reader1);

    TextFileReader reader2 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "1"), null);
    List<String> splitData2 = TestUtils.readData(reader2);

    TextFileReader reader3 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "2"), null);
    List<String> splitData3 = TestUtils.readData(reader3);

    assertThat(splitData1.size() + splitData2.size() + splitData3.size(), is(DATA09ARRAY.length));
  }
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
    for (int i = 0; i<45000; i++) {
      TestUtils.writeData(writer, DATA09ARRAY, false);
    }
    TestUtils.writeData(writer, DATA09ARRAY, true);

    TextFileReader reader1 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "0"), Codecs.GZIP.getCodecInfo());
    List<String> splitData1 = TestUtils.readData(reader1);

    TextFileReader reader2 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "1"), Codecs.GZIP.getCodecInfo());
    List<String> splitData2 = TestUtils.readData(reader2);

    assertThat(splitData1.size() + splitData2.size(), is(450010));
  }
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
    writer.setFileNamingStrategy(fileNamingStrategy);
    writer.setInWritingSuffix(".tmp");
    writer.afterPropertiesSet();
    TestUtils.writeData(writer, dataArray);

    TextFileReader reader1 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "data-0"), null);
    List<String> splitData1 = TestUtils.readData(reader1);

    TextFileReader reader2 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "data-1"), null);
    List<String> splitData2 = TestUtils.readData(reader2);

    TextFileReader reader3 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "data-2"), null);
    List<String> splitData3 = TestUtils.readData(reader3);

    assertThat(splitData1.size() + splitData2.size() + splitData3.size(), is(3));
  }
View Full Code Here
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
    writer.setFileNamingStrategy(fileNamingStrategy);
    writer.setInWritingSuffix(".tmp");
    writer.afterPropertiesSet();
    TestUtils.writeData(writer, dataArray);

    TextFileReader reader1 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "data-0.gzip"), Codecs.GZIP.getCodecInfo());
    List<String> splitData1 = TestUtils.readData(reader1);

    TextFileReader reader2 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "data-1.gzip"), Codecs.GZIP.getCodecInfo());
    List<String> splitData2 = TestUtils.readData(reader2);

    TextFileReader reader3 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "data-2.gzip"), Codecs.GZIP.getCodecInfo());
    List<String> splitData3 = TestUtils.readData(reader3);

    assertThat(splitData1.size() + splitData2.size() + splitData3.size(), is(3));
  }
View Full Code Here
267
268
269
270
271
272
273
274
    // restore configuration after writer lifecycle has been started
    TestUtils.setField("configuration", writer, getConfiguration());
    TestUtils.writeData(writer, dataArray);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, dataArray);
  }
View Full Code Here
295
296
297
298
299
300
301
302
    // restore configuration after writer lifecycle has been started
    TestUtils.setField("configuration", writer, getConfiguration());
    TestUtils.writeData(writer, dataArray);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, dataArray);
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
    TestUtils.writeData(writer, new String[] { DATA11 }, false);
    Thread.sleep(3000);
    TestUtils.writeData(writer, new String[] { DATA12 }, true);
    Thread.sleep(3000);

    TextFileReader reader1 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "0"), null);
    List<String> splitData1 = TestUtils.readData(reader1);

    TextFileReader reader2 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "1"), null);
    List<String> splitData2 = TestUtils.readData(reader2);

    TextFileReader reader3 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "2"), null);
    List<String> splitData3 = TestUtils.readData(reader3);

    assertThat(splitData1.size() + splitData2.size() + splitData3.size(), is(3));
  }
View Full Code Here
TOP

Related Classes of org.springframework.data.hadoop.store.input.TextFileReader

  • org.apache.hadoop.io.Text
  • org.apache.hadoop.util.LineReader
  • org.springframework.data.hadoop.store.PartitionTextFileWriterTests
  • org.springframework.data.hadoop.store.RawStreamStoreTests
  • org.springframework.data.hadoop.store.TextFileStoreAppendCtxTests
  • org.springframework.data.hadoop.store.TextFileStoreAppendTests
  • org.springframework.data.hadoop.store.TextFileStoreCtxTests
  • org.springframework.data.hadoop.store.TextFileStoreSplitTests
  • org.springframework.data.hadoop.store.TextFileStoreTests
  • org.springframework.yarn.batch.item.DataStoreItemReaderTests
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.