Package org.springframework.data.hadoop.store.input

Examples of org.springframework.data.hadoop.store.input.TextFileReader

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    Splitter splitter = new StaticLengthSplitter(getConfiguration(), 110l);
    List<Split> inputSplits = splitter.getSplits(testDefaultPath);
    assertNotNull(inputSplits);
    assertThat(inputSplits.size(), is(3));

    TextFileReader reader1 = new TextFileReader(getConfiguration(), testDefaultPath, null, inputSplits.get(0), null);
    List<String> readData1 = TestUtils.readData(reader1);
    assertNotNull(readData1);

    TextFileReader reader2 = new TextFileReader(getConfiguration(), testDefaultPath, null, inputSplits.get(1), null);
    List<String> readData2 = TestUtils.readData(reader2);
    assertNotNull(readData2);

    TextFileReader reader3 = new TextFileReader(getConfiguration(), testDefaultPath, null, inputSplits.get(2), null);
    List<String> readData3 = TestUtils.readData(reader3);
    assertNotNull(readData3);

    assertThat(readData1.size()+readData2.size()+readData3.size(), is(30));
  }
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    Splitter splitter = new StaticLengthSplitter(getConfiguration(), 30l);
    List<Split> inputSplits = splitter.getSplits(testDefaultPath);
    assertNotNull(inputSplits);
    assertThat(inputSplits.size(), is(3));

    TextFileReader reader1 = new TextFileReader(getConfiguration(), testDefaultPath, Codecs.BZIP2.getCodecInfo(), inputSplits.get(0), null);
    List<String> readData1 = TestUtils.readData(reader1);
    assertNotNull(readData1);

    TextFileReader reader2 = new TextFileReader(getConfiguration(), testDefaultPath, Codecs.BZIP2.getCodecInfo(), inputSplits.get(1), null);
    List<String> readData2 = TestUtils.readData(reader2);
    assertNotNull(readData2);

    TextFileReader reader3 = new TextFileReader(getConfiguration(), testDefaultPath, Codecs.BZIP2.getCodecInfo(), inputSplits.get(2), null);
    List<String> readData3 = TestUtils.readData(reader3);
    assertNotNull(readData3);

    assertThat(readData1.size()+readData2.size()+readData3.size(), is(30));
  }
View Full Code Here
70
71
72
73
74
75
76
77
    writer.write(dataArray[0], headers);
    writer.flush();
    writer.close();

    TextFileReader reader = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "foo/" + nowYYYYMM + "/0_hash/jee_list/10_range/bar"), null);
    TestUtils.readDataAndAssert(reader, dataArray);
  }
View Full Code Here
93
94
95
96
97
98
99
100
    writer.write(dataArray[0], message);
    writer.flush();
    writer.close();

    TextFileReader reader = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "foo/" + nowYYYYMM + "/bar"), null);
    TestUtils.readDataAndAssert(reader, dataArray);
  }
View Full Code Here
114
115
116
117
118
119
120
121
    writer.write(dataArray[0], null);
    writer.flush();
    writer.close();

    TextFileReader reader = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "bar"), null);
    TestUtils.readDataAndAssert(reader, dataArray);
  }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
    writer.flush();
    writer.close();

    String nowYYYYMM = new SimpleDateFormat("yyyy/MM").format(new Date());

    TextFileReader reader1 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "foo/" + nowYYYYMM + "/0"), null);
    List<String> splitData1 = TestUtils.readData(reader1);

    TextFileReader reader2 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "foo/" + nowYYYYMM + "/1"), null);
    List<String> splitData2 = TestUtils.readData(reader2);

    TextFileReader reader3 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "foo/" + nowYYYYMM + "/2"), null);
    List<String> splitData3 = TestUtils.readData(reader3);

    assertThat(splitData1.size() + splitData2.size() + splitData3.size(), is(DATA09ARRAY.length));
  }
View Full Code Here
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
    writer.write(dataArray3[2], "customer3");
    writer.flush();
    writer.close();

    // /tmp/TextFilePartitionedWriterTests/default/customer1
    TextFileReader reader1 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "customer1"), null);
    TestUtils.readDataAndAssert(reader1, dataArray1);

    // /tmp/TextFilePartitionedWriterTests/default/customer2
    TextFileReader reader2 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "customer2"), null);
    TestUtils.readDataAndAssert(reader2, dataArray2);

    // /tmp/TextFilePartitionedWriterTests/default/customer3
    TextFileReader reader3 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "customer3"), null);
    TestUtils.readDataAndAssert(reader3, dataArray3);
  }
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
    writer.write(dataArray3[2]);
    writer.flush();
    writer.close();

    // /tmp/TextFilePartitionedWriterTests/default/customer1
    TextFileReader reader1 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "customer1"), null);
    TestUtils.readDataAndAssert(reader1, dataArray1);

    // /tmp/TextFilePartitionedWriterTests/default/customer2
    TextFileReader reader2 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "customer2"), null);
    TestUtils.readDataAndAssert(reader2, dataArray2);

    // /tmp/TextFilePartitionedWriterTests/default/customer3
    TextFileReader reader3 = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "customer3"), null);
    TestUtils.readDataAndAssert(reader3, dataArray3);
  }
View Full Code Here
57
58
59
60
61
62
63
64
    String[] dataArray = new String[] { DATA10 };

    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    TestUtils.writeData(writer, dataArray);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, dataArray);
  }
View Full Code Here
66
67
68
69
70
71
72
73
  @Test
  public void testWriteReadTextManyLines() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    TestUtils.writeData(writer, DATA09ARRAY);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
  }
View Full Code Here
TOP

Related Classes of org.springframework.data.hadoop.store.input.TextFileReader

  • org.apache.hadoop.io.Text
  • org.apache.hadoop.util.LineReader
  • org.springframework.data.hadoop.store.PartitionTextFileWriterTests
  • org.springframework.data.hadoop.store.RawStreamStoreTests
  • org.springframework.data.hadoop.store.TextFileStoreAppendCtxTests
  • org.springframework.data.hadoop.store.TextFileStoreAppendTests
  • org.springframework.data.hadoop.store.TextFileStoreCtxTests
  • org.springframework.data.hadoop.store.TextFileStoreSplitTests
  • org.springframework.data.hadoop.store.TextFileStoreTests
  • org.springframework.yarn.batch.item.DataStoreItemReaderTests
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.