Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.LocalFileSystem$LocalFSFileInputStream

255
256
257
258
259
260
261
262
263
264
265
        assertThat(file.exists(), is(true));
        return file;
    }

    private <T> List<T> load(OrcFileFormat<T> format, File file) throws IOException, InterruptedException {
        LocalFileSystem fs = FileSystem.getLocal(format.getConf());
        ModelInput<T> input = format.createInput(
                format.getSupportedType(),
                fs, new Path(file.toURI()),
                0, file.length(),
                new Counter());
View Full Code Here
278
279
280
281
282
283
284
285
286
        }
        String path = conf.get(KEY_LOCAL_TEMPDIR);
        if (path == null) {
            return null;
        }
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path result = fs.makeQualified(new Path(path));
        return result;
    }
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
        List<String> names = consumeLibraryNames(rest);
        if (names.isEmpty()) {
            return Collections.emptyList();
        }
        List<Path> results = new ArrayList<Path>();
        LocalFileSystem local = FileSystem.getLocal(configuration);
        for (String name : names) {
            Path path = new Path(name);
            FileSystem fs;
            if (path.toUri().getScheme() == null) {
                fs = local;
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
     * @throws Exception if failed
     */
    @Test
    public void input() throws Exception {
        final int count = 10000;
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class, Text.class);
        try {
            LongWritable k = new LongWritable();
            Text v = new Text();
            for (int i = 0; i < count; i++) {
                k.set(i);
                v.set("Hello, world at " + i);
                writer.append(k, v);
            }
        } finally {
            writer.close();
        }

        ModelInput<StringOption> in = format.createInput(
                StringOption.class,
                fs,
                path,
                0,
                fs.getFileStatus(path).getLen(),
                new Counter());
        try {
            StringOption value = new StringOption();
            for (int i = 0; i < count; i++) {
                String answer = "Hello, world at " + i;
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
     */
    @Test
    public void input_fragment() throws Exception {
        final int count = 30000;
        Random rand = new Random();
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class, Text.class);
        try {
            LongWritable k = new LongWritable();
            Text v = new Text();
            for (int i = 0; i < count; i++) {
                k.set(i);
                v.set("Hello, world at " + i);
                writer.append(k, v);
            }
        } finally {
            writer.close();
        }

        long fileLen = fs.getFileStatus(path).getLen();
        StringOption value = new StringOption();
        for (int attempt = 0; attempt < 5; attempt++) {
            int index = 0;
            long offset = 0;
            while (offset < fileLen) {
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
            buf.append("Hello, world!");
        }
        Text record = new Text(buf.toString());

        final int count = 5;
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class, Text.class);
        try {
            LongWritable k = new LongWritable();
            Text v = new Text();
            for (int i = 0; i < count; i++) {
                k.set(i);
                v.set(record);
                writer.append(k, v);
            }
        } finally {
            writer.close();
        }

        long fileLen = fs.getFileStatus(path).getLen();
        StringOption value = new StringOption();
        int index = 0;
        long offset = 0;
        while (offset < fileLen) {
            long length = SequenceFile.SYNC_INTERVAL * 2;
 
View Full Code Here
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
     * Test for input empty file.
     * @throws Exception if failed
     */
    @Test
    public void input_empty() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        fs.create(path).close();
        ModelInput<StringOption> in = format.createInput(
                StringOption.class,
                fs,
                path,
                0,
                fs.getFileStatus(path).getLen(),
                new Counter());
        try {
            assertThat("eof", in.readTo(new StringOption()), is(false));
        } finally {
            in.close();
View Full Code Here
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
     * Test for input invalid file.
     * @throws Exception if failed
     */
    @Test(expected = IOException.class)
    public void input_invalid() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        FSDataOutputStream output = fs.create(path);
        try {
            output.writeUTF("Hello, world!");
        } finally {
            output.close();
        }
        ModelInput<StringOption> in = format.createInput(
                StringOption.class,
                fs,
                path,
                0,
                fs.getFileStatus(path).getLen(),
                new Counter());
        in.close();
    }
View Full Code Here
272
273
274
275
276
277
278
279
280
281
282
     */
    @SuppressWarnings("deprecation")
    @Test
    public void output() throws Exception {
        final int count = 10000;
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter());
        try {
            StringOption value = new StringOption();
            for (int i = 0; i < count; i++) {
View Full Code Here
307
308
309
310
311
312
313
314
315
316
317
     * compressed output.
     * @throws Exception if failed
     */
    @Test
    public void output_compressed() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        ModelOutput<StringOption> out = format.codec(new DefaultCodec())
            .createOutput(StringOption.class, fs, path, new Counter());
        try {
            out.write(new StringOption("Hello, world!"));
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.LocalFileSystem$LocalFSFileInputStream

  • cascading.flow.hadoop.util.HadoopUtil
  • cascading.flow.tez.util.TezUtil
  • com.asakusafw.directio.hive.orc.OrcFileFormatTest
  • com.asakusafw.directio.hive.parquet.ParquetFileFormatTest
  • com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest
  • com.datasalt.pangool.solr.SolrRecordWriter
  • com.ngdata.sep.impl.SepConsumerIT
  • org.apache.accumulo.server.ServerConstantsTest
  • org.apache.accumulo.server.tabletserver.InMemoryMap
  • org.apache.accumulo.server.tabletserver.InMemoryMap$MemoryDataSource
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.