Package org.apache.hadoop.io.file.tfile.TFile

Examples of org.apache.hadoop.io.file.tfile.TFile.Writer

84
85
86
87
88
89
90
  public void setUp() throws IOException {
    conf = new Configuration();
    path = new Path(ROOT, outputFile);
    fs = path.getFileSystem(conf);
    out = fs.create(path);
    writer = new Writer(out, BLOCK_SIZE, compression, comparator, conf);
  }
View Full Code Here
359
360
361
362
363
364
365
366
367
368
369
    if (skip)
      return;
    closeOutput();
    out = fs.create(path);
    try {
      writer = new Writer(out, BLOCK_SIZE, "BAD", comparator, conf);
      Assert.fail("Error on handling invalid compression codecs.");
    }
    catch (Exception e) {
      // noop, expecting exceptions
      // e.printStackTrace();
View Full Code Here
558
559
560
561
562
563
564
565
566
567
568
    closeOutput();
    out = fs.create(path);
    out.write(123);

    try {
      writer = new Writer(out, BLOCK_SIZE, compression, comparator, conf);
      Assert.fail("Failed to catch file write not at position 0.");
    }
    catch (Exception e) {
      // noop, expecting exceptions
    }
View Full Code Here
277
278
279
280
281
282
283
284
285
286
287
   * test none codecs
   */
  void basicWithSomeCodec(String codec) throws IOException {
    Path ncTFile = new Path(ROOT, "basic.tfile");
    FSDataOutputStream fout = createFSOutput(ncTFile);
    Writer writer = new Writer(fout, minBlockSize, codec, "memcmp", conf);
    writeRecords(writer);
    fout.close();
    FSDataInputStream fin = fs.open(ncTFile);
    Reader reader =
        new Reader(fs.open(ncTFile), fs.getFileStatus(ncTFile).getLen(), conf);
View Full Code Here
330
331
332
333
334
335
336
337
338
339
340
341
342
  // unsorted with some codec
  void unsortedWithSomeCodec(String codec) throws IOException {
    Path uTfile = new Path(ROOT, "unsorted.tfile");
    FSDataOutputStream fout = createFSOutput(uTfile);
    Writer writer = new Writer(fout, minBlockSize, codec, null, conf);
    writeRecords(writer);
    writer.close();
    fout.close();
    FSDataInputStream fin = fs.open(uTfile);
    Reader reader =
        new Reader(fs.open(uTfile), fs.getFileStatus(uTfile).getLen(), conf);
View Full Code Here
415
416
417
418
419
420
421
422
423
424
425
426
427
  // test meta blocks for tfiles
  public void testMetaBlocks() throws IOException {
    Path mFile = new Path(ROOT, "meta.tfile");
    FSDataOutputStream fout = createFSOutput(mFile);
    Writer writer = new Writer(fout, minBlockSize, "none", null, conf);
    someTestingWithMetaBlock(writer, "none");
    writer.close();
    fout.close();
    FSDataInputStream fin = fs.open(mFile);
    Reader reader = new Reader(fin, fs.getFileStatus(mFile).getLen(), conf);
    someReadingWithMetaBlock(reader);
    fs.delete(mFile, true);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
  public void setUp() throws IOException {
    conf = new Configuration();
    path = new Path(ROOT, outputFile);
    fs = path.getFileSystem(conf);
    out = fs.create(path);
    writer = new Writer(out, BLOCK_SIZE, compression, null, conf);
    writer.append("keyZ".getBytes(), "valueZ".getBytes());
    writer.append("keyM".getBytes(), "valueM".getBytes());
    writer.append("keyN".getBytes(), "valueN".getBytes());
    writer.append("keyA".getBytes(), "valueA".getBytes());
    closeOutput();
View Full Code Here
53
54
55
56
57
58
59
     */
    public TFileRecordWriter(Path file, String codec, Configuration conf)
                    throws IOException {
        FileSystem fs = file.getFileSystem(conf);
        fileOut = fs.create(file, false);
        writer = new Writer(fileOut, 1024 * 1024, codec, null, conf);
    }
View Full Code Here
90
91
92
93
94
95
96
  @Before
  public void setUp() throws IOException {
    path = new Path(ROOT, outputFile);
    fs = path.getFileSystem(conf);
    out = fs.create(path);
    writer = new Writer(out, BLOCK_SIZE, compression, comparator, conf);
  }
View Full Code Here
364
365
366
367
368
369
370
371
372
373
374
    if (skip)
      return;
    closeOutput();
    out = fs.create(path);
    try {
      writer = new Writer(out, BLOCK_SIZE, "BAD", comparator, conf);
      Assert.fail("Error on handling invalid compression codecs.");
    } catch (Exception e) {
      // noop, expecting exceptions
      // e.printStackTrace();
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.file.tfile.TFile.Writer

  • org.apache.hadoop.io.file.tfile.TestTFile
  • org.apache.hadoop.io.file.tfile.TestTFileByteArrays
  • org.apache.hadoop.io.file.tfile.TestTFileComparator2
  • org.apache.hadoop.io.file.tfile.TestTFileComparators
  • org.apache.hadoop.io.file.tfile.TestTFileSeek
  • org.apache.hadoop.io.file.tfile.TestTFileSplit
  • org.apache.hadoop.io.file.tfile.TestTFileStreams
  • org.apache.hadoop.io.file.tfile.TestTFileUnsortedByteArrays
  • org.apache.pig.impl.io.TFileRecordWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.