Examples of LzopInputStream

Examples of org.anarres.lzo.LzopInputStream

219
220
221
222
223
224
225
226
227
228
229
230
231
    boolean indexingSucceeded = false;
    try {
      is = fs.open(lzoFile);
      os = fs.create(tmpOutputFile);
      // Solely for reading the header
      LzopInputStream lzis = new LzopInputStream(is);
      int numCompressedChecksums = lzis.getCompressedChecksumCount();
      int numDecompressedChecksums = lzis.getUncompressedChecksumCount();

      while (true) {
        // read and ignore, we just want to get to the next int
        int uncompressedBlockSize = is.readInt();
        if (uncompressedBlockSize == 0) {
View Full Code Here

Examples of org.anarres.lzo.LzopInputStream

51
52
53
54
55
56
57
58
59
60
61
62
63
    FileSystem fs = lzoFile.getFileSystem(conf);
    rawInputStream = fs.open(lzoFile);

    // Creating the LzopInputStream here just reads the lzo header for us, nothing more.
    // We do the rest of our input off of the raw stream is.
    LzopInputStream lzis = new LzopInputStream(rawInputStream);

    // This must be called AFTER createInputStream is called, because createInputStream
    // is what reads the header, which has the checksum information.  Otherwise getChecksumsCount
    // erroneously returns zero, and all block offsets will be wrong.
    numCompressedChecksums = lzis.getCompressedChecksumCount();
    numDecompressedChecksums = lzis.getUncompressedChecksumCount();
  }
View Full Code Here

Examples of org.anarres.lzo.hadoop.codec.LzopInputStream

28
29
30
31
32
33
34
35
36
37
38
    public void testInputFile() throws Exception {
        try {
            File dir = LzopFileTest.getDataDirectory();
            File file = new File(dir, "input.txt.lzo");
            FileInputStream fi = new FileInputStream(file);
            LzopInputStream ci = new LzopInputStream(fi);
            ByteArrayOutputStream bo = new ByteArrayOutputStream((int) (file.length() * 2));
            bo.reset();
            long start = System.currentTimeMillis();
            IOUtils.copy(ci, bo);
            long end = System.currentTimeMillis();
View Full Code Here

Examples of org.anarres.lzo.hadoop.codec.LzopInputStream

50
51
52
53
54
55
56
57
58
59
60
    public void testLineReader() throws Exception {
        try {
            File dir = LzopFileTest.getDataDirectory();
            File file = new File(dir, "input.txt.lzo");
            FileInputStream fi = new FileInputStream(file);
            LzopInputStream ci = new LzopInputStream(fi);
            LineRecordReader reader = new LineRecordReader(ci, 0L, 9999L, 9999);
            LongWritable key = new LongWritable();
            Text value = new Text();
            while (reader.next(key, value)) {
                LOG.info("key=" + key + "; value=" + value);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.