Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.PureJavaCrc32

2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
                              "self=" + sock.getLocalSocketAddress() +
                              ", remote=" + sock.getRemoteSocketAddress() +
                              " for file " + file +
                              " for block " + blockId);
      }
      DataChecksum checksum = DataChecksum.newDataChecksum( in , new PureJavaCrc32());
      //Warning when we get CHECKSUM_NULL?

      // Read the first chunk offset.
      long firstChunkOffset = in.readLong();
View Full Code Here
5033
5034
5035
5036
5037
5038
5039
                              "multiple of io.bytes.per.checksum");

      }
      checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
                                              bytesPerChecksum,
                                              new PureJavaCrc32());
    }
View Full Code Here
117
118
119
120
121
122
123
124
    this.verifyChecksum = verifyChecksum;
    this.metrics = metrics;

     // create a checksum checker
     if (this.verifyChecksum) {
       this.checker = new PureJavaCrc32();
     }
  }
View Full Code Here
47
48
49
50
51
52
53
        numPartitions * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH);
    entries = buf.asLongBuffer();
  }

  public SpillRecord(Path indexFileName, JobConf job) throws IOException {
    this(indexFileName, job, new PureJavaCrc32());
  }
View Full Code Here
109
110
111
112
113
114
115
  /**
   * Write this spill record to the location provided.
   */
  public void writeToFile(Path loc, JobConf job)
      throws IOException {
    writeToFile(loc, job, new PureJavaCrc32());
  }
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
      assertEquals(len1, raf.length());
    } else {
      int bytesPerChecksum = conf.getInt("io.bytes.per.checksum", 512);
      int checksumSize = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
          bytesPerChecksum,
          new PureJavaCrc32()).getChecksumSize();
      assertEquals(raf.length(), BlockInlineChecksumReader.getFileLengthFromBlockSize(
          len1, bytesPerChecksum, checksumSize));
    }
    raf.setLength(0);
    raf.close();
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
        byte[] version = new byte[CHECKSUM_VERSION.length];
        sums.readFully(version);
        if (!Arrays.equals(version, CHECKSUM_VERSION))
          throw new IOException("Not a checksum file: "+sumFile);
        this.bytesPerSum = sums.readInt();
        set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4);
      } catch (FileNotFoundException e) {         // quietly ignore
        set(fs.verifyChecksum, null, 1, 0);
      } catch (IOException e) {                   // loudly ignore
        LOG.warn("Problem opening checksum file: "+ file +
                 ".  Ignoring exception: " +
View Full Code Here
330
331
332
333
334
335
336
337
338
339
340
                          short replication,
                          long blockSize,
                          int bytesPerChecksum,
                          Progressable progress)
      throws IOException {
      super(new PureJavaCrc32(), bytesPerChecksum, 4, null);
      this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize,
                                         replication, blockSize, progress);
      int sumBufferSize = fs.getSumBufferSize(bytesPerChecksum, bufferSize);
      this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), true,
                                               sumBufferSize, replication,
View Full Code Here
48
49
50
51
52
53
54
55
56
57
   * @param len The length of the input stream including checksum bytes.
   */
  public IFileInputStream(InputStream in, long len) {
    this.in = in;
    sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
        Integer.MAX_VALUE, new PureJavaCrc32());
    checksumSize = sum.getChecksumSize();
    length = len;
    dataLength = length - checksumSize;
  }
View Full Code Here
45
46
47
48
49
50
51
52
   * @param out
   */
  public IFileOutputStream(OutputStream out) {
    super(out);
    sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
        Integer.MAX_VALUE, new PureJavaCrc32());
    barray = new byte[sum.getChecksumSize()];
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.util.PureJavaCrc32

  • org.apache.hadoop.examples.terasort.GenSort
  • org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSInputChecker
  • org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer
  • org.apache.hadoop.fs.ChecksumFs$ChecksumFSInputChecker
  • org.apache.hadoop.fs.ChecksumFs$ChecksumFSOutputSummer
  • org.apache.hadoop.hdfs.BlockReader
  • org.apache.hadoop.hdfs.BlockReaderAccelerator
  • org.apache.hadoop.hdfs.DFSClient$BlockReader
  • org.apache.hadoop.hdfs.DFSClient$DFSOutputStream
  • org.apache.hadoop.hdfs.DFSOutputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.