Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.PureJavaCrc32

118
119
120
121
122
123
124
  /**
   * Write this spill record to the location provided.
   */
  public void writeToFile(Path loc, JobConf job)
      throws IOException {
    writeToFile(loc, job, new PureJavaCrc32());
  }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
        byte[] version = new byte[CHECKSUM_VERSION.length];
        sums.readFully(version);
        if (!Arrays.equals(version, CHECKSUM_VERSION))
          throw new IOException("Not a checksum file: "+sumFile);
        this.bytesPerSum = sums.readInt();
        set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4);
      } catch (FileNotFoundException e) {         // quietly ignore
        set(fs.verifyChecksum, null, 1, 0);
      } catch (IOException e) {                   // loudly ignore
        LOG.warn("Problem opening checksum file: "+ file +
                 ".  Ignoring exception: " , e);
View Full Code Here
378
379
380
381
382
383
384
385
386
387
388
                          int bufferSize,
                          short replication,
                          long blockSize,
                          Progressable progress)
      throws IOException {
      super(new PureJavaCrc32(), fs.getBytesPerSum(), 4);
      int bytesPerSum = fs.getBytesPerSum();
      this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize,
                                         replication, blockSize, progress);
      int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize);
      this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), true,
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
        sums.readFully(version);
        if (!Arrays.equals(version, CHECKSUM_VERSION)) {
          throw new IOException("Not a checksum file: "+sumFile);
        }
        this.bytesPerSum = sums.readInt();
        set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4);
      } catch (FileNotFoundException e) {         // quietly ignore
        set(fs.verifyChecksum, null, 1, 0);
      } catch (IOException e) {                   // loudly ignore
        LOG.warn("Problem opening checksum file: "+ file +
                 ".  Ignoring exception: " , e);
View Full Code Here
324
325
326
327
328
329
330
331
332
333
334
      final EnumSet<CreateFlag> createFlag,
      final FsPermission absolutePermission, final int bufferSize,
      final short replication, final long blockSize,
      final Progressable progress, final int bytesPerChecksum,
      final boolean createParent) throws IOException {
      super(new PureJavaCrc32(), fs.getBytesPerSum(), 4);

      this.datas = fs.getRawFs().createInternal(file, createFlag,
          absolutePermission, bufferSize, replication, blockSize, progress,
           bytesPerChecksum,  createParent);
     
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
        byte[] version = new byte[CHECKSUM_VERSION.length];
        sums.readFully(version);
        if (!Arrays.equals(version, CHECKSUM_VERSION))
          throw new IOException("Not a checksum file: "+sumFile);
        this.bytesPerSum = sums.readInt();
        set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4);
      } catch (FileNotFoundException e) {         // quietly ignore
        set(fs.verifyChecksum, null, 1, 0);
      } catch (IOException e) {                   // loudly ignore
        LOG.warn("Problem opening checksum file: "+ file +
                 ".  Ignoring exception: " +
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
        sums.readFully(version);
        if (!Arrays.equals(version, CHECKSUM_VERSION)) {
          throw new IOException("Not a checksum file: "+sumFile);
        }
        this.bytesPerSum = sums.readInt();
        set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4);
      } catch (FileNotFoundException e) {         // quietly ignore
        set(fs.verifyChecksum, null, 1, 0);
      } catch (IOException e) {                   // loudly ignore
        LOG.warn("Problem opening checksum file: "+ file +
                 ".  Ignoring exception: " , e);
View Full Code Here
333
334
335
336
337
338
339
340
341
342
343
      final EnumSet<CreateFlag> createFlag,
      final FsPermission absolutePermission, final int bufferSize,
      final short replication, final long blockSize,
      final Progressable progress, final ChecksumOpt checksumOpt,
      final boolean createParent) throws IOException {
      super(new PureJavaCrc32(), fs.getBytesPerSum(), 4);

      // checksumOpt is passed down to the raw fs. Unless it implements
      // checksum impelemts internally, checksumOpt will be ignored.
      // If the raw fs does checksum internally, we will end up with
      // two layers of checksumming. i.e. checksumming checksum file.
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
        byte[] version = new byte[CHECKSUM_VERSION.length];
        sums.readFully(version);
        if (!Arrays.equals(version, CHECKSUM_VERSION))
          throw new IOException("Not a checksum file: "+sumFile);
        this.bytesPerSum = sums.readInt();
        set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4);
      } catch (FileNotFoundException e) {         // quietly ignore
        set(fs.verifyChecksum, null, 1, 0);
      } catch (IOException e) {                   // loudly ignore
        LOG.warn("Problem opening checksum file: "+ file +
                 ".  Ignoring exception: " , e);
View Full Code Here
377
378
379
380
381
382
383
384
385
386
387
                          int bufferSize,
                          short replication,
                          long blockSize,
                          Progressable progress)
      throws IOException {
      super(new PureJavaCrc32(), fs.getBytesPerSum(), 4);
      int bytesPerSum = fs.getBytesPerSum();
      this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize,
                                         replication, blockSize, progress);
      int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize);
      this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), true,
View Full Code Here
TOP

Related Classes of org.apache.hadoop.util.PureJavaCrc32

  • org.apache.hadoop.examples.terasort.GenSort
  • org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSInputChecker
  • org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer
  • org.apache.hadoop.fs.ChecksumFs$ChecksumFSInputChecker
  • org.apache.hadoop.fs.ChecksumFs$ChecksumFSOutputSummer
  • org.apache.hadoop.hdfs.BlockReader
  • org.apache.hadoop.hdfs.BlockReaderAccelerator
  • org.apache.hadoop.hdfs.DFSClient$BlockReader
  • org.apache.hadoop.hdfs.DFSClient$DFSOutputStream
  • org.apache.hadoop.hdfs.DFSOutputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.