Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.PureJavaCrc32

1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
    return value;
  }

  private DFSOutputStream(DFSClient dfsClient, String src, long blockSize, Progressable progress,
      int bytesPerChecksum, short replication) throws IOException {
    super(new PureJavaCrc32(), bytesPerChecksum, 4);
    this.dfsClient = dfsClient;
    this.src = src;
    this.blockSize = blockSize;
    this.blockReplication = replication;
    this.progress = progress;
View Full Code Here
54
55
56
57
58
59
60
    this(indexFileName, job, null);
  }

  public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner)
    throws IOException {
    this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
  }
View Full Code Here
117
118
119
120
121
122
123
  /**
   * Write this spill record to the location provided.
   */
  public void writeToFile(Path loc, JobConf job)
      throws IOException {
    writeToFile(loc, job, new PureJavaCrc32());
  }
View Full Code Here
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
    @SuppressWarnings("deprecation")
    public Reader(DataInputStream in, StreamLimiter limiter,
        int logVersion) {
      this.logVersion = logVersion;
      if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) {
        this.checksum = new PureJavaCrc32();
      } else {
        this.checksum = null;
      }

      if (this.checksum != null) {
View Full Code Here
2210
2211
2212
2213
2214
2215
2216
    private final DataOutputBuffer buf;
    private final Checksum checksum;

    public Writer(DataOutputBuffer out) {
      this.buf = out;
      this.checksum = new PureJavaCrc32();
    }
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
            getFSImage().dirIterator(FSImage.NameNodeDirType.IMAGE);
    List<Long> checksums = new ArrayList<Long>();
    while (iter.hasNext()) {
      StorageDirectory sd = iter.next();
      File fsImage = FSImage.getImageFile(sd, FSImage.NameNodeFile.IMAGE);
      PureJavaCrc32 crc = new PureJavaCrc32();
      FileInputStream in = new FileInputStream(fsImage);
      byte[] buff = new byte[4096];
      int read = 0;
      while ((read = in.read(buff)) != -1) {
       crc.update(buff, 0, read);
      }
      long val = crc.getValue();
      checksums.add(val);
    }
    assertTrue("Not enough fsimage copies in MiniDFSCluster " +
               "to test parallel write", checksums.size() > 1);
    for (int i = 1; i < checksums.size(); i++) {
View Full Code Here
1332
1333
1334
1335
1336
1337
1338
    private final DataOutputBuffer buf;
    private final Checksum checksum;

    public Writer(DataOutputBuffer out) {
      this.buf = out;
      this.checksum = new PureJavaCrc32();
    }
View Full Code Here
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
     */
    @SuppressWarnings("deprecation")
    public Reader(DataInputStream in, int logVersion) {
      this.logVersion = logVersion;
      if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) {
        this.checksum = new PureJavaCrc32();
      } else {
        this.checksum = null;
      }

      if (this.checksum != null) {
View Full Code Here
830
831
832
833
834
835
836
837
838
839
840
    } finally {
      IOUtils.closeStream(instr);
    }

    // compute crc of partial chunk from data read in the block file.
    partialCrc = new PureJavaCrc32();
    partialCrc.update(buf, 0, sizePartialChunk);
    LOG.info("Read in partial CRC chunk from disk for block " + block);

    // paranoia! verify that the pre-computed crc matches what we
    // recalculated just now
View Full Code Here
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
    return value;
  }

  private DFSOutputStream(DFSClient dfsClient, String src, long blockSize, Progressable progress,
      int bytesPerChecksum, short replication) throws IOException {
    super(new PureJavaCrc32(), bytesPerChecksum, 4);
    this.dfsClient = dfsClient;
    this.conf = dfsClient.conf;
    this.src = src;
    this.blockSize = blockSize;
    this.blockReplication = replication;
View Full Code Here
TOP

Related Classes of org.apache.hadoop.util.PureJavaCrc32

  • org.apache.hadoop.examples.terasort.GenSort
  • org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSInputChecker
  • org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer
  • org.apache.hadoop.fs.ChecksumFs$ChecksumFSInputChecker
  • org.apache.hadoop.fs.ChecksumFs$ChecksumFSOutputSummer
  • org.apache.hadoop.hdfs.BlockReader
  • org.apache.hadoop.hdfs.BlockReaderAccelerator
  • org.apache.hadoop.hdfs.DFSClient$BlockReader
  • org.apache.hadoop.hdfs.DFSClient$DFSOutputStream
  • org.apache.hadoop.hdfs.DFSOutputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.