Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.PureJavaCrc32

677
678
679
680
681
682
683
684
685
686
687
    } finally {
      IOUtils.closeStream(instr);
    }

    // compute crc of partial chunk from data read in the block file.
    partialCrc = new PureJavaCrc32();
    partialCrc.update(buf, 0, sizePartialChunk);
    LOG.info("Read in partial CRC chunk from disk for block " + block);

    // paranoia! verify that the pre-computed crc matches what we
    // recalculated just now
View Full Code Here
50
51
52
53
54
55
56
    entries = buf.asLongBuffer();
  }

  public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner)
  throws IOException {
    this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
  }
View Full Code Here
114
115
116
117
118
119
120
  /**
   * Write this spill record to the location provided.
   */
  public void writeToFile(Path loc, JobConf job)
      throws IOException {
    writeToFile(loc, job, new PureJavaCrc32());
  }
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
        byte[] version = new byte[CHECKSUM_VERSION.length];
        sums.readFully(version);
        if (!Arrays.equals(version, CHECKSUM_VERSION))
          throw new IOException("Not a checksum file: "+sumFile);
        this.bytesPerSum = sums.readInt();
        set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4);
      } catch (FileNotFoundException e) {         // quietly ignore
        set(fs.verifyChecksum, null, 1, 0);
      } catch (IOException e) {                   // loudly ignore
        LOG.warn("Problem opening checksum file: "+ file +
                 ".  Ignoring exception: " +
View Full Code Here
329
330
331
332
333
334
335
336
337
338
339
                          int bufferSize,
                          short replication,
                          long blockSize,
                          Progressable progress)
      throws IOException {
      super(new PureJavaCrc32(), fs.getBytesPerSum(), 4);
      int bytesPerSum = fs.getBytesPerSum();
      this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize,
                                         replication, blockSize, progress);
      int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize);
      this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), true,
View Full Code Here
2213
2214
2215
2216
2217
2218
2219
    private final DataOutputBuffer buf;
    private final Checksum checksum;

    public Writer(DataOutputBuffer out) {
      this.buf = out;
      this.checksum = new PureJavaCrc32();
    }
View Full Code Here
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
     */
    @SuppressWarnings("deprecation")
    public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) {
      this.logVersion = logVersion;
      if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) {
        this.checksum = new PureJavaCrc32();
      } else {
        this.checksum = null;
      }

      if (this.checksum != null) {
View Full Code Here
504
505
506
507
508
509
510
511
512
513
514
                            "self=" + sock.getLocalSocketAddress() +
                            ", remote=" + sock.getRemoteSocketAddress() +
                            " for file " + file +
                            " for block " + blockId);
    }
    DataChecksum checksum = DataChecksum.newDataChecksum( in , new PureJavaCrc32());
    //Warning when we get CHECKSUM_NULL?

    // Read the first chunk offset.
    long firstChunkOffset = in.readLong();
View Full Code Here
976
977
978
979
980
981
982
                            "multiple of io.bytes.per.checksum");

    }
    checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
                                            bytesPerChecksum,
                                            new PureJavaCrc32());
  }
View Full Code Here
638
639
640
641
642
643
644
645
646
647
648
649
      indexFile.delete();
    }
    indexFile.createNewFile();
    FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
        new Path(indexFile.getAbsolutePath()));
    Checksum crc = new PureJavaCrc32();
    crc.reset();
    CheckedOutputStream chk = new CheckedOutputStream(output, crc);
    String msg = "Writing new index file. This file will be used only " +
        "for the testing.";
    chk.write(Arrays.copyOf(msg.getBytes(),
        MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
View Full Code Here
TOP

Related Classes of org.apache.hadoop.util.PureJavaCrc32

  • org.apache.hadoop.examples.terasort.GenSort
  • org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSInputChecker
  • org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer
  • org.apache.hadoop.fs.ChecksumFs$ChecksumFSInputChecker
  • org.apache.hadoop.fs.ChecksumFs$ChecksumFSOutputSummer
  • org.apache.hadoop.hdfs.BlockReader
  • org.apache.hadoop.hdfs.BlockReaderAccelerator
  • org.apache.hadoop.hdfs.DFSClient$BlockReader
  • org.apache.hadoop.hdfs.DFSClient$DFSOutputStream
  • org.apache.hadoop.hdfs.DFSOutputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.