Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

org.apache.hadoop.hdfs.util.DataTransferThrottler
a class to throttle the data transfers. This class is thread safe. It can be shared by multiple threads. The parameter bandwidthPerSec specifies the total bandwidth shared by threads.

      LOG.warn("Could not open verfication log. " +
               "Verification times are not stored.");
    }
    
    synchronized (this) {
      throttler = new DataTransferThrottler(200, MAX_SCAN_RATE);
    }
  }

View Full Code Here

    long transferBandwidth = conf.getLong(
        HdfsConstants.DFS_IMAGE_TRANSFER_RATE_KEY,
        HdfsConstants.DFS_IMAGE_TRANSFER_RATE_DEFAULT);


    if (transferBandwidth > 0) {
      this.imageTransferThrottler = new DataTransferThrottler(transferBandwidth);
    }
  }

View Full Code Here

   */
  public final static DataTransferThrottler getThrottler(Configuration conf) {
    long transferBandwidth = 
      conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
                   DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }

View Full Code Here

        ImageServlet.setVerificationHeadersForGet(response, editFile);
        ImageServlet.setFileNameHeaders(response, editFile);
        editFileIn = new FileInputStream(editFile);
      }
      
      DataTransferThrottler throttler = ImageServlet.getThrottler(conf);


      // send edits
      TransferFsImage.copyFileToStream(response.getOutputStream(), editFile,
          editFileIn, throttler);

View Full Code Here

    FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
    long bandwidthPerSec = 1024*1024L;
    final long TOTAL_BYTES =6*bandwidthPerSec; 
    long bytesToSend = TOTAL_BYTES; 
    long start = Time.now();
    DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec);
    long totalBytes = 0L;
    long bytesSent = 1024*512L; // 0.5MB
    throttler.throttle(bytesSent);
    bytesToSend -= bytesSent;
    bytesSent = 1024*768L; // 0.75MB
    throttler.throttle(bytesSent);
    bytesToSend -= bytesSent;
    try {
      Thread.sleep(1000);
    } catch (InterruptedException ignored) {}
    throttler.throttle(bytesToSend);
    long end = Time.now();
    assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec);
  }

View Full Code Here

   */
  private final DataTransferThrottler getThrottler(Configuration conf) {
    long transferBandwidth = 
      conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
                   DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }

View Full Code Here

   */
  public final static DataTransferThrottler getThrottler(Configuration conf) {
    long transferBandwidth = 
      conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
                   DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }

View Full Code Here

   */
  public final static DataTransferThrottler getThrottler(Configuration conf) {
    long transferBandwidth = 
      conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
                   DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }

View Full Code Here

   */
  private final DataTransferThrottler getThrottler(Configuration conf) {
    long transferBandwidth = 
      conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
                   DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }

View Full Code Here

    FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
    long bandwidthPerSec = 1024*1024L;
    final long TOTAL_BYTES =6*bandwidthPerSec; 
    long bytesToSend = TOTAL_BYTES; 
    long start = Util.now();
    DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec);
    long totalBytes = 0L;
    long bytesSent = 1024*512L; // 0.5MB
    throttler.throttle(bytesSent);
    bytesToSend -= bytesSent;
    bytesSent = 1024*768L; // 0.75MB
    throttler.throttle(bytesSent);
    bytesToSend -= bytesSent;
    try {
      Thread.sleep(1000);
    } catch (InterruptedException ignored) {}
    throttler.throttle(bytesToSend);
    long end = Util.now();
    assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec);
  }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.hdfs.util.DataTransferThrottler

org.apache.hadoop.hdfs.qjournal.server.GetJournalEditServlet

org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner

org.apache.hadoop.hdfs.server.datanode.DataBlockScanner

org.apache.hadoop.hdfs.server.datanode.TestBlockReplacement

org.apache.hadoop.hdfs.server.namenode.FSImage

org.apache.hadoop.hdfs.server.namenode.GetImageServlet

org.apache.hadoop.hdfs.server.namenode.ImageServlet

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.