Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.DataTransferThrottler


          i++;
        }
        Path[] srcs = new Path[argv.length-1-i];
        for (int j=0 ; i < argv.length-1 ;)
          srcs[j++] = new Path(argv[i++]);
        DataTransferThrottler throttler =
          rate > 0L ? new DataTransferThrottler(rate) : null;
        copyFromLocal(srcs, argv[i++], validate, throttler);
      } else if ("-moveFromLocal".equals(cmd)) {
        Path[] srcs = new Path[argv.length-2];
        for (int j=0 ; i < argv.length-1 ;)
          srcs[j++] = new Path(argv[i++]);
        moveFromLocal(srcs, argv[i++]);
      } else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
        long rate = -1L;
        if (argv[i].equals("-rate")) {
          i++;
          rate = Long.parseLong(argv[i]);
          if (rate <= 0L) {
            throw new IllegalArgumentException(
                "IO Throttler bandwidth must be positive. The input rate is invalid: " +
                rate);
          }
          i++;
        }
        DataTransferThrottler throttler =
          rate > 0L ? new DataTransferThrottler(rate) : null;
        copyToLocal(argv, i, throttler);
      } else if ("-getmerge".equals(cmd)) {
        if (argv.length>i+2)
          copyMergeToLocal(argv[i++], new Path(argv[i++]), Boolean.parseBoolean(argv[i++]));
        else
View Full Code Here


      LOG.warn("Could not open verfication log. " +
               "Verification times are not stored.");
    }
   
    synchronized (this) {
      throttler = new DataTransferThrottler(MAX_SCAN_RATE);
    }
  }
View Full Code Here

        FileChannel fc = rp.getChannel();
        fc.position(position);
        fStream = new BufferedInputStream(fStream);
      }
     
      DataTransferThrottler throttler = GetImageServlet.getThrottler(conf, false);

      // send edits
      TransferFsImage.getFileServerForPartialFiles(response.getOutputStream(),
          editFile.getAbsolutePath(), fStream, throttler, position,
          lengthToSend);
View Full Code Here

            targets.length - 1, targets, "");
        header.writeVersionAndOpCode(out);
        header.write(out);

        // send data & checksum
        DataTransferThrottler trottler = null;
        if (dataTransferMaxRate > 0) {
          trottler = new DataTransferThrottler(dataTransferMaxRate);
        }
        blockSender.sendBlock(out, baseStream, trottler);

        // no response necessary
        LOG.info(getDatanodeInfo() + ":Transmitted block " + b + " at " + srcNamespaceId + " to " + curTarget);
View Full Code Here

    FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
    long bandwidthPerSec = 1024*1024L;
    final long TOTAL_BYTES =6*bandwidthPerSec;
    long bytesToSend = TOTAL_BYTES;
    long start = Util.now();
    DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec);
    long totalBytes = 0L;
    long bytesSent = 1024*512L; // 0.5MB
    throttler.throttle(bytesSent);
    bytesToSend -= bytesSent;
    bytesSent = 1024*768L; // 0.75MB
    throttler.throttle(bytesSent);
    bytesToSend -= bytesSent;
    try {
      Thread.sleep(1000);
    } catch (InterruptedException ignored) {}
    throttler.throttle(bytesToSend);
    long end = Util.now();
    assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec);
  }
View Full Code Here

      return null;
    }
    long transferBandwidth =
      conf.getLong(HdfsConstants.DFS_IMAGE_TRANSFER_RATE_KEY,
          HdfsConstants.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.util.DataTransferThrottler

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.