Examples of com.facebook.LinkBench.stats.LatencyStats

com.facebook.LinkBench.stats.LatencyStats
Class used to track and compute latency statistics, particularly percentiles. Times are divided into buckets, with counts maintained per bucket. The division into buckets is based on typical latencies for database operations: most are in the range of 0.1ms to 100ms. we have 0.1ms-granularity buckets up to 1ms, then 1ms-granularity from 1-100ms, then 100ms-granularity, and then 1s-granularity.


      LinkStoreTestBase.serialLoad(rng, logger, props, getStoreHandle(false));
      serialLoadNodes(rng, logger, props, getStoreHandle(false));


      DummyLinkStore reqStore = getStoreHandle(false);
      LatencyStats latencyStats = new LatencyStats(1);
      RequestProgress tracker = new RequestProgress(logger, requests, timeLimit, 1, 10000);


      // Test both link and node requests
      LinkBenchRequest requester = new LinkBenchRequest(reqStore,
               reqStore, props, latencyStats, System.out, tracker, rng, 0, 1);
      tracker.startTimer();


      requester.run();


      latencyStats.displayLatencyStats();
      latencyStats.printCSVStats(System.out, true);


      assertEquals(requests, reqStore.adds + reqStore.updates + reqStore.deletes +
          reqStore.countLinks + reqStore.multigetLinks + reqStore.getLinkLists +
          reqStore.addNodes + reqStore.updateNodes + reqStore.deleteNodes +
          reqStore.getNodes);

View Full Code Here

    storeHandle.initialize(props, Phase.LOAD, 0);
    storeHandle.resetNodeStore(testDB, ConfigUtil.getLong(props,
                                                  Config.MIN_ID));
    storeHandle.close(); // Close before passing to loader


    LatencyStats stats = new LatencyStats(1);
    NodeLoader loader = new NodeLoader(props, logger, storeHandle, rng,
                                       stats, System.out, 0);
    loader.run();
  }

View Full Code Here




    boolean genNodes = ConfigUtil.getBool(props, Config.GENERATE_NODES);
    int nTotalLoaders = genNodes ? nLinkLoaders + 1 : nLinkLoaders;


    LatencyStats latencyStats = new LatencyStats(nTotalLoaders);
    List<Runnable> loaders = new ArrayList<Runnable>(nTotalLoaders);


    LoadProgress loadTracker = LoadProgress.create(logger, props);
    for (int i = 0; i < nLinkLoaders; i++) {
      LinkStore linkStore = createLinkStore();


      bulkLoad = bulkLoad && linkStore.bulkLoadBatchSize() > 0;
      LinkBenchLoad l = new LinkBenchLoad(linkStore, props, latencyStats,
              csvStreamFile, i, maxid1 == startid1 + 1, chunk_q, loadTracker);
      loaders.add(l);
    }


    if (genNodes) {
      logger.info("Will generate graph nodes during loading");
      int loaderId = nTotalLoaders - 1;
      NodeStore nodeStore = createNodeStore(null);
      Random rng = new Random(masterRandom.nextLong());
      loaders.add(new NodeLoader(props, logger, nodeStore, rng,
          latencyStats, csvStreamFile, loaderId));
    }
    enqueueLoadWork(chunk_q, startid1, maxid1, nLinkLoaders,
                    new Random(masterRandom.nextLong()));
    // run loaders
    loadTracker.startTimer();
    long loadTime = concurrentExec(loaders);


    long expectedNodes = maxid1 - startid1;
    long actualLinks = 0;
    long actualNodes = 0;
    for (final Runnable l:loaders) {
      if (l instanceof LinkBenchLoad) {
        actualLinks += ((LinkBenchLoad)l).getLinksLoaded();
      } else {
        assert(l instanceof NodeLoader);
        actualNodes += ((NodeLoader)l).getNodesLoaded();
      }
    }


    latencyStats.displayLatencyStats();


    if (csvStatsFile != null) {
      latencyStats.printCSVStats(csvStatsFile, true);
    }


    double loadTime_s = (loadTime/1000.0);
    logger.info(String.format("LOAD PHASE COMPLETED. " +
        " Loaded %d nodes (Expected %d)." +

View Full Code Here

    int nrequesters = ConfigUtil.getInt(props, Config.NUM_REQUESTERS);
    if (nrequesters == 0) {
      logger.info("NO REQUEST PHASE CONFIGURED. ");
      return;
    }
    LatencyStats latencyStats = new LatencyStats(nrequesters);
    List<LinkBenchRequest> requesters = new LinkedList<LinkBenchRequest>();


    RequestProgress progress = LinkBenchRequest.createProgress(logger, props);


    Random masterRandom = createMasterRNG(props, Config.REQUEST_RANDOM_SEED);


    // create requesters
    for (int i = 0; i < nrequesters; i++) {
      Stores stores = initStores();
      LinkBenchRequest l = new LinkBenchRequest(stores.linkStore,
              stores.nodeStore, props, latencyStats, csvStreamFile,
              progress, new Random(masterRandom.nextLong()), i, nrequesters);
      requesters.add(l);
    }
    progress.startTimer();
    // run requesters
    concurrentExec(requesters);
    long finishTime = System.currentTimeMillis();
    // Calculate duration accounting for warmup time
    long benchmarkTime = finishTime - progress.getBenchmarkStartTime();


    long requestsdone = 0;
    int abortedRequesters = 0;
    // wait for requesters
    for (LinkBenchRequest requester: requesters) {
      requestsdone += requester.getRequestsDone();
      if (requester.didAbort()) {
        abortedRequesters++;
      }
    }


    latencyStats.displayLatencyStats();


    if (csvStatsFile != null) {
      latencyStats.printCSVStats(csvStatsFile, true);
    }


    logger.info("REQUEST PHASE COMPLETED. " + requestsdone +
                 " requests done in " + (benchmarkTime/1000) + " seconds." +
                 " Requests/second = " + (1000*requestsdone)/benchmarkTime);

View Full Code Here

      Random rng = createRNG();


      serialLoad(rng, logger, props, getStoreHandle(false));


      DummyLinkStore reqStore = getStoreHandle(false);
      LatencyStats latencyStats = new LatencyStats(1);
      RequestProgress tracker = new RequestProgress(logger, requests, timeLimit, 0, 1000);


      LinkBenchRequest requester = new LinkBenchRequest(reqStore,
                      null, props, latencyStats, System.out, tracker, rng,
                      0, 1);
      tracker.startTimer();
      requester.run();
      latencyStats.displayLatencyStats();
      latencyStats.printCSVStats(System.out, true);


      assertEquals(requests, reqStore.adds + reqStore.updates + reqStore.deletes +
          reqStore.countLinks + reqStore.multigetLinks + reqStore.getLinkLists);
      // Check that the proportion of operations is roughly right - within 1%
      // For now, updates are actually implemented as add operations

View Full Code Here

      serialLoad(rng, logger, props, getStoreHandle(false));
      RequestProgress tracker = new RequestProgress(logger, requests, timeLimit, 2, 1000);


      DummyLinkStore reqStore = getStoreHandle(false);
      LinkBenchRequest requester = new LinkBenchRequest(reqStore, null,
                      props, new LatencyStats(1), System.out, tracker,
                      rng, 0, 1);


      long startTime = System.currentTimeMillis();
      tracker.startTimer();
      requester.run();

View Full Code Here

      serialLoad(rng, logger, props, getStoreHandle(false));
      RequestProgress tracker = new RequestProgress(logger, requests, timeLimit, 0, 1000);


      DummyLinkStore reqStore = getStoreHandle(false);
      reqStore.setRangeLimit(rangeLimit); // Small limit for testing
      LatencyStats latencyStats = new LatencyStats(1);
      LinkBenchRequest requester = new LinkBenchRequest(reqStore, null,
                      props, latencyStats, System.out, tracker, rng, 0, 1);


      tracker.startTimer();
      requester.run();
      latencyStats.displayLatencyStats();


      assertEquals(requests, reqStore.getLinkLists);


      double actualPHistory = reqStore.getLinkListsHistory /
                              (double) reqStore.getLinkLists;

View Full Code Here

   * @throws IOException
   * @throws Exception
   */
  static void serialLoad(Random rng, Logger logger, Properties props,
      DummyLinkStore store) throws IOException, Exception {
    LatencyStats latencyStats = new LatencyStats(1);


    /* Load up queue with work */
    BlockingQueue<LoadChunk>  chunk_q = new LinkedBlockingQueue<LoadChunk>();
    long startId = ConfigUtil.getLong(props, Config.MIN_ID);
    long idCount = ConfigUtil.getLong(props, Config.MAX_ID) - startId;

View Full Code Here

                    IntWritable nloaders,
                    OutputCollector<IntWritable, LongWritable> output,
                    Reporter reporter) throws IOException {
      ConfigUtil.setupLogging(props, null);
      LinkStore store = initStore(Phase.LOAD, loaderid.get());
      LatencyStats latencyStats = new LatencyStats(nloaders.get());


      long maxid1 = ConfigUtil.getLong(props, Config.MAX_ID);
      long startid1 = ConfigUtil.getLong(props, Config.MIN_ID);


      LoadProgress prog_tracker = LoadProgress.create(

View Full Code Here

                    IntWritable nrequesters,
                    OutputCollector<IntWritable, LongWritable> output,
                    Reporter reporter) throws IOException {
      ConfigUtil.setupLogging(props, null);
      LinkStore store = initStore(Phase.REQUEST, requesterid.get());
      LatencyStats latencyStats = new LatencyStats(nrequesters.get());
      RequestProgress progress =
                              LinkBenchRequest.createProgress(logger, props);
      progress.startTimer();
      // TODO: Don't support NodeStore yet
      final LinkBenchRequest requester =

View Full Code Here

TOP

Related Classes of com.facebook.LinkBench.stats.LatencyStats

com.facebook.LinkBench.GraphStoreTestBase

com.facebook.LinkBench.LinkBenchDriver

com.facebook.LinkBench.LinkBenchDriverMR$LoadMapper

com.facebook.LinkBench.LinkBenchDriverMR$RequestMapper

com.facebook.LinkBench.LinkStoreTestBase

org.apache.log4j.Logger

java.text.DecimalFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.