Examples of org.apache.mahout.common.TimingStatistics

org.apache.mahout.common.TimingStatistics

    doDeserializeBenchmark(RAND_SPARSE_VECTOR, "/tmp/randsparse-vector");
    doDeserializeBenchmark(SEQ_SPARSE_VECTOR, "/tmp/seqsparse-vector");
  }


  private void doDeserializeBenchmark(String name, String pathString) throws IOException {
    TimingStatistics stats = new TimingStatistics();
    TimingStatistics.Call call = stats.newCall(mark.leadTimeUsec);
    SequenceFileValueIterator<Writable> iterator = new SequenceFileValueIterator<Writable>(new Path(pathString), true,
        new Configuration());
    while (iterator.hasNext()) {
      iterator.next();
      call.end();
      call = stats.newCall(mark.leadTimeUsec);
    }
    iterator.close();
    mark.printStats(stats, DESERIALIZE, name);
  }

View Full Code Here

    }
    ClassifierContext classifier = new ClassifierContext(algorithm, datastore);
    classifier.initialize();
    ResultAnalyzer resultAnalyzer = new ResultAnalyzer(classifier.getLabels(),
        params.get("defaultCat"));
    final TimingStatistics totalStatistics = new TimingStatistics();
    if (subdirs != null) {


      for (File file : subdirs) {
        log.info("--------------");
        log.info("Testing: " + file);
        String correctLabel = file.getName().split(".txt")[0];
        final TimingStatistics operationStats = new TimingStatistics();


        long lineNum = 0;
        for (String line : new FileLineIterable(new File(file.getPath()),
            Charset.forName(params.get("encoding")), false)) {


          Map<String, List<String>> document = new NGrams(line, Integer
              .parseInt(params.get("gramSize"))).generateNGrams();
          for (Map.Entry<String, List<String>> stringListEntry : document
              .entrySet()) {
            List<String> strings = stringListEntry.getValue();
            TimingStatistics.Call call = operationStats.newCall();
            TimingStatistics.Call outercall = totalStatistics.newCall();
            ClassifierResult classifiedLabel = classifier.classifyDocument(
                strings.toArray(new String[strings.size()]), params
                    .get("defaultCat"));
            call.end();
            outercall.end();
            boolean correct = resultAnalyzer.addInstance(correctLabel,
                classifiedLabel);
            if (verbose) {
              // We have one document per line
              log.info("Line Number: " + lineNum + " Line(30): "
                  + (line.length() > 30 ? line.substring(0, 30) : line)
                  + " Expected Label: " + correctLabel + " Classified Label: "
                  + classifiedLabel.getLabel() + " Correct: " + correct);
            }
            // log.info("{} {}", correctLabel, classifiedLabel);


          }
          lineNum++;
        }
        log.info("{}\t{}\t{}/{}", new Object[] { correctLabel,
            resultAnalyzer.getConfusionMatrix().getAccuracy(correctLabel),
            resultAnalyzer.getConfusionMatrix().getCorrect(correctLabel),
            resultAnalyzer.getConfusionMatrix().getTotal(correctLabel) });
        log.info("{}", operationStats.toString());
      }


    }
    log.info("{}", totalStatistics.toString());
    log.info(resultAnalyzer.summarize());

View Full Code Here

    }
    implStats.set(implId, TAB_PATTERN.split(info));
  }
  
  public void createBenchmark() {
    TimingStatistics stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        vectors[0][i] = new DenseVector(randomVectors.get(i));
        call.end();
      }
    }
    printStats(stats, "Create (copy)", "DenseVector");
    
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        vectors[1][i] = new RandomAccessSparseVector(randomVectors.get(i));
        call.end();
      }
    }
    printStats(stats, "Create (copy)", "RandSparseVector");
    
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        vectors[2][i] = new SequentialAccessSparseVector(randomVectors.get(i));
        call.end();
      }
    }
    printStats(stats, "Create (copy)", "SeqSparseVector");

View Full Code Here

    }
    call.end();
  }


  public void incrementalCreateBenchmark() {
    TimingStatistics stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        vectors[0][i] = new DenseVector(cardinality);
        buildVectorIncrementally(stats, i, vectors[0][i], false);
      }
    }
    printStats(stats, "Create (incrementally)", "DenseVector");


    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        vectors[1][i] = new RandomAccessSparseVector(cardinality);
        buildVectorIncrementally(stats, i, vectors[1][i], false);
      }
    }
    printStats(stats, "Create (incrementally)", "RandSparseVector");


//    stats = new TimingStatistics();
//    for (int l = 0; l < loop; l++) {
//      for (int i = 0; i < numVectors; i++) {
//        vectors[2][i] = new SequentialAccessSparseVector(cardinality);
//        buildVectorIncrementally(stats, i, vectors[2][i], false);
//      }
//    }
//    printStats(stats, "Create (incrementally)", "SeqSparseVector");
    
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numClusters; i++) {
        clusters[i] = new RandomAccessSparseVector(cardinality);
        buildVectorIncrementally(stats, i, clusters[i], false);
      }

View Full Code Here

    }
    printStats(stats, "Create (incrementally)", "Clusters");
  }
  
  public void cloneBenchmark() {
    TimingStatistics stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        vectors[0][i] = vectors[0][i].clone();
        call.end();
      }
    }
    printStats(stats, "Clone", "DenseVector");
    
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        vectors[1][i] = vectors[1][i].clone();
        call.end();
      }
    }
    printStats(stats, "Clone", "RandSparseVector");
    
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        vectors[2][i] = vectors[2][i].clone();
        call.end();
      }
    }
    printStats(stats, "Clone", "SeqSparseVector");

View Full Code Here

      new Path("/tmp/dense-vector"), IntWritable.class, VectorWritable.class);


    Writable one = new IntWritable(0);
    VectorWritable vec = new VectorWritable();
    
    TimingStatistics stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        vec.set(vectors[0][i]);
        writer.append(one, vec);
        call.end();
      }
    }
    writer.close();
    printStats(stats, "Serialize", "DenseVector");
    
    writer = new SequenceFile.Writer(fs, conf,
      new Path("/tmp/randsparse-vector"), IntWritable.class, VectorWritable.class);
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        vec.set(vectors[1][i]);
        writer.append(one, vec);
        call.end();
      }
    }
    writer.close();
    printStats(stats, "Serialize", "RandSparseVector");
    
    writer = new SequenceFile.Writer(fs, conf,
      new Path("/tmp/seqsparse-vector"), IntWritable.class, VectorWritable.class);
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        vec.set(vectors[2][i]);
        writer.append(one, vec);
        call.end();
      }
    }

View Full Code Here

    doDeserializeBenchmark("RandSparseVector", "/tmp/randsparse-vector");
    doDeserializeBenchmark("SeqSparseVector", "/tmp/seqsparse-vector");
  }


  private void doDeserializeBenchmark(String name, String pathString) throws IOException {
    TimingStatistics stats = new TimingStatistics();
    TimingStatistics.Call call = stats.newCall();
    Iterator<?> iterator = new SequenceFileValueIterator<Writable>(new Path(pathString), true, new Configuration());
    while (iterator.hasNext()) {
      iterator.next();
      call.end();
      call = stats.newCall();
    }
    printStats(stats, "Deserialize", name);
  }

View Full Code Here

    printStats(stats, "Deserialize", name);
  }
  
  public void dotBenchmark() {
    double result = 0;
    TimingStatistics stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        result += vectors[0][i].dot(vectors[0][(i + 1) % numVectors]);
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, "DotProduct", "DenseVector", "sum = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        result += vectors[1][i].dot(vectors[1][(i + 1) % numVectors]);
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, "DotProduct", "RandSparseVector", "sum = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        result += vectors[2][i].dot(vectors[2][(i + 1) % numVectors]);
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, "DotProduct", "SeqSparseVector", "sum = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        result += vectors[0][i].dot(vectors[1][(i + 1) % numVectors]);
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, "DotProduct", "Dense.fn(Rand)", "sum = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        result += vectors[0][i].dot(vectors[2][(i + 1) % numVectors]);
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, "DotProduct", "Dense.fn(Seq)", "sum = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        result += vectors[1][i].dot(vectors[0][(i + 1) % numVectors]);
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, "DotProduct", "Rand.fn(Dense)", "sum = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        result += vectors[1][i].dot(vectors[2][(i + 1) % numVectors]);
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, "DotProduct", "Rand.fn(Seq)", "sum = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        result += vectors[2][i].dot(vectors[0][(i + 1) % numVectors]);
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, "DotProduct", "Seq.fn(Dense)", "sum = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        result += vectors[2][i].dot(vectors[1][(i + 1) % numVectors]);
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode

View Full Code Here

        clusterDistances.setQuick(i, j, distance);
      }
    }


    long distanceCalculations = 0;
    TimingStatistics stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      TimingStatistics.Call call = stats.newCall();
      for (int i = 0; i < numVectors; i++) {
        Vector vector = vectors[1][i];
        double minDistance = Double.MAX_VALUE;
        for (int k = 0; k < numClusters; k++) {
          double distance = measure.distance(vector, clusters[k]);
          distanceCalculations++;
          if (distance < minDistance) {
            minDistance = distance;
          }
        }
      }
      call.end();
    }
    printStats(stats,
               measure.getClass().getName(),
               "Closest center without Elkan's trick",
               "distanceCalculations = " + distanceCalculations);




    distanceCalculations = 0;
    stats = new TimingStatistics();
    Random rand = RandomUtils.getRandom();
    //rand.setSeed(System.currentTimeMillis());
    for (int l = 0; l < loop; l++) {
      TimingStatistics.Call call = stats.newCall();
      for (int i = 0; i < numVectors; i++) {
        Vector vector = vectors[1][i];
        int closestCentroid = rand.nextInt(numClusters);
        double dist = measure.distance(vector, clusters[closestCentroid]);
        distanceCalculations++;

View Full Code Here

               "distanceCalculations = " + distanceCalculations);
  }


  public void distanceMeasureBenchmark(DistanceMeasure measure) {
    double result = 0;
    TimingStatistics stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        double minDistance = Double.MAX_VALUE;
        for (int u = 0; u < opsPerUnit; u++) {
          double distance = measure.distance(vectors[0][i], vectors[0][u]);
          if (distance < minDistance) {
            minDistance = distance;
          }
        }
        result += minDistance;
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, measure.getClass().getName(), "DenseVector", "minDistance = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        double minDistance = Double.MAX_VALUE;
        for (int u = 0; u < opsPerUnit; u++) {
          double distance = measure.distance(vectors[1][i], vectors[1][u]);
          if (distance < minDistance) {
            minDistance = distance;
          }
        }
        result += minDistance;
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, measure.getClass().getName(), "RandSparseVector", "minDistance = " + result
                                                                                + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        double minDistance = Double.MAX_VALUE;
        for (int u = 0; u < opsPerUnit; u++) {
          double distance = measure.distance(vectors[2][i], vectors[2][u]);
          if (distance < minDistance) {
            minDistance = distance;
          }
        }
        result += minDistance;
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, measure.getClass().getName(), "SeqSparseVector", "minDistance = " + result
                                                                                    + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        double minDistance = Double.MAX_VALUE;
        for (int u = 0; u < opsPerUnit; u++) {
          double distance = measure.distance(vectors[0][i], vectors[1][u]);
          if (distance < minDistance) {
            minDistance = distance;
          }
        }
        result += minDistance;
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, measure.getClass().getName(), "Dense.fn(Rand)", "minDistance = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        double minDistance = Double.MAX_VALUE;
        for (int u = 0; u < opsPerUnit; u++) {
          double distance = measure.distance(vectors[0][i], vectors[2][u]);
          if (distance < minDistance) {
            minDistance = distance;
          }
        }
        result += minDistance;
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, measure.getClass().getName(), "Dense.fn(Seq)", "minDistance = " + result
                                                                                + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        double minDistance = Double.MAX_VALUE;
        for (int u = 0; u < opsPerUnit; u++) {
          double distance = measure.distance(vectors[1][i], vectors[0][u]);
          if (distance < minDistance) {
            minDistance = distance;
          }
        }
        result += minDistance;
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, measure.getClass().getName(), "Rand.fn(Dense)", "minDistance = " + result
                                                                                    + ' ');
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        double minDistance = Double.MAX_VALUE;
        for (int u = 0; u < opsPerUnit; u++) {
          double distance = measure.distance(vectors[1][i], vectors[2][u]);
          if (distance < minDistance) {
            minDistance = distance;
          }
        }
        result += minDistance;
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, measure.getClass().getName(), "Rand.fn(Seq)", "minDistance = " + result + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        double minDistance = Double.MAX_VALUE;
        for (int u = 0; u < opsPerUnit; u++) {
          double distance = measure.distance(vectors[2][i], vectors[0][u]);
          if (distance < minDistance) {
            minDistance = distance;
          }
        }
        result += minDistance;
        call.end();
      }
    }
    // print result to prevent hotspot from eliminating deadcode
    printStats(stats, measure.getClass().getName(), "Seq.fn(Dense)", "minDistance = " + result
                                                                                + ' ');
    result = 0;
    stats = new TimingStatistics();
    for (int l = 0; l < loop; l++) {
      for (int i = 0; i < numVectors; i++) {
        TimingStatistics.Call call = stats.newCall();
        double minDistance = Double.MAX_VALUE;
        for (int u = 0; u < opsPerUnit; u++) {
          double distance = measure.distance(vectors[2][i], vectors[1][u]);
          if (distance < minDistance) {
            minDistance = distance;

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.mahout.common.TimingStatistics

org.apache.mahout.benchmark.BenchmarkRunner

org.apache.mahout.benchmark.ClosestCentroidBenchmark

org.apache.mahout.benchmark.SerializationBenchmark

org.apache.mahout.benchmark.VectorBenchmarks

org.apache.mahout.classifier.bayes.TestClassifier

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.