/*
Copyright 2012 Christian Prause and Fraunhofer FIT
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package net.sf.collabreview.evaluation;
import net.sf.collabreview.core.Artifact;
import net.sf.collabreview.core.ArtifactIdentifier;
import net.sf.collabreview.core.CollabReview;
import net.sf.collabreview.core.configuration.*;
import net.sf.collabreview.core.filter.ConstantFilter;
import net.sf.collabreview.core.filter.Filter;
import net.sf.collabreview.core.filter.PartitioningFilter;
import net.sf.collabreview.core.toolbox.Correlation;
import net.sf.collabreview.core.users.Author;
import net.sf.collabreview.measurements.weight.LineCountArtifactWeightFactory;
import net.sf.collabreview.measurements.weight.SizeArtifactWeightFactory;
import net.sf.collabreview.measurements.weight.UnityWeightFactory;
import net.sf.collabreview.reputation.*;
import net.sf.collabreview.responsibility.*;
import net.sf.collabreview.web.ExtensionServlet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.*;
/**
* K-fold cross validation of reputation scores (originally in MSR2010 class but I decided to want to have it available
* in all instances of CollabReview and runnable from the web).
*
* @author Christian Prause (prause)
* @date 2011-10-31 10:07
*/
public class ReputationValidation extends ExtensionServlet implements Runnable {
/**
* Apache commons logging logger for class ReputationValidation.
*/
private static final Log logger = LogFactory.getLog(ReputationValidation.class);
private ConfigurationConfigurableCollabReview collabReview;
/**
* The ConfigurationData according to which the original CollabReview instance was configured.
* This can be adjusted to run different evaluations.
*/
private ProgrammableConfigurationData configurationData;
/**
* The Thread in which the evaluation is executed.
*/
private Thread worker = null;
/**
* To keep the user informed of what is currently happening.
*/
private String statusInfo = null;
/**
* The number of partitions in the k-fold cross validation.
*/
private int partitionCount = 7;
/**
* Restrict evaluation to a subset of all non-obsolete artifacts.
*/
private PartitioningFilter filter = null;
/**
* The validation heavily relies on the computation of reputation values.
* Whenever a new ReputationMetricManager is needed, it is created with this factory.
*/
private ReputationMetricManagerFactory reputationMetricManagerFactory;
private String resultString = "";
@Override
protected void configure(CollabReview collabReview, ConfigurationData config) {
this.configurationData = new ProgrammableConfigurationData(AutoConfigurator.getFirstFoundConfigurationData());
//((ProgrammableConfigurationData) configurationData.getSubElement("reputation")).setValue("ensureAllScoresReady", "false");
if (config.getSubElement("filter") != null) {
filter = new PartitioningFilter(Filter.readFilter(config.getSubElement("filter")));
} else {
filter = new PartitioningFilter(new ConstantFilter(true));
}
if (config.getValue("partitions") != null) {
partitionCount = Integer.parseInt(config.getValue("partitions"));
}
}
@Override
protected void destroy() {
if (collabReview != null) {
collabReview.shutdown();
}
}
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
String theForm = "<form method=\"post\">";
if (worker == null) {
theForm += "<select name=\"responsibility\" size=\"5\">\n" +
" <option>" + AuthorshipArtifactResponsibilityFactory.class.getName() + "</option>\n" +
" <option>" + EditorArtifactResponsibilityFactory.class.getName() + "</option>\n" +
" <option>" + InvertedEditorArtifactResponsibilityFactory.class.getName() + "</option>\n" +
" <option>" + JavaPackageIsAuthorResponsibilityFactory.class.getName() + "</option>\n" +
" <option>" + LatestContributorResponsibilityFactory.class.getName() + "</option>\n" +
" <option>" + UnitAuthorArtifactResponsibilityFactory.class.getName() + "</option>\n" +
" <option>" + UnitEditorArtifactResponsibilityFactory.class.getName() + "</option>\n" +
" </select><br/>";
theForm += "<select name=\"weight\" size=\"3\">\n" +
" <option>" + LineCountArtifactWeightFactory.class.getName() + "</option>\n" +
" <option>" + SizeArtifactWeightFactory.class.getName() + "</option>\n" +
" <option>" + UnityWeightFactory.class.getName() + "</option>\n" +
" </select><br/>";
theForm += "<input name=\"run\" value=\"run\" type=\"submit\"/>";
} else {
theForm += "<input name=\"refresh\" value=\"refresh\" type=\"submit\"/>";
}
theForm += "</form><br/>\n";
assert response != null;
response.setContentType("text/html");
response.setCharacterEncoding("utf-8");
PrintWriter out = response.getWriter();
out.printf("<html>" +
"<title>" +
"ReputationValidation" +
"</title>" +
"<body>" +
"Worker thread: %s<br/>", worker);
out.printf(theForm);
out.printf("Status info: <b>" + statusInfo + "</b><br/><hr/>");
out.printf("Partition count: %d (%d) of %d artifacts<br/>", filter.getPartitions().size(), partitionCount, collabReview == null ? -1 : listArtifacts().size());
out.printf("Filter: " + filter.toString() + "<br/>");
out.printf("Authorship: " + (collabReview == null ? "n/a" : collabReview.getMeasurementsManager().getArtifactResponsibility().getClass().toString()) + "<br/>");
out.printf("Weight: " + (collabReview == null ? "n/a" : collabReview.getMeasurementsManager().getArtifactWeight().getClass().toString()) + "<br/>");
out.printf("<hr/>" +
"Results: <br/><pre>%s</pre>" +
"</body>" +
"</html>",
resultString);
}
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException {
String responsibility = request.getParameter("responsibility");
if (responsibility != null) {
assert configurationData.getSubElement("artifactResponsibilityFactoryConfiguration") != null;
((ProgrammableConfigurationData) configurationData.getSubElement("artifactResponsibilityFactoryConfiguration")).setValue("class", responsibility);
}
String weight = request.getParameter("weight");
if (weight != null) {
assert configurationData.getSubElement("artifactWeightFactoryConfiguration") != null;
((ProgrammableConfigurationData) configurationData.getSubElement("artifactWeightFactoryConfiguration")).setValue("class", weight);
}
if (request.getParameter("run") != null && worker == null) {
startEvaluation();
}
doGet(request, response);
}
private synchronized void startEvaluation() {
if (worker != null) {
logger.warn("Cannot start evaluation thread, there is already a worker thread");
return;
}
logger.info("Starting worker");
worker = new Thread(this, "RepuVal");
worker.setDaemon(true);
worker.setPriority(Thread.MIN_PRIORITY);
worker.start();
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
@Override
public void run() {
try {
doEvaluationWork();
statusInfo = "completed";
} catch (Throwable t) {
logger.error("A problem occurred when running evaluation", t);
statusInfo = "terminated with exception";
} finally {
worker = null;
}
}
private void reconfigureCollabReview() {
if (collabReview != null) {
collabReview.shutdown();
collabReview = null;
}
collabReview = new ConfigurationConfigurableCollabReview();
Configurator configurator = new Configurator();
configurator.setConfigurationData(configurationData);
try {
configurator.configure(collabReview, false);
} catch (Exception e) {
logger.error("Failed to configure the evaluation collabReview instance", e);
}
reputationMetricManagerFactory = new ReputationMetricManagerFactory();
reputationMetricManagerFactory.configure(configurationData.getSubElement("reputation"));
reputationMetricManagerFactory.setCollabReview(collabReview);
reputationMetricManagerFactory.setDefaultFilter(filter);
}
private void setStatusInfo(String newStatus) {
logger.debug("New status: " + newStatus);
statusInfo = newStatus;
}
private void doEvaluationWork() throws Throwable {
setStatusInfo("started");
resultString = "";
setStatusInfo("configuring CollabReview instance");
reconfigureCollabReview();
setStatusInfo("partitioning artifacts");
filter.createPartitioning(partitionCount, listArtifacts());
developerReputations();
computeGlobalCorrelation();
computeCorrelations();
setStatusInfo("finishing");
}
private void developerReputations() {
setStatusInfo("printing developer reputations");
String names = "";
String scoreString = "";
ReputationMetric rm = collabReview.getReputationMetricManager().findReputationMetric("quality");
// put developer in a new Map that is sorted alphabetically
TreeMap<String, Float> scores = new TreeMap<String, Float>();
for (String name : rm.getAuthorScores().keySet()) {
scores.put(name, rm.getAuthorScore(name));
}
for (String name : scores.keySet()) {
names += "\"" + name + "\",";
scoreString += String.format(Locale.US, "%" + (name.length() + 2) + ".2f,", scores.get(name));
}
resultString += names + "\n" + scoreString + "\n";
}
private void computeGlobalCorrelation() {
setStatusInfo("computing one-shot correlations");
Correlation globalResults = correlateQualityPredictions(listArtifacts(), collabReview.getReputationMetricManager());
globalResults.sortByX();
resultString += String.format(Locale.US, "One-shot Pearson r=%.2f, one-shot Spearman rho=%.2f\n", globalResults.getPearsonCorrelation(), globalResults.getSpearmanCorrelation());
String assessedValues = "";
String predictedValues = "";
for (Correlation.RowData data : globalResults.listRowData()) {
assessedValues += String.format(Locale.US, "%6.2f,", data.getX());
predictedValues += String.format(Locale.US, "%6.2f,", data.getY());
}
resultString += "Assessed values: " + assessedValues + "\n";
resultString += "Predicted values: " + predictedValues + "\n";
}
private void computeCorrelations() throws Exception {
// compute the cross validation
setStatusInfo("computing correlations in cross validation");
double spearman = 0;
double pearson = 0;
double top20Recall = 0;
double top20Precision = 0;
double bad20Recall = 0;
double bad20Precision = 0;
double top10Recall = 0;
double top10Precision = 0;
double bad10Recall = 0;
double bad10Precision = 0;
double top5Recall = 0;
double top5Precision = 0;
double bad5Recall = 0;
double bad5Precision = 0;
for (int i = 0; i < partitionCount; i++) {
setStatusInfo("validation iteration " + i);
ReputationMetricManager rmm = computePartitionReputations(i);
Correlation localResults = correlateQualityPredictions(filter.getPartitions().get(i), rmm);
resultString += String.format(Locale.US, "Partition %d (%d artifacts): " +
"Pearson r=%+.2f, Spearman r=%+.2f, " +
"Top10Recall=%.2f, Top10Precision=%.2f, " +
"Bad10Recall=%.2f, Bad10Precision=%.2f, " +
"\n",
i, filter.getPartitions().get(i).size(),
localResults.getPearsonCorrelation(), localResults.getSpearmanCorrelation(),
localResults.topPercentRecall(0.1), localResults.topPercentPrecision(0.1),
localResults.badPercentRecall(0.1), localResults.badPercentPrecision(0.1)
);
spearman += localResults.getSpearmanCorrelation() / partitionCount;
pearson += localResults.getPearsonCorrelation() / partitionCount;
top20Recall += localResults.topPercentRecall(0.2) / partitionCount;
top20Precision += localResults.topPercentRecall(0.2) / partitionCount;
bad20Recall += localResults.badPercentRecall(0.2) / partitionCount;
bad20Precision += localResults.badPercentPrecision(0.2) / partitionCount;
top10Recall += localResults.topPercentRecall(0.1) / partitionCount;
top10Precision += localResults.topPercentRecall(0.1) / partitionCount;
bad10Recall += localResults.badPercentRecall(0.1) / partitionCount;
bad10Precision += localResults.badPercentPrecision(0.1) / partitionCount;
top5Recall += localResults.topPercentRecall(0.05) / partitionCount;
top5Precision += localResults.topPercentRecall(0.05) / partitionCount;
bad5Recall += localResults.badPercentRecall(0.05) / partitionCount;
bad5Precision += localResults.badPercentPrecision(0.05) / partitionCount;
// clean up
rmm.shutdown();
}
resultString += String.format(Locale.US, "Average Pearson r=%.2f, average Spearman rho=%.2f\n", pearson, spearman);
resultString += String.format(Locale.US, "20%%: Top Recall %.2f Top Precision %.2f Bad Recall %.2f Bad Precision %.2f\n", top20Recall, top20Precision, bad20Recall, bad20Precision);
resultString += String.format(Locale.US, "10%%: Top Recall %.2f Top Precision %.2f Bad Recall %.2f Bad Precision %.2f\n", top10Recall, top10Precision, bad10Recall, bad10Precision);
resultString += String.format(Locale.US, " 5%%: Top Recall %.2f Top Precision %.2f Bad Recall %.2f Bad Precision %.2f\n", top5Recall, top5Precision, bad5Recall, bad5Precision);
}
private ReputationMetricManager computePartitionReputations(int i) throws Exception {
filter.setSelectedPartition(i);
filter.setInverted(true);
return reputationMetricManagerFactory.create();
}
private Correlation correlateQualityPredictions(Collection<ArtifactIdentifier> partition, ReputationMetricManager rmm) {
Correlation correlation = new Correlation();
// predict the quality of the artifacts in the partition using the reputation results from rmm
for (ArtifactIdentifier aid : partition) {
Artifact artifact = collabReview.getRepository().getArtifact(aid);
double predicted = predictQuality(artifact, rmm.findReputationMetric("quality"));
Float assessedFloat = collabReview.getMeasurementsManager().getArtifactQualityAssessor().assessQuality(aid);
assert assessedFloat != null;
double assessed = assessedFloat;
//correlation.addRow(assessed, predicted, collabReview.getMeasurementsManager().getArtifactWeight().measure(artifact));
correlation.addRow(assessed, predicted, 1);
}
return correlation;
}
private double predictQuality(Artifact a, ReputationMetric rm) {
NavigableMap<Author, Float> responsibilities = collabReview.getMeasurementsManager().getArtifactResponsibility().listResponsibilities(a);
double sum = 0;
for (Author author : responsibilities.keySet()) {
float responsibility = responsibilities.get(author);
Float score = rm.getAuthorScore(author.getName());
double summand = score != null ? responsibility * rm.getAuthorScore(author.getName()) : 0;
//logger.debug("Author " + author + ": " + responsibility + ", summand=" + summand);
sum += summand;
}
return sum;
}
private Collection<ArtifactIdentifier> listArtifacts() {
return filter.getChildFilter().preFilterAll(collabReview.getRepository().listNonObsoleteArtifacts(new Date()));
}
}