learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

                if (gr_NewToOrig != null)

                {

                  for(SampleData sample:result.samples)

                    gr_NewToOrig.add(sample.referenceLearner.getValue(),sample.actualLearner.getValue());

                }

               

                for(SampleData sample:result.samples)

                  if (sample.referenceLearner.getValue() > 0)

                    gr_QualityForNumberOfTraces.add(traceQuantity+"",sample.actualLearner.getValue()/sample.referenceLearner.getValue());

              }

              if (gr_PairQuality != null)

              {

                synchronized(pairQualityCounter)

                {

                  learnerRunner.setSelectionID(branch+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

                for(SampleData sample:result.samples)

                  for(Entry<String,ScoresForGraph> score:sample.miscGraphs.entrySet())

                    gr_StructuralForDifferentLearners.add(score.getKey(),score.getValue().differenceStructural.getValue());

             

                for(SampleData sample:result.samples)

                  for(Entry<String,ScoresForGraph> score:sample.miscGraphs.entrySet())

                    gr_BCRForDifferentLearners.add(score.getKey(),score.getValue().differenceBCR.getValue());

                gr_BCRForDifferentLearners.drawInteractive(gr);gr_StructuralForDifferentLearners.drawInteractive(gr);

              }

             

            }

            catch(Exception ex)

            {

              IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

              if (executorService != null) { executorService.shutdownNow();executorService = null; }

              throw e;

            }

      }

    if (gr_BCRForDifferentLearners != null) gr_BCRForDifferentLearners.drawPdf(gr);if (gr_StructuralForDifferentLearners != null) gr_StructuralForDifferentLearners.drawPdf(gr);

    // Inference from a few traces

    final int traceQuantityToUse = traceQuantity;

    SquareBagPlot gr_StructuralDiff = new SquareBagPlot("Structural score, EDSM,>=2","Structural Score, EDSM-Markov learner",new File(branch+"_"+(traceQuantityToUse/2)+"_trace_structuraldiff.pdf"),0,1,true);

    SquareBagPlot gr_BCR = new SquareBagPlot("BCR, EDSM,>=2","BCR, EDSM-Markov learner",new File(branch+"_"+(traceQuantityToUse/2)+"_trace_bcr.pdf"),0.5,1,true);

    RBagPlot gr_BCR_singletons = new RBagPlot("%% states identified by singletons","BCR Score, EDSM-Markov learner",new File(branch+"_"+(traceQuantityToUse/2)+"_trace_bcr_singletons.pdf"));

    RBagPlot gr_BCR_states = new RBagPlot("number of states in reference","BCR Score, EDSM-Markov learner",new File(branch+"_"+(traceQuantityToUse/2)+"_trace_bcr_numberofstates.pdf"));

    ScatterPlot gr_ImprovementPerState = new ScatterPlot("State number", "BCR, improvement",new File(branch+"_"+(traceQuantityToUse/2)+"_bcr_statenumber.pdf"));

    for(final boolean onlyPositives:new boolean[]{true})

      for(final double alphabetMultiplier:new double[]{alphabetMultiplierMax})

      {

            final int totalTaskNumber = traceQuantityToUse;

            try

            {

              int numberOfTasks = 0;

              for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

                for(int sample=0;sample<samplesPerFSM;++sample)

                {

                  LearnerRunner learnerRunner = new LearnerRunner(states,sample,totalTaskNumber+numberOfTasks,traceQuantityToUse, config, converter);

                  learnerRunner.setOnlyUsePositives(onlyPositives);

                  learnerRunner.setAlphabetMultiplier(alphabetMultiplier);

                  learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);learnerRunner.setChunkLen(chunkSize);

                  learnerRunner.setSelectionID(branch+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

                for(SampleData sample:result.samples)

                  gr_StructuralDiff.add(sample.referenceLearner.differenceStructural.getValue(),sample.actualLearner.differenceStructural.getValue());

             

                for(SampleData sample:result.samples)

                {

                  gr_BCR.add(sample.referenceLearner.differenceBCR.getValue(),sample.actualLearner.differenceBCR.getValue());

                  gr_BCR_singletons.add((double)sample.fractionOfStatesIdentifiedBySingletons,sample.actualLearner.differenceBCR.getValue());

                  gr_BCR_states.add((double)sample.stateNumber,sample.actualLearner.differenceBCR.getValue());

                  if (sample.referenceLearner.differenceBCR.getValue() > 0)

                  gr_ImprovementPerState.add((double)sample.stateNumber,sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

                }

              }

              gr_StructuralDiff.drawInteractive(gr);gr_BCR.drawInteractive(gr);gr_BCR_singletons.drawInteractive(gr);gr_BCR_states.drawInteractive(gr);gr_ImprovementPerState.drawInteractive(gr);

            }

            catch(Exception ex)

            {

              IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

              if (executorService != null) { executorService.shutdownNow();executorService = null; }

              throw e;

            }

      }

      if (gr_StructuralDiff != null) gr_StructuralDiff.drawPdf(gr);

      if (gr_BCR != null) gr_BCR.drawPdf(gr);

      if (gr_BCR_singletons != null) gr_BCR_singletons.drawPdf(gr);

      if (gr_BCR_states != null) gr_BCR_states.drawPdf(gr);

      if (gr_ImprovementPerState != null) gr_ImprovementPerState.drawPdf(gr);

    final RBoxPlot<String> gr_BCRImprovementForDifferentAlphabetSize = new RBoxPlot<String>("alphabet multiplier","improvement, BCR",new File(branch+"BCR_vs_alphabet.pdf"));

    final RBoxPlot<String> gr_BCRForDifferentAlphabetSize = new RBoxPlot<String>("alphabet multiplier","BCR",new File(branch+"BCR_absolute_vs_alphabet.pdf"));

    final RBoxPlot<String> gr_StructuralImprovementForDifferentAlphabetSize = new RBoxPlot<String>("alphabet multiplier","improvement, structural",new File(branch+"structural_vs_alphabet.pdf"));

    final RBoxPlot<String> gr_StructuralForDifferentAlphabetSize = new RBoxPlot<String>("alphabet multiplier","structural",new File(branch+"structural_absolute_vs_alphabet.pdf"));

    final RBoxPlot<String> gr_MarkovAccuracyForDifferentAlphabetSize = new RBoxPlot<String>("alphabet multiplier","Markov accuracy",new File(branch+"markov_accuracy_vs_alphabet.pdf"));

    // Same experiment but with different alphabet size

    for(final boolean onlyPositives:new boolean[]{true})

      for(final double alphabetMultiplier:new double[]{alphabetMultiplierMax/16,alphabetMultiplierMax/8,alphabetMultiplierMax/4,alphabetMultiplierMax/2,alphabetMultiplierMax,alphabetMultiplierMax*2,alphabetMultiplierMax*4})

      {

        String selection;

          selection = branch+";EVALUATION;"+

              ";onlyPositives="+onlyPositives+";";

        final int totalTaskNumber = traceQuantity;

        try

        {

          int numberOfTasks = 0;

          for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

            for(int sample=0;sample<samplesPerFSM;++sample)

            {

              LearnerRunner learnerRunner = new LearnerRunner(states,sample,totalTaskNumber+numberOfTasks,traceQuantity, config, converter);

              learnerRunner.setOnlyUsePositives(onlyPositives);

              learnerRunner.setTracesAlphabetMultiplier(alphabetMultiplierMax);

              learnerRunner.setAlphabetMultiplier(alphabetMultiplier);

              learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);learnerRunner.setChunkLen(chunkSize);

              learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

              runner.submit(learnerRunner);

              ++numberOfTasks;

            }

          for(int count=0;count < numberOfTasks;++count)

          {

            ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

            for(SampleData sample:result.samples)

            {

              if (sample.referenceLearner.differenceBCR.getValue() > 0)

              {

                gr_BCRImprovementForDifferentAlphabetSize.add(String.format("%.2f",alphabetMultiplier),sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

                gr_BCRForDifferentAlphabetSize.add(String.format("%.2f",alphabetMultiplier),sample.actualLearner.differenceBCR.getValue());

              }

              if (sample.referenceLearner.differenceStructural.getValue() > 0)

              {

                gr_StructuralImprovementForDifferentAlphabetSize.add(String.format("%.2f",alphabetMultiplier),sample.actualLearner.differenceStructural.getValue()/sample.referenceLearner.differenceStructural.getValue());

                gr_StructuralForDifferentAlphabetSize.add(String.format("%.2f",alphabetMultiplier),sample.actualLearner.differenceStructural.getValue());

              }

              gr_MarkovAccuracyForDifferentAlphabetSize.add(String.format("%.2f,P",alphabetMultiplier),(double)sample.markovPrecision,"green");

              gr_MarkovAccuracyForDifferentAlphabetSize.add(String.format("%.2f,R",alphabetMultiplier),(double)sample.markovRecall,"blue");

            }

          }

          gr_BCRImprovementForDifferentAlphabetSize.drawInteractive(gr);gr_BCRForDifferentAlphabetSize.drawInteractive(gr);gr_MarkovAccuracyForDifferentAlphabetSize.drawInteractive(gr);

          gr_StructuralImprovementForDifferentAlphabetSize.drawInteractive(gr);gr_StructuralForDifferentAlphabetSize.drawInteractive(gr);

        }

        catch(Exception ex)

        {

          IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

          if (executorService != null) { executorService.shutdownNow();executorService = null; }

          throw e;

        }

      }

      if (gr_BCRImprovementForDifferentAlphabetSize != null) gr_BCRImprovementForDifferentAlphabetSize.drawPdf(gr);

      if (gr_StructuralImprovementForDifferentAlphabetSize != null) gr_StructuralImprovementForDifferentAlphabetSize.drawPdf(gr);

      if (gr_BCRForDifferentAlphabetSize != null) gr_BCRForDifferentAlphabetSize.drawPdf(gr);

      if (gr_StructuralForDifferentAlphabetSize != null) gr_StructuralForDifferentAlphabetSize.drawPdf(gr);

      if (gr_MarkovAccuracyForDifferentAlphabetSize != null) gr_MarkovAccuracyForDifferentAlphabetSize.drawPdf(gr);

      // Same experiment but with different number of sequences.

      final RBoxPlot<Integer> gr_BCRImprovementForDifferentNrOfTraces = new RBoxPlot<Integer>("nr of traces","improvement, BCR",new File(branch+"BCR_vs_tracenumber.pdf"));

      final RBoxPlot<Integer> gr_BCRForDifferentNrOfTraces = new RBoxPlot<Integer>("nr of traces","BCR",new File(branch+"BCR_absolute_vs_tracenumber.pdf"));

      final RBoxPlot<Integer> gr_StructuralImprovementForDifferentNrOfTraces = new RBoxPlot<Integer>("nr of traces","improvement, structural",new File(branch+"structural_vs_tracenumber.pdf"));

      final RBoxPlot<Integer> gr_StructuralForDifferentNrOfTraces = new RBoxPlot<Integer>("nr of traces","structural",new File(branch+"structural_absolute_vs_tracenumber.pdf"));

      for(final boolean onlyPositives:new boolean[]{true})

        for(final double alphabetMultiplier:new double[]{alphabetMultiplierMax})

          for(int traceNum=2;traceNum<traceQuantity*3;traceNum+=2)

          {

            String selection;

              selection = branch+";EVALUATION;"+

                  ";onlyPositives="+onlyPositives+";";

            final int totalTaskNumber = traceNum;

            try

            {

              int numberOfTasks = 0;

              for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

                for(int sample=0;sample<samplesPerFSM;++sample)

                {

                  LearnerRunner learnerRunner = new LearnerRunner(states,sample,totalTaskNumber+numberOfTasks,traceNum, config, converter);

                  learnerRunner.setOnlyUsePositives(onlyPositives);

                  learnerRunner.setAlphabetMultiplier(alphabetMultiplier);

                  learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);learnerRunner.setChunkLen(chunkSize);

                  learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

               

                for(SampleData sample:result.samples)

                {

                  if (sample.referenceLearner.differenceBCR.getValue() > 0)

                  {

                    // we'll generate both positives and negatives; in the considered experiments, only positives are used hence half the number of sequences are actually being learnt from.

                    gr_BCRImprovementForDifferentNrOfTraces.add(traceNum/2,sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

                    gr_BCRForDifferentNrOfTraces.add(traceNum/2,sample.actualLearner.differenceBCR.getValue());

                  }

                  if (sample.referenceLearner.differenceStructural.getValue() > 0)

                  {

                    gr_StructuralImprovementForDifferentNrOfTraces.add(traceNum/2,sample.actualLearner.differenceStructural.getValue()/sample.referenceLearner.differenceStructural.getValue());

                    gr_StructuralForDifferentNrOfTraces.add(traceNum/2,sample.actualLearner.differenceStructural.getValue());

                  }

                }

              }

              gr_BCRForDifferentNrOfTraces.drawInteractive(gr);gr_StructuralForDifferentNrOfTraces.drawInteractive(gr);

            }

            catch(Exception ex)

            {

              IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

              if (executorService != null) { executorService.shutdownNow();executorService = null; }

              throw e;

            }

          }

      if (gr_BCRImprovementForDifferentNrOfTraces != null) gr_BCRImprovementForDifferentNrOfTraces.drawPdf(gr);

      if (gr_BCRForDifferentNrOfTraces != null) gr_BCRForDifferentNrOfTraces.drawPdf(gr);

      if (gr_StructuralImprovementForDifferentNrOfTraces != null) gr_StructuralImprovementForDifferentNrOfTraces.drawPdf(gr);

      if (gr_StructuralForDifferentNrOfTraces != null) gr_StructuralForDifferentNrOfTraces.drawPdf(gr);

      // Same experiment but with different number of sequences, both positive and negative.

      final RBoxPlot<Integer> gr_BCRImprovementForDifferentNrOfTracesPosNeg = new RBoxPlot<Integer>("nr of traces","improvement, BCR",new File(branch+"BCR_vs_tracenumber_posneg.pdf"));

      final RBoxPlot<Integer> gr_BCRForDifferentNrOfTracesPosNeg = new RBoxPlot<Integer>("nr of traces","BCR",new File(branch+"BCR_absolute_vs_tracenumber_posneg.pdf"));

      final RBoxPlot<Integer> gr_StructuralImprovementForDifferentNrOfTracesPosNeg = new RBoxPlot<Integer>("nr of traces","improvement, structural",new File(branch+"structural_vs_tracenumber_posneg.pdf"));

      final RBoxPlot<Integer> gr_StructuralForDifferentNrOfTracesPosNeg = new RBoxPlot<Integer>("nr of traces","structural",new File(branch+"structural_absolute_vs_tracenumber_posneg.pdf"));

      for(final boolean onlyPositives:new boolean[]{false})

        for(final double alphabetMultiplier:new double[]{alphabetMultiplierMax})

          for(int traceNum=2;traceNum<traceQuantity*3;traceNum+=2)

          {

            String selection;

            selection = branch+";EVALUATION;"+

                ";onlyPositives="+onlyPositives+";";

            final int totalTaskNumber = traceNum;

            try

            {

              int numberOfTasks = 0;

              for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

                for(int sample=0;sample<samplesPerFSM;++sample)

                {

                  LearnerRunner learnerRunner = new LearnerRunner(states,sample,totalTaskNumber+numberOfTasks,traceNum, config, converter);

                  learnerRunner.setOnlyUsePositives(onlyPositives);

                  learnerRunner.setAlphabetMultiplier(alphabetMultiplier);

                  learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);learnerRunner.setChunkLen(chunkSize);

                  learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

               

                for(SampleData sample:result.samples)

                {

                  if (sample.referenceLearner.differenceBCR.getValue() > 0)

                  {

                    gr_BCRImprovementForDifferentNrOfTracesPosNeg.add(traceNum,sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

                    gr_BCRForDifferentNrOfTracesPosNeg.add(traceNum,sample.actualLearner.differenceBCR.getValue());

                  }

                  if (sample.referenceLearner.differenceStructural.getValue() > 0)

                  {

                    gr_StructuralImprovementForDifferentNrOfTracesPosNeg.add(traceNum,sample.actualLearner.differenceStructural.getValue()/sample.referenceLearner.differenceStructural.getValue());

                    gr_StructuralForDifferentNrOfTracesPosNeg.add(traceNum,sample.actualLearner.differenceStructural.getValue());

                  }

                }

              }

              gr_BCRForDifferentNrOfTracesPosNeg.drawInteractive(gr);gr_StructuralForDifferentNrOfTracesPosNeg.drawInteractive(gr);

            }

            catch(Exception ex)

            {

              IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

              if (executorService != null) { executorService.shutdownNow();executorService = null; }

              throw e;

            }

          }

    if (gr_BCRImprovementForDifferentNrOfTracesPosNeg != null) gr_BCRImprovementForDifferentNrOfTracesPosNeg.drawPdf(gr);

    if (gr_BCRForDifferentNrOfTracesPosNeg != null) gr_BCRForDifferentNrOfTracesPosNeg.drawPdf(gr);

    if (gr_StructuralImprovementForDifferentNrOfTracesPosNeg != null) gr_StructuralImprovementForDifferentNrOfTracesPosNeg.drawPdf(gr);

    if (gr_StructuralForDifferentNrOfTracesPosNeg != null) gr_StructuralForDifferentNrOfTracesPosNeg.drawPdf(gr);

    // Same experiment but with different trace length but the same number of sequences

    final RBoxPlot<Double> gr_BCRImprovementForDifferentLengthOfTraces = new RBoxPlot<Double>("trace length multiplier","improvement, BCR",new File(branch+"BCR_vs_tracelength.pdf"));

    final RBoxPlot<Double> gr_BCRForDifferentLengthOfTraces = new RBoxPlot<Double>("trace length multiplier","BCR",new File(branch+"BCR_absolute_vs_tracelength.pdf"));

    final RBoxPlot<Double> gr_StructuralImprovementForDifferentLengthOfTraces = new RBoxPlot<Double>("trace length multiplier","improvement, structural",new File(branch+"structural_vs_tracelength.pdf"));

    final RBoxPlot<Double> gr_StructuralForDifferentLengthOfTraces = new RBoxPlot<Double>("trace length multiplier","structural",new File(branch+"structural_absolute_vs_tracelength.pdf"));

    final RBoxPlot<Double> gr_TransitionCoverageForDifferentLengthOfTraces = new RBoxPlot<Double>("trace length multiplier","transition coverage",new File(branch+"transitionCoverage_vs_tracelength.pdf"));

    for(final boolean onlyPositives:new boolean[]{true})

      for(double traceMultiplier=0.125;traceMultiplier<10;traceMultiplier*=2.)

        {

          String selection;

            selection = branch+";EVALUATION;"+

                ";onlyPositives="+onlyPositives+";";

          final int totalTaskNumber = traceQuantity;

          try

          {

            int numberOfTasks = 0;

            for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

              for(int sample=0;sample<samplesPerFSM;++sample)

              {

                LearnerRunner learnerRunner = new LearnerRunner(states,sample,totalTaskNumber+numberOfTasks,traceQuantity, config, converter);

                learnerRunner.setAlphabetMultiplier(alphabetMultiplierMax);

                learnerRunner.setOnlyUsePositives(onlyPositives);

                learnerRunner.setTraceLengthMultiplier(traceMultiplier);learnerRunner.setChunkLen(chunkSize);

                learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                runner.submit(learnerRunner);

                ++numberOfTasks;

              }

            for(int count=0;count < numberOfTasks;++count)

            {

              ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

             

              for(SampleData sample:result.samples)

              {

                if (sample.referenceLearner.differenceBCR.getValue() > 0)

                {

                  gr_BCRImprovementForDifferentLengthOfTraces.add(traceMultiplier,sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

                  gr_BCRForDifferentLengthOfTraces.add(traceMultiplier,sample.actualLearner.differenceBCR.getValue());

                }

                if (sample.referenceLearner.differenceStructural.getValue() > 0)

                {

                  gr_StructuralImprovementForDifferentLengthOfTraces.add(traceMultiplier,sample.actualLearner.differenceStructural.getValue()/sample.referenceLearner.differenceStructural.getValue());

                  gr_StructuralForDifferentLengthOfTraces.add(traceMultiplier,sample.actualLearner.differenceStructural.getValue());

                }

                gr_TransitionCoverageForDifferentLengthOfTraces.add(traceMultiplier,(double)sample.transitionsSampled);

              }

            }

           

            gr_BCRForDifferentLengthOfTraces.drawInteractive(gr);gr_StructuralForDifferentLengthOfTraces.drawInteractive(gr);gr_TransitionCoverageForDifferentLengthOfTraces.drawInteractive(gr);

          }

          catch(Exception ex)

          {

            IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

            if (executorService != null) { executorService.shutdownNow();executorService = null; }

            throw e;

          }

        }

    if (gr_BCRImprovementForDifferentLengthOfTraces != null) gr_BCRImprovementForDifferentLengthOfTraces.drawPdf(gr);

    if (gr_BCRForDifferentLengthOfTraces != null) gr_BCRForDifferentLengthOfTraces.drawPdf(gr);

    if (gr_StructuralImprovementForDifferentLengthOfTraces != null) gr_StructuralImprovementForDifferentLengthOfTraces.drawPdf(gr);

    if (gr_StructuralForDifferentLengthOfTraces != null) gr_StructuralForDifferentLengthOfTraces.drawPdf(gr);

    if (gr_TransitionCoverageForDifferentLengthOfTraces != null) gr_TransitionCoverageForDifferentLengthOfTraces.drawPdf(gr);

    // Same experiment but with different prefix length but the same number of sequences and their length

    final RBoxPlot<Integer> gr_BCRImprovementForDifferentPrefixlen = new RBoxPlot<Integer>("length of prefix","improvement, BCR",new File(branch+"BCR_vs_prefixLength.pdf"));

    final RBoxPlot<Integer> gr_BCRForDifferentPrefixlen = new RBoxPlot<Integer>("length of prefix","BCR",new File(branch+"BCR_absolute_vs_prefixLength.pdf"));

    final RBoxPlot<Integer> gr_StructuralImprovementForDifferentPrefixlen = new RBoxPlot<Integer>("length of prefix","improvement, structural",new File(branch+"structural_vs_prefixLength.pdf"));

    final RBoxPlot<Integer> gr_StructuralForDifferentPrefixlen = new RBoxPlot<Integer>("length of prefix","structural",new File(branch+"structural_absolute_vs_prefixLength.pdf"));

    final RBoxPlot<String> gr_MarkovAccuracyForDifferentPrefixlen = new RBoxPlot<String>("length of prefix","Markov accuracy",new File(branch+"markov_accuracy_vs_prefixLength.pdf"));

    for(final boolean onlyPositives:new boolean[]{true})

      for(int prefixLength=1;prefixLength<6;++prefixLength)

        {

          String selection;

            selection = branch+";EVALUATION;"+

                ";onlyPositives="+onlyPositives+";";

            final int totalTaskNumber = traceQuantity;

            try

            {

              int numberOfTasks = 0;

              for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

                for(int sample=0;sample<samplesPerFSM;++sample)

                {

                  LearnerRunner learnerRunner = new LearnerRunner(states,sample,totalTaskNumber+numberOfTasks,traceQuantity, config, converter);

                  learnerRunner.setAlphabetMultiplier(alphabetMultiplierMax);

                  learnerRunner.setOnlyUsePositives(onlyPositives);

                  learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);learnerRunner.setChunkLen(prefixLength+1);

                  learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

               

                for(SampleData sample:result.samples)

                {

                  if (sample.referenceLearner.differenceBCR.getValue() > 0)

                  {

                    gr_BCRImprovementForDifferentPrefixlen.add(prefixLength,sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

                    gr_BCRForDifferentPrefixlen.add(prefixLength,sample.actualLearner.differenceBCR.getValue());

                  }

                  if (sample.referenceLearner.differenceStructural.getValue() > 0)

                  {

                    gr_StructuralImprovementForDifferentPrefixlen.add(prefixLength,sample.actualLearner.differenceStructural.getValue()/sample.referenceLearner.differenceStructural.getValue());

                    gr_StructuralForDifferentPrefixlen.add(prefixLength,sample.actualLearner.differenceStructural.getValue());

                  }

                  gr_MarkovAccuracyForDifferentPrefixlen.add(""+prefixLength+",P",(double)sample.markovPrecision,"green");

                  gr_MarkovAccuracyForDifferentPrefixlen.add(""+prefixLength+",R",(double)sample.markovRecall,"blue");

                }

              }

              gr_BCRForDifferentPrefixlen.drawInteractive(gr);gr_StructuralForDifferentPrefixlen.drawInteractive(gr);gr_MarkovAccuracyForDifferentPrefixlen.drawInteractive(gr);

            }

            catch(Exception ex)

            {

                learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                runner.submit(learnerRunner);

                ++numberOfTasks;

              }

            for(int count=0;count < numberOfTasks;++count)

            {

              ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

              samples.addAll(result.samples);

            }

          }

          catch(Exception ex)

          {

            IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

            if (executorService != null) { executorService.shutdown();executorService = null; }

            throw e;

          }

          int nonZeroes = 0;

          long numberOfValues = 0;

          System.out.println("number of instances: "+dataCollector.trainingData.numInstances());

          int freqData[] = new int[dataCollector.attributesOfAnInstance.length];

          for(int i=0;i<dataCollector.trainingData.numInstances();++i)

            for(int attrNum=0;attrNum<dataCollector.attributesOfAnInstance.length;++attrNum)

            {

              assert dataCollector.attributesOfAnInstance[attrNum].index() == attrNum;

              if (dataCollector.trainingData.instance(i).stringValue(attrNum) != WekaDataCollector.ZERO)

              {

                ++freqData[attrNum];++numberOfValues;

              }

            }

          for(int attrNum=0;attrNum<dataCollector.attributesOfAnInstance.length;++attrNum)

            if (freqData[attrNum]>0)

              ++nonZeroes;

         

          System.out.println("Total instances: "+dataCollector.trainingData.numInstances()+" with "+dataCollector.attributesOfAnInstance.length+" attributes, non-zeroes are "+nonZeroes+" with average of "+((double)numberOfValues)/nonZeroes);

          Arrays.sort(freqData);

          int numOfcolumns=20;

          int stepWidth = dataCollector.attributesOfAnInstance.length/numOfcolumns;

         

          final RBoxPlot<Long> gr_HistogramOfAttributeValues = new RBoxPlot<Long>("Attributes","Number of values",new File("attributes_use"+selection+".pdf"));

          for(int i=0;i<numOfcolumns;++i)

          {

            int columnData=0;

            for(int j=i*stepWidth;j<(i+1)*stepWidth;++j)

              if (j < dataCollector.attributesOfAnInstance.length)

                columnData+=freqData[j];

           

            gr_HistogramOfAttributeValues.add(new Long(numOfcolumns-i),new Double(columnData>0?Math.log10(columnData):0));

          }

          //gr_HistogramOfAttributeValues.drawInteractive(gr);

          gr_HistogramOfAttributeValues.drawPdf(gr);

          /*

          // write arff

          FileWriter wekaInstances = null;

          String whereToWrite = "qualityLearner_"+selection+".arff";

          try

          {

            wekaInstances = new FileWriter(whereToWrite);

            // This chunk is almost verbatim from Weka's Instances.toString()

            wekaInstances.append(Instances.ARFF_RELATION).append(" ").append(Utils.quote(dataCollector.trainingData.relationName())).append("\n\n");

              for (int i = 0; i < dataCollector.trainingData.numAttributes(); i++) {

                wekaInstances.append(dataCollector.trainingData.attribute(i).toString()).append("\n");

              }

              wekaInstances.append("\n").append(Instances.ARFF_DATA).append("\n");

              for (int i = 0; i < dataCollector.trainingData.numInstances(); i++) {

                wekaInstances.append(dataCollector.trainingData.instance(i).toString());

                  if (i < dataCollector.trainingData.numInstances() - 1) {

                    wekaInstances.append('\n');

                  }

                }

          }

          catch(Exception ex)

          {

            Helper.throwUnchecked("failed to create a file with training data for "+whereToWrite, ex);

          }

          finally

          {

            if (wekaInstances != null)

              try {

                wekaInstances.close();

              } catch (IOException e) {

                // ignore this, we are not proceeding anyway due to an earlier exception so whether the file was actually written does not matter

              }

          }

          */

          // Run the evaluation

          final weka.classifiers.trees.REPTree repTree = new weka.classifiers.trees.REPTree();repTree.setMaxDepth(4);

          //repTree.setNoPruning(true);// since we only use the tree as a classifier (as a conservative extension of what is currently done) and do not actually look at it, elimination of pruning is not a problem.

          // As part of learning, we also prune some of the nodes where the ratio of correctly-classified pairs to those incorrectly classified is comparable.

          // The significant advantage of not pruning is that the result is no longer sensitive to the order of elements in the tree and hence does not depend on the order in which elements have been obtained by concurrent threads.

          //final weka.classifiers.lazy.IB1 ib1 = new weka.classifiers.lazy.IB1();

          //final weka.classifiers.trees.J48 classifier = new weka.classifiers.trees.J48();

          final Classifier classifier = repTree;

          classifier.buildClassifier(dataCollector.trainingData);

          System.out.println("Entries in the classifier: "+dataCollector.trainingData.numInstances());

          System.out.println(classifier);

          dataCollector=null;// throw all the training data away.

         

          {// serialise the classifier, this is the only way to store it.

            OutputStream os = new FileOutputStream(selection+".ser");

            ObjectOutputStream oo = new ObjectOutputStream(os);

                      oo.writeObject(classifier);

                      os.close();

          }

                   

          for(final boolean selectingRed:new boolean[]{false})

          for(final boolean classifierToBlockAllMergers:new boolean[]{true})

          //for(final boolean zeroScoringAsRed:(classifierToBlockAllMergers?new boolean[]{true,false}:new boolean[]{false}))// where we are not using classifier to rule out all mergers proposed by pair selection, it does not make sense to use two values configuring this classifier.

          for(final double threshold:new double[]{1})

          {

            final boolean zeroScoringAsRed = false;

            selection = "TRUNK;EVALUATION;"+"ifDepth="+ifDepth+";threshold="+threshold+// ";useUnique="+useUnique+";onlyPositives="+onlyPositives+

                ";selectingRed="+selectingRed+";classifierToBlockAllMergers="+classifierToBlockAllMergers+";zeroScoringAsRed="+zeroScoringAsRed+";traceQuantity="+traceQuantity+";lengthMultiplier="+lengthMultiplier+";trainingDataMultiplier="+trainingDataMultiplier+";";

            final int totalTaskNumber = traceQuantity;

            final RBoxPlot<Long> gr_PairQuality = new RBoxPlot<Long>("Correct v.s. wrong","%%",new File("percentage_score"+selection+".pdf"));

            final RBoxPlot<String> gr_QualityForNumberOfTraces = new RBoxPlot<String>("traces","%%",new File("quality_traces"+selection+".pdf"));

            SquareBagPlot gr_NewToOrig = new SquareBagPlot("orig score","score with learnt selection",new File("new_to_orig"+selection+".pdf"),0,1,true);

            final Map<Long,TrueFalseCounter> pairQualityCounter = new TreeMap<Long,TrueFalseCounter>();

            try

            {

              int numberOfTasks = 0;

              for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

                for(int sample=0;sample<samplesPerFSM;++sample)

                {

                  LearnerRunner learnerRunner = new LearnerRunner(dataCollector,states,sample,totalTaskNumber+numberOfTasks,traceQuantity, config, converter)

                  {

                    @Override

                    public LearnerThatCanClassifyPairs createLearner(LearnerEvaluationConfiguration evalCnf,LearnerGraph argReferenceGraph,@SuppressWarnings("unused") WekaDataCollector argDataCollector,  LearnerGraph argInitialPTA)

                    {

                      LearnerThatUsesWekaResults l = new LearnerThatUsesWekaResults(ifDepth,evalCnf,argReferenceGraph,classifier,argInitialPTA);

                      if (gr_PairQuality != null)

                        l.setPairQualityCounter(pairQualityCounter);

                     

                      l.setUseClassifierForRed(selectingRed);l.setUseClassifierToChooseNextRed(classifierToBlockAllMergers);

                      l.setBlacklistZeroScoringPairs(zeroScoringAsRed);

                      l.setThreshold(threshold);

                      return l;

                    }

                   

                  };

                  learnerRunner.setPickUniqueFromInitial(useUnique);learnerRunner.setEvaluateAlsoUsingReferenceLearner(true);

                  learnerRunner.setOnlyUsePositives(onlyPositives);learnerRunner.setIfdepth(ifDepth);learnerRunner.setLengthMultiplier(lengthMultiplier);

                  learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

                if (gr_NewToOrig != null)

                {

                  for(SampleData sample:result.samples)

                    gr_NewToOrig.add(sample.referenceLearner.getValue(),sample.actualLearner.getValue());

                }

               

                for(SampleData sample:result.samples)

                  if (sample.referenceLearner.getValue() > 0)

                    gr_QualityForNumberOfTraces.add(traceQuantity+"",sample.actualLearner.getValue()/sample.referenceLearner.getValue());

              }

              if (gr_PairQuality != null)

              {

                synchronized(pairQualityCounter)

                {

     

      MachineGenerator mg = new MachineGenerator(states, 40, states/10);

      int mutationsPerStage = (states/2) / 2;

      //System.out.print("\n"+states+": ");

      for(int mutationStage = 0;mutationStage<mutationStages;mutationStage++)

      {

        for(int experiment=0;experiment<experimentsPerMutationCategory;experiment++)

        {

          ExperimentResult outcome = new ExperimentResult();

          while(!outcome.experimentValid)

          {

            int mutations = mutationsPerStage * (mutationStage+1);

            LearnerGraphND origGraph = mg.nextMachine(alphabet, experiment,config,converter);

            GraphMutator<List<CmpVertex>,LearnerGraphNDCachedData> mutator = new GraphMutator<List<CmpVertex>,LearnerGraphNDCachedData>(origGraph,r);

            mutator.mutate(mutations);

            LearnerGraphND origAfterRenaming = new LearnerGraphND(origGraph.config);

            Map<CmpVertex,CmpVertex> origToNew = copyStatesAndTransitions(origGraph,origAfterRenaming);

            LearnerGraphND mutated = (LearnerGraphND)mutator.getMutated();

            Set<Transition> appliedMutations = new HashSet<Transition>();

            for(Transition tr:mutator.getDiff())

            {

              CmpVertex renamedFrom = origToNew.get(tr.getFrom());if (renamedFrom == null) renamedFrom = tr.getFrom();

              CmpVertex renamedTo = origToNew.get(tr.getTo());if (renamedTo == null) renamedTo = tr.getTo();

              appliedMutations.add(new Transition(renamedFrom,renamedTo,tr.getLabel()));

            }

           

            final double perfectLowToHigh=0.7,perfectThreshold=0.5;

           

            // These experiments are only run for the maximal number of states

            if (graphComplexity == graphComplexityMax-1)

            {

              for(double k:new double[]{0,.2,.4,.6,.8,.95})

              {

                config.setAttenuationK(k);

                config.setGdKeyPairThreshold(0.5);

                config.setGdLowToHighRatio(perfectLowToHigh);

                config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

                linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

               

                gr_Diff_k.add(k, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

               

              }

              for(double threshold:new double[]{0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95})

              {

                config.setGdKeyPairThreshold(threshold);

                config.setGdLowToHighRatio(perfectLowToHigh);

                config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

                linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

               

                gr_Diff_threshold.add(threshold, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

              }

              for(double lowtohigh:new double[]{0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95})

              {

                config.setGdKeyPairThreshold(perfectThreshold);

                config.setGdLowToHighRatio(lowtohigh);

                config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

                linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

               

                gr_Diff_lowtohigh.add(lowtohigh, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

              }

             

             

              for(int i=0;i<pairThreshold.length;++i)

                for(double ratio:lowToHigh)

                {

                  config.setGdKeyPairThreshold(pairThreshold[i]);

                  config.setGdLowToHighRatio(ratio);

                  config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

                  linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

                 

                  gr_Diff_thresholdslowhigh.add(new Pair<Double,Double>(ratio,pairThreshold[i]), outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

                }

            }

            config.setAttenuationK(0.5);

            config.setGdKeyPairThreshold(perfectThreshold);

            config.setGdLowToHighRatio(perfectLowToHigh);

            config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

            linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

           

            if (!skip)

            {

              LearnerGraph fromDet = null, toDet = null;

              try {

                fromDet = mergeAndDeterminize(origAfterRenaming);

                toDet = mergeAndDeterminize(mutated);

              } catch (IncompatibleStatesException e) {

                Helper.throwUnchecked("failed to build a deterministic graph from a nondet one", e);

              }

              languageDiff(fromDet,toDet,states, graphComplexity,outcome);

            }

            outcome.experimentValid = true;

            gr_Diff_States.add(states, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

            gr_W_States.add(states,outcome.getValue(DOUBLE_V.ACCURACY_W));

            Pair<Integer,Integer> mutations_states = new Pair<Integer,Integer>(mutationStage+1,states);

            Pair<Integer,Integer> states_mutations = new Pair<Integer,Integer>(states,mutationStage+1);

            gr_DiffGD_StatesLevel.add(mutations_states, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

                learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                runner.submit(learnerRunner);

                ++numberOfTasks;

              }

            for(int count=0;count < numberOfTasks;++count)

            {

              ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

              samples.addAll(result.samples);

            }

          }

          catch(Exception ex)

          {

            IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

            if (executorService != null) { executorService.shutdown();executorService = null; }

            throw e;

          }

          int nonZeroes = 0;

          long numberOfValues = 0;

          System.out.println("number of instances: "+dataCollector.trainingData.numInstances());

          int freqData[] = new int[dataCollector.attributesOfAnInstance.length];

          for(int i=0;i<dataCollector.trainingData.numInstances();++i)

            for(int attrNum=0;attrNum<dataCollector.attributesOfAnInstance.length;++attrNum)

            {

              assert dataCollector.attributesOfAnInstance[attrNum].index() == attrNum;

              if (dataCollector.trainingData.instance(i).stringValue(attrNum) != WekaDataCollector.ZERO)

              {

                ++freqData[attrNum];++numberOfValues;

              }

            }

          for(int attrNum=0;attrNum<dataCollector.attributesOfAnInstance.length;++attrNum)

            if (freqData[attrNum]>0)

              ++nonZeroes;

         

          System.out.println("Total instances: "+dataCollector.trainingData.numInstances()+" with "+dataCollector.attributesOfAnInstance.length+" attributes, non-zeroes are "+nonZeroes+" with average of "+((double)numberOfValues)/nonZeroes);

          Arrays.sort(freqData);

          int numOfcolumns=20;

          int stepWidth = dataCollector.attributesOfAnInstance.length/numOfcolumns;

         

          final RBoxPlot<Long> gr_HistogramOfAttributeValues = new RBoxPlot<Long>("Attributes","Number of values",new File("attributes_use"+selection+".pdf"));

          for(int i=0;i<numOfcolumns;++i)

          {

            int columnData=0;

            for(int j=i*stepWidth;j<(i+1)*stepWidth;++j)

              if (j < dataCollector.attributesOfAnInstance.length)

                columnData+=freqData[j];

           

            gr_HistogramOfAttributeValues.add(new Long(numOfcolumns-i),new Double(columnData>0?Math.log10(columnData):0));

          }

          //gr_HistogramOfAttributeValues.drawInteractive(gr);

          gr_HistogramOfAttributeValues.drawPdf(gr);

          /*

          // write arff

          FileWriter wekaInstances = null;

          String whereToWrite = "qualityLearner_"+selection+".arff";

          try

          {

            wekaInstances = new FileWriter(whereToWrite);

            // This chunk is almost verbatim from Weka's Instances.toString()

            wekaInstances.append(Instances.ARFF_RELATION).append(" ").append(Utils.quote(dataCollector.trainingData.relationName())).append("\n\n");

              for (int i = 0; i < dataCollector.trainingData.numAttributes(); i++) {

                wekaInstances.append(dataCollector.trainingData.attribute(i).toString()).append("\n");

              }

              wekaInstances.append("\n").append(Instances.ARFF_DATA).append("\n");

              for (int i = 0; i < dataCollector.trainingData.numInstances(); i++) {

                wekaInstances.append(dataCollector.trainingData.instance(i).toString());

                  if (i < dataCollector.trainingData.numInstances() - 1) {

                    wekaInstances.append('\n');

                  }

                }

          }

          catch(Exception ex)

          {

            Helper.throwUnchecked("failed to create a file with training data for "+whereToWrite, ex);

          }

          finally

          {

            if (wekaInstances != null)

              try {

                wekaInstances.close();

              } catch (IOException e) {

                // ignore this, we are not proceeding anyway due to an earlier exception so whether the file was actually written does not matter

              }

          }

          */

          // Run the evaluation

          final weka.classifiers.trees.REPTree repTree = new weka.classifiers.trees.REPTree();repTree.setMaxDepth(4);

          //repTree.setNoPruning(true);// since we only use the tree as a classifier (as a conservative extension of what is currently done) and do not actually look at it, elimination of pruning is not a problem.

          // As part of learning, we also prune some of the nodes where the ratio of correctly-classified pairs to those incorrectly classified is comparable.

          // The significant advantage of not pruning is that the result is no longer sensitive to the order of elements in the tree and hence does not depend on the order in which elements have been obtained by concurrent threads.

          //final weka.classifiers.lazy.IB1 ib1 = new weka.classifiers.lazy.IB1();

          //final weka.classifiers.trees.J48 classifier = new weka.classifiers.trees.J48();

          final Classifier classifier = repTree;

          classifier.buildClassifier(dataCollector.trainingData);

          System.out.println("Entries in the classifier: "+dataCollector.trainingData.numInstances());

          System.out.println(classifier);

          dataCollector=null;// throw all the training data away.

         

          {// serialise the classifier, this is the only way to store it.

            OutputStream os = new FileOutputStream(selection+".ser");

            ObjectOutputStream oo = new ObjectOutputStream(os);

                      oo.writeObject(classifier);

                      os.close();

          }

                   

          for(final boolean selectingRed:new boolean[]{false})

          for(final boolean classifierToBlockAllMergers:new boolean[]{true})

          //for(final boolean zeroScoringAsRed:(classifierToBlockAllMergers?new boolean[]{true,false}:new boolean[]{false}))// where we are not using classifier to rule out all mergers proposed by pair selection, it does not make sense to use two values configuring this classifier.

          for(final double threshold:new double[]{1})

          {

            final boolean zeroScoringAsRed = false;

            selection = "TRUNK;EVALUATION;"+"ifDepth="+ifDepth+";threshold="+threshold+// ";useUnique="+useUnique+";onlyPositives="+onlyPositives+

                ";selectingRed="+selectingRed+";classifierToBlockAllMergers="+classifierToBlockAllMergers+";zeroScoringAsRed="+zeroScoringAsRed+";traceQuantity="+traceQuantity+";lengthMultiplier="+lengthMultiplier+";trainingDataMultiplier="+trainingDataMultiplier+";";

            final int totalTaskNumber = traceQuantity;

            final RBoxPlot<Long> gr_PairQuality = new RBoxPlot<Long>("Correct v.s. wrong","%%",new File("percentage_score"+selection+".pdf"));

            final RBoxPlot<String> gr_QualityForNumberOfTraces = new RBoxPlot<String>("traces","%%",new File("quality_traces"+selection+".pdf"));

            SquareBagPlot gr_NewToOrig = new SquareBagPlot("orig score","score with learnt selection",new File("new_to_orig"+selection+".pdf"),0,1,true);

            final Map<Long,TrueFalseCounter> pairQualityCounter = new TreeMap<Long,TrueFalseCounter>();

            try

            {

              int numberOfTasks = 0;

              for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

                for(int sample=0;sample<samplesPerFSM;++sample)

                {

                  LearnerRunner learnerRunner = new LearnerRunner(dataCollector,states,sample,totalTaskNumber+numberOfTasks,traceQuantity, config, converter)

                  {

                    @Override

                    public LearnerThatCanClassifyPairs createLearner(LearnerEvaluationConfiguration evalCnf,LearnerGraph argReferenceGraph,@SuppressWarnings("unused") WekaDataCollector argDataCollector,  LearnerGraph argInitialPTA)

                    {

                      LearnerThatUsesWekaResults l = new LearnerThatUsesWekaResults(ifDepth,evalCnf,argReferenceGraph,classifier,argInitialPTA);

                      if (gr_PairQuality != null)

                        l.setPairQualityCounter(pairQualityCounter);

                     

                      l.setUseClassifierForRed(selectingRed);l.setUseClassifierToChooseNextRed(classifierToBlockAllMergers);

                      l.setBlacklistZeroScoringPairs(zeroScoringAsRed);

                      l.setThreshold(threshold);

                      return l;

                    }

                   

                  };

                  learnerRunner.setPickUniqueFromInitial(useUnique);learnerRunner.setEvaluateAlsoUsingReferenceLearner(true);

                  learnerRunner.setOnlyUsePositives(onlyPositives);learnerRunner.setIfdepth(ifDepth);learnerRunner.setLengthMultiplier(lengthMultiplier);

                  learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

                if (gr_NewToOrig != null)

                {

                  for(SampleData sample:result.samples)

                    gr_NewToOrig.add(sample.referenceLearner.getValue(),sample.actualLearner.getValue());

                }

               

                for(SampleData sample:result.samples)

                  if (sample.referenceLearner.getValue() > 0)

                    gr_QualityForNumberOfTraces.add(traceQuantity+"",sample.actualLearner.getValue()/sample.referenceLearner.getValue());

              }

              if (gr_PairQuality != null)

              {

                synchronized(pairQualityCounter)

                {

      {

        switch(phase)

        {

        case RUN_STANDALONE:

          for(int count=taskCounterFromPreviousSubExperiment;count < taskCounter;++count)

          {

            RESULT result = runner.take().get();// this will throw an exception if any of the tasks failed.

            if (result == null)

              throw new IllegalArgumentException("experiment "+taskToRun+" did not complete or returned null");

            handlerForExperimentResults.processSubResult(result,this);plotAllGraphs(nameToGraph.values(),count);

          }

          plotAllGraphs(nameToGraph.values(),-1);

          break;

        case COUNT_TASKS:

          break;

                learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                runner.submit(learnerRunner);

                ++numberOfTasks;

              }

            for(int count=0;count < numberOfTasks;++count)

            {

              ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

              samples.addAll(result.samples);

            }

          }

          catch(Exception ex)

          {

            IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

            if (executorService != null) { executorService.shutdown();executorService = null; }

            throw e;

          }

          int nonZeroes = 0;

          long numberOfValues = 0;

          System.out.println("number of instances: "+dataCollector.trainingData.numInstances());

          int freqData[] = new int[dataCollector.attributesOfAnInstance.length];

          for(int i=0;i<dataCollector.trainingData.numInstances();++i)

            for(int attrNum=0;attrNum<dataCollector.attributesOfAnInstance.length;++attrNum)

            {

              assert dataCollector.attributesOfAnInstance[attrNum].index() == attrNum;

              if (dataCollector.trainingData.instance(i).stringValue(attrNum) != WekaDataCollector.ZERO)

              {

                ++freqData[attrNum];++numberOfValues;

              }

            }

          for(int attrNum=0;attrNum<dataCollector.attributesOfAnInstance.length;++attrNum)

            if (freqData[attrNum]>0)

              ++nonZeroes;

         

          System.out.println("Total instances: "+dataCollector.trainingData.numInstances()+" with "+dataCollector.attributesOfAnInstance.length+" attributes, non-zeroes are "+nonZeroes+" with average of "+((double)numberOfValues)/nonZeroes);

          Arrays.sort(freqData);

          int numOfcolumns=20;

          int stepWidth = dataCollector.attributesOfAnInstance.length/numOfcolumns;

         

          final RBoxPlot<Long> gr_HistogramOfAttributeValues = new RBoxPlot<Long>("Attributes","Number of values",new File("attributes_use"+selection+".pdf"));

          for(int i=0;i<numOfcolumns;++i)

          {

            int columnData=0;

            for(int j=i*stepWidth;j<(i+1)*stepWidth;++j)

              if (j < dataCollector.attributesOfAnInstance.length)

                columnData+=freqData[j];

           

            gr_HistogramOfAttributeValues.add(new Long(numOfcolumns-i),new Double(columnData>0?Math.log10(columnData):0));

          }

          //gr_HistogramOfAttributeValues.drawInteractive(gr);

          gr_HistogramOfAttributeValues.drawPdf(gr);

          /*

          // write arff

          FileWriter wekaInstances = null;

          String whereToWrite = "qualityLearner_"+selection+".arff";

          try

          {

            wekaInstances = new FileWriter(whereToWrite);

            // This chunk is almost verbatim from Weka's Instances.toString()

            wekaInstances.append(Instances.ARFF_RELATION).append(" ").append(Utils.quote(dataCollector.trainingData.relationName())).append("\n\n");

              for (int i = 0; i < dataCollector.trainingData.numAttributes(); i++) {

                wekaInstances.append(dataCollector.trainingData.attribute(i).toString()).append("\n");

              }

              wekaInstances.append("\n").append(Instances.ARFF_DATA).append("\n");

              for (int i = 0; i < dataCollector.trainingData.numInstances(); i++) {

                wekaInstances.append(dataCollector.trainingData.instance(i).toString());

                  if (i < dataCollector.trainingData.numInstances() - 1) {

                    wekaInstances.append('\n');

                  }

                }

          }

          catch(Exception ex)

          {

            Helper.throwUnchecked("failed to create a file with training data for "+whereToWrite, ex);

          }

          finally

          {

            if (wekaInstances != null)

              try {

                wekaInstances.close();

              } catch (IOException e) {

                // ignore this, we are not proceeding anyway due to an earlier exception so whether the file was actually written does not matter

              }

          }

          */

          // Run the evaluation

          final weka.classifiers.trees.REPTree repTree = new weka.classifiers.trees.REPTree();repTree.setMaxDepth(4);

          //repTree.setNoPruning(true);// since we only use the tree as a classifier (as a conservative extension of what is currently done) and do not actually look at it, elimination of pruning is not a problem.

          // As part of learning, we also prune some of the nodes where the ratio of correctly-classified pairs to those incorrectly classified is comparable.

          // The significant advantage of not pruning is that the result is no longer sensitive to the order of elements in the tree and hence does not depend on the order in which elements have been obtained by concurrent threads.

          //final weka.classifiers.lazy.IB1 ib1 = new weka.classifiers.lazy.IB1();

          //final weka.classifiers.trees.J48 classifier = new weka.classifiers.trees.J48();

          final Classifier classifier = repTree;

          classifier.buildClassifier(dataCollector.trainingData);

          System.out.println("Entries in the classifier: "+dataCollector.trainingData.numInstances());

          System.out.println(classifier);

          dataCollector=null;// throw all the training data away.

         

          {// serialise the classifier, this is the only way to store it.

            OutputStream os = new FileOutputStream(selection+".ser");

            ObjectOutputStream oo = new ObjectOutputStream(os);

                      oo.writeObject(classifier);

                      os.close();

          }

                   

          for(final boolean selectingRed:new boolean[]{false})

          for(final boolean classifierToBlockAllMergers:new boolean[]{true})

          //for(final boolean zeroScoringAsRed:(classifierToBlockAllMergers?new boolean[]{true,false}:new boolean[]{false}))// where we are not using classifier to rule out all mergers proposed by pair selection, it does not make sense to use two values configuring this classifier.

          for(final double threshold:new double[]{1})

          {

            final boolean zeroScoringAsRed = false;

            selection = "TRUNK;EVALUATION;"+"ifDepth="+ifDepth+";threshold="+threshold+// ";useUnique="+useUnique+";onlyPositives="+onlyPositives+

                ";selectingRed="+selectingRed+";classifierToBlockAllMergers="+classifierToBlockAllMergers+";zeroScoringAsRed="+zeroScoringAsRed+";traceQuantity="+traceQuantity+";lengthMultiplier="+lengthMultiplier+";trainingDataMultiplier="+trainingDataMultiplier+";";

            final int totalTaskNumber = traceQuantity;

            final RBoxPlot<Long> gr_PairQuality = new RBoxPlot<Long>("Correct v.s. wrong","%%",new File("percentage_score"+selection+".pdf"));

            final RBoxPlot<String> gr_QualityForNumberOfTraces = new RBoxPlot<String>("traces","%%",new File("quality_traces"+selection+".pdf"));

            SquareBagPlot gr_NewToOrig = new SquareBagPlot("orig score","score with learnt selection",new File("new_to_orig"+selection+".pdf"),0,1,true);

            final Map<Long,TrueFalseCounter> pairQualityCounter = new TreeMap<Long,TrueFalseCounter>();

            try

            {

              int numberOfTasks = 0;

              for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

                for(int sample=0;sample<samplesPerFSM;++sample)

                {

                  LearnerRunner learnerRunner = new LearnerRunner(dataCollector,states,sample,totalTaskNumber+numberOfTasks,traceQuantity, config, converter)

                  {

                    @Override

                    public LearnerThatCanClassifyPairs createLearner(LearnerEvaluationConfiguration evalCnf,LearnerGraph argReferenceGraph,@SuppressWarnings("unused") WekaDataCollector argDataCollector,  LearnerGraph argInitialPTA)

                    {

                      LearnerThatUsesWekaResults l = new LearnerThatUsesWekaResults(ifDepth,evalCnf,argReferenceGraph,classifier,argInitialPTA);

                      if (gr_PairQuality != null)

                        l.setPairQualityCounter(pairQualityCounter);

                     

                      l.setUseClassifierForRed(selectingRed);l.setUseClassifierToChooseNextRed(classifierToBlockAllMergers);

                      l.setBlacklistZeroScoringPairs(zeroScoringAsRed);

                      l.setThreshold(threshold);

                      return l;

                    }

                   

                  };

                  learnerRunner.setPickUniqueFromInitial(useUnique);learnerRunner.setEvaluateAlsoUsingReferenceLearner(true);

                  learnerRunner.setOnlyUsePositives(onlyPositives);learnerRunner.setIfdepth(ifDepth);learnerRunner.setLengthMultiplier(lengthMultiplier);

                  learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

                if (gr_NewToOrig != null)

                {

                  for(SampleData sample:result.samples)

                    gr_NewToOrig.add(sample.differenceForReferenceLearner.getValue(),sample.difference.getValue());

                }

               

                for(SampleData sample:result.samples)

                  if (sample.differenceForReferenceLearner.getValue() > 0)

                    gr_QualityForNumberOfTraces.add(traceQuantity+"",sample.difference.getValue()/sample.differenceForReferenceLearner.getValue());

              }

              if (gr_PairQuality != null)

              {

                synchronized(pairQualityCounter)

                {

      runners[i]=new ErlangRuntime();processNames[i]=""+currentNumber++;runners[i].startErlang(processNames[i], 0);