learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                runner.submit(learnerRunner);

                ++numberOfTasks;

              }

            for(int count=0;count < numberOfTasks;++count)

            {

              ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

              samples.addAll(result.samples);

            }

          }

          catch(Exception ex)

          {

            IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

            if (executorService != null) { executorService.shutdown();executorService = null; }

            throw e;

          }

          int nonZeroes = 0;

          long numberOfValues = 0;

          System.out.println("number of instances: "+dataCollector.trainingData.numInstances());

          int freqData[] = new int[dataCollector.attributesOfAnInstance.length];

          for(int i=0;i<dataCollector.trainingData.numInstances();++i)

            for(int attrNum=0;attrNum<dataCollector.attributesOfAnInstance.length;++attrNum)

            {

              assert dataCollector.attributesOfAnInstance[attrNum].index() == attrNum;

              if (dataCollector.trainingData.instance(i).stringValue(attrNum) != WekaDataCollector.ZERO)

              {

                ++freqData[attrNum];++numberOfValues;

              }

            }

          for(int attrNum=0;attrNum<dataCollector.attributesOfAnInstance.length;++attrNum)

            if (freqData[attrNum]>0)

              ++nonZeroes;

         

          System.out.println("Total instances: "+dataCollector.trainingData.numInstances()+" with "+dataCollector.attributesOfAnInstance.length+" attributes, non-zeroes are "+nonZeroes+" with average of "+((double)numberOfValues)/nonZeroes);

          Arrays.sort(freqData);

          int numOfcolumns=20;

          int stepWidth = dataCollector.attributesOfAnInstance.length/numOfcolumns;

         

          final RBoxPlot<Long> gr_HistogramOfAttributeValues = new RBoxPlot<Long>("Attributes","Number of values",new File("attributes_use"+selection+".pdf"));

          for(int i=0;i<numOfcolumns;++i)

          {

            int columnData=0;

            for(int j=i*stepWidth;j<(i+1)*stepWidth;++j)

              if (j < dataCollector.attributesOfAnInstance.length)

                columnData+=freqData[j];

           

            gr_HistogramOfAttributeValues.add(new Long(numOfcolumns-i),new Double(columnData>0?Math.log10(columnData):0));

          }

          //gr_HistogramOfAttributeValues.drawInteractive(gr);

          gr_HistogramOfAttributeValues.drawPdf(gr);

          /*

          // write arff

          FileWriter wekaInstances = null;

          String whereToWrite = "qualityLearner_"+selection+".arff";

          try

          {

            wekaInstances = new FileWriter(whereToWrite);

            // This chunk is almost verbatim from Weka's Instances.toString()

            wekaInstances.append(Instances.ARFF_RELATION).append(" ").append(Utils.quote(dataCollector.trainingData.relationName())).append("\n\n");

              for (int i = 0; i < dataCollector.trainingData.numAttributes(); i++) {

                wekaInstances.append(dataCollector.trainingData.attribute(i).toString()).append("\n");

              }

              wekaInstances.append("\n").append(Instances.ARFF_DATA).append("\n");

              for (int i = 0; i < dataCollector.trainingData.numInstances(); i++) {

                wekaInstances.append(dataCollector.trainingData.instance(i).toString());

                  if (i < dataCollector.trainingData.numInstances() - 1) {

                    wekaInstances.append('\n');

                  }

                }

          }

          catch(Exception ex)

          {

            Helper.throwUnchecked("failed to create a file with training data for "+whereToWrite, ex);

          }

          finally

          {

            if (wekaInstances != null)

              try {

                wekaInstances.close();

              } catch (IOException e) {

                // ignore this, we are not proceeding anyway due to an earlier exception so whether the file was actually written does not matter

              }

          }

          */

          // Run the evaluation

          final weka.classifiers.trees.REPTree repTree = new weka.classifiers.trees.REPTree();repTree.setMaxDepth(4);

          //repTree.setNoPruning(true);// since we only use the tree as a classifier (as a conservative extension of what is currently done) and do not actually look at it, elimination of pruning is not a problem.

          // As part of learning, we also prune some of the nodes where the ratio of correctly-classified pairs to those incorrectly classified is comparable.

          // The significant advantage of not pruning is that the result is no longer sensitive to the order of elements in the tree and hence does not depend on the order in which elements have been obtained by concurrent threads.

          //final weka.classifiers.lazy.IB1 ib1 = new weka.classifiers.lazy.IB1();

          //final weka.classifiers.trees.J48 classifier = new weka.classifiers.trees.J48();

          final Classifier classifier = repTree;

          classifier.buildClassifier(dataCollector.trainingData);

          System.out.println("Entries in the classifier: "+dataCollector.trainingData.numInstances());

          System.out.println(classifier);

          dataCollector=null;// throw all the training data away.

         

          {// serialise the classifier, this is the only way to store it.

            OutputStream os = new FileOutputStream(selection+".ser");

            ObjectOutputStream oo = new ObjectOutputStream(os);

                      oo.writeObject(classifier);

                      os.close();

          }

                   

          for(final boolean selectingRed:new boolean[]{false})

          for(final boolean classifierToBlockAllMergers:new boolean[]{true})

          //for(final boolean zeroScoringAsRed:(classifierToBlockAllMergers?new boolean[]{true,false}:new boolean[]{false}))// where we are not using classifier to rule out all mergers proposed by pair selection, it does not make sense to use two values configuring this classifier.

          for(final double threshold:new double[]{1})

          {

            final boolean zeroScoringAsRed = false;

            selection = "TRUNK;EVALUATION;"+"ifDepth="+ifDepth+";threshold="+threshold+// ";useUnique="+useUnique+";onlyPositives="+onlyPositives+

                ";selectingRed="+selectingRed+";classifierToBlockAllMergers="+classifierToBlockAllMergers+";zeroScoringAsRed="+zeroScoringAsRed+";traceQuantity="+traceQuantity+";lengthMultiplier="+lengthMultiplier+";trainingDataMultiplier="+trainingDataMultiplier+";";

            final int totalTaskNumber = traceQuantity;

            final RBoxPlot<Long> gr_PairQuality = new RBoxPlot<Long>("Correct v.s. wrong","%%",new File("percentage_score"+selection+".pdf"));

            final RBoxPlot<String> gr_QualityForNumberOfTraces = new RBoxPlot<String>("traces","%%",new File("quality_traces"+selection+".pdf"));

            SquareBagPlot gr_NewToOrig = new SquareBagPlot("orig score","score with learnt selection",new File("new_to_orig"+selection+".pdf"),0,1,true);

            final Map<Long,TrueFalseCounter> pairQualityCounter = new TreeMap<Long,TrueFalseCounter>();

            try

            {

              int numberOfTasks = 0;

              for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

                for(int sample=0;sample<samplesPerFSM;++sample)

                {

                  LearnerRunner learnerRunner = new LearnerRunner(dataCollector,states,sample,totalTaskNumber+numberOfTasks,traceQuantity, config, converter)

                  {

                    @Override

                    public LearnerThatCanClassifyPairs createLearner(LearnerEvaluationConfiguration evalCnf,LearnerGraph argReferenceGraph,@SuppressWarnings("unused") WekaDataCollector argDataCollector,  LearnerGraph argInitialPTA)

                    {

                      LearnerThatUsesWekaResults l = new LearnerThatUsesWekaResults(ifDepth,evalCnf,argReferenceGraph,classifier,argInitialPTA);

                      if (gr_PairQuality != null)

                        l.setPairQualityCounter(pairQualityCounter);

                     

                      l.setUseClassifierForRed(selectingRed);l.setUseClassifierToChooseNextRed(classifierToBlockAllMergers);

                      l.setBlacklistZeroScoringPairs(zeroScoringAsRed);

                      l.setThreshold(threshold);

                      return l;

                    }

                   

                  };

                  learnerRunner.setPickUniqueFromInitial(useUnique);learnerRunner.setEvaluateAlsoUsingReferenceLearner(true);

                  learnerRunner.setOnlyUsePositives(onlyPositives);learnerRunner.setIfdepth(ifDepth);learnerRunner.setLengthMultiplier(lengthMultiplier);

                  learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

                if (gr_NewToOrig != null)

                {

                  for(SampleData sample:result.samples)

                    gr_NewToOrig.add(sample.differenceForReferenceLearner.getValue(),sample.difference.getValue());

                }

               

                for(SampleData sample:result.samples)

                  if (sample.differenceForReferenceLearner.getValue() > 0)

                    gr_QualityForNumberOfTraces.add(traceQuantity+"",sample.difference.getValue()/sample.differenceForReferenceLearner.getValue());

              }

              if (gr_PairQuality != null)

              {

                synchronized(pairQualityCounter)

                {

          int alphabet = states/2;

         

          MachineGenerator mg = new MachineGenerator(states, 40, states/10);

          int mutationsPerStage = (states/2) / 2;

          for(int mutationStage = 0;mutationStage<mutationStages;mutationStage++)

          {

            for(int experiment=0;experiment<experimentsPerMutationCategory;experiment++)

            {

              ExperimentResult outcome = new ExperimentResult();

              final double perfectLowToHigh=0.7,perfectThreshold=0.5;

   

              config.setAttenuationK(0.5);

              config.setGdKeyPairThreshold(perfectThreshold);

              config.setGdLowToHighRatio(perfectLowToHigh);

              config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

   

              int mutations = mutationsPerStage * (mutationStage+1);

              LearnerGraphND origGraph = mg.nextMachine(alphabet, experiment,config,converter);

              GraphMutator<List<CmpVertex>,LearnerGraphNDCachedData> mutator = new GraphMutator<List<CmpVertex>,LearnerGraphNDCachedData>(origGraph,new Random(mutationStage*experimentsPerMutationCategory+experiment));

              mutator.mutate(mutations);

              LearnerGraphND origAfterRenaming = new LearnerGraphND(origGraph.config);

              Map<CmpVertex,CmpVertex> origToNew = copyStatesAndTransitions(origGraph,origAfterRenaming);

              LearnerGraphND mutated = (LearnerGraphND)mutator.getMutated();

              mutated.setName(origAfterRenaming.getName()+"_mutated");origAfterRenaming.setName(origAfterRenaming.getName()+"_orig");

              Set<Transition> appliedMutations = new HashSet<Transition>();

              for(Transition tr:mutator.getDiff())

              {

                CmpVertex renamedFrom = origToNew.get(tr.getFrom());if (renamedFrom == null) renamedFrom = tr.getFrom();

                CmpVertex renamedTo = origToNew.get(tr.getTo());if (renamedTo == null) renamedTo = tr.getTo();

                appliedMutations.add(new Transition(renamedFrom,renamedTo,tr.getLabel()));

              }

             

              linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

             

              LearnerGraph fromDet = null, toDet = null;

              try {

                fromDet = mergeAndDeterminize(origAfterRenaming);

                toDet = mergeAndDeterminize(mutated);

              } catch (IncompatibleStatesException e) {

                Helper.throwUnchecked("failed to build a deterministic graph from a nondet one", e);

              }

              languageDiff(fromDet,toDet,states, graphComplexity,outcome);

             

              // zero the times.

              outcome.setValue(LONG_V.DURATION_GD, 0);

              outcome.setValue(LONG_V.DURATION_RAND, 0);

              outcome.setValue(LONG_V.DURATION_W, 0);

             

              outcome.experimentValid = true;

              processor.process(outcome);

             

            }

          }

        }

      }

      finally

          int alphabet = states/2;

         

          MachineGenerator mg = new MachineGenerator(states, 40, states/10);

          int mutationsPerStage = (states/2) / 2;

          for(int mutationStage = 0;mutationStage<mutationStages;mutationStage++)

          {

            for(int experiment=0;experiment<experimentsPerMutationCategory;experiment++)

            {

              ExperimentResult outcome = new ExperimentResult();

              final double perfectLowToHigh=0.7,perfectThreshold=0.5;

   

              config.setAttenuationK(0.5);

              config.setGdKeyPairThreshold(perfectThreshold);

              config.setGdLowToHighRatio(perfectLowToHigh);

              config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

   

              int mutations = mutationsPerStage * (mutationStage+1);

              LearnerGraphND origGraph = mg.nextMachine(alphabet, experiment,config,converter);

              GraphMutator<List<CmpVertex>,LearnerGraphNDCachedData> mutator = new GraphMutator<List<CmpVertex>,LearnerGraphNDCachedData>(origGraph,new Random(mutationStage*experimentsPerMutationCategory+experiment));

              mutator.mutate(mutations);

              LearnerGraphND origAfterRenaming = new LearnerGraphND(origGraph.config);

              Map<CmpVertex,CmpVertex> origToNew = copyStatesAndTransitions(origGraph,origAfterRenaming);

              LearnerGraphND mutated = (LearnerGraphND)mutator.getMutated();

              mutated.setName(origAfterRenaming.getName()+"_mutated");origAfterRenaming.setName(origAfterRenaming.getName()+"_orig");

              Set<Transition> appliedMutations = new HashSet<Transition>();

              for(Transition tr:mutator.getDiff())

              {

                CmpVertex renamedFrom = origToNew.get(tr.getFrom());if (renamedFrom == null) renamedFrom = tr.getFrom();

                CmpVertex renamedTo = origToNew.get(tr.getTo());if (renamedTo == null) renamedTo = tr.getTo();

                appliedMutations.add(new Transition(renamedFrom,renamedTo,tr.getLabel()));

              }

             

              linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

             

              LearnerGraph fromDet = null, toDet = null;

              try {

                fromDet = mergeAndDeterminize(origAfterRenaming);

                toDet = mergeAndDeterminize(mutated);

              } catch (IncompatibleStatesException e) {

                Helper.throwUnchecked("failed to build a deterministic graph from a nondet one", e);

              }

              languageDiff(fromDet,toDet,states, graphComplexity,outcome);

             

              // zero the times.

              outcome.setValue(LONG_V.DURATION_GD, 0);

              outcome.setValue(LONG_V.DURATION_RAND, 0);

              outcome.setValue(LONG_V.DURATION_W, 0);

             

              outcome.experimentValid = true;

              processor.process(outcome);

             

            }

          }

        }

      }

      finally

            learnerRunner.setSelectionID(branch+"_states"+states+"_sample"+sample);

            runner.submit(learnerRunner);

            ++numberOfTasks;

          }

        for(int count=0;count < numberOfTasks;++count)

        {

          ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

          for(SampleData sample:result.samples)

            for(Entry<String,ScoresForGraph> score:sample.miscGraphs.entrySet())

              gr_StructuralForDifferentLearners.add(score.getKey(),score.getValue().differenceStructural.getValue());

       

          for(SampleData sample:result.samples)

            for(Entry<String,ScoresForGraph> score:sample.miscGraphs.entrySet())

              gr_BCRForDifferentLearners.add(score.getKey(),score.getValue().differenceBCR.getValue());

          gr_BCRForDifferentLearners.drawInteractive(gr);gr_StructuralForDifferentLearners.drawInteractive(gr);

        }

       

      }

      catch(Exception ex)

      {

        IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

        if (executorService != null) { executorService.shutdownNow();executorService = null; }

        throw e;

      }

    }

    if (gr_BCRForDifferentLearners != null) gr_BCRForDifferentLearners.drawPdf(gr);if (gr_StructuralForDifferentLearners != null) gr_StructuralForDifferentLearners.drawPdf(gr);

   

    for(final boolean useCentreVertex:new boolean[]{true,false})

    for(final boolean useDifferentScoringNearRoot:new boolean[]{true,false})

    for(final boolean mergeIdentifiedPathsAfterInference:new boolean[]{true,false})

    for(final boolean useClassifyToOrderPairs:new boolean[]{true,false})

     

    for(final int traceQuantity:new int[]{10})

    for(final double traceLengthMultiplier:new double[]{1})

      {

       

        final int traceQuantityToUse = traceQuantity;

       

        String selection = "c="+useCentreVertex+

            ";r="+useDifferentScoringNearRoot+

            ";m="+mergeIdentifiedPathsAfterInference+

            ";o="+useClassifyToOrderPairs+

            ";traceQuantity="+traceQuantity+";traceLengthMultiplier="+traceLengthMultiplier+";"+";alphabetMultiplier="+alphabetMultiplier+";";

        SquareBagPlot gr_StructuralDiff = new SquareBagPlot("Structural score, Sicco","Structural Score, EDSM-Markov learner",new File(branch+"_"+selection+"_trace_structuraldiff.pdf"),0,1,true);

        SquareBagPlot gr_BCR = new SquareBagPlot("BCR, Sicco","BCR, EDSM-Markov learner",new File(branch+"_"+selection+"_trace_bcr.pdf"),0.5,1,true);   

            try

            {

              int numberOfTasks = 0;

              for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

                for(int sample=0;sample<samplesPerFSM;++sample)

                {

                  LearnerRunner learnerRunner = new LearnerRunner(states,sample,numberOfTasks,traceQuantityToUse, config, converter);

                  learnerRunner.setOnlyUsePositives(onlyPositives);

                  learnerRunner.setAlphabetMultiplier(alphabetMultiplier);

                  learnerRunner.setTraceLengthMultiplier(traceLengthMultiplier);

                  learnerRunner.setChunkLen(chunkSize);

                  learnerRunner.setSelectionID(selection);

                  learnerRunner.setlearningParameters(useCentreVertex, useDifferentScoringNearRoot, mergeIdentifiedPathsAfterInference, useClassifyToOrderPairs);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              ProgressIndicator progress = new ProgressIndicator(new Date()+" evaluating "+numberOfTasks+" tasks for learning whole graphs", numberOfTasks);

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

                for(SampleData sample:result.samples)

                  gr_StructuralDiff.add(sample.referenceLearner.differenceStructural.getValue(),sample.actualLearner.differenceStructural.getValue());

             

                for(SampleData sample:result.samples)

                {

                  gr_BCR.add(sample.referenceLearner.differenceBCR.getValue(),sample.actualLearner.differenceBCR.getValue());

                }

                progress.next();

              }

              gr_StructuralDiff.drawInteractive(gr);

              gr_BCR.drawInteractive(gr);

            }

            catch(Exception ex)

            {

              IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

              if (executorService != null) { executorService.shutdownNow();executorService = null; }

              throw e;

            }

            if (gr_StructuralDiff != null) gr_StructuralDiff.drawPdf(gr);

            if (gr_BCR != null) gr_BCR.drawPdf(gr);

      }

*/

    for(final int preset: new int[]{0,4})//0,1,2})

    {

       

      final int traceQuantityToUse = traceQuantity;

      long comparisonsPerformed = 0;

      String selection = "preset="+preset+";quantity="+traceQuantity+";tracelen="+traceLengthMultiplierMax+";statesMax="+(minStateNumber+rangeOfStateNumbers-stateNumberIncrement)+";alphabetMult="+alphabetMultiplierMax+";";

      SquareBagPlot gr_StructuralDiff = new SquareBagPlot("Structural score, Sicco","Structural Score, EDSM-Markov learner",new File(branch+"_"+preset+"_"+(minStateNumber+rangeOfStateNumbers-stateNumberIncrement)+"_trace_structuraldiff.pdf"),0,1,true);

      SquareBagPlot gr_BCR = new SquareBagPlot("BCR, Sicco","BCR, EDSM-Markov learner",new File(branch+"_"+preset+"_"+(minStateNumber+rangeOfStateNumbers-stateNumberIncrement)+"_trace_bcr.pdf"),0.5,1,true);   

      try

      {

        int numberOfTasks = 0;

        for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

          for(int sample=0;sample<samplesPerFSM;++sample)

          {

            LearnerRunner learnerRunner = new LearnerRunner(states,sample,numberOfTasks,traceQuantityToUse, config, converter);

            learnerRunner.setOnlyUsePositives(onlyPositives);

            learnerRunner.setAlphabetMultiplier(alphabetMultiplierMax);

            learnerRunner.setTracesAlphabetMultiplier(alphabetMultiplierMax);

            learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);

            learnerRunner.setChunkLen(chunkSize);

            learnerRunner.setSelectionID(selection);

            learnerRunner.setPresetLearningParameters(preset);

            learnerRunner.setDisableInconsistenciesInMergers(false);

            runner.submit(learnerRunner);

            ++numberOfTasks;

          }

        for(int count=0;count < numberOfTasks;++count)

        {

          ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

          for(SampleData sample:result.samples)

            gr_StructuralDiff.add(sample.referenceLearner.differenceStructural.getValue(),sample.actualLearner.differenceStructural.getValue());

       

          for(SampleData sample:result.samples)

          {

            gr_BCR.add(sample.referenceLearner.differenceBCR.getValue(),sample.actualLearner.differenceBCR.getValue());

            comparisonsPerformed+=sample.comparisonsPerformed;

          }

         

          if (count % 10 == 0)

          {

            gr_StructuralDiff.drawInteractive(gr);

            gr_BCR.drawInteractive(gr);

          }

        }

        gr_StructuralDiff.drawInteractive(gr);

        gr_BCR.drawInteractive(gr);

        System.out.println("\nLOG of comparisons performed: "+Math.log10(comparisonsPerformed)+"\n");

      }

      catch(Exception ex)

      {

        IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

        if (executorService != null) { executorService.shutdownNow();executorService = null; }

        throw e;

      }

      if (gr_StructuralDiff != null) gr_StructuralDiff.drawPdf(gr);

      if (gr_BCR != null) gr_BCR.drawPdf(gr);

    }

    final int presetForBestResults = 0;

   

    final int traceQuantityToUse = traceQuantity;

    {

      SquareBagPlot gr_StructuralDiffWithoutInconsistencies = new SquareBagPlot("Structural score, Sicco","Structural Score, EDSM-Markov learner",new File(branch+"_noinconsistencies_trace_structuraldiff.pdf"),0,1,true);

      SquareBagPlot gr_BCRWithoutInconsistencies = new SquareBagPlot("BCR, Sicco","BCR, EDSM-Markov learner",new File(branch+"_noinconsistencies_trace_bcr.pdf"),0.5,1,true);   

      String selection = "noinconsistencies;quantity="+traceQuantity+";tracelen="+traceLengthMultiplierMax+";alphabetMult="+alphabetMultiplierMax+";";

      long comparisonsPerformed = 0;

      try

      {

        int numberOfTasks = 0;

        for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

          for(int sample=0;sample<samplesPerFSM;++sample)

          {

            LearnerRunner learnerRunner = new LearnerRunner(states,sample,numberOfTasks,traceQuantityToUse, config, converter);

            learnerRunner.setOnlyUsePositives(onlyPositives);

            learnerRunner.setAlphabetMultiplier(alphabetMultiplierMax);

            learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);

            learnerRunner.setChunkLen(chunkSize);

            learnerRunner.setSelectionID(selection);

            learnerRunner.setPresetLearningParameters(presetForBestResults);

            learnerRunner.setDisableInconsistenciesInMergers(true);

            runner.submit(learnerRunner);

            ++numberOfTasks;

          }

        for(int count=0;count < numberOfTasks;++count)

        {

          ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

          for(SampleData sample:result.samples)

            gr_StructuralDiffWithoutInconsistencies.add(sample.referenceLearner.differenceStructural.getValue(),sample.actualLearner.differenceStructural.getValue());

       

          for(SampleData sample:result.samples)

          {

            gr_BCRWithoutInconsistencies.add(sample.referenceLearner.differenceBCR.getValue(),sample.actualLearner.differenceBCR.getValue());

            comparisonsPerformed+=sample.comparisonsPerformed;

          }

         

          if (count % 10 == 0)

          {

            gr_StructuralDiffWithoutInconsistencies.drawInteractive(gr);

            gr_BCRWithoutInconsistencies.drawInteractive(gr);

          }

        }

        gr_StructuralDiffWithoutInconsistencies.drawInteractive(gr);

        gr_BCRWithoutInconsistencies.drawInteractive(gr);

        System.out.println("\nLOG of comparisons performed: "+Math.log10(comparisonsPerformed)+"\n");

      }

      catch(Exception ex)

      {

        IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

        if (executorService != null) { executorService.shutdownNow();executorService = null; }

        throw e;

      }

      if (gr_StructuralDiffWithoutInconsistencies != null) gr_StructuralDiffWithoutInconsistencies.drawPdf(gr);

      if (gr_BCRWithoutInconsistencies != null) gr_BCRWithoutInconsistencies.drawPdf(gr);

    }

    // Same experiment but with different number of sequences.

    final RBoxPlot<Integer> gr_BCRImprovementForDifferentNrOfTracesWithNegatives = new RBoxPlot<Integer>("nr of traces","improvement, BCR",new File(branch+"WithNegatives_BCR_vs_tracenumber.pdf"));

    final RBoxPlot<Integer> gr_BCRForDifferentNrOfTracesWithNegatives = new RBoxPlot<Integer>("nr of traces","BCR",new File(branch+"WithNegatives_BCR_absolute_vs_tracenumber.pdf"));

    final RBoxPlot<Integer> gr_StructuralImprovementForDifferentNrOfTracesWithNegatives = new RBoxPlot<Integer>("nr of traces","improvement, structural",new File(branch+"WithNegatives_structural_vs_tracenumber.pdf"));

    final RBoxPlot<Integer> gr_StructuralForDifferentNrOfTracesWithNegatives = new RBoxPlot<Integer>("nr of traces","structural",new File(branch+"WithNegatives_structural_absolute_vs_tracenumber.pdf"));

     

    for(final int traceNum:new int[]{2,4,6,8,10})

    {

      String selection = "number_of_traces="+traceNum;

      try

      {

        int numberOfTasks = 0;

        for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

          for(int sample=0;sample<samplesPerFSM;++sample)

          {

            LearnerRunner learnerRunner = new LearnerRunner(states,sample,numberOfTasks,traceNum, config, converter);

            learnerRunner.setOnlyUsePositives(false);

            learnerRunner.setAlphabetMultiplier(alphabetMultiplierMax);

            learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);

            learnerRunner.setChunkLen(chunkSize);

            learnerRunner.setSelectionID(selection);

            learnerRunner.setPresetLearningParameters(presetForBestResults);

            runner.submit(learnerRunner);

            ++numberOfTasks;

          }

        for(int count=0;count < numberOfTasks;++count)

        {

          ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

         

          for(SampleData sample:result.samples)

          {

            if (sample.referenceLearner.differenceBCR.getValue() > 0)

            {

              // we'll generate both positives and negatives; in the considered experiments, only positives are used hence half the number of sequences are actually being learnt from.

              gr_BCRImprovementForDifferentNrOfTracesWithNegatives.add(traceNum,sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

              gr_BCRForDifferentNrOfTracesWithNegatives.add(traceNum,sample.actualLearner.differenceBCR.getValue());

            }

            if (sample.referenceLearner.differenceStructural.getValue() > 0)

            {

              gr_StructuralImprovementForDifferentNrOfTracesWithNegatives.add(traceNum,sample.actualLearner.differenceStructural.getValue()/sample.referenceLearner.differenceStructural.getValue());

              gr_StructuralForDifferentNrOfTracesWithNegatives.add(traceNum,sample.actualLearner.differenceStructural.getValue());

            }

          }

        }

        gr_BCRForDifferentNrOfTracesWithNegatives.drawInteractive(gr);gr_StructuralForDifferentNrOfTracesWithNegatives.drawInteractive(gr);

      }

      catch(Exception ex)

      {

        IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

        if (executorService != null) { executorService.shutdownNow();executorService = null; }

        throw e;

      }

    }

    if (gr_BCRImprovementForDifferentNrOfTracesWithNegatives != null) gr_BCRImprovementForDifferentNrOfTracesWithNegatives.drawPdf(gr);

    if (gr_BCRForDifferentNrOfTracesWithNegatives != null) gr_BCRForDifferentNrOfTracesWithNegatives.drawPdf(gr);

    if (gr_StructuralImprovementForDifferentNrOfTracesWithNegatives != null) gr_StructuralImprovementForDifferentNrOfTracesWithNegatives.drawPdf(gr);

    if (gr_StructuralForDifferentNrOfTracesWithNegatives != null) gr_StructuralForDifferentNrOfTracesWithNegatives.drawPdf(gr);

    // Same experiment but with different number of sequences.

    final RBoxPlot<Integer> gr_BCRImprovementForDifferentNrOfTraces = new RBoxPlot<Integer>("nr of traces","improvement, BCR",new File(branch+"BCR_vs_tracenumber.pdf"));

    final RBoxPlot<Integer> gr_BCRForDifferentNrOfTraces = new RBoxPlot<Integer>("nr of traces","BCR",new File(branch+"BCR_absolute_vs_tracenumber.pdf"));

    final RBoxPlot<Integer> gr_StructuralImprovementForDifferentNrOfTraces = new RBoxPlot<Integer>("nr of traces","improvement, structural",new File(branch+"structural_vs_tracenumber.pdf"));

    final RBoxPlot<Integer> gr_StructuralForDifferentNrOfTraces = new RBoxPlot<Integer>("nr of traces","structural",new File(branch+"structural_absolute_vs_tracenumber.pdf"));

     

    for(final int traceNum:new int[]{2,4,6,8,10})

    {

      String selection = "number_of_traces="+traceNum;

      try

      {

        int numberOfTasks = 0;

        for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

          for(int sample=0;sample<samplesPerFSM;++sample)

          {

            LearnerRunner learnerRunner = new LearnerRunner(states,sample,numberOfTasks,traceNum, config, converter);

            learnerRunner.setOnlyUsePositives(onlyPositives);

            learnerRunner.setAlphabetMultiplier(alphabetMultiplierMax);

            learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);

            learnerRunner.setChunkLen(chunkSize);

            learnerRunner.setSelectionID(selection);

            learnerRunner.setPresetLearningParameters(presetForBestResults);

            runner.submit(learnerRunner);

            ++numberOfTasks;

          }

        for(int count=0;count < numberOfTasks;++count)

        {

          ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

         

          for(SampleData sample:result.samples)

          {

            if (sample.referenceLearner.differenceBCR.getValue() > 0)

            {

              // we'll generate both positives and negatives; in the considered experiments, only positives are used hence half the number of sequences are actually being learnt from.

              gr_BCRImprovementForDifferentNrOfTraces.add(traceNum/2,sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

              gr_BCRForDifferentNrOfTraces.add(traceNum/2,sample.actualLearner.differenceBCR.getValue());

            }

            if (sample.referenceLearner.differenceStructural.getValue() > 0)

            {

              gr_StructuralImprovementForDifferentNrOfTraces.add(traceNum/2,sample.actualLearner.differenceStructural.getValue()/sample.referenceLearner.differenceStructural.getValue());

              gr_StructuralForDifferentNrOfTraces.add(traceNum/2,sample.actualLearner.differenceStructural.getValue());

            }

          }

        }

        gr_BCRForDifferentNrOfTraces.drawInteractive(gr);gr_StructuralForDifferentNrOfTraces.drawInteractive(gr);

      }

      catch(Exception ex)

      {

        IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

        if (executorService != null) { executorService.shutdownNow();executorService = null; }

        throw e;

      }

    }

    if (gr_BCRImprovementForDifferentNrOfTraces != null) gr_BCRImprovementForDifferentNrOfTraces.drawPdf(gr);

    if (gr_BCRForDifferentNrOfTraces != null) gr_BCRForDifferentNrOfTraces.drawPdf(gr);

    if (gr_StructuralImprovementForDifferentNrOfTraces != null) gr_StructuralImprovementForDifferentNrOfTraces.drawPdf(gr);

    if (gr_StructuralForDifferentNrOfTraces != null) gr_StructuralForDifferentNrOfTraces.drawPdf(gr);

    // Same experiment but with different trace length but the same number of sequences

    final RBoxPlot<Double> gr_BCRImprovementForDifferentLengthOfTraces = new RBoxPlot<Double>("trace length multiplier","improvement, BCR",new File(branch+"BCR_vs_tracelength.pdf"));

    final RBoxPlot<Double> gr_BCRForDifferentLengthOfTraces = new RBoxPlot<Double>("trace length multiplier","BCR",new File(branch+"BCR_absolute_vs_tracelength.pdf"));

    final RBoxPlot<Double> gr_StructuralImprovementForDifferentLengthOfTraces = new RBoxPlot<Double>("trace length multiplier","improvement, structural",new File(branch+"structural_vs_tracelength.pdf"));

    final RBoxPlot<Double> gr_StructuralForDifferentLengthOfTraces = new RBoxPlot<Double>("trace length multiplier","structural",new File(branch+"structural_absolute_vs_tracelength.pdf"));

    final RBoxPlot<Double> gr_TransitionCoverageForDifferentLengthOfTraces = new RBoxPlot<Double>("trace length multiplier","transition coverage",new File(branch+"transitionCoverage_vs_tracelength.pdf"));

    for(final int traceNum:new int[]{10})

      for(double traceLengthMultiplierToUse=0.125;traceLengthMultiplierToUse<4;traceLengthMultiplierToUse*=2.)

      {

        String selection="traceLengthMultiplier="+traceLengthMultiplierToUse;

        try

        {

          int numberOfTasks = 0;

          for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

            for(int sample=0;sample<samplesPerFSM;++sample)

            {

              LearnerRunner learnerRunner = new LearnerRunner(states,sample,numberOfTasks,traceNum, config, converter);

              learnerRunner.setOnlyUsePositives(onlyPositives);

              learnerRunner.setAlphabetMultiplier(alphabetMultiplierMax);

              learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierToUse);

              learnerRunner.setChunkLen(chunkSize);

              learnerRunner.setSelectionID(selection);

              learnerRunner.setPresetLearningParameters(presetForBestResults);

              runner.submit(learnerRunner);

              ++numberOfTasks;

            }

          for(int count=0;count < numberOfTasks;++count)

          {

            ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

           

            for(SampleData sample:result.samples)

            {

              if (sample.referenceLearner.differenceBCR.getValue() > 0)

              {

                gr_BCRImprovementForDifferentLengthOfTraces.add(traceLengthMultiplierToUse,sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

                gr_BCRForDifferentLengthOfTraces.add(traceLengthMultiplierToUse,sample.actualLearner.differenceBCR.getValue());

              }

              if (sample.referenceLearner.differenceStructural.getValue() > 0)

              {

                gr_StructuralImprovementForDifferentLengthOfTraces.add(traceLengthMultiplierToUse,sample.actualLearner.differenceStructural.getValue()/sample.referenceLearner.differenceStructural.getValue());

                gr_StructuralForDifferentLengthOfTraces.add(traceLengthMultiplierToUse,sample.actualLearner.differenceStructural.getValue());

              }

              gr_TransitionCoverageForDifferentLengthOfTraces.add(traceLengthMultiplierToUse,(double)sample.transitionsSampled);

            }

          }

         

          gr_BCRForDifferentLengthOfTraces.drawInteractive(gr);gr_StructuralForDifferentLengthOfTraces.drawInteractive(gr);gr_TransitionCoverageForDifferentLengthOfTraces.drawInteractive(gr);

        }

        catch(Exception ex)

        {

          IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

          if (executorService != null) { executorService.shutdownNow();executorService = null; }

          throw e;

        }

      }

    if (gr_BCRImprovementForDifferentLengthOfTraces != null) gr_BCRImprovementForDifferentLengthOfTraces.drawPdf(gr);

    if (gr_BCRForDifferentLengthOfTraces != null) gr_BCRForDifferentLengthOfTraces.drawPdf(gr);

    if (gr_StructuralImprovementForDifferentLengthOfTraces != null) gr_StructuralImprovementForDifferentLengthOfTraces.drawPdf(gr);

    if (gr_StructuralForDifferentLengthOfTraces != null) gr_StructuralForDifferentLengthOfTraces.drawPdf(gr);

    if (gr_TransitionCoverageForDifferentLengthOfTraces != null) gr_TransitionCoverageForDifferentLengthOfTraces.drawPdf(gr);

    final RBoxPlot<Integer> gr_BCRImprovementForDifferentPrefixlen = new RBoxPlot<Integer>("length of prefix","improvement, BCR",new File(branch+"BCR_vs_prefixLength.pdf"));

    final RBoxPlot<Integer> gr_BCRForDifferentPrefixlen = new RBoxPlot<Integer>("length of prefix","BCR",new File(branch+"BCR_absolute_vs_prefixLength.pdf"));

    final RBoxPlot<Integer> gr_StructuralImprovementForDifferentPrefixlen = new RBoxPlot<Integer>("length of prefix","improvement, structural",new File(branch+"structural_vs_prefixLength.pdf"));

    final RBoxPlot<Integer> gr_StructuralForDifferentPrefixlen = new RBoxPlot<Integer>("length of prefix","structural",new File(branch+"structural_absolute_vs_prefixLength.pdf"));

    final RBoxPlot<String> gr_MarkovAccuracyForDifferentPrefixlen = new RBoxPlot<String>("length of prefix","Markov accuracy",new File(branch+"markov_accuracy_vs_prefixLength.pdf"));

    for(int prefixLength=1;prefixLength<3;++prefixLength)

    {

     

      String selection="prefix Length ="+prefixLength;

      try

      {

        int numberOfTasks = 0;

        for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

          for(int sample=0;sample<samplesPerFSM;++sample)

          {

            LearnerRunner learnerRunner = new LearnerRunner(states,sample,numberOfTasks,traceQuantity, config, converter);

            learnerRunner.setOnlyUsePositives(onlyPositives);

            learnerRunner.setAlphabetMultiplier(alphabetMultiplierMax);

            learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);

            learnerRunner.setChunkLen(prefixLength+1);

            learnerRunner.setSelectionID(selection);

            learnerRunner.setPresetLearningParameters(presetForBestResults);

            runner.submit(learnerRunner);

            ++numberOfTasks;

          }

        for(int count=0;count < numberOfTasks;++count)

        {

          ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

         

          for(SampleData sample:result.samples)

          {

            if (sample.referenceLearner.differenceBCR.getValue() > 0)

            {

              gr_BCRImprovementForDifferentPrefixlen.add(prefixLength,sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

              gr_BCRForDifferentPrefixlen.add(prefixLength,sample.actualLearner.differenceBCR.getValue());

            }

            if (sample.referenceLearner.differenceStructural.getValue() > 0)

            {

              gr_StructuralImprovementForDifferentPrefixlen.add(prefixLength,sample.actualLearner.differenceStructural.getValue()/sample.referenceLearner.differenceStructural.getValue());

              gr_StructuralForDifferentPrefixlen.add(prefixLength,sample.actualLearner.differenceStructural.getValue());

            }

            gr_MarkovAccuracyForDifferentPrefixlen.add(""+prefixLength+",P",(double)sample.markovPrecision,"green");

            gr_MarkovAccuracyForDifferentPrefixlen.add(""+prefixLength+",R",(double)sample.markovRecall,"blue");

          }

        }

        gr_BCRForDifferentPrefixlen.drawInteractive(gr);gr_StructuralForDifferentPrefixlen.drawInteractive(gr);gr_MarkovAccuracyForDifferentPrefixlen.drawInteractive(gr);

      }

      catch(Exception ex)

      {

        IllegalArgumentException e = new IllegalArgumentException("failed to compute, the problem is: "+ex);e.initCause(ex);

        if (executorService != null) { executorService.shutdownNow();executorService = null; }

        throw e;

      }

       

    }

    if (gr_BCRImprovementForDifferentPrefixlen != null) gr_BCRImprovementForDifferentPrefixlen.drawPdf(gr);

    if (gr_BCRForDifferentPrefixlen != null) gr_BCRForDifferentPrefixlen.drawPdf(gr);

    if (gr_StructuralImprovementForDifferentPrefixlen != null) gr_StructuralImprovementForDifferentPrefixlen.drawPdf(gr);

    if (gr_StructuralForDifferentPrefixlen != null) gr_StructuralForDifferentPrefixlen.drawPdf(gr);

    if (gr_MarkovAccuracyForDifferentPrefixlen != null) gr_MarkovAccuracyForDifferentPrefixlen.drawPdf(gr);

    final RBoxPlot<String> gr_BCRImprovementForDifferentAlphabetSize = new RBoxPlot<String>("alphabet multiplier","improvement, BCR",new File(branch+"BCR_vs_alphabet.pdf"));

    final RBoxPlot<String> gr_BCRForDifferentAlphabetSize = new RBoxPlot<String>("alphabet multiplier","BCR",new File(branch+"BCR_absolute_vs_alphabet.pdf"));

    final RBoxPlot<String> gr_StructuralImprovementForDifferentAlphabetSize = new RBoxPlot<String>("alphabet multiplier","improvement, structural",new File(branch+"structural_vs_alphabet.pdf"));

    final RBoxPlot<String> gr_StructuralForDifferentAlphabetSize = new RBoxPlot<String>("alphabet multiplier","structural",new File(branch+"structural_absolute_vs_alphabet.pdf"));

    final RBoxPlot<String> gr_MarkovAccuracyForDifferentAlphabetSize = new RBoxPlot<String>("alphabet multiplier","Markov accuracy",new File(branch+"markov_accuracy_vs_alphabet.pdf"));

    // Same experiment but with different alphabet size

    for(final double alphabetMultiplierActual:new double[]{alphabetMultiplierMax/4,alphabetMultiplierMax/2,alphabetMultiplierMax,alphabetMultiplierMax*2,alphabetMultiplierMax*4})

    {

      String selection="alphabet_size="+alphabetMultiplierActual;

      try

      {

        int numberOfTasks = 0;

        for(int states=minStateNumber;states < minStateNumber+rangeOfStateNumbers;states+=stateNumberIncrement)

          for(int sample=0;sample<samplesPerFSM;++sample)

          {

            LearnerRunner learnerRunner = new LearnerRunner(states,sample,numberOfTasks,traceQuantity, config, converter);

            learnerRunner.setOnlyUsePositives(onlyPositives);

            learnerRunner.setTracesAlphabetMultiplier(alphabetMultiplierMax);

            learnerRunner.setAlphabetMultiplier(alphabetMultiplierActual);

            learnerRunner.setTraceLengthMultiplier(traceLengthMultiplierMax);learnerRunner.setChunkLen(chunkSize);

            learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

            learnerRunner.setPresetLearningParameters(presetForBestResults);

            runner.submit(learnerRunner);

            ++numberOfTasks;

          }

        for(int count=0;count < numberOfTasks;++count)

        {

          ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

          for(SampleData sample:result.samples)

          {

            if (sample.referenceLearner.differenceBCR.getValue() > 0)

            {

              gr_BCRImprovementForDifferentAlphabetSize.add(String.format("%.2f",alphabetMultiplierActual),sample.actualLearner.differenceBCR.getValue()/sample.referenceLearner.differenceBCR.getValue());

              gr_BCRForDifferentAlphabetSize.add(String.format("%.2f",alphabetMultiplierActual),sample.actualLearner.differenceBCR.getValue());

            }

            if (sample.referenceLearner.differenceStructural.getValue() > 0)

            {

              gr_StructuralImprovementForDifferentAlphabetSize.add(String.format("%.2f",alphabetMultiplierActual),sample.actualLearner.differenceStructural.getValue()/sample.referenceLearner.differenceStructural.getValue());

              gr_StructuralForDifferentAlphabetSize.add(String.format("%.2f",alphabetMultiplierActual),sample.actualLearner.differenceStructural.getValue());

            }

            gr_MarkovAccuracyForDifferentAlphabetSize.add(String.format("%.2f,P",alphabetMultiplierActual),(double)sample.markovPrecision,"green");

            gr_MarkovAccuracyForDifferentAlphabetSize.add(String.format("%.2f,R",alphabetMultiplierActual),(double)sample.markovRecall,"blue");

          }

        }

        gr_BCRImprovementForDifferentAlphabetSize.drawInteractive(gr);gr_BCRForDifferentAlphabetSize.drawInteractive(gr);gr_MarkovAccuracyForDifferentAlphabetSize.drawInteractive(gr);

        gr_StructuralImprovementForDifferentAlphabetSize.drawInteractive(gr);gr_StructuralForDifferentAlphabetSize.drawInteractive(gr);

      }

      catch(Exception ex)

  private ErlangModule(Configuration config) throws IOException {

    final File f = config.getErlangSourceFile();

    name = ErlangRunner.getName(f, ERL.MOD);

    sourceFolder = f.getParentFile();

    // launch Erlang by calling a test method.

    ErlangRunner.getRunner().call(

        new OtpErlangObject[] { new OtpErlangAtom("echo2Tuple"),

            new OtpErlangAtom("aaa") }, "echo2Tuple");

    // Compile and typecheck the module...

    ErlangRunner.compileErl(f, ErlangRunner.getRunner());

    sigs = new TreeMap<String, FuncSignature>();ignoredFunctions = new TreeSet<String>();ignoredBehaviours = new TreeSet<String>();

    File pltFile = new File(ErlangRunner.getName(f, ERL.PLT));

    // Almost the same arguments for dialyzer and typer, the first argument

    // determines which of the two to run.

    OtpErlangObject otpArgs[] = new OtpErlangObject[] {

        null, // either Dialyzer or typer

        new OtpErlangList(new OtpErlangObject[] { new OtpErlangString(

            ErlangRunner.getName(f, ERL.BEAM)) }),

        new OtpErlangString(ErlangRunner.getName(f, ERL.PLT)),

        new OtpErlangList(new OtpErlangObject[] { new OtpErlangString(

            ErlangRunner.getName(f, ERL.ERL)) }),

        new OtpErlangAtom("types") };

    if (!pltFile.canRead() || f.lastModified() > pltFile.lastModified()) {// rebuild the PLT file since the source was modified or the plt file does not exist

      pltFile.delete();

      otpArgs[0] = new OtpErlangAtom("dialyzer");

      ErlangRunner.getRunner().call(otpArgs, "Could not run dialyzer");

    }

    // Typer always has to be run

    otpArgs[0] = new OtpErlangAtom("typer");

    OtpErlangTuple response = null;

    try

    {

      response = ErlangRunner.getRunner().call(otpArgs,"Could not run typer");

     

    }

    catch(ErlangThrownException ex)

    {

      pltFile.delete();

      otpArgs[0] = new OtpErlangAtom("dialyzer");

      ErlangRunner.getRunner().call(otpArgs, "Could not run dialyzer");

      otpArgs[0] = new OtpErlangAtom("typer");

      response = ErlangRunner.getRunner().call(otpArgs,"Could not run typer for the second time");     

    }

    OtpErlangList analysisResults = (OtpErlangList) response.elementAt(1);

    Assert.assertEquals(1, analysisResults.arity());

    OtpErlangTuple fileDetails = (OtpErlangTuple) analysisResults.elementAt(0);

    OtpErlangList typeInformation = (OtpErlangList) fileDetails.elementAt(3);

    for (int i = 0; i < typeInformation.arity(); ++i) {

      //System.out.print("\n" + typeInformation.elementAt(i).toString());

      OtpErlangTuple functionDescr = (OtpErlangTuple) typeInformation.elementAt(i);

      if (functionDescr.arity() > 3)

      {

        FuncSignature s = new FuncSignature(config,typeInformation.elementAt(i), null);

        sigs.put(s.getQualifiedName(), s);

      }

      else

      {// if not a function signature, it is an error message. The first two elements are function name and arity, we add this module name.

        String fullName = FuncSignature.qualifiedNameFromFunction(getName(),

            ((OtpErlangAtom)functionDescr.elementAt(0)).atomValue(),

            ((OtpErlangLong)functionDescr.elementAt(1)).longValue());

        ignoredFunctions.add( fullName );

        System.out.println("Ignoring: "+fullName+", "+functionDescr.elementAt(2));

      }

    }

   

    behaviour = OTPBehaviour.obtainDeclaredBehaviour(f, this,ignoredBehaviours);

  }

                  learnerRunner.setSelectionID(selection+"_states"+states+"_sample"+sample);

                  runner.submit(learnerRunner);

                  ++numberOfTasks;

                }

              for(int count=0;count < numberOfTasks;++count)

              {

                ThreadResult result = runner.take().get();// this will throw an exception if any of the tasks failed.

                if (gr_NewToOrig != null)

                {

                  for(SampleData sample:result.samples)

                    gr_NewToOrig.add(sample.referenceLearner.getValue(),sample.actualLearner.getValue());

                }

               

                for(SampleData sample:result.samples)

                  if (sample.referenceLearner.getValue() > 0)

                    gr_QualityForNumberOfTraces.add(traceQuantity+"",sample.actualLearner.getValue()/sample.referenceLearner.getValue());

              }

              if (gr_PairQuality != null)

              {

                synchronized(pairQualityCounter)

                {

     

      MachineGenerator mg = new MachineGenerator(states, 40, states/10);

      int mutationsPerStage = (states/2) / 2;

      //System.out.print("\n"+states+": ");

      for(int mutationStage = 0;mutationStage<mutationStages;mutationStage++)

      {

        for(int experiment=0;experiment<experimentsPerMutationCategory;experiment++)

        {

          ExperimentResult outcome = new ExperimentResult();

          while(!outcome.experimentValid)

          {

            int mutations = mutationsPerStage * (mutationStage+1);

            LearnerGraphND origGraph = mg.nextMachine(alphabet, experiment,config,converter);

            GraphMutator<List<CmpVertex>,LearnerGraphNDCachedData> mutator = new GraphMutator<List<CmpVertex>,LearnerGraphNDCachedData>(origGraph,r);

            mutator.mutate(mutations);

            LearnerGraphND origAfterRenaming = new LearnerGraphND(origGraph.config);

            Map<CmpVertex,CmpVertex> origToNew = copyStatesAndTransitions(origGraph,origAfterRenaming);

            LearnerGraphND mutated = (LearnerGraphND)mutator.getMutated();

            Set<Transition> appliedMutations = new HashSet<Transition>();

            for(Transition tr:mutator.getDiff())

            {

              CmpVertex renamedFrom = origToNew.get(tr.getFrom());if (renamedFrom == null) renamedFrom = tr.getFrom();

              CmpVertex renamedTo = origToNew.get(tr.getTo());if (renamedTo == null) renamedTo = tr.getTo();

              appliedMutations.add(new Transition(renamedFrom,renamedTo,tr.getLabel()));

            }

           

            final double perfectLowToHigh=0.7,perfectThreshold=0.5;

           

            // These experiments are only run for the maximal number of states

            if (graphComplexity == graphComplexityMax-1)

            {

              for(double k:new double[]{0,.2,.4,.6,.8,.95})

              {

                config.setAttenuationK(k);

                config.setGdKeyPairThreshold(0.5);

                config.setGdLowToHighRatio(perfectLowToHigh);

                config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

                linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

               

                gr_Diff_k.add(k, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

               

              }

              for(double threshold:new double[]{0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95})

              {

                config.setGdKeyPairThreshold(threshold);

                config.setGdLowToHighRatio(perfectLowToHigh);

                config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

                linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

               

                gr_Diff_threshold.add(threshold, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

              }

              for(double lowtohigh:new double[]{0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95})

              {

                config.setGdKeyPairThreshold(perfectThreshold);

                config.setGdLowToHighRatio(lowtohigh);

                config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

                linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

               

                gr_Diff_lowtohigh.add(lowtohigh, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

              }

             

             

              for(int i=0;i<pairThreshold.length;++i)

                for(double ratio:lowToHigh)

                {

                  config.setGdKeyPairThreshold(pairThreshold[i]);

                  config.setGdLowToHighRatio(ratio);

                  config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

                  linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

                 

                  gr_Diff_thresholdslowhigh.add(new Pair<Double,Double>(ratio,pairThreshold[i]), outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

                }

            }

            config.setAttenuationK(0.5);

            config.setGdKeyPairThreshold(perfectThreshold);

            config.setGdLowToHighRatio(perfectLowToHigh);

            config.setGdPropagateDet(false);// this is to ensure that if we removed a transition 0 from to a state and then added one from that state to a different one, det-propagation will not force the two very different states into a key-pair relation.

            linearDiff(origAfterRenaming,mutated, appliedMutations,outcome);

           

            if (!skip)

            {

              LearnerGraph fromDet = null, toDet = null;

              try {

                fromDet = mergeAndDeterminize(origAfterRenaming);

                toDet = mergeAndDeterminize(mutated);

              } catch (IncompatibleStatesException e) {

                Helper.throwUnchecked("failed to build a deterministic graph from a nondet one", e);

              }

              languageDiff(fromDet,toDet,states, graphComplexity,outcome);

            }

            outcome.experimentValid = true;

            gr_Diff_States.add(states, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

            gr_W_States.add(states,outcome.getValue(DOUBLE_V.ACCURACY_W));

            Pair<Integer,Integer> mutations_states = new Pair<Integer,Integer>(mutationStage+1,states);

            Pair<Integer,Integer> states_mutations = new Pair<Integer,Integer>(states,mutationStage+1);

            gr_DiffGD_StatesLevel.add(mutations_states, outcome.getValue(DOUBLE_V.OBTAINED_TO_EXPECTED));

      {

        int states=initStates+graphComplexity*20;

        int alphabet = states/2;

        MachineGenerator mg = new MachineGenerator(states, 40, states/10);

       

        for(int experiment=0;experiment<experimentsPerCategory;experiment++)

        {

          LearnerGraphND generatedFSM = mg.nextMachine(alphabet, experiment,config, converter);

          if (generatedFSM.getInit() != generatedFSM.transitionMatrix.entrySet().iterator().next().getKey())

            throw new RuntimeException("first state is not the initial state");

         

          //Visualiser.updateFrame(origGraph, null);

          randomFSM = new FileWriter(GlobalConfiguration.getConfiguration().getProperty(G_PROPERTIES.RESOURCES)+File.separator+"randomFSM/fsm_"+states+"_"+experiment+".x_");

          for(Entry<CmpVertex,Map<Label,List<CmpVertex>>> entry:generatedFSM.transitionMatrix.entrySet())

            for(Entry<Label,List<CmpVertex>> targets:entry.getValue().entrySet())

              for(CmpVertex targetState:generatedFSM.getTargets(targets.getValue()))

              {

                randomFSM.write(entry.getKey().getStringId());randomFSM.write(' ');

                randomFSM.write(targetState.getStringId());randomFSM.write(' ');

                randomFSM.write(targets.getKey().toString());randomFSM.write('\n');

              }

          randomFSM.close();randomFSM = null;

        }

      }

    }

    catch(Exception ex)

    {