* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*   http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing,

* software distributed under the License is distributed on an

* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

* KIND, either express or implied.  See the License for the

* specific language governing permissions and limitations

* under the License.

*/

//import org.chboston.cnlp.ctakes.relationextractor.eval.RelationExtractorEvaluation;

//import org.chboston.cnlp.ctakes.relationextractor.ae.ModifierExtractorAnnotator;

 

  private static final String YTEX_NEGATION_DESCRIPTOR = "ytex.uima.NegexAnnotator";

  enum Corpus {SHARP_SEED, SHARP_STRATIFIED, MIPACQ, I2B2, NEGEX}

  public static class Options {

    @Option(

        name = "--train-dir",

        usage = "specify the directory containing the XMI training files (for example, /NLP/Corpus/Relations/mipacq/xmi/train)",

        required = false)

    public String trainDirectory;

   

    @Option(

        name = "--test-dir",

        usage = "specify the directory containing the XMI testing files (for example, /NLP/Corpus/Relations/mipacq/xmi/test)",

        required = false)

    public File testDirectory;

   

    @Option(

            name = "--dev-dir",

            usage = "if running --preprocess, store the XMI development files here",

            required = false)

        public File devDirectory;

    @Option(

        name = "--models-dir",

        usage = "specify the directory where the models will be placed",

        required = false)

    public File modelsDirectory;

   

    @Option(

            name = "--evaluation-output-dir",

            usage = "specify the directory where the evaluation output xmi files will go",

            required = false)

    public File evaluationOutputDirectory;

       

    @Option(

            name = "--ignore-polarity",

            usage = "specify whether polarity processing should be ignored (true or false). default: false",

            required = false)

    public boolean ignorePolarity = false; // note that this is reversed from the "ignore" statement

       

    @Option(

            name = "--ignore-conditional",

            usage = "specify whether conditional processing should be ignored (true or false). default: false",

            required = false)

    public boolean ignoreConditional = false;

       

    @Option(

            name = "--ignore-uncertainty",

            usage = "specify whether uncertainty processing should be ignored (true or false). default: false",

            required = false)

    public boolean ignoreUncertainty = false;

       

    @Option(

            name = "--ignore-subject",

            usage = "specify whether subject processing should be ignored (true or false). default: false",

            required = false,

            handler=BooleanOptionHandler.class)

    public boolean ignoreSubject = false;

       

    @Option(

            name = "--ignore-generic",

            usage = "specify whether generic processing should be ignored (true or false). default: false",

            required = false)

    public boolean ignoreGeneric = false;

       

    // srh adding 2/20/13

    @Option(

            name = "--ignore-history",

            usage = "specify whether 'history of' processing should be run (true or false). default: false",

            required = false)

    public boolean ignoreHistory = false;

       

    @Option(

            name = "--cross-validation",

            usage = "ignore the test set and run n-fold cross-validation. default: n=2",

            required = false)

    public Integer crossValidationFolds;

   

    @Option(

            name = "--train-only",

            usage = "do not test a model, build one from xmi output and store in --models-dir",

            required = false)

    public boolean trainOnly = false;

    @Option(

            name = "--test-only",

            usage = "do not train a model, use the one specified in --models-dir",

            required = false)

    public boolean testOnly = false;

    @Option(

            name = "--preprocess-only",

            usage = "run preprocessing pipeline on a SHARP-style corpus, specify root directory",

            required = false)

    public File preprocessDir;

    @Option(

            name = "--no-cleartk",

            usage = "run the version of the assertion module released with cTAKES 2.5",

            required = false)

    public boolean noCleartk = false;

   

    @Option(

        name = "--print-errors",

        usage = "Flag to have test method print out error context for misclassified examples",

        required = false)

    public boolean printErrors = false;

    @Option(

        name = "--print-instances",

        usage = "Flag to have test method print out lots of info for statistical significance testing",

        required = false)

    public File printInstances;

    @Option(

        name = "--eval-only",

        usage = "Evaluate a CASes (supply the directory as an argument) with both system and gold in them.",

        required = false)

    public boolean evalOnly;

    @Option(

        name = "--ytex-negation",

        usage = "Use the negation detection from ytex, which is based on a more recent NegEx than the original cTAKES used." +

        " Note that using this requires adding the directory for YTEX_NEGATION_DESCRIPTOR to the classpath as well" +

        " as the annotator class itself, since ytex is under a different license than Apache cTAKES.",

        required = false)

    public boolean useYtexNegation;

    @Option(

        name = "--feature-selection",

        usage = "Takes an argument: the Chi^2 feature selection threshold",

        required = false)

    public Float featureSelectionThreshold = null;

    @Option(

        name = "--kernel-params",

        usage = "Set of parameters to pass to kernel (libsvm)",

        required = false)

    public String kernelParams = null;

   

    @Option(

        name = "--use-tmp",

        usage = "Whether to put trained models into a temp directory (e.g., for a grid search)",

        required = false)

    public boolean useTmp = false;

   

    @Option(

        name = "--corpus",

        usage = "What corpus to read for pre-processing",

        required = false)

    public Corpus corpus = Corpus.SHARP_SEED;

   

    @Option(

        name = "--feats",

        usage = "What feature configuration to use",

        required = false)

    public FEATURE_CONFIG featConfig = FEATURE_CONFIG.ALL_SYN;

    @Option(

        name = "--feda",

        usage = "Domain adaptation -- for each semicolon-separated directory in train-dir, creates a domain-specific feature space",

        required = false)

    public boolean feda = false;

  }

 

  protected ArrayList<String> annotationTypes;

  private Class<? extends AssertionCleartkAnalysisEngine> classifierAnnotatorClass;

 

  private Class<? extends DataWriter<String>> dataWriterClass;

 

  private File evaluationOutputDirectory;

  static public String evaluationLogFilePath;

  static private File evaluationLogFile;

  static private BufferedWriter evaluationLogFileOut;

  static {

    evaluationLogFilePath = "eval_"+new Date().toString().replaceAll(" ","_") + ".txt";

  }

  static public boolean useEvaluationLogFile = false;

  private boolean ignoreAnatomicalSites = false;

  protected static Options options = new Options();

 

  public static void main(String[] args) throws Exception {

   

    System.out.println("Started assertion module at " + new Date());

   

    //Options options = new Options();

    resetOptions();

    CmdLineParser parser = new CmdLineParser(options);

    parser.parseArgument(args);

    if (useEvaluationLogFile && evaluationLogFileOut == null) {

      evaluationLogFile = new File(evaluationLogFilePath);

      evaluationLogFileOut = new BufferedWriter(new FileWriter(evaluationLogFile), 32768);

    }

   

//    options.runSubject = false;

//    System.err.println("forcing skipping of generic processing!!!");

//    options.runGeneric = false;

//    System.err.println("forcing skipping of polarity processing!!!");

//    options.runPolarity = false;

//    System.err.println("forcing skipping of uncertainty processing!!!");

//    options.runUncertainty = false;

//    System.err.println("forcing skipping of conditional processing!!!");

//    options.runConditional = false;

    printOptionsForDebugging(options);

    List<File> trainFiles = new ArrayList<File>();

    if (null != options.trainDirectory) {

      String[] dirs = options.trainDirectory.split("[;:]");

      for (String dir : dirs) {

        File trainDir = new File(dir);

        if (trainDir.listFiles()!=null) {

          for (File f : trainDir.listFiles()) {

            trainFiles.add(f);

          }

        }

      }

    }

    //File modelsDir = new File("models/modifier");

    File modelsDir = options.modelsDirectory;

    if(options.useTmp){

      File tempModelDir = new File(options.modelsDirectory, "temp");

      tempModelDir.mkdirs();

      File curModelDir = File.createTempFile("assertion", null, tempModelDir);

      curModelDir.delete();

      curModelDir.mkdir();

      modelsDir = curModelDir;

    }

   

    File evaluationOutputDirectory = options.evaluationOutputDirectory;

    // determine the type of classifier to be trained

//    Class<? extends DataWriterFactory<String>> dataWriterFactoryClass = DefaultMaxentDataWriterFactory.class;

//    Class<? extends DataWriterFactory<String>> dataWriterFactoryClass = DefaultMultiClassLIBSVMDataWriterFactory.class;

   

    // TODO Class<? extends DataWriterFactory<String>> dataWriterFactoryClass = DefaultDataWriterFactory.class;

    //

    // A DataWriterFactory that creates a data writer from the class given by

    // PARAM_DATA_WRITER_CLASS_NAME and the directory given by

    // DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY.

    //

    // DefaultMaxentDataWriterFactory is deprecated and says to use DefaultDattaWriterFactory

    // with MaxentDataWriter.

   

    Class<? extends AssertionCleartkAnalysisEngine> annotatorClass = AssertionCleartkAnalysisEngine.class;

    //String [] annotationTypes = { "polarity", "conditional", "uncertainty", "subject", "generic" };

    ArrayList<String> annotationTypes = new ArrayList<String>();

    if (!options.ignorePolarity) { annotationTypes.add("polarity"); }

    if (!options.ignoreConditional) { annotationTypes.add("conditional"); }

    if (!options.ignoreUncertainty) { annotationTypes.add("uncertainty"); }

    if (!options.ignoreSubject) { annotationTypes.add("subject"); }

    if (!options.ignoreGeneric) { annotationTypes.add("generic"); }

    if (!options.ignoreHistory) { annotationTypes.add("historyOf"); }

   

    String[] kernelParams = null;

    if(options.kernelParams != null){

      kernelParams = options.kernelParams.split("\\s+");

    }else{

      kernelParams = new String[]{"-t", "0", "-c", "1"};

    }

    Class<? extends DataWriter<String>> dw = null;

    if(options.featConfig == FEATURE_CONFIG.STK || options.featConfig == FEATURE_CONFIG.PTK){

      throw new UnsupportedOperationException("This requires cleartk-2.0 which");

    }

    dw = LIBSVMStringOutcomeDataWriter.class;

   

    AssertionEvaluation evaluation = new AssertionEvaluation(

        modelsDir,

        evaluationOutputDirectory,

        annotationTypes,

        annotatorClass,

        dw,

        kernelParams

//        "0",

//       TKLIBSVMStringOutcomeDataWriter.class,

//        "-c",

//        "1"

//        "-t",

//        "5",

//        "-C",

//        "+",

//        "-L",

//        "0.4",

//        "-N",

//        "3",

//        "-S",

//        "0"

      

//        "-w0",

//        "100.0",

//        "-w1",

//        "1.0"

//        "100",

//        "2"

        );

   

    // if preprocessing, don't do anything else

    if(options.preprocessDir!=null ) {

      preprocess(options.preprocessDir);

    }

   

    // run cross-validation

    else if(options.crossValidationFolds != null) {

      // run n-fold cross-validation

      List<Map<String, AnnotationStatisticsCompact>> foldStats = evaluation.crossValidation(trainFiles, options.crossValidationFolds);

      //AnnotationStatisticsCompact overallStats = AnnotationStatisticsCompact.addAll(foldStats);

      Map<String, AnnotationStatisticsCompact> overallStats = new TreeMap<String, AnnotationStatisticsCompact>();

     

      for (String currentAnnotationType : annotationTypes)

      {

        AnnotationStatisticsCompact currentAnnotationStatisticsCompact = new AnnotationStatisticsCompact();

        overallStats.put(currentAnnotationType, currentAnnotationStatisticsCompact);

      }

      for (Map<String, AnnotationStatisticsCompact> singleFoldMap : foldStats)

      {

        for (String currentAnnotationType : annotationTypes)

        {

          AnnotationStatisticsCompact currentFoldStatistics = singleFoldMap.get(currentAnnotationType);

          overallStats.get(currentAnnotationType).addAll(currentFoldStatistics);

        }

      }

     

      AssertionEvaluation.printScore(overallStats,  "CROSS FOLD OVERALL");

     

    }

   

    // run train and test

    else {

      // train on the entire training set and evaluate on the test set

      List<File> testFiles;

      if (options.evalOnly) {

        testFiles = Arrays.asList(options.evaluationOutputDirectory.listFiles());

        logger.debug("evalOnly using files in directory " + evaluationOutputDirectory.getName() + " aka " + evaluationOutputDirectory.getCanonicalPath());

      } else if (options.trainOnly){

        testFiles = new ArrayList<File>();

      } else {

        testFiles = Arrays.asList(options.testDirectory.listFiles());

      }

     

      if (!options.testOnly && !options.evalOnly) {

        CollectionReader trainCollectionReader = evaluation.getCollectionReader(trainFiles);

        evaluation.train(trainCollectionReader, modelsDir);

      }

     

      // run testing

      if (!options.trainOnly) {

        if (testFiles==null || testFiles.size()==0) {

          throw new RuntimeException("testFiles = " + testFiles + " testFiles.size() = " + (testFiles==null ? "null": testFiles.size())) ;

        }

        logger.debug("testFiles.size() = " + testFiles.size());

        CollectionReader testCollectionReader = evaluation.getCollectionReader(testFiles);

        Map<String, AnnotationStatisticsCompact> stats = evaluation.test(testCollectionReader, modelsDir);

        AssertionEvaluation.printScore(stats,  modelsDir!=null? modelsDir.getAbsolutePath() : "no_model");

      }

    }

    if(options.useTmp && modelsDir != null){

      FileUtils.deleteRecursive(modelsDir);

    }

    System.out.println("Finished assertion module at " + new Date());

   

  }

 

  private static void resetOptions() {

    options.ignoreConditional = false;

    options.ignoreGeneric = false;

    options.ignoreHistory = false;

    options.ignorePolarity = false;

    options.ignoreSubject = false;

    options.ignoreUncertainty = false;

   

    options.trainOnly = false;

    options.testOnly = false;

    options.noCleartk = false;

    options.printErrors = false;

    options.printInstances = null;

    options.evalOnly = false;

   

    options.evaluationOutputDirectory = null;

    options.trainDirectory = null;

    options.testDirectory = null;

    options.devDirectory = null;

    options.modelsDirectory = null;

    options.preprocessDir = null;

   

    options.crossValidationFolds = null;

  }

  {

  String message;

  message = String.format(

    "Printing options: %n" +

    "training dir: %s%n" +

      "test dir: %s%n" +

      "model dir: %s%n" +

      "preprocess dir: %s%n" +

      "evaluation output dir: %s%n" +

      "cross-validation: %d%n" +

      "ignore polarity: %b%n" +

      "ignore conditional: %b%n" +

      "ignore uncertainty: %b%n" +

      "ignore subject: %b%n" +

      "ignore generic: %b%n" +

      "ignore history: %b%n" +

      "train only: %b%n" +

      "test only: %b%n" +

      "eval only: %b%n" +

      //"crossValidationFolds: %s%n" +

      "noCleartk: %b%n" +

      "%n%n",

      options.trainDirectory, // just a String so no need to check for null because not using getAbsolutePath()

         (options.testDirectory != null) ? options.testDirectory.getAbsolutePath() : "",

      (options.modelsDirectory!=null) ? options.modelsDirectory.getAbsolutePath() : "",

       (options.preprocessDir!=null) ? options.preprocessDir.getAbsolutePath() : "",

      (options.evaluationOutputDirectory!=null) ? options.evaluationOutputDirectory.getAbsolutePath() : "",

      options.crossValidationFolds,

      options.ignorePolarity,

      options.ignoreConditional,

      options.ignoreUncertainty,

      options.ignoreSubject,

      options.ignoreGeneric,

      options.ignoreHistory,

    options.trainOnly,

    options.testOnly,

    options.evalOnly,

    //(options.crossValidationFolds != null) ? options.crossValidationFolds.intValue()+"" : "",

    options.noCleartk

      );

  logger.info(message);

  }

  {

      for (Map.Entry<String, AnnotationStatisticsCompact> currentEntry : map.entrySet())

    {

        String annotationType = currentEntry.getKey();

        AnnotationStatisticsCompact stats = currentEntry.getValue();

       

        System.out.format("directory: \"%s\"; assertion type: %s%n%s%n%s%n%n",

          directory,

          annotationType.toUpperCase(),

          options.testDirectory,

          stats.toString());

       

        try {

          if (useEvaluationLogFile) {

            evaluationLogFileOut.write(

                String.format("%s\t%f\t%s\t%s\t%s",

                    annotationType,

                    options.featureSelectionThreshold,

                    options.modelsDirectory.getName(),

                    options.testDirectory.toString(),

                    stats.toTsv())

                );

            evaluationLogFileOut.flush();

          }

    } catch (IOException e) {

      // TODO Auto-generated catch block

      e.printStackTrace();

    }

    }

  }

  private String[] trainingArguments;

  public AssertionEvaluation(

      File modelDirectory,

      File evaluationOutputDirectory,

      ArrayList<String> annotationTypes,

      Class<? extends AssertionCleartkAnalysisEngine> classifierAnnotatorClass,

      Class<? extends DataWriter<String>> dataWriterClass,

      String... trainingArguments

      ) {

    super(modelDirectory);

   

    this.annotationTypes = annotationTypes;

    this.classifierAnnotatorClass = classifierAnnotatorClass;

    this.dataWriterClass = dataWriterClass;

    this.trainingArguments = trainingArguments;

    this.evaluationOutputDirectory = evaluationOutputDirectory;

  }

  @Override

  public CollectionReader getCollectionReader(List<File> items)

      throws ResourceInitializationException {

    String[] paths = new String[items.size()];

    for (int i = 0; i < paths.length; ++i) {

      paths[i] = items.get(i).getPath();

    }

    return CollectionReaderFactory.createCollectionReader(

        XMIReader.class,

        TypeSystemDescriptionFactory.createTypeSystemDescriptionFromPath(),

        XMIReader.PARAM_FILES,

        paths);

  }

  public static void preprocess(File rawDir ) throws ResourceInitializationException, UIMAException, IOException {

    File preprocessedDir = null;

    if (options.trainDirectory.split("[;]").length>1) {

      throw new IOException("Assertion preprocess wants to write to one train directory, but you've supplied multiple: " + options.trainDirectory);

    }

    preprocessedDir = new File(options.trainDirectory);

   

    if(options.corpus == Corpus.I2B2){

      GoldEntityAndAttributeReaderPipelineForSeedCorpus.readI2B2Challenge2010(rawDir, preprocessedDir);

    }else if(options.corpus == Corpus.MIPACQ){

      GoldEntityAndAttributeReaderPipelineForSeedCorpus.readMiPACQ(rawDir, preprocessedDir, options.testDirectory, options.devDirectory);

    }else if(options.corpus == Corpus.NEGEX){

      GoldEntityAndAttributeReaderPipelineForSeedCorpus.readNegexTestSet(rawDir, preprocessedDir);

    }else if(options.corpus == Corpus.SHARP_STRATIFIED){

      GoldEntityAndAttributeReaderPipelineForSeedCorpus.readSharpStratifiedUmls(

          rawDir, preprocessedDir, options.testDirectory, options.devDirectory);

    } else if(options.corpus == Corpus.SHARP_SEED){

      GoldEntityAndAttributeReaderPipelineForSeedCorpus.readSharpSeedUmls(

          rawDir, preprocessedDir, options.testDirectory, options.devDirectory);

    } else{

      throw new ResourceInitializationException("No corpus type specified!", new Object[]{rawDir});

    }

  }

 

  @Override

  public void train(CollectionReader collectionReader, File directory) throws Exception {

    AggregateBuilder builder = new AggregateBuilder();

   

    //builder.add(AnalysisEngineFactory.createPrimitiveDescription(ReplaceCTakesEntityMentionsAndModifiersWithGold.class));

//    AnalysisEngineDescription assertionDescription = AssertionCleartkAnalysisEngine.getDescription(

//        CleartkAnnotator.PARAM_DATA_WRITER_FACTORY_CLASS_NAME,

//        //MultiClassLIBSVMDataWriterFactory.class.getName(),

//        MaxentStringOutcomeDataWriter.class.getName(),

//        DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,

//        directory.getPath());

//    builder.add(assertionDescription);

   

    AnalysisEngineDescription documentIdPrinterAnnotator = AnalysisEngineFactory.createPrimitiveDescription(DocumentIdPrinterAnalysisEngine.class);

    builder.add(documentIdPrinterAnnotator);

   

    AnalysisEngineDescription goldCopierIdentifiedAnnotsAnnotator = AnalysisEngineFactory.createPrimitiveDescription(ReferenceIdentifiedAnnotationsSystemToGoldCopier.class);

    builder.add(goldCopierIdentifiedAnnotsAnnotator);

   

    AnalysisEngineDescription goldCopierSupportingAnnotsAnnotator = AnalysisEngineFactory.createPrimitiveDescription(ReferenceSupportingAnnotationsSystemToGoldCopier.class);

    builder.add(goldCopierSupportingAnnotsAnnotator);

   

    AnalysisEngineDescription assertionAttributeClearerAnnotator = AnalysisEngineFactory.createPrimitiveDescription(ReferenceAnnotationsSystemAssertionClearer.class);

    builder.add(assertionAttributeClearerAnnotator);

   

//        "org/mitre/medfacts/zoner/section_regex.xml";

//    AnalysisEngineDescription zonerAnnotator =

//        AnalysisEngineFactory.createPrimitiveDescription(ZoneAnnotator.class,

//            ZoneAnnotator.PARAM_SECTION_REGEX_FILE_URI,

//            generalSectionRegexFileUri

//            );

//    builder.add(zonerAnnotator);

//

//    String mayoSectionRegexFileUri =

//        "org/mitre/medfacts/uima/mayo_sections.xml";

//    AnalysisEngineDescription mayoZonerAnnotator =

//        AnalysisEngineFactory.createPrimitiveDescription(ZoneAnnotator.class,

//            ZoneAnnotator.PARAM_SECTION_REGEX_FILE_URI,

//            mayoSectionRegexFileUri

//            );

//    builder.add(mayoZonerAnnotator);

 

//    URL assertionCuePhraseLookupAnnotatorDescriptorUrl1 = this.getClass().getClassLoader().getResource("org/apache/ctakes/dictionary/lookup/AssertionCuePhraseDictionaryLookupAnnotator.xml");

//    logger.info(String.format("assertionCuePhraseLookupAnnotatorDescriptorUrl1 (slashes): %s", assertionCuePhraseLookupAnnotatorDescriptorUrl1));

//    URL assertionCuePhraseLookupAnnotatorDescriptorUrl2 = this.getClass().getClassLoader().getResource("org.apache.ctakes.dictionary.lookup.AssertionCuePhraseDictionaryLookupAnnotator.xml");

//    logger.info(String.format("assertionCuePhraseLookupAnnotatorDescriptorUrl2 (periods): %s", assertionCuePhraseLookupAnnotatorDescriptorUrl2));

//

//   

//    AnalysisEngineDescription cuePhraseLookupAnnotator =

//        AnalysisEngineFactory.createAnalysisEngineDescription("org/apache/ctakes/dictionary/lookup/AssertionCuePhraseDictionaryLookupAnnotator");

//    builder.add(cuePhraseLookupAnnotator);

    // Set up Feature Selection parameters

    Float featureSelectionThreshold = options.featureSelectionThreshold;

    Class<? extends DataWriter> dataWriterClassFirstPass = getDataWriterClass();

    if (options.featureSelectionThreshold==null) {

      featureSelectionThreshold = 0f;

    }

   

    // Add each assertion Analysis Engine to the pipeline!

    builder.add(AnalysisEngineFactory.createPrimitiveDescription(AlternateCuePhraseAnnotator.class, new Object[]{}));

   

    if (!options.ignorePolarity)

    {

      AnalysisEngineDescription polarityAnnotator;

      if (options.useYtexNegation) {

         polarityAnnotator = AnalysisEngineFactory.createAnalysisEngineDescription(YTEX_NEGATION_DESCRIPTOR);

      } else {

        if (options.feda) {

          polarityAnnotator = AnalysisEngineFactory.createPrimitiveDescription(PolarityFedaCleartkAnalysisEngine.class);

            ConfigurationParameterFactory.addConfigurationParameters(

                polarityAnnotator,

                AssertionCleartkAnalysisEngine.FILE_TO_DOMAIN_MAP,

                options.trainDirectory

                );

        } else {

          // default: cleartk-based polarity, no domain adaptation

          polarityAnnotator = AnalysisEngineFactory.createPrimitiveDescription(PolarityCleartkAnalysisEngine.class); //,  this.additionalParamemters);

          }

        ConfigurationParameterFactory.addConfigurationParameters(

            polarityAnnotator,

            AssertionCleartkAnalysisEngine.PARAM_GOLD_VIEW_NAME,

            AssertionEvaluation.GOLD_VIEW_NAME,

//            this.dataWriterFactoryClass.getName(),

            DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,

            dataWriterClassFirstPass,

            DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,

            new File(directory, "polarity").getPath(),

            AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_URI,

            PolarityCleartkAnalysisEngine.createFeatureSelectionURI(new File(directory, "polarity")),

            AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_THRESHOLD,

            featureSelectionThreshold,

            AssertionCleartkAnalysisEngine.PARAM_FEATURE_CONFIG,

            options.featConfig

            );

      }

    builder.add(polarityAnnotator);

    }

    if (!options.ignoreConditional)

    {

      AnalysisEngineDescription conditionalAnnotator = AnalysisEngineFactory.createPrimitiveDescription(ConditionalCleartkAnalysisEngine.class); //,  this.additionalParamemters);

      ConfigurationParameterFactory.addConfigurationParameters(

          conditionalAnnotator,

          AssertionCleartkAnalysisEngine.PARAM_GOLD_VIEW_NAME,

          AssertionEvaluation.GOLD_VIEW_NAME,

//          this.dataWriterFactoryClass.getName(),

          DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,

      dataWriterClassFirstPass,

          DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,

          new File(directory, "conditional").getPath(),

      AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_URI,

      ConditionalCleartkAnalysisEngine.createFeatureSelectionURI(new File(directory, "conditional")),

      AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_THRESHOLD,

      featureSelectionThreshold

          );

      builder.add(conditionalAnnotator);

    }

    if (!options.ignoreUncertainty)

    {

      AnalysisEngineDescription uncertaintyAnnotator = AnalysisEngineFactory.createPrimitiveDescription(UncertaintyCleartkAnalysisEngine.class); //,  this.additionalParamemters);

      ConfigurationParameterFactory.addConfigurationParameters(

          uncertaintyAnnotator,

          AssertionCleartkAnalysisEngine.PARAM_GOLD_VIEW_NAME,

          AssertionEvaluation.GOLD_VIEW_NAME,

//          this.dataWriterFactoryClass.getName(),

          DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,

      dataWriterClassFirstPass,

          DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,

          new File(directory, "uncertainty").getPath(),

      AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_URI,

      UncertaintyCleartkAnalysisEngine.createFeatureSelectionURI(new File(directory, "uncertainty")),

      AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_THRESHOLD,

      featureSelectionThreshold,

      AssertionCleartkAnalysisEngine.PARAM_FEATURE_CONFIG,

      options.featConfig

          );

      builder.add(uncertaintyAnnotator);

    }

    if (!options.ignoreSubject)

    {

      AnalysisEngineDescription subjectAnnotator = AnalysisEngineFactory.createPrimitiveDescription(SubjectCleartkAnalysisEngine.class); //,  this.additionalParamemters);

      ConfigurationParameterFactory.addConfigurationParameters(

          subjectAnnotator,

          AssertionCleartkAnalysisEngine.PARAM_GOLD_VIEW_NAME,

          AssertionEvaluation.GOLD_VIEW_NAME,

//          this.dataWriterFactoryClass.getName(),

          DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,

      dataWriterClassFirstPass,

          DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,

          new File(directory, "subject").getPath(),

      AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_URI,

      SubjectCleartkAnalysisEngine.createFeatureSelectionURI(new File(directory, "subject")),

      AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_THRESHOLD,

      featureSelectionThreshold

          );

      builder.add(subjectAnnotator);

    }

    if (!options.ignoreGeneric)

    {

    AnalysisEngineDescription genericAnnotator = AnalysisEngineFactory.createPrimitiveDescription(GenericCleartkAnalysisEngine.class); //,  this.additionalParamemters);

    ConfigurationParameterFactory.addConfigurationParameters(

        genericAnnotator,

        AssertionCleartkAnalysisEngine.PARAM_GOLD_VIEW_NAME,

        AssertionEvaluation.GOLD_VIEW_NAME,

//        this.dataWriterFactoryClass.getName(),

        DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,

      dataWriterClassFirstPass,

        DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,

        new File(directory, "generic").getPath(),

      AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_URI,

      GenericCleartkAnalysisEngine.createFeatureSelectionURI(new File(directory, "generic")),

      AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_THRESHOLD,

      featureSelectionThreshold

        );

    builder.add(genericAnnotator);

    }

   

    // 2/20/13 srh adding

    if (!options.ignoreHistory) {

      AnalysisEngineDescription historyAnnotator = AnalysisEngineFactory.createPrimitiveDescription(HistoryCleartkAnalysisEngine.class);

      ConfigurationParameterFactory.addConfigurationParameters(

          historyAnnotator,

          AssertionCleartkAnalysisEngine.PARAM_GOLD_VIEW_NAME,

          AssertionEvaluation.GOLD_VIEW_NAME,

//          this.dataWriterFactoryClass.getName(),

          DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,

      dataWriterClassFirstPass,

          DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,

          new File(directory, "historyOf").getPath(),

        AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_URI,

        HistoryCleartkAnalysisEngine.createFeatureSelectionURI(new File(directory, "historyOf")),

        AssertionCleartkAnalysisEngine.PARAM_FEATURE_SELECTION_THRESHOLD,

        featureSelectionThreshold

          );

      builder.add(historyAnnotator);

    }

    AnalysisEngineDescription classifierAnnotator = AnalysisEngineFactory.createPrimitiveDescription(

        this.classifierAnnotatorClass,

        this.additionalParameters);

    ConfigurationParameterFactory.addConfigurationParameters(

        classifierAnnotator,

        RelationExtractorAnnotator.PARAM_GOLD_VIEW_NAME,

        RelationExtractorEvaluation.GOLD_VIEW_NAME,

        CleartkAnnotator.PARAM_DATA_WRITER_FACTORY_CLASS_NAME,

        this.dataWriterFactoryClass.getName(),

        DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,

        directory.getPath());

    builder.add(classifierAnnotator);

*/   

   

    SimplePipeline.runPipeline(collectionReader,  builder.createAggregateDescription());

   

    //HideOutput hider = new HideOutput();

    for (String currentAssertionAttribute : annotationTypes)

    {

      File currentDirectory = new File(directory, currentAssertionAttribute);

      trainAndPackage(currentAssertionAttribute, currentDirectory, trainingArguments);

    }

    //hider.restoreOutput();

  }

  @Override

  protected Map<String, AnnotationStatisticsCompact> test(CollectionReader collectionReader, File directory)

      throws Exception {

//        GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH,

//        new File(directory, "model.jar").getPath()));

    AggregateBuilder builder = new AggregateBuilder();

   

    // directory is such as /cTAKES/workspaces/Apache-cTAKES-trunk/ctakes/ctakes-assertion/sharp_data/model/eval.model

   

    AnalysisEngineDescription goldCopierAnnotator = AnalysisEngineFactory.createPrimitiveDescription(ReferenceIdentifiedAnnotationsSystemToGoldCopier.class);

    builder.add(goldCopierAnnotator);

   

    AnalysisEngineDescription assertionAttributeClearerAnnotator = AnalysisEngineFactory.createPrimitiveDescription(ReferenceAnnotationsSystemAssertionClearer.class);

    builder.add(assertionAttributeClearerAnnotator);

   

    if ( options.noCleartk ) {

      addExternalAttributeAnnotatorsToAggregate(builder);

    } else {

      addCleartkAttributeAnnotatorsToAggregate(directory, builder);

    }

    if (options.evalOnly && evaluationOutputDirectory != null) {

      // short circuit any other stuff in the pipeline

      builder = new AggregateBuilder();

     

      // uimafit find available type systems on classpath

      TypeSystemDescription typeSystemDescription = TypeSystemDescriptionFactory.createTypeSystemDescription();

     

        AnalysisEngineDescription noOp =

        AnalysisEngineFactory.createPrimitiveDescription(

              NoOpAnnotator.class,

              typeSystemDescription);

      builder.add(noOp);

     

        AnalysisEngineDescription mergeGold =

        AnalysisEngineFactory.createPrimitiveDescription(org.apache.ctakes.assertion.eval.MergeGoldViewFromOneCasIntoInitialViewOfAnotherCas.class, typeSystemDescription);

      builder.add(mergeGold);

     

    } else if (evaluationOutputDirectory!=null)  {

        AnalysisEngineDescription xwriter =

        AnalysisEngineFactory.createPrimitiveDescription(

              XWriter.class,

              AssertionComponents.CTAKES_CTS_TYPE_SYSTEM_DESCRIPTION,

              XWriter.PARAM_OUTPUT_DIRECTORY_NAME,

              evaluationOutputDirectory,

              XWriter.PARAM_XML_SCHEME_NAME,

              XWriter.XMI,

              XWriter.PARAM_FILE_NAMER_CLASS_NAME,

              CtakesFileNamer.class.getName());

        builder.add(xwriter);

    }

   

    //SimplePipeline.runPipeline(collectionReader,  builder.createAggregateDescription());

    //AnalysisEngineDescription aggregateDescription = builder.createAggregateDescription();

   

    AnalysisEngine aggregate = builder.createAggregate();

   

    AnnotationStatisticsCompact polarityStats = new AnnotationStatisticsCompact();

    AnnotationStatisticsCompact conditionalStats = new AnnotationStatisticsCompact();

    AnnotationStatisticsCompact uncertaintyStats = new AnnotationStatisticsCompact();

    AnnotationStatisticsCompact subjectStats = new AnnotationStatisticsCompact();

    AnnotationStatisticsCompact genericStats = new AnnotationStatisticsCompact();

    AnnotationStatisticsCompact historyStats = new AnnotationStatisticsCompact();  // srh 3/6/13

   

    Map<String, AnnotationStatisticsCompact> map = new TreeMap<String, AnnotationStatisticsCompact>();

    if (!options.ignorePolarity)

    {

      map.put("polarity",  polarityStats);

    }

    if (!options.ignoreConditional)

    {

      map.put("conditional",  conditionalStats);

    }

    if (!options.ignoreUncertainty)

    {

      map.put("uncertainty",  uncertaintyStats);

    }

    if (!options.ignoreSubject)

    {

      map.put("subject", subjectStats);

    }

    if (!options.ignoreGeneric)

    {

      map.put("generic", genericStats);

    }

   

    // srh 3/6/13

    if (!options.ignoreHistory)

    {

      map.put("historyOf", historyStats);

    }

    // run on existing output that has both system (or instance gathering) and gold

    for (JCas jCas : new JCasIterable(collectionReader, aggregate)) {

     

     

      JCas goldView;

      try {

        goldView = jCas.getView(GOLD_VIEW_NAME);

      } catch (CASException e) {

      logger.info("jCas.getViewName() = " + jCas.getViewName());

        throw new AnalysisEngineProcessException(e);

      }

      String documentId = DocumentIDAnnotationUtil.getDocumentID(jCas);

      System.out.format("document id: %s%n", documentId);

     

      Collection<IdentifiedAnnotation> goldEntitiesAndEvents = new ArrayList<IdentifiedAnnotation>();

      if ( !ignoreAnatomicalSites ) {

        Collection<EntityMention> goldEntities = JCasUtil.select(goldView, EntityMention.class);

        goldEntitiesAndEvents.addAll(goldEntities);

      }

      Collection<EventMention> goldEvents = JCasUtil.select(goldView, EventMention.class);

      goldEntitiesAndEvents.addAll(goldEvents);

      // System.out.format("gold entities: %d%ngold events: %d%n%n", goldEntities.size(), goldEvents.size());

     

      if (goldEntitiesAndEvents.size()==0) {

        // gold annotations might have been read in as just IdentifiedAnnotation annotations

        // since no EventMentio or EntityMention annotations were found, ok to just try IdentifiedAnnotation

        // without concern for using some twice.

          Collection<IdentifiedAnnotation> identifiedAnnotations = JCasUtil.select(goldView, IdentifiedAnnotation.class);

          goldEntitiesAndEvents.addAll(identifiedAnnotations);

       

       

      }

     

      Collection<IdentifiedAnnotation> systemEntitiesAndEvents = new ArrayList<IdentifiedAnnotation>();

      if ( !ignoreAnatomicalSites ) {

        Collection<EntityMention> systemEntities = JCasUtil.select(jCas, EntityMention.class);

        systemEntitiesAndEvents.addAll(systemEntities);

      }

      Collection<EventMention> systemEvents = JCasUtil.select(jCas, EventMention.class);

      systemEntitiesAndEvents.addAll(systemEvents);

     

      if (!options.ignorePolarity)

      {

        polarityStats.add(goldEntitiesAndEvents, systemEntitiesAndEvents,

        AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToSpan(),

        AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToFeatureValue("polarity"));

        if(options.printErrors){

          printErrors(jCas, goldEntitiesAndEvents, systemEntitiesAndEvents, "polarity", CONST.NE_POLARITY_NEGATION_PRESENT, Integer.class);

        }

        if(options.printInstances!=null){

          printInstances(jCas, goldEntitiesAndEvents, systemEntitiesAndEvents, "polarity", CONST.NE_POLARITY_NEGATION_PRESENT, Integer.class, options.printInstances);

        }

      }

      if (!options.ignoreConditional)

      {

        conditionalStats.add(goldEntitiesAndEvents, systemEntitiesAndEvents,

        AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToSpan(),

        AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToFeatureValue("conditional"));

        if(options.printErrors){

          printErrors(jCas, goldEntitiesAndEvents, systemEntitiesAndEvents, "conditional", CONST.NE_CONDITIONAL_TRUE, Boolean.class);

        }

      }

      if (!options.ignoreUncertainty)

      {

        uncertaintyStats.add(goldEntitiesAndEvents, systemEntitiesAndEvents,

        AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToSpan(),

        AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToFeatureValue("uncertainty"));

        if(options.printErrors){

          printErrors(jCas, goldEntitiesAndEvents, systemEntitiesAndEvents, "uncertainty", CONST.NE_UNCERTAINTY_PRESENT, Integer.class);

        }

      }

      if (!options.ignoreSubject)

      {

        subjectStats.add(goldEntitiesAndEvents, systemEntitiesAndEvents,

        AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToSpan(),

        AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToFeatureValue("subject"));

        if(options.printErrors){

          printErrors(jCas, goldEntitiesAndEvents, systemEntitiesAndEvents, "subject", null, CONST.ATTR_SUBJECT_PATIENT.getClass());

        }

      }

      if (!options.ignoreGeneric)

      {

        genericStats.add(goldEntitiesAndEvents, systemEntitiesAndEvents,

        AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToSpan(),

        AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToFeatureValue("generic"));

        if(options.printErrors){

          printErrors(jCas, goldEntitiesAndEvents, systemEntitiesAndEvents, "generic", CONST.NE_GENERIC_TRUE, Boolean.class);

        }

      }

     

      // srh 3/6/13

      if (!options.ignoreHistory)

      {

        historyStats.add(goldEntitiesAndEvents, systemEntitiesAndEvents,

            AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToSpan(),

            AnnotationStatisticsCompact.<IdentifiedAnnotation>annotationToFeatureValue("historyOf"));

        if(options.printErrors){

          printErrors(jCas, goldEntitiesAndEvents, systemEntitiesAndEvents, "historyOf", CONST.NE_HISTORY_OF_PRESENT, Integer.class);

        }

      }

     

    }

    return map;

  }

  protected void trainAndPackage(String currentAssertionAttribute, File directory, String[] arguments) throws Exception {

    if (options.featureSelectionThreshold!=null) {

      // Extracting features and writing instances

      Iterable<Instance<String>> instances = InstanceStream.loadFromDirectory(directory);

      // Collect MinMax stats for feature normalization

      FeatureSelection<String> featureSelection;

      if (currentAssertionAttribute.equals("polarity")) {

        // TODO: parameterize the thresholds

        featureSelection = PolarityCleartkAnalysisEngine.createFeatureSelection(options.featureSelectionThreshold);

        featureSelection.train(instances);

        featureSelection.save(PolarityCleartkAnalysisEngine.createFeatureSelectionURI(directory));

      }

      else if (currentAssertionAttribute.equals("uncertainty")) {

        // TODO: parameterize the thresholds

        featureSelection = UncertaintyCleartkAnalysisEngine.createFeatureSelection(options.featureSelectionThreshold);

        featureSelection.train(instances);

        featureSelection.save(UncertaintyCleartkAnalysisEngine.createFeatureSelectionURI(directory));

      }

      else if (currentAssertionAttribute.equals("conditional")) {

        // TODO: parameterize the thresholds

        featureSelection = ConditionalCleartkAnalysisEngine.createFeatureSelection(options.featureSelectionThreshold);

        featureSelection.train(instances);

        featureSelection.save(ConditionalCleartkAnalysisEngine.createFeatureSelectionURI(directory));

      }

      else if (currentAssertionAttribute.equals("subject")) {

        // TODO: parameterize the thresholds

        featureSelection = SubjectCleartkAnalysisEngine.createFeatureSelection(options.featureSelectionThreshold);

        featureSelection.train(instances);

        featureSelection.save(SubjectCleartkAnalysisEngine.createFeatureSelectionURI(directory));

      }

      else if (currentAssertionAttribute.equals("generic")) {

        // TODO: parameterize the thresholds

        featureSelection = GenericCleartkAnalysisEngine.createFeatureSelection(options.featureSelectionThreshold);

        featureSelection.train(instances);

        featureSelection.save(GenericCleartkAnalysisEngine.createFeatureSelectionURI(directory));

      }

      else if (currentAssertionAttribute.equals("historyOf")) {

        // TODO: parameterize the thresholds

        featureSelection = HistoryCleartkAnalysisEngine.createFeatureSelection(options.featureSelectionThreshold);

        featureSelection.train(instances);

        featureSelection.save(HistoryCleartkAnalysisEngine.createFeatureSelectionURI(directory));

      }

      else {

        featureSelection = null;

      }

        // now write in the libsvm format

//        LIBLINEARStringOutcomeDataWriter dataWriter = new LIBLINEARStringOutcomeDataWriter(directory);

      Constructor c = this.dataWriterClass.getConstructor(File.class);

        DataWriter dataWriter = (DataWriter) c.newInstance(directory);

       

        // try filtering

        for (Instance<String> instance : instances) {

          dataWriter.write(featureSelection.transform(instance));

        }

        dataWriter.finish();

    }

    // train models based on instances

    JarClassifierBuilder.trainAndPackage(directory, arguments);

  }

 

  protected Class<? extends DataWriter> getDataWriterClass()

      throws ResourceInitializationException {

    return (options.featureSelectionThreshold!=null)

        ? InstanceDataWriter.class

        : this.dataWriterClass;

  }

 

  private static boolean DEBUG = false;

  private static void printViewNames(String message, JCas jcas) {

 

    Iterator<JCas> viewIter;

  try {

    viewIter = jcas.getViewIterator();

  } catch (CASException e) {

    e.printStackTrace();

    return;

  }

    while (viewIter.hasNext()) {

      JCas view = viewIter.next();

      String viewName = view.getViewName();

      logger.debug(message + " View name " + viewName);

      int numIndexedAnnotations = view.getAnnotationIndex().size();

      logger.debug(message + "  has " + numIndexedAnnotations + " indexed annotations");

      if (viewName.toLowerCase().contains("gold")) {

            if (DEBUG) printAnnotations(IdentifiedAnnotation.type, view);