Examples of opennlp.tools.tokenize.TokenSampleStream

opennlp.tools.tokenize.TokenSampleStream
This class is a stream filter which reads in string encoded samples and creates {@link TokenSample}s out of them. The input string sample is tokenized if a whitespace or the special separator chars occur.
Sample:
"token1 token2 token3token4"
The tokens token1 and token2 are separated by a whitespace, token3 and token3 are separated by the special character sequence, in this case the default split sequence.
The sequence must be unique in the input string and is not escaped.

          mLogger.log(Level.INFO, "Using addional training data file: " + additionalTrainingDataFile); 
        }
        
        additionalTrainingDataIn = new FileInputStream(additionalTrainingDataFile);
        
        ObjectStream<TokenSample> additionalSamples = new TokenSampleStream(
            new PlainTextByLineStream(new InputStreamReader(additionalTrainingDataIn, additionalTrainingDataEncoding)));
        
        samples = ObjectStreamUtils.createObjectStream(samples, additionalSamples);
      }

View Full Code Here

    FileInputStream sampleDataIn = CmdLineUtil.openInFile(sampleDataFile);


    ObjectStream<String> lineStream = new PlainTextByLineStream(sampleDataIn
        .getChannel(), encoding);


    return new TokenSampleStream(lineStream);
  }

View Full Code Here

          mLogger.log(Level.INFO, "Using addional training data file: " + additionalTrainingDataFile); 
        }
        
        additionalTrainingDataIn = new FileInputStream(additionalTrainingDataFile);
        
        ObjectStream<TokenSample> additionalSamples = new TokenSampleStream(
            new PlainTextByLineStream(new InputStreamReader(additionalTrainingDataIn, additionalTrainingDataEncoding)));
        
        samples = ObjectStreamUtils.createObjectStream(samples, additionalSamples);
      }

View Full Code Here

    FileInputStream sampleDataIn = CmdLineUtil.openInFile(params.getData());


    ObjectStream<String> lineStream = new PlainTextByLineStream(sampleDataIn.getChannel(),
        params.getEncoding());


    return new TokenSampleStream(lineStream);
  }

View Full Code Here

      lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
    } catch (IOException ex) {
      CmdLineUtil.handleCreateObjectStreamError(ex);
    }


    return new TokenSampleStream(lineStream);
  }

View Full Code Here

          mLogger.log(Level.INFO, "Using addional training data file: " + additionalTrainingDataFile);
        }


        additionalTrainingDataIn = new FileInputStream(additionalTrainingDataFile);


        ObjectStream<TokenSample> additionalSamples = new TokenSampleStream(
            new PlainTextByLineStream(new InputStreamReader(additionalTrainingDataIn, additionalTrainingDataEncoding)));


        samples = ObjectStreamUtils.createObjectStream(samples, additionalSamples);
      }

View Full Code Here

    FileInputStream sampleDataIn = CmdLineUtil.openInFile(sampleDataFile);


    ObjectStream<String> lineStream = new PlainTextByLineStream(sampleDataIn
        .getChannel(), encoding);


    return new TokenSampleStream(lineStream);
  }

View Full Code Here

TOP

Related Classes of opennlp.tools.tokenize.TokenSampleStream

opennlp.tools.cmdline.tokenizer.TokenizerTrainerTool

opennlp.tools.formats.TokenSampleStreamFactory

opennlp.uima.tokenize.TokenizerTrainer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.