Package org.sf.mustru.filters

Examples of org.sf.mustru.filters.StarHandler

23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
{ PropertyConfigurator.configure (Constants.LOG4J_FILE);  }
public void ttestPdfFile()
{
  System.out.println("Started testPdfFile");
  Properties props = null; StarHandler sh = null;
  try { props = new Properties(); props.load(new FileInputStream(Constants.FILTER_FILE)); sh = new StarHandler(props); }
  catch (IOException e) { throw new RuntimeException("Could not read filtersFile + " + e.getMessage() ); }
 
  String filename = "/home/manuk/html/akr/ebooks/MYSQLmanual-a4.pdf";
  IndexableDoc doc = new IndexableDoc();
  sh.getDocument(filename, doc);
  int contentSizeLimit = Constants.DOC_LENGTH_MAXLIMIT;
  int textSize = doc.getContents().length();
  if (textSize > contentSizeLimit )
       doc.setContents( (new StringBuffer( doc.getContents().substring(0, contentSizeLimit ) ) ) );
 
View Full Code Here
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
  public static void main(String[] args)
  {
   //*-- get a generic document handler to extract text
   System.out.println("Started TestFilters...");
   setUp();
   Properties props = null; StarHandler sh = null;
   try { props = new Properties(); props.load(new FileInputStream(Constants.FILTER_FILE)); sh = new StarHandler(props); }
   catch (IOException e) { throw new RuntimeException("Could not read filtersFile + " + e.getMessage() ); }
 
   for (int i = 0; i < sampleFiles.length; i++)
   { String filename = sampleDir + sampleFiles[i];
     IndexableDoc doc = new IndexableDoc();
     sh.getDocument(filename, doc);
     String contents = doc.getContents().toString();
     contents = (contents.length() > 2000) ? contents.substring(0, 2000): contents;
     System.out.println("File: " + sampleFiles[i] + " has content: " + contents);
   }
   System.out.println("Finished TestFilters");
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
  */
public void run()
{     
  //*-- read the filters file and get a generic handler to extract text 
  fileReadTime -=  new Date().getTime();
  Properties props = null; StarHandler sh = null;
  try { props = new Properties(); props.load(new BufferedInputStream(new FileInputStream(Constants.FILTER_FILE)));
  sh = new StarHandler(props); }
  catch (IOException e) { crawlManager.cleanUp("Could not read filtersFile " + e.getMessage()); }
  if (sh == null) crawlManager.cleanUp("The generic handler to filter files was not found");

  //*-- read the document types file to classify documents into types - text, image, audio, book, letter, etc.
  Properties dprops = null;
  try { dprops = new Properties(); dprops.load(new BufferedInputStream(new FileInputStream(Constants.DOCTYPES_FILE))); }
  catch (IOException e)  { crawlManager.cleanUp("Could not read doctypesFile " + e.getMessage()); }

  //*-- read the media suffixes and associated file types properties file
  Properties mprops = new Properties();
  try { mprops.load(new BufferedInputStream(new FileInputStream( Constants.MEDIATYPES_FILE))); }
  catch ( IOException e) { crawlManager.cleanUp("Could not read media types file " + e.getMessage()); }
  fileReadTime += new Date().getTime();

  currentDoc = crawlManager.getCurrentDoc(threadNum);

  long startMemory = Runtime.getRuntime().freeMemory();
  logger.info("Free memory at start of scan: " + startMemory);

  //*-- scan the list of files and process depending on thread number
  String iDocument = ""; IndexableDoc idoc = null; boolean completed = false;
  FILELOOP: while (!completed)
  {
   //*-- distribute the load evenly
   if ( (++currentDoc % numThreads) != threadNum )  continue FILELOOP;

   //*-- for a restart skip documents scanned earlier
   if (currentDoc < crawlConfig.getStartPosition()) continue FILELOOP;

   //*-- terminate if all files have been processed
   if (currentDoc >= crawlManager.totalNumFiles() ) { completed = true; break FILELOOP; }

   //*-- periodically synchronize the indexes
   syncTime -= new Date().getTime();
   long freeMemory = Runtime.getRuntime().freeMemory();
   //  if (freeMemory < startMemory)  mergeIndexes(false);
   if ( ( (currentDoc % 1000) == 0) && (docsProcessed > 0) ) mergeIndexes(false);
   syncTime += new Date().getTime();          
   try
   {
    //*-- check if the document was indexed prior to the modification date of the file
    iDocument = crawlManager.getFileName(currentDoc); initDoc(iDocument, tempDoc);
    //logger.info(currentDoc + ": Started File: " + iDocument + "<----");
    if (checkIndexed(iDocument)) continue FILELOOP;

    //*-- extract the text from the document and create an indexable document
    docFilterTime -= new Date().getTime();
    sh.getDocument(iDocument, tempDoc);
    docFilterTime += new Date().getTime();

    //*-- get the signature of the document text and check for duplicates
    if (duplicateDoc()) continue FILELOOP;

View Full Code Here
TOP

Related Classes of org.sf.mustru.filters.StarHandler

  • org.sf.mustru.crawl.CrawlThread
  • org.sf.mustru.test.TestFilters
  • org.sf.mustru.test.TestSentenceExtraction
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.