Package org.archive.util.iterator

Examples of org.archive.util.iterator.LineReadingIterator

160
161
162
163
164
165
166
167
168
169
170
171
172
     */
    private static int populatePersistEnvFromLog(BufferedReader persistLogReader, StoredSortedMap<String,Map> historyMap)
    throws UnsupportedEncodingException, DatabaseException {
        int count = 0;

        Iterator<String> iter = new LineReadingIterator(persistLogReader);
        while (iter.hasNext()) {
            String line = iter.next();
            if (line.length() == 0) {
                continue;
            }
            String[] splits = line.split(" ");
            if (splits.length != 2) {
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
     */
    protected void announceSeedsFromReader(BufferedReader reader, CountDownLatch latchOrNull) {
        String s;
        Iterator<String> iter =
            new RegexLineIterator(
                    new LineReadingIterator(reader),
                    RegexLineIterator.COMMENT_LINE,
                    RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT,
                    RegexLineIterator.ENTRY);

        int count = 0;
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
        int count = 0;
       
        if(source.isFile()) {
            // scan log, writing to database
            BufferedReader br = ArchiveUtils.getBufferedReader(source);
            Iterator<String> iter = new LineReadingIterator(br);
            while(iter.hasNext()) {
                String line = (String) iter.next();
                String[] splits = line.split("\\s");
                String uri = splits[0];
                if(!uri.matches("\\w+:.*")) {
                    // prepend "http://"
                    uri = "http://"+uri;
View Full Code Here
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
        boolean asSeeds = !params.isNull("asSeeds");
        boolean scopeScheduleds = !params.isNull("scopeScheduleds");
        DecideRule scope = scopeScheduleds ? getScope() : null;
        try {
            br = new BufferedReader(new InputStreamReader(new FileInputStream(path)));
            Iterator<String> iter = new RegexLineIterator(new LineReadingIterator(br),
                RegexLineIterator.COMMENT_LINE, extractor, output);
            while(iter.hasNext()) {
                try {
                   
                    CrawlURI curi = CrawlURI.fromHopsViaString(((String)iter.next()));
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
            reader = new InputStreamReader(conn.getInputStream());
        }
        reader = new BufferedReader(reader);
        Iterator<String> iter =
            new RegexLineIterator(
                    new LineReadingIterator((BufferedReader) reader),
                    RegexLineIterator.COMMENT_LINE,
                    RegexLineIterator.TRIMMED_ENTRY_TRAILING_COMMENT,
                    RegexLineIterator.ENTRY);
        while (iter.hasNext()) {
            String[] entry = ((String) iter.next()).split("\\s+");
View Full Code Here
TOP

Related Classes of org.archive.util.iterator.LineReadingIterator

  • org.archive.crawler.frontier.AbstractFrontier
  • org.archive.crawler.frontier.precedence.PrecedenceLoader
  • org.archive.crawler.processor.LexicalCrawlMapper
  • org.archive.modules.recrawl.PersistProcessor
  • org.archive.modules.seeds.TextSeedModule
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.