Package it.unimi.dsi.io

Examples of it.unimi.dsi.io.FastBufferedReader

153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
    final Collection<? extends CharSequence> collection;
    if ( termFile == null ) {
      ArrayList<MutableString> termList = new ArrayList<MutableString>();
      final ProgressLogger pl = new ProgressLogger();
      pl.itemsName = "URIs";
      final LineIterator termIterator = new LineIterator( new FastBufferedReader( new InputStreamReader( System.in, "UTF-8" ), bufferSize ), pl );
     
      pl.start( "Reading URIs..." );
      MutableString uri;
      while( termIterator.hasNext() ) {
        uri = termIterator.next();
        if ( uniqueURIs ) makeUnique( filter, uri );
        termList.add( uri.copy() );
      }
      pl.done();
     
      collection = termList;
    }
    else {
      if ( uniqueURIs ) {
        // Create temporary file with unique URIs
        final ProgressLogger pl = new ProgressLogger();
        pl.itemsName = "URIs";
        pl.start( "Copying URIs..." );
        final LineIterator termIterator = new LineIterator( new FastBufferedReader( new InputStreamReader( new FileInputStream( termFile ) ), bufferSize ), pl );
        File temp = File.createTempFile( URLMPHVirtualDocumentResolver.class.getName(), ".uniqueuris" );
        temp.deleteOnExit();
        termFile = temp.toString();
        final FastBufferedOutputStream outputStream = new FastBufferedOutputStream( new FileOutputStream( termFile ), bufferSize );
        MutableString uri;
View Full Code Here
324
325
326
327
328
329
330
331
332
333
334
            int d = resolver.resolve( fragment.documentSpecifier() );

            if ( d != -1 ) {
              if ( map != null ) d = map[ d ];
              if ( maxDoc[ i ] < d ) maxDoc[ i ] = d;
              currMaxPos[ i ][ d ] = processDocument( document.wordReader( field[ i ] ).setReader( new FastBufferedReader( fragment.text() ) ), d, currMaxPos[ i ][ d ], termMap[ i ],
                  index[ i ].termProcessor )
                  + gap;
            }
          }
          break;
View Full Code Here
93
94
95
96
97
98
99
100
          result.add( new Anchor( new MutableString( entry.getKey().toString() ), new MutableString( entry.getValue() ) ) );
        return result;
      }
      public CharSequence title() { return null; }
      public CharSequence uri() { return null; }
      public WordReader wordReader( int field ) { return new FastBufferedReader(); }
    };
  }
View Full Code Here
85
86
87
88
89
90
91
92
93
        if ( field != 0 ) throw new IndexOutOfBoundsException();
        return NullReader.getInstance();
      }

      public WordReader wordReader( final int field ) {
        return new FastBufferedReader();
      }
    };
  }
View Full Code Here
89
90
91
92
93
94
95
96
97
        if ( firstByte == -1 ) return NullReader.getInstance();
        return new CharArrayReader( new char[] { (char)firstByte } );
      }

      public WordReader wordReader( final int field ) {
        return new FastBufferedReader();
      }
    };
  }
View Full Code Here
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
  public Requirement(SimpleQuery seQuery) {
    for (SimpleQuery.Component c : seQuery.getSequence()) {
      if (c instanceof SimpleQuery.Term) {
        // FIXME: uses the same default reader as MG4J, but should be
        // adapted!
        FastBufferedReader reader = new FastBufferedReader();
        SimpleQuery.Term term = (SimpleQuery.Term) c;
        reader.setReader(new StringReader(term.getTerm()));
        MutableString word = new MutableString();
        MutableString nonWord = new MutableString();
        try {
          while (reader.next(word, nonWord)) {
            if (word.length() > 0)
              terms.add(new Term(term.getOperator(), word
                  .toString()));
          }
        } catch (IOException e) {
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
    composedBuilder
        .add(this.textExtractor = new StructuredTextExtractor());
    parser.setCallback(composedBuilder.compose());

    this.wordReader = new FastBufferedReader();
    text = new char[DEFAULT_BUFFER_SIZE];

    if (collectionType == null)
      collectionType = CollectionType.WARC018;
View Full Code Here
347
348
349
350
351
352
353
354
355
356
357
358
359
    public Object content(final int field) throws IOException {
      ensureFieldIndex(field);
      ensureParsed();
      switch (field) {
      case 0:
                return new FastBufferedReader(textExtractor.text);
      case 1:
        return new FastBufferedReader(textExtractor.title);
      case 2:
        return anchorExtractor.anchors;
      default:
        throw new IllegalArgumentException();
      }
View Full Code Here
108
109
110
111
112
113
114
115
                TRECParsingFactory.ELEMENT_FIRST,
                TRECParsingFactory.ELEMENT_SECOND
        );
        parser.setCallback(composedBuilder.compose());

        this.wordReader = new FastBufferedReader();
        text = new char[DEFAULT_BUFFER_SIZE];
    }
View Full Code Here
355
356
357
358
359
360
361
362
363
364
365
366
367
        public Object content(final int field) throws IOException {
            ensureFieldIndex(field);
            ensureParsed();
            switch (field) {
                case 0:
                    return new FastBufferedReader(textExtractor.text);
                case 1:
                    return new FastBufferedReader(textExtractor.title);
                case 2:
                    return anchorExtractor.anchors;
                default:
                    throw new IllegalArgumentException();
            }
View Full Code Here
TOP

Related Classes of it.unimi.dsi.io.FastBufferedReader

  • it.unimi.dsi.mg4j.document.FirstByteDocumentFactory
  • it.unimi.dsi.mg4j.document.HtmlDocumentFactory
  • it.unimi.dsi.mg4j.document.HtmlDocumentFactory$HtmlDocument
  • it.unimi.dsi.mg4j.document.IdentityDocumentFactory
  • it.unimi.dsi.mg4j.document.JavamailDocumentCollection
  • it.unimi.dsi.mg4j.document.JavamailDocumentCollection$JavamailDocumentFactory
  • it.unimi.dsi.mg4j.document.MapVirtualDocumentCollection
  • it.unimi.dsi.mg4j.document.NullDocumentFactory
  • it.unimi.dsi.mg4j.document.PdfDocumentFactory
  • it.unimi.dsi.mg4j.document.SimpleCompressedDocumentCollection
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.