Package it.unimi.dsi.io

Examples of it.unimi.dsi.io.SegmentedInputStream

276
277
278
279
280
281
282
  public InputStream stream( final int n ) throws IOException {
    // Creates a Segmented Input Stream with only one segment in (the requested one).
    ensureDocumentIndex( n );
    IOUtils.closeQuietly( lastStream );
    final TRECDocumentDescriptor descr = descriptors.get( n );
    return lastStream = new SegmentedInputStream( openFileStream( file[ descr.fileIndex ] ), descr.toSegments() );
  }
View Full Code Here
354
355
356
357
358
359
360
361
362
363
364
        /*
         * We create the segmented input stream with all just collected
         * descriptors
         */
        siStream = new SegmentedInputStream( openFileStream( file[ currentFileIndex ] ) );

        do {
          siStream.addBlock( currentDescriptor.toSegments() );
          if ( ! descriptorIterator.hasNext() ) break;
          currentDescriptor = descriptorIterator.next();
View Full Code Here
817
818
819
820
821
822
823
824
825
826
827
        // check if we can read the substreams from the markers
        for (Iterator<long[]> iterator = markers.iterator(); iterator.hasNext(); ) {
            long[] ls = iterator.next();
            stream =
                    new BlockCompressedInputStream(new File(filename));
            SegmentedInputStream sis = new SegmentedInputStream(stream, ls[0], ls[1]);
            in = new DataInputStream(sis);
            while ((warcRecord = readNextWarcRecord(in)) != null) {
                System.out.println(warcRecord.toString());
                WarcHTMLResponseRecord w = new WarcHTMLResponseRecord(warcRecord);
                if (w.isHTMLResponse()) {
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
        SegmentedDocumentDescriptor currentDescriptor = firstNextDescriptor != null ? firstNextDescriptor
                : descriptorIterator.next();
        int currentFileIndex = currentDescriptor.fileIndex;

        // We create the segmented input stream with all just collected descriptors
        siStream = new SegmentedInputStream(collection.openFileStream(collection.files[currentFileIndex]));

        do {
            siStream.addBlock(currentDescriptor.toSegments());
            if (!descriptorIterator.hasNext())
                break;
View Full Code Here
196
197
198
199
200
201
202
        if (lastStream != null)
            lastStream.close();

        // FIXME: not efficient at all
        final SegmentedDocumentDescriptor descr = descriptors.get(n);
        return new SegmentedInputStream(openFileStream(files[descr.fileIndex]), descr.toSegments());
    }
View Full Code Here
TOP

Related Classes of it.unimi.dsi.io.SegmentedInputStream

  • it.unimi.dsi.mg4j.document.TRECDocumentCollection
  • net.bpiwowar.mg4j.extensions.segmented.SegmentedDocumentCollection
  • net.bpiwowar.mg4j.extensions.segmented.SegmentedDocumentIterator
  • net.bpiwowar.mg4j.extensions.warc.WarcRecord
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.