Examples of it.unimi.dsi.fastutil.io.FastBufferedInputStream

   * {@link IndexServer#GET_INDEX_READER}, &hellip;).
   */
  public RemoteIndexServerConnection( final SocketAddress address, final byte command ) throws IOException {
    socket = new Socket();
    socket.connect( address );
    inputStream = new DataInputStream( new FastBufferedInputStream( socket.getInputStream() ) );
    outputStream = new DataOutputStream( new FastBufferedOutputStream( socket.getOutputStream() ) );
    outputStream.writeByte( command );
    outputStream.flush();
  }

    
    public ServerThread( final Socket socket, final InputStream stream ) throws IOException {
      super( socket );
      //this.remotedInputStream = (FileInputStream)stream;
      // TODO: which buffer size?
      this.remotedInputStream = new FastBufferedInputStream( stream );
    }

    }


    towerData = new TowerData();
    tempFile = File.createTempFile( "MG4J", ".data" );
    tempFile.deleteOnExit();
    cacheDataIn = new FastBufferedInputStream( new FileInputStream( tempFile ) );
    cacheDataOut = new CachingOutputBitStream( tempFile, tempBufferSize );
    cacheDataLength = new int[ two2h ];
    cachePositionsLength = new long[ two2h + 1 ];
    cachePointer = new OutputBitStream[ two2h ];
    cachePointerByte = new FastByteArrayOutputStream[ two2h ];

      System.err.println();*/
    }


    towerData = new TowerData();
    tempFile = File.createTempFile( "MG4J", ".data" );
    cacheDataIn = new FastBufferedInputStream( new FileInputStream( tempFile ) );
    cacheDataOut = new CachingOutputBitStream( tempFile, tempBufferSize );
    cacheDataLength = new int[ two2h ];
    cachePointer = new OutputBitStream[ two2h ];
    cachePointerByte = new FastByteArrayOutputStream[ two2h ];

      docOffsets = loadOffsetsSuccinctly( basename + DOCUMENT_OFFSETS_EXTENSION, documents, new File( basename + DOCUMENTS_EXTENSION ).length() * Byte.SIZE + 1 );
      termOffsets = loadOffsetsSuccinctly( basename + TERM_OFFSETS_EXTENSION, terms, new File( basename + TERMS_EXTENSION ).length() + 1 );
      nonTermOffsets = nonTerms < 0 ? null : loadOffsetsSuccinctly( basename + NONTERM_OFFSETS_EXTENSION, nonTerms, new File( basename + NONTERMS_EXTENSION ).length() + 1 );


      documentsInputBitStream = documentsByteBufferInputStream != null ? new InputBitStream( documentsByteBufferInputStream ) : new InputBitStream( basename + DOCUMENTS_EXTENSION );
      termsInputStream = new FastBufferedInputStream( termsByteBufferInputStream != null ? termsByteBufferInputStream : new FileInputStream( basename + TERMS_EXTENSION ) );
      nonTermsInputStream = exact ? new FastBufferedInputStream( nonTermsByteBufferInputStream != null ? nonTermsByteBufferInputStream : new FileInputStream( basename + NONTERMS_EXTENSION ) ) : null;
      zipFile = hasNonText ? new ZipFile(  basename + ZipDocumentCollection.ZIP_EXTENSION ) : null;
      fileOpenOk = true;
    }
    catch( IOException e ) {
      // We leave the possibility for a filename() to fix the problem and load the right files.

  
  public Document document( int index ) throws IOException {
    ensureDocumentIndex( index );
    ensureFiles();
    documentsInputBitStream.position( docOffsets.getLong( index ) );
    final DataInputStream nonTextDataInputStream = hasNonText ? new DataInputStream( new FastBufferedInputStream( zipFile.getInputStream( zipFile.getEntry( Integer.toString( index ) ) ) ) ) : null;
    final MutableString uri = readSelfDelimitedUtf8String( documentsInputBitStream, new MutableString() );
    final MutableString title = readSelfDelimitedUtf8String( documentsInputBitStream, new MutableString() );


    return new AbstractDocument() {
      final MutableString fieldContent = new MutableString();

    new File( basename + DOCUMENTS_EXTENSION ).delete();
    newDocumentsFile.renameTo( new File( basename + DOCUMENTS_EXTENSION ) );
    newDocumentsObs = null;
    invTermPerm = invNonTermPerm = null;
    
    FastBufferedInputStream termsStream = new FastBufferedInputStream( new FileInputStream( basename + TERMS_EXTENSION ) ) ;
    MutableString term[] = new MutableString[ (int)collection.terms ];
    for( int i = 0; i < term.length; i++ ) term[ i ] = new MutableString().readSelfDelimUTF8( termsStream );
    termsStream.close();


    new FastBufferedOutputStream( new FileOutputStream( basename + TERMS_EXTENSION ) );
  }

    long currStart, currStop, currInter, oldPos;
    boolean pastHeader = false, startedBlock = false;


    LOGGER.debug( "Processing file " + fileIndex + " (" + file[ fileIndex ] + ")" );


    FastBufferedInputStream fbis = new FastBufferedInputStream( is, bufferSize );


    currStart = 0; // make java compiler happy.
    currInter = 0;
    oldPos = 0;
    
    int l;
    
    while ( ( l = fbis.readLine( buffer ) ) != -1 ) {
      if ( l == buffer.length ) {
        // We filled the buffer, which means we have a very very long line. Let's skip it.
        while ( ( l = fbis.readLine( buffer ) ) == buffer.length );
      }
      else {
        if ( !startedBlock && equals( buffer, l, DOC_OPEN ) ) {
          currStart = oldPos;
          startedBlock = true; // Start of the current block (includes <DOC> marker)
        }
        else if ( startedBlock && equals( buffer, l, DOC_CLOSE ) ) {
          currStop = oldPos;
          if ( DEBUG ) LOGGER.debug( "Setting markers <" + currStart + "," + currInter + ", " + currStop + ">" );
          descriptors.add( new TRECDocumentDescriptor( fileIndex, currStart, currInter, currStop ) );
          startedBlock = pastHeader = false;
        }
        else if ( startedBlock && !pastHeader && equals( buffer, l, DOCHDR_CLOSE ) ) {
          currInter = fbis.position();
          pastHeader = true;
        }
        oldPos = fbis.position();
      }
    }


    fbis.close();
  }

    pl.start( "Scanning files..." );
    
    // Scan files and retrieve page pointers
    for( String f : file ) {
      p.clear();
      final FastBufferedInputStream fbis = gzipped ? new FastBufferedInputStream( new GZIPInputStream( new FileInputStream( f ) ) ) : new FastBufferedInputStream( new FileInputStream( f ) );
      long position;
      for(;;) {
        position = fbis.position();
        if ( readLine( fbis ) == -1 ) break;
        if ( startsWith( lineBuffer, DOC_MARKER ) ) p.add( position );
        if ( phrase && startsWith( lineBuffer, SENTENCE_MARKER ) ) p.add( position );
      }
      
      count += p.size();
      p.add( fbis.position() );
      fbis.close();
      
      pointers.add( new EliasFanoMonotoneLongBigList( p ) );
      firstDocument[ pointers.size() ] = count;
      
      pl.update();

  @Override
  public DocumentIterator iterator() throws IOException {
    return new AbstractDocumentIterator() {
      private int index = 0;
      private int f = 0;
      private FastBufferedInputStream fbis = new FastBufferedInputStream( new FileInputStream( file[ 0 ] ) );
      
      public void close() throws IOException {
        super.close();
        if( fbis != null ) {
          fbis.close();
          fbis = null;
        }
      }


      public Document nextDocument() throws IOException {
        if ( index == size ) return null;
        if ( index == firstDocument[ f + 1 ] ) {
          fbis.close();
          fbis = new FastBufferedInputStream( new FileInputStream( file[ ++f ] ) );
        }
        
        readDocument( index, f, fbis );
        return document( index++ );
      }

Examples of it.unimi.dsi.fastutil.io.FastBufferedInputStream

Related Classes of it.unimi.dsi.fastutil.io.FastBufferedInputStream