Examples of it.unimi.dsi.fastutil.io.FastBufferedOutputStream

it.unimi.dsi.fastutil.io.FastBufferedOutputStream
Lightweight, unsynchronized output stream buffering class with {@linkplain MeasurableStream measurability} and{@linkplain RepositionableStream repositionability}.
This class provides buffering for output streams, but it does so with purposes and an internal logic that are radically different from the ones adopted in {@link java.io.BufferedOutputStream}. The main features follow.
- All methods are unsychronized.
- As an additional feature, this class implements the {@link RepositionableStream} and {@link MeasurableStream} interfaces. An instance of this class will try to cast the underlying byte stream to a {@link RepositionableStream} and to fetch byreflection the {@link java.nio.channels.FileChannel} underlying the givenoutput stream, in this order. If either reference can be successfully fetched, you can use {@link #position(long)} to reposition the stream.Much in the same way, an instance of this class will try to cast the the underlying byte stream to a {@link MeasurableStream}, and if this operation is successful, or if a {@link java.nio.channels.FileChannel} canbe detected, then {@link #position()} and {@link #length()} will work as expected.
@since 4.4

   */
  public RemoteIndexServerConnection( final SocketAddress address, final byte command ) throws IOException {
    socket = new Socket();
    socket.connect( address );
    inputStream = new DataInputStream( new FastBufferedInputStream( socket.getInputStream() ) );
    outputStream = new DataOutputStream( new FastBufferedOutputStream( socket.getOutputStream() ) );
    outputStream.writeByte( command );
    outputStream.flush();
  }

View Full Code Here

    
    final PrintWriter[] localTerms = new PrintWriter[ numIndices ]; 
    final int numTerms[] = new int[ numIndices ];
    final FastBufferedReader terms = new FastBufferedReader( new InputStreamReader( new FileInputStream( inputBasename + DiskBasedIndex.TERMS_EXTENSION ), "UTF-8" ) );
    
    for( int i = 0; i < numIndices; i++ ) localTerms[ i ] = new PrintWriter( new OutputStreamWriter( new FastBufferedOutputStream( new FileOutputStream( localBasename[ i ] + DiskBasedIndex.TERMS_EXTENSION ) ), "UTF-8" ) );


    // The current term
    final MutableString currTerm = new MutableString();
    
    pl.itemsName = "terms";

View Full Code Here

    for( int i = 0; i < numIndices; i++ ) {
      localIndexStream[ i ] = new OutputBitStream( localBasename[ i ] + DiskBasedIndex.INDEX_EXTENSION, bufferSize );
      if ( isHighPerformance ) localPositionsStream[ i ] = new OutputBitStream( localBasename[ i ] + DiskBasedIndex.POSITIONS_EXTENSION, bufferSize );
      localFrequencies[ i ] = new OutputBitStream( localBasename[ i ] + DiskBasedIndex.FREQUENCIES_EXTENSION );
      localGlobCounts[ i ] = new OutputBitStream( localBasename[ i ] + DiskBasedIndex.GLOBCOUNTS_EXTENSION );
      localTerms[ i ] = new PrintWriter( new OutputStreamWriter( new FastBufferedOutputStream( new FileOutputStream( localBasename[ i ] + DiskBasedIndex.TERMS_EXTENSION ) ), "UTF-8" ) );
      localOffsets[ i ] = new OutputBitStream( localBasename[ i ] + DiskBasedIndex.OFFSETS_EXTENSION );
      if ( posNumBits != null ) localPosNumBits[ i ] = new OutputBitStream( localBasename[ i ] + DiskBasedIndex.POSITIONS_NUMBER_OF_BITS_EXTENSION );
      localOffsets[ i ].writeGamma( 0 );
    }

View Full Code Here

    if ( ASSERTS ) assert numTerms == termMap.size();
    if ( ! indexingIsVirtual ) sizes.close();


    // We sort the terms appearing in the batch and write them on disk.
    ObjectArrays.quickSort( termArray );
    final PrintWriter pw = new PrintWriter( new OutputStreamWriter( new FastBufferedOutputStream( new FileOutputStream( batchBasename + DiskBasedIndex.TERMS_EXTENSION ), bufferSize ), "UTF-8" ) );
    for ( MutableString t : termArray ) t.println( pw );
    pw.close();


    try {
      final OutputBitStream frequencies = new OutputBitStream( batchBasename + DiskBasedIndex.FREQUENCIES_EXTENSION );

View Full Code Here

  }
    
  public void open( final CharSequence suffix ) throws IOException {
    basenameSuffix = basename + suffix;
    documentsOutputBitStream = new OutputBitStream( basenameSuffix + SimpleCompressedDocumentCollection.DOCUMENTS_EXTENSION );
    termsOutputStream = new CountingOutputStream( new FastBufferedOutputStream( new FileOutputStream( basenameSuffix + SimpleCompressedDocumentCollection.TERMS_EXTENSION ) ) );
    nonTermsOutputStream = exact ? new CountingOutputStream( new FastBufferedOutputStream( new FileOutputStream( basenameSuffix + SimpleCompressedDocumentCollection.NONTERMS_EXTENSION ) ) ) : null;
    documentOffsetsObs = new OutputBitStream( basenameSuffix + SimpleCompressedDocumentCollection.DOCUMENT_OFFSETS_EXTENSION );
    termOffsetsObs = new OutputBitStream( basenameSuffix + SimpleCompressedDocumentCollection.TERM_OFFSETS_EXTENSION );
    nonTermOffsetsObs = exact? new OutputBitStream( basenameSuffix + SimpleCompressedDocumentCollection.NONTERM_OFFSETS_EXTENSION ) : null;
    fieldContent = new IntArrayList();


    if ( hasNonText ) nonTextZipDataOutputStream = new DataOutputStream( nonTextZipOutputStream = new ZipOutputStream( new FastBufferedOutputStream( new FileOutputStream( basenameSuffix + ZipDocumentCollection.ZIP_EXTENSION ) ) ) );


    terms.clear();
    terms.trim( Scan.INITIAL_TERM_MAP_SIZE );
    if ( exact ) {
      nonTerms.clear();

View Full Code Here

    Matcher matcher = pattern.matcher( "" );
    
    final int h = jsapResult.getInt( "h" );
    final String basename = jsapResult.getString( "basename" );
    final PrintStream printStream[] = new PrintStream[ h + 1 ];
    for( int i = 0; i <= h; i++ ) printStream[ i ] = new PrintStream( new FastBufferedOutputStream( new FileOutputStream( basename + "-" + i + DiskBasedIndex.STATS_EXTENSION ) ) );
    
    final MutableString line = new MutableString();
    final FastBufferedReader reader = new FastBufferedReader( new InputStreamReader( System.in ) );
    
    int l, start;

View Full Code Here

    FastBufferedInputStream termsStream = new FastBufferedInputStream( new FileInputStream( basename + TERMS_EXTENSION ) ) ;
    MutableString term[] = new MutableString[ (int)collection.terms ];
    for( int i = 0; i < term.length; i++ ) term[ i ] = new MutableString().readSelfDelimUTF8( termsStream );
    termsStream.close();


    new FastBufferedOutputStream( new FileOutputStream( basename + TERMS_EXTENSION ) );
  }

View Full Code Here

    Util.ensureLog4JIsConfigured();


    final DocumentIterator documentIterator = documentSequence.iterator();


    Document document;
    FastBufferedOutputStream uriStream = null, titleStream = null;
    
    if ( jsapResult.userSpecified( "uris" ) ) uriStream = new FastBufferedOutputStream( new FileOutputStream( jsapResult.getString( "uris" ) ) );
    if ( jsapResult.userSpecified( "titles" ) ) titleStream = new FastBufferedOutputStream( new FileOutputStream( jsapResult.getString( "titles" ) ) );
    
    MutableString s = new MutableString();


    ProgressLogger progressLogger = new ProgressLogger( LOGGER, jsapResult.getLong( "logInterval" ), "documents" );
    if ( documentSequence instanceof DocumentCollection ) progressLogger.expectedUpdates = ((DocumentCollection)documentSequence).size();
    progressLogger.start( "Scanning..." );
    
    while( ( document = documentIterator.nextDocument() ) != null ) {
      if ( uriStream != null ) {
        s.replace( document.uri() );
        s.replace( LINE_TERMINATORS, SPACES );
        s.writeUTF8( uriStream );
        uriStream.write( '\n' );
      }
      if ( titleStream != null ) {
        s.replace( document.title() );
        s.replace( LINE_TERMINATORS, SPACES );
        s.writeUTF8( titleStream );
        titleStream.write( '\n' );
      }
      progressLogger.lightUpdate();
    }
    
    progressLogger.done();
    if ( uriStream != null ) uriStream.close();
    if ( titleStream != null ) titleStream.close();
  }

View Full Code Here

      if ( part.length > 2 ) System.err.println( "Wrong argument(s) to command" );
      else {
        if ( output != System.out )
          output.close();
        try {
          output = part.length == 1 ? System.out : new PrintStream( new FastBufferedOutputStream( new FileOutputStream( part[ 1 ] ) ) );
        }
        catch ( FileNotFoundException e ) {
          System.err.println( "Cannot create file " + part[ 1 ] );
          output = System.out;
        }

View Full Code Here

        pl.start( "Copying URIs..." );
        final LineIterator termIterator = new LineIterator( new FastBufferedReader( new InputStreamReader( new FileInputStream( termFile ) ), bufferSize ), pl );
        File temp = File.createTempFile( URLMPHVirtualDocumentResolver.class.getName(), ".uniqueuris" );
        temp.deleteOnExit();
        termFile = temp.toString();
        final FastBufferedOutputStream outputStream = new FastBufferedOutputStream( new FileOutputStream( termFile ), bufferSize );
        MutableString uri;
        while( termIterator.hasNext() ) {
          uri = termIterator.next();
          makeUnique( filter, uri );
          uri.writeUTF8( outputStream );
          outputStream.write( '\n' );
        }
        pl.done();
        outputStream.close();
      }
      collection = new FileLinesCollection( termFile, "UTF-8" );
    }
    LOGGER.debug( "Building function..." );
    final int width = jsapResult.getInt( "width" );

View Full Code Here

0 1

TOP

Related Classes of it.unimi.dsi.fastutil.io.FastBufferedOutputStream

it.unimi.dsi.mg4j.document.SimpleCompressedDocumentCollection

it.unimi.dsi.mg4j.document.SimpleCompressedDocumentCollectionBuilder

it.unimi.dsi.mg4j.index.remote.RemoteIndexServerConnection

it.unimi.dsi.mg4j.query.Query

it.unimi.dsi.mg4j.test.SelectHeights

it.unimi.dsi.mg4j.tool.PartitionLexically

it.unimi.dsi.mg4j.tool.Scan

it.unimi.dsi.mg4j.tool.ScanMetadata

it.unimi.dsi.mg4j.tool.URLMPHVirtualDocumentResolver

org.archive.io.CrawlerJournal

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.