Package org.elasticsearch.river.wikipedia.bzip2

Examples of org.elasticsearch.river.wikipedia.bzip2.CBZip2InputStream$Data


                    if ( istream.read() != magic[ i ] )
                    {
                        throw new ArchiverException( "Invalid bz2 file." + file.toString() );
                    }
                }
                return new CBZip2InputStream( istream );
            }
            return istream;
        }
View Full Code Here


                        if ( istream.read() != magic[ i ] )
                        {
                            throw new ArchiverException( "Invalid bz2 file." + file.toString() );
                        }
                    }
                    return new CBZip2InputStream( istream );
                }
            }
            return istream;
        }
View Full Code Here

                        if ( istream.read() != magic[ i ] )
                        {
                            throw new ArchiverException( "Invalid bz2 file." + file.toString() );
                        }
                    }
                    return new CBZip2InputStream( istream );
                }
            }
            return istream;
        }
View Full Code Here

            br = new BufferedReader(new InputStreamReader(new GZIPInputStream(wikiXMLFile.openStream()), "UTF-8"));
        } else if (wikiXMLFile.toExternalForm().endsWith(".bz2")) {
            InputStream fis = wikiXMLFile.openStream();
            byte[] ignoreBytes = new byte[2];
            fis.read(ignoreBytes); //"B", "Z" bytes from commandline tools
            br = new BufferedReader(new InputStreamReader(new CBZip2InputStream(fis), "UTF-8"));
        } else {
            br = new BufferedReader(new InputStreamReader(wikiXMLFile.openStream(), "UTF-8"));
        }

        return new InputSource(br);
View Full Code Here

            br = new BufferedReader(new InputStreamReader(new GZIPInputStream(wikiXMLFile.openStream()), "UTF-8"));
        } else if (wikiXMLFile.toExternalForm().endsWith(".bz2")) {
            InputStream fis = wikiXMLFile.openStream();
            byte[] ignoreBytes = new byte[2];
            fis.read(ignoreBytes); //"B", "Z" bytes from commandline tools
            br = new BufferedReader(new InputStreamReader(new CBZip2InputStream(fis), "UTF-8"));
        } else {
            br = new BufferedReader(new InputStreamReader(wikiXMLFile.openStream(), "UTF-8"));
        }

        return new InputSource(br);
View Full Code Here

      Configuration configuration = project.getConfiguration();

      if (configuration != null)
      {
        Data data = configuration.getData();

        if (data != null)
        {
          ModuleDependencies moduleDependencies = data.getModuleDependencies();

          if (moduleDependencies != null)
          {
            dependencies = moduleDependencies.getDependency();
          }
View Full Code Here

      boolean eof = false;
      int numCompBytes;
      byte[] ubuff = new byte[40000];
      byte[] obuff = new byte[40000];
      try {
        CBZip2InputStream cbzip2 = new CBZip2InputStream();
        while (!eof) {
          try {
            numCompBytes = inputRaf.readInt();
            if (numCompBytes == -1) {
              if (log.isDebugEnabled()) log.debug("  done: numCompBytes=-1 ");
              break;
            }
          } catch (EOFException ee) {
            log.warn("  got EOFException ");
            break; // assume this is ok
          }

          if (log.isDebugEnabled()) {
            log.debug("reading compressed bytes " + numCompBytes + " input starts at " + inputRaf.getFilePointer() + "; output starts at " + outputRaf.getFilePointer());
          }
          /*
          * For some stupid reason, the last block seems to
          * have the number of bytes negated.  So, we just
          * assume that any negative number (other than -1)
          * is the last block and go on our merry little way.
          */
          if (numCompBytes < 0) {
            if (log.isDebugEnabled()) log.debug("last block?" + numCompBytes);
            numCompBytes = -numCompBytes;
            eof = true;
          }
          byte[] buf = new byte[numCompBytes];
          inputRaf.readFully(buf);
          ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);

          //CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
          cbzip2.setStream(bis);
          int total = 0;
          int nread;
          /*
          while ((nread = cbzip2.read(ubuff)) != -1) {
            dout2.write(ubuff, 0, nread);
            total += nread;
          }
          */
          try {
            while ((nread = cbzip2.read(ubuff)) != -1) {
              if (total + nread > obuff.length) {
                byte[] temp = obuff;
                obuff = new byte[temp.length * 2];
                System.arraycopy(temp, 0, obuff, 0, temp.length);
              }
View Full Code Here

    boolean eof = false;
    int numCompBytes;
    byte[] ubuff = new byte[40000];
    byte[] obuff = new byte[40000];
    try {
      CBZip2InputStream cbzip2 = new CBZip2InputStream();
      while (!eof) {

        try {
          numCompBytes = raf2.readInt();
          if (numCompBytes == -1) {
            if (debug) log.debug("  done: numCompBytes=-1 ");
            break;
          }
        } catch (EOFException ee) {
          if (debug) log.debug("  got EOFException ");
          break; // assume this is ok
        }

        if (debug) {
          log.debug("reading compressed bytes " + numCompBytes + " input starts at " + raf2.getFilePointer() + "; output starts at " + dout2.getFilePointer());
        }
        /*
        * For some stupid reason, the last block seems to
        * have the number of bytes negated.  So, we just
        * assume that any negative number (other than -1)
        * is the last block and go on our merry little way.
        */
        if (numCompBytes < 0) {
          if (debug) log.debug("last block?" + numCompBytes);
          numCompBytes = -numCompBytes;
          eof = true;
        }
        byte[] buf = new byte[numCompBytes];
        raf2.readFully(buf);
        ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);

        //CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
        cbzip2.setStream(bis);
        int total = 0;
        int nread;
        /*
        while ((nread = cbzip2.read(ubuff)) != -1) {
          dout2.write(ubuff, 0, nread);
          total += nread;
        }
        */
        try {
          while ((nread = cbzip2.read(ubuff)) != -1) {
            if (total + nread > obuff.length) {
              byte[] temp = obuff;
              obuff = new byte[temp.length * 2];
              System.arraycopy(temp, 0, obuff, 0, temp.length);
            }
View Full Code Here

        buf.get(header);
        byte[] out = new byte[offset+uncomplen];
        System.arraycopy(header, 0, out, 0, offset);


        CBZip2InputStream cbzip2 = new CBZip2InputStream();

        int numCompBytes = buf.remaining();
        byte[] bufc = new byte[numCompBytes];
        buf.get(bufc, 0, numCompBytes);

        ByteArrayInputStream bis = new ByteArrayInputStream(bufc, 2, numCompBytes - 2);

        //CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
        cbzip2.setStream(bis);
        int total = 0;
        int nread;
        byte[] ubuff = new byte[40000];
        byte[] obuff = new byte[40000];
        try {
            while ((nread = cbzip2.read(ubuff)) != -1) {
              if (total + nread > obuff.length) {
                byte[] temp = obuff;
                obuff = new byte[temp.length * 2];
                System.arraycopy(temp, 0, obuff, 0, temp.length);
              }
View Full Code Here

        in = new ZipInputStream(new FileInputStream(filename));
        copy(in, fout, 100000);
        if (debugCompress) System.out.println("unzipped " + filename + " to " + uncompressedFile);

      } else if (suffix.equalsIgnoreCase("bz2")) {
        in = new CBZip2InputStream(new FileInputStream(filename), true);
        copy(in, fout, 100000);
        if (debugCompress) System.out.println("unbzipped " + filename + " to " + uncompressedFile);

      } else if (suffix.equalsIgnoreCase("gzip") || suffix.equalsIgnoreCase("gz")) {
View Full Code Here

TOP

Related Classes of org.elasticsearch.river.wikipedia.bzip2.CBZip2InputStream$Data

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.