Examples of net.nutch.io.UTF8

net.nutch.io.UTF8
A WritableComparable for strings that uses the UTF8 encoding.
Also includes utilities for efficiently reading and writing UTF-8. @author Doug Cutting

            LOG.fine(getName() + " got value #" + id);


          Call call = (Call)calls.remove(new Integer(id));
          boolean isError = in.readBoolean();     // read if error
          if (isError) {
            UTF8 utf8 = new UTF8();
            utf8.readFields(in);                  // read error string
            call.error = utf8.toString();
            call.value = null;
          } else {
            Writable value = makeValue();
            value.readFields(in);                 // read value
            call.value = value;

View Full Code Here

   */
  public void run() {
    BitSet bits = new BitSet(reader.maxDoc());
    AllHitsCollector ahc = new AllHitsCollector(bits);
    boolean doDelete = false;
    UTF8 url = new UTF8();
    for (int i = 0; i < queries.length; i++) {
      LOG.info(dr + "Processing query: " + queries[i].toString());
      bits.clear();
      try {
        searcher.search(queries[i], ahc);

View Full Code Here

          DataOutputStream out = call.connection.out;
          synchronized (out) {
            out.writeInt(call.id);                // write call id
            out.writeBoolean(error!=null);        // write error flag
            if (error != null)
              value = new UTF8(error);
            value.write(out);                     // write value
            out.flush();
          }


        } catch (Exception e) {

View Full Code Here

        //    public boolean isHidden()
        // so you can not rely on returned list for these operations.
        NDFSFile[] nfiles = new NDFSFile[files.length];
        for (int i = 0; i < files.length; i++) {
            long len = files[i].length();
            UTF8 name = new UTF8(files[i].toString());
            NDFSFileInfo info = new NDFSFileInfo(name, len, len, files[i].isDirectory());
            nfiles[i] = new NDFSFile(info);
        }
        return nfiles;
    }

View Full Code Here


        /** Keys are longs. */
        public Writable createKey() { return new LongWritable(); }


        /** Values are lines. */
        public Writable createValue() { return new UTF8(); }


        /** Read a line. */
        public boolean next(Writable key, Writable value) throws IOException {
          long pos = in.getPos();
          if (pos >= end)

View Full Code Here

      nfs.delete(sortedFile);
      SequenceFile.Writer seqWriter = new SequenceFile.Writer(nfs,
              unsortedFile.toString(), UTF8.class, LongWritable.class);
      FetchListEntry fle;
      LongWritable rec = new LongWritable();
      UTF8 url = new UTF8();
      String urlString;
      while (fetcherReader.next(fo) != null) {
        fle = fo.getFetchListEntry();
        urlString = fle.getPage().getURL().toString();
        rec.set(recNo);
        url.set(urlString);
        seqWriter.append(url, rec);
        recNo++;
      }
      seqWriter.close();
      // sort the SequenceFile

View Full Code Here

TOP

Related Classes of net.nutch.io.UTF8

net.nutch.fs.LocalFileSystem

net.nutch.ipc.Client$Connection

net.nutch.ipc.Server$Handler

net.nutch.mapReduce.TextInputFormat

net.nutch.segment.SegmentReader

net.nutch.tools.PruneIndexTool

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.