Examples of org.apache.hadoop.io.UTF8

org.apache.hadoop.io.UTF8
A WritableComparable for strings that uses the UTF8 encoding.
Also includes utilities for efficiently reading and writing UTF-8. @deprecated replaced by Text

  /** 
   * Add set replication record to edit log
   */
  void logSetReplication(String src, short replication) {
    logEdit(OP_SET_REPLICATION, 
            new UTF8(src), 
            FSEditLog.toLogReplication(replication));
  }

View Full Code Here

  void logRemoveDatanode(DatanodeID nodeID) {
    logEdit(OP_DATANODE_REMOVE, new DatanodeID(nodeID), null);
  }
  
  static UTF8 toLogReplication(short replication) {
    return new UTF8(Short.toString(replication));
  }

View Full Code Here

    // concatenate strings
    if (field.startsWith("s:")) {
      String sSum = "";
      while (values.hasNext())
        sSum += ((UTF8) values.next()).toString() + ";";
      output.collect(key, new UTF8(sSum));
      reporter.setStatus("finished " + field + " ::host = " + hostName);
      return;
    }
    // sum long values
    if (field.startsWith("f:")) {
      float fSum = 0;
      while (values.hasNext())
        fSum += Float.parseFloat(((UTF8) values.next()).toString());
      output.collect(key, new UTF8(String.valueOf(fSum)));
      reporter.setStatus("finished " + field + " ::host = " + hostName);
      return;
    }
    // sum long values
    if (field.startsWith("l:")) {
      long lSum = 0;
      while (values.hasNext()) {
        lSum += Long.parseLong(((UTF8) values.next()).toString());
      }
      output.collect(key, new UTF8(String.valueOf(lSum)));
    }
    reporter.setStatus("finished " + field + " ::host = " + hostName);
  }

View Full Code Here

        //
        //           write nBytes of data using randomDataGenerator to numFiles
        //
        ArrayList<UTF8> testfilesList = new ArrayList<UTF8>();
        byte[] buffer = new byte[bufferSize];
        UTF8 testFileName = null;
        for (int iFileNumber = 0; iFileNumber < numFiles; iFileNumber++) {
          testFileName = new UTF8("/f" + iFileNumber);
          testfilesList.add(testFileName);
          OutputStream nos = dfsClient.create(testFileName, false);
          try {
            for (long nBytesWritten = 0L;
                 nBytesWritten < nBytes;

View Full Code Here

  public static class Map extends MapReduceBase implements Mapper {
    public void map(WritableComparable key, Writable value,
                    OutputCollector output, Reporter reporter) throws IOException 
    {
      String line = value.toString();
      output.collect(new UTF8(process(line)), new UTF8(""));    
    }

View Full Code Here

  public static class Reduce extends MapReduceBase implements Reducer {
    public void reduce(WritableComparable key, Iterator values,
                       OutputCollector output, Reporter reporter) throws IOException 
    {
      while(values.hasNext()) {
        output.collect(key, new UTF8(values.next().toString()));
      }
    }

View Full Code Here

      needToSave = (imgVersion != FSConstants.LAYOUT_VERSION);


      // read file info
      short replication = FSNamesystem.getFSNamesystem().getDefaultReplication();
      for (int i = 0; i < numFiles; i++) {
        UTF8 name = new UTF8();
        name.readFields(in);
        // version 0 does not support per file replication
        if (!(imgVersion >= 0)) {
          replication = in.readShort(); // other versions do
          replication = FSEditLog.adjustReplication(replication);
        }

View Full Code Here

                                FSDirectory.INode root, 
                                DataOutputStream out) throws IOException {
    String fullName = "";
    if (root.getParent() != null) {
      fullName = parentPrefix + "/" + root.getLocalName();
      new UTF8(fullName).write(out);
      out.writeShort(root.getReplication());
      if (root.isDir()) {
        out.writeInt(0);
      } else {
        int nrBlocks = root.getBlocks().length;

View Full Code Here

            fos = null;
          }
          continue;
        }
        if (fos == null) {
          fos = dfs.create(new UTF8(target.toString() + "/" + chain), true);
          if (fos != null) chain++;
        }
        if (fos == null) {
          LOG.warn(errmsg + ": could not store chain " + chain);
          // perhaps we should bail out here...
          // return;
          continue;
        }
        
        // copy the block. It's a pity it's not abstracted from DFSInputStream ...
        try {
          copyBlock(dfs, lblock, fos);
        } catch (Exception e) {
          e.printStackTrace();
          // something went wrong copying this block...
          LOG.warn(" - could not copy block " + lblock.getBlock().getBlockName() + " to " + target);
          fos.flush();
          fos.close();
          fos = null;
        }
      }
      if (fos != null) fos.close();
      LOG.warn("\n - moved corrupted file " + file.getPath() + " to /lost+found");
      dfs.delete(new UTF8(file.getPath()));
    }  catch (Exception e) {
      e.printStackTrace();
      LOG.warn(errmsg + ": " + e.getMessage());
    }
  }

View Full Code Here

  }
  
  private void lostFoundInit(DFSClient dfs) {
    lfInited = true;
    try {
      UTF8 lfName = new UTF8("/lost+found");
      // check that /lost+found exists
      if (!dfs.exists(lfName)) {
        lfInitedOk = dfs.mkdirs(lfName);
        lostFound = lfName;
      } else        if (!dfs.isDirectory(lfName)) {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.UTF8

org.apache.hadoop.benchmarks.mapred.BenchmarkMapper

org.apache.hadoop.benchmarks.mapred.BenchmarkReducer

org.apache.hadoop.dfs.ClusterTestDFS

org.apache.hadoop.dfs.ClusterTestDFSNamespaceLogging

org.apache.hadoop.dfs.DatanodeRegistration

org.apache.hadoop.dfs.DataStorage

org.apache.hadoop.dfs.DFSck

org.apache.hadoop.dfs.FSEditLog

org.apache.hadoop.dfs.FSImage

org.apache.hadoop.dfs.NamenodeFsck

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.