Examples of org.apache.hadoop.io.UTF8

org.apache.hadoop.io.UTF8
A WritableComparable for strings that uses the UTF8 encoding.
Also includes utilities for efficiently reading and writing UTF-8. @deprecated replaced by Text

    return Short.parseShort(ustr.toString());
  }


  /** write the long value */
  static void writeShortAsString(short value, DataOutputStream out) throws IOException {
    UTF8 ustr = TL_DATA.get().U_STR;
    ustr.set(toLogShort(value), true);
    ustr.write(out);
  }

View Full Code Here

  }
  
  // Same comments apply for this method as for readString()
  @SuppressWarnings("deprecation")
  public static byte[] readBytes(DataInputStream in) throws IOException {
    UTF8 ustr = TL_DATA.get().U_STR;
    ustr.readFields(in);
    int len = ustr.getLength();
    byte[] bytes = new byte[len];
    System.arraycopy(ustr.getBytes(), 0, bytes, 0, len);
    return bytes;
  }

View Full Code Here

   * @throws IOException
   */
  @SuppressWarnings("deprecation")
  public static byte[][] readPathComponents(DataInputStream in)
      throws IOException {
    UTF8 ustr = TL_DATA.get().U_STR;
    
    ustr.readFields(in);
    return DFSUtil.bytes2byteArray(ustr.getBytes(),
      ustr.getLength(), (byte) Path.SEPARATOR_CHAR);
  }

View Full Code Here

    assertTrue(Arrays.equals(stringResults, new String[]{"foo","bar"}));


    stringResults = proxy.echo((String[])null);
    assertTrue(Arrays.equals(stringResults, null));


    UTF8 utf8Result = (UTF8)proxy.echo(new UTF8("hello world"));
    assertEquals(new UTF8("hello world"), utf8Result );


    utf8Result = (UTF8)proxy.echo((UTF8)null);
    assertEquals(null, utf8Result);


    int intResult = proxy.add(1, 2);

View Full Code Here

   * @param path starting point
   * @return result of checking
   * @throws Exception
   */
  public Result fsck(String path) throws Exception {
    DFSFileInfo[] files = dfs.listFiles(new UTF8(path));
    Result res = new Result();
    res.setReplication(conf.getInt("dfs.replication", 3));
    for (int i = 0; i < files.length; i++) {
      check(files[i], res);
    }

View Full Code Here

  private void check(DFSFileInfo file, Result res) throws Exception {
    if (file.isDir()) {
      if (showFiles)
        System.out.println(file.getPath() + " <dir>");
      res.totalDirs++;
      DFSFileInfo[] files = dfs.listFiles(new UTF8(file.getPath()));
      for (int i = 0; i < files.length; i++) {
        check(files[i], res);
      }
      return;
    }
    res.totalFiles++;
    res.totalSize += file.getLen();
    LocatedBlock[] blocks = dfs.namenode.open(file.getPath());
    res.totalBlocks += blocks.length;
    if (showFiles) {
      System.out.print(file.getPath() + " " + file.getLen() + ", " + blocks.length + " block(s): ");
    } else {
      System.out.print('.');
      System.out.flush();
      if (res.totalFiles % 100 == 0) System.out.println();
    }
    int missing = 0;
    long missize = 0;
    StringBuffer report = new StringBuffer();
    for (int i = 0; i < blocks.length; i++) {
      Block block = blocks[i].getBlock();
      long id = block.getBlockId();
      DatanodeInfo[] locs = blocks[i].getLocations();
      if (locs.length > res.replication) res.overReplicatedBlocks += (locs.length - res.replication);
      if (locs.length < res.replication && locs.length > 0) res.underReplicatedBlocks += (res.replication - locs.length);
      report.append(i + ". " + id + " len=" + block.getNumBytes());
      if (locs == null || locs.length == 0) {
        report.append(" MISSING!");
        res.addMissing(block.getBlockName(), block.getNumBytes());
        missing++;
        missize += block.getNumBytes();
      } else {
        report.append(" repl=" + locs.length);
        if (showLocations) {
          StringBuffer sb = new StringBuffer("[");
          for (int j = 0; j < locs.length; j++) {
            if (j > 0) sb.append(", ");
            sb.append(locs[j]);
          }
          sb.append(']');
          report.append(" " + sb.toString());
        }
      }
      report.append('\n');
    }
    if (missing > 0) {
      if (!showFiles)
        System.out.println("\nMISSING " + missing + " blocks of total size " + missize + " B");
      res.corruptFiles++;
      switch (fixing) {
        case FIXING_NONE: // do nothing
          System.err.println("\n - ignoring corrupted " + file.getPath());
          break;
        case FIXING_MOVE:
          System.err.println("\n - moving to /lost+found: " + file.getPath());
          lostFoundMove(file, blocks);
          break;
        case FIXING_DELETE:
          System.err.println("\n - deleting corrupted " + file.getPath());
          dfs.delete(new UTF8(file.getPath()));
      }
    }
    if (showFiles) {
      if (missing > 0) {
        System.out.println(" MISSING " + missing + " blocks of total size " + missize + " B");

View Full Code Here

      lostFoundInit();
    }
    if (!lfInitedOk) {
      return;
    }
    UTF8 target = new UTF8(lostFound.toString() + file.getPath());
    String errmsg = "Failed to move " + file.getPath() + " to /lost+found";
    try {
      if (!dfs.mkdirs(target)) {
        System.err.println(errmsg);
        return;
      }
      // create chains
      int chain = 0;
      FSOutputStream fos = null;
      for (int i = 0; i < blocks.length; i++) {
        LocatedBlock lblock = blocks[i];
        DatanodeInfo[] locs = lblock.getLocations();
        if (locs == null || locs.length == 0) {
          if (fos != null) {
            fos.flush();
            fos.close();
            fos = null;
          }
          continue;
        }
        if (fos == null) {
          fos = dfs.create(new UTF8(target.toString() + "/" + chain), true);
          if (fos != null) chain++;
        }
        if (fos == null) {
          System.err.println(errmsg + ": could not store chain " + chain);
          // perhaps we should bail out here...
          // return;
          continue;
        }
        
        // copy the block. It's a pity it's not abstracted from DFSInputStream ...
        try {
          copyBlock(lblock, fos);
        } catch (Exception e) {
          e.printStackTrace();
          // something went wrong copying this block...
          System.err.println(" - could not copy block " + lblock.getBlock().getBlockName() + " to " + target);
          fos.flush();
          fos.close();
          fos = null;
        }
      }
      if (fos != null) fos.close();
      System.err.println("\n - moved corrupted file " + file.getPath() + " to /lost+found");
      dfs.delete(new UTF8(file.getPath()));
    } catch (Exception e) {
      e.printStackTrace();
      System.err.println(errmsg + ": " + e.getMessage());
    }
  }

View Full Code Here

  }


  private void lostFoundInit() {
    lfInited = true;
    try {
      UTF8 lfName = new UTF8("/lost+found");
      // check that /lost+found exists
      if (!dfs.exists(lfName)) {
        lfInitedOk = dfs.mkdirs(lfName);
        lostFound = lfName;
      } else if (!dfs.isDirectory(lfName)) {

View Full Code Here

    assertTrue(Arrays.equals(stringResults, new String[]{"foo","bar"}));


    stringResults = proxy.echo((String[])null);
    assertTrue(Arrays.equals(stringResults, null));


    UTF8 utf8Result = (UTF8)proxy.echo(new UTF8("hello world"));
    assertEquals(utf8Result, new UTF8("hello world"));


    utf8Result = (UTF8)proxy.echo((UTF8)null);
    assertEquals(utf8Result, null);


    int intResult = proxy.add(1, 2);

View Full Code Here

    assertTrue(Arrays.equals(stringResults, new String[]{"foo","bar"}));


    stringResults = proxy.echo((String[])null);
    assertTrue(Arrays.equals(stringResults, null));


    UTF8 utf8Result = (UTF8)proxy.echo(new UTF8("hello world"));
    assertEquals(utf8Result, new UTF8("hello world"));


    utf8Result = (UTF8)proxy.echo((UTF8)null);
    assertEquals(utf8Result, null);


    int intResult = proxy.add(1, 2);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.UTF8

org.apache.hadoop.benchmarks.mapred.BenchmarkMapper

org.apache.hadoop.benchmarks.mapred.BenchmarkReducer

org.apache.hadoop.dfs.ClusterTestDFS

org.apache.hadoop.dfs.ClusterTestDFSNamespaceLogging

org.apache.hadoop.dfs.DatanodeRegistration

org.apache.hadoop.dfs.DataStorage

org.apache.hadoop.dfs.DFSck

org.apache.hadoop.dfs.FSEditLog

org.apache.hadoop.dfs.FSImage

org.apache.hadoop.dfs.NamenodeFsck

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.