Package org.apache.hadoop.hdfs

Examples of org.apache.hadoop.hdfs.DFSClient


    testClientRetryWithFailover(op);
  }
 
  @Test (timeout=60000)
  public void testRename2() throws Exception {
    final DFSClient client = genClientWithDummyHandler();
    AtMostOnceOp op = new Rename2Op(client, "/file1", "/file2");
    testClientRetryWithFailover(op);
  }
View Full Code Here


    testClientRetryWithFailover(op);
  }
 
  @Test (timeout=60000)
  public void testConcat() throws Exception {
    final DFSClient client = genClientWithDummyHandler();
    AtMostOnceOp op = new ConcatOp(client, new Path("/test/file"), 5);
    testClientRetryWithFailover(op);
  }
View Full Code Here

    testClientRetryWithFailover(op);
  }
 
  @Test (timeout=60000)
  public void testDelete() throws Exception {
    final DFSClient client = genClientWithDummyHandler();
    AtMostOnceOp op = new DeleteOp(client, "/testfile");
    testClientRetryWithFailover(op);
  }
View Full Code Here

    testClientRetryWithFailover(op);
  }
 
  @Test (timeout=60000)
  public void testCreateSymlink() throws Exception {
    final DFSClient client = genClientWithDummyHandler();
    AtMostOnceOp op = new CreateSymlinkOp(client, "/testfile", "/testlink");
    testClientRetryWithFailover(op);
  }
View Full Code Here

    testClientRetryWithFailover(op);
  }
 
  @Test (timeout=60000)
  public void testUpdatePipeline() throws Exception {
    final DFSClient client = genClientWithDummyHandler();
    AtMostOnceOp op = new UpdatePipelineOp(client, "/testfile");
    testClientRetryWithFailover(op);
  }
View Full Code Here

    }
  }
 
  private void copyBlocksToLostFound(String parent, HdfsFileStatus file,
        LocatedBlocks blocks) throws IOException {
    final DFSClient dfs = new DFSClient(NameNode.getAddress(conf), conf);
    final String fullName = file.getFullName(parent);
    OutputStream fos = null;
    try {
      if (!lfInited) {
        lostFoundInit(dfs);
      }
      if (!lfInitedOk) {
        throw new IOException("failed to initialize lost+found");
      }
      String target = lostFound + fullName;
      if (hdfsPathExists(target)) {
        LOG.warn("Fsck: can't copy the remains of " + fullName + " to " +
          "lost+found, because " + target + " already exists.");
        return;
      }
      if (!namenode.getRpcServer().mkdirs(
          target, file.getPermission(), true)) {
        throw new IOException("failed to create directory " + target);
      }
      // create chains
      int chain = 0;
      boolean copyError = false;
      for (LocatedBlock lBlk : blocks.getLocatedBlocks()) {
        LocatedBlock lblock = lBlk;
        DatanodeInfo[] locs = lblock.getLocations();
        if (locs == null || locs.length == 0) {
          if (fos != null) {
            fos.flush();
            fos.close();
            fos = null;
          }
          continue;
        }
        if (fos == null) {
          fos = dfs.create(target + "/" + chain, true);
          if (fos == null) {
            throw new IOException("Failed to copy " + fullName +
                " to /lost+found: could not store chain " + chain);
          }
          chain++;
        }
       
        // copy the block. It's a pity it's not abstracted from DFSInputStream ...
        try {
          copyBlock(dfs, lblock, fos);
        } catch (Exception e) {
          LOG.error("Fsck: could not copy block " + lblock.getBlock() +
              " to " + target, e);
          fos.flush();
          fos.close();
          fos = null;
          internalError = true;
          copyError = true;
        }
      }
      if (copyError) {
        LOG.warn("Fsck: there were errors copying the remains of the " +
          "corrupted file " + fullName + " to /lost+found");
      } else {
        LOG.info("Fsck: copied the remains of the corrupted file " +
          fullName + " to /lost+found");
      }
    } catch (Exception e) {
      LOG.error("copyBlocksToLostFound: error processing " + fullName, e);
      internalError = true;
    } finally {
      if (fos != null) fos.close();
      dfs.close();
    }
  }
View Full Code Here

                                                 InterruptedException {
    return
      user.doAs(new PrivilegedExceptionAction<DFSClient>() {
        @Override
        public DFSClient run() throws IOException {
          return new DFSClient(NetUtils.createSocketAddr(addr), conf);
        }
      });
  }
View Full Code Here

    if (nnAddr == null){
      out.print(JspHelper.NAMENODE_ADDRESS + " url param is null");
      return;
    }
   
    DFSClient dfs = getDFSClient(ugi, nnAddr, conf);
    String target = dir;
    final HdfsFileStatus targetStatus = dfs.getFileInfo(target);
    if (targetStatus == null) { // not exists
      out.print("<h3>File or directory : " + target + " does not exist</h3>");
      JspHelper.printGotoForm(out, namenodeInfoPort, tokenString, target,
          nnAddr);
    } else {
      if (!targetStatus.isDir()) { // a file
        List<LocatedBlock> blocks = dfs.getNamenode().getBlockLocations(dir, 0, 1)
            .getLocatedBlocks();

        LocatedBlock firstBlock = null;
        DatanodeInfo[] locations = null;
        if (blocks.size() > 0) {
          firstBlock = blocks.get(0);
          locations = firstBlock.getLocations();
        }
        if (locations == null || locations.length == 0) {
          out.print("Empty file");
        } else {
          DatanodeInfo chosenNode = JspHelper.bestNode(firstBlock, conf);
          String fqdn = canonicalize(chosenNode.getIpAddr());
          int datanodePort = chosenNode.getXferPort();
          String redirectLocation = HttpConfig.getSchemePrefix() + fqdn + ":"
              + chosenNode.getInfoPort() + "/browseBlock.jsp?blockId="
              + firstBlock.getBlock().getBlockId() + "&blockSize="
              + firstBlock.getBlock().getNumBytes() + "&genstamp="
              + firstBlock.getBlock().getGenerationStamp() + "&filename="
              + URLEncoder.encode(dir, "UTF-8") + "&datanodePort="
              + datanodePort + "&namenodeInfoPort=" + namenodeInfoPort
              + JspHelper.getDelegationTokenUrlParam(tokenString)
              + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
          resp.sendRedirect(redirectLocation);
        }
        return;
      }
      // directory
      // generate a table and dump the info
      String[] headings = { "Name", "Type", "Size", "Replication",
          "Block Size", "Modification Time", "Permission", "Owner", "Group" };
      out.print("<h3>Contents of directory ");
      JspHelper.printPathWithLinks(dir, out, namenodeInfoPort, tokenString,
          nnAddr);
      out.print("</h3><hr>");
      JspHelper.printGotoForm(out, namenodeInfoPort, tokenString, dir, nnAddr);
      out.print("<hr>");

      File f = new File(dir);
      String parent;
      if ((parent = f.getParent()) != null)
        out.print("<a href=\"" + req.getRequestURL() + "?dir=" + parent
            + "&namenodeInfoPort=" + namenodeInfoPort
            + JspHelper.getDelegationTokenUrlParam(tokenString)
            + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr)
            + "\">Go to parent directory</a><br>");

      DirectoryListing thisListing =
        dfs.listPaths(target, HdfsFileStatus.EMPTY_NAME);
      if (thisListing == null || thisListing.getPartialListing().length == 0) {
        out.print("Empty directory");
      } else {
        JspHelper.addTableHeader(out);
        int row = 0;
        JspHelper.addTableRow(out, headings, row++);
        String cols[] = new String[headings.length];
        do {
          HdfsFileStatus[] files = thisListing.getPartialListing();
          for (int i = 0; i < files.length; i++) {
            String localFileName = files[i].getLocalName();
            // Get the location of the first block of the file
            if (!files[i].isDir()) {
              cols[1] = "file";
              cols[2] = StringUtils.byteDesc(files[i].getLen());
              cols[3] = Short.toString(files[i].getReplication());
              cols[4] = StringUtils.byteDesc(files[i].getBlockSize());
            } else {
              cols[1] = "dir";
              cols[2] = "";
              cols[3] = "";
              cols[4] = "";
            }
            String datanodeUrl = req.getRequestURL() + "?dir="
              + URLEncoder.encode(files[i].getFullName(target), "UTF-8")
              + "&namenodeInfoPort=" + namenodeInfoPort
              + JspHelper.getDelegationTokenUrlParam(tokenString)
              + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
            cols[0] = "<a href=\"" + datanodeUrl + "\">"
              + localFileName + "</a>";
            cols[5] = lsDateFormat.format(new Date((files[i]
              .getModificationTime())));
            cols[6] = files[i].getPermission().toString();
            cols[7] = files[i].getOwner();
            cols[8] = files[i].getGroup();
            JspHelper.addTableRow(out, cols, row++);
          }
          if (!thisListing.hasMore()) {
            break;
          }
          thisListing = dfs.listPaths(target, thisListing.getLastName());
        } while (thisListing != null);
        JspHelper.addTableFooter(out);
      }
    }
    out.print("<br><a href=\"" + HttpConfig.getSchemePrefix()
        + canonicalize(nnAddr) + ":"
        + namenodeInfoPort + "/dfshealth.jsp\">Go back to DFS home</a>");
    dfs.close();
  }
View Full Code Here

      out.print("Invalid input");
      return;
    }
    long blockSize = Long.parseLong(blockSizeStr);

    final DFSClient dfs = getDFSClient(ugi, nnAddr, conf);
    List<LocatedBlock> blocks = dfs.getNamenode().getBlockLocations(filename, 0,
        Long.MAX_VALUE).getLocatedBlocks();
    // Add the various links for looking at the file contents
    // URL for downloading the full file
    String downloadUrl = HttpConfig.getSchemePrefix() + req.getServerName() + ":"
        + req.getServerPort() + "/streamFile" + ServletUtil.encodePath(filename)
        + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr, true)
        + JspHelper.getDelegationTokenUrlParam(tokenString);
    out.print("<a name=\"viewOptions\"></a>");
    out.print("<a href=\"" + downloadUrl + "\">Download this file</a><br>");

    DatanodeInfo chosenNode;
    // URL for TAIL
    LocatedBlock lastBlk = blocks.get(blocks.size() - 1);
    try {
      chosenNode = JspHelper.bestNode(lastBlk, conf);
    } catch (IOException e) {
      out.print(e.toString());
      dfs.close();
      return;
    }
    String fqdn = canonicalize(chosenNode.getIpAddr());
    String tailUrl = HttpConfig.getSchemePrefix() + fqdn + ":" + chosenNode.getInfoPort()
        + "/tail.jsp?filename=" + URLEncoder.encode(filename, "UTF-8")
        + "&namenodeInfoPort=" + namenodeInfoPort
        + "&chunkSizeToView=" + chunkSizeToView
        + JspHelper.getDelegationTokenUrlParam(tokenString)
        + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr)
        + "&referrer=" + URLEncoder.encode(
            req.getRequestURL() + "?" + req.getQueryString(), "UTF-8");
    out.print("<a href=\"" + tailUrl + "\">Tail this file</a><br>");

    out.print("<form action=\"/browseBlock.jsp\" method=GET>");
    out.print("<b>Chunk size to view (in bytes, up to file's DFS block size): </b>");
    out.print("<input type=\"hidden\" name=\"blockId\" value=\"" + blockId
        + "\">");
    out.print("<input type=\"hidden\" name=\"blockSize\" value=\"" + blockSize
        + "\">");
    out.print("<input type=\"hidden\" name=\"startOffset\" value=\""
        + startOffset + "\">");
    out.print("<input type=\"hidden\" name=\"filename\" value=\"" + filename
        + "\">");
    out.print("<input type=\"hidden\" name=\"genstamp\" value=\"" + genStamp
        + "\">");
    out.print("<input type=\"hidden\" name=\"datanodePort\" value=\""
        + datanodePort + "\">");
    out.print("<input type=\"hidden\" name=\"namenodeInfoPort\" value=\""
        + namenodeInfoPort + "\">");
    out.print("<input type=\"hidden\" name=\"" + JspHelper.NAMENODE_ADDRESS
        + "\" value=\"" + nnAddr + "\">");
    out.print("<input type=\"text\" name=\"chunkSizeToView\" value="
        + chunkSizeToView + " size=10 maxlength=10>");
    out.print("&nbsp;&nbsp;<input type=\"submit\" name=\"submit\" value=\"Refresh\">");
    out.print("</form>");
    out.print("<hr>");
    out.print("<a name=\"blockDetails\"></a>");
    out.print("<B>Total number of blocks: " + blocks.size() + "</B><br>");
    // generate a table and dump the info
    out.println("\n<table>");
   
    String nnCanonicalName = canonicalize(nnAddr);
    for (LocatedBlock cur : blocks) {
      out.print("<tr>");
      final String blockidstring = Long.toString(cur.getBlock().getBlockId());
      blockSize = cur.getBlock().getNumBytes();
      out.print("<td>" + blockidstring + ":</td>");
      DatanodeInfo[] locs = cur.getLocations();
      for (int j = 0; j < locs.length; j++) {
        String datanodeAddr = locs[j].getXferAddr();
        datanodePort = locs[j].getXferPort();
        fqdn = canonicalize(locs[j].getIpAddr());
        String blockUrl = HttpConfig.getSchemePrefix() + fqdn + ":" + locs[j].getInfoPort()
            + "/browseBlock.jsp?blockId=" + blockidstring
            + "&blockSize=" + blockSize
            + "&filename=" + URLEncoder.encode(filename, "UTF-8")
            + "&datanodePort=" + datanodePort
            + "&genstamp=" + cur.getBlock().getGenerationStamp()
            + "&namenodeInfoPort=" + namenodeInfoPort
            + "&chunkSizeToView=" + chunkSizeToView
            + JspHelper.getDelegationTokenUrlParam(tokenString)
            + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);

        String blockInfoUrl = HttpConfig.getSchemePrefix() + nnCanonicalName + ":"
            + namenodeInfoPort
            + "/block_info_xml.jsp?blockId=" + blockidstring;
        out.print("<td>&nbsp</td><td><a href=\"" + blockUrl + "\">"
            + datanodeAddr + "</a></td><td>"
            + "<a href=\"" + blockInfoUrl + "\">View Block Info</a></td>");
      }
      out.println("</tr>");
    }
    out.println("</table>");
    out.print("<hr>");
    out.print("<br><a href=\"" + HttpConfig.getSchemePrefix()
        + nnCanonicalName + ":"
        + namenodeInfoPort + "/dfshealth.jsp\">Go back to DFS home</a>");
    dfs.close();
  }
View Full Code Here

    if (blockId == null) {
      out.print("Invalid input (blockId absent)");
      return;
    }
   
    final DFSClient dfs = getDFSClient(ugi, nnAddr, conf);

    String bpid = null;
    Token<BlockTokenIdentifier> blockToken = BlockTokenSecretManager.DUMMY_TOKEN;
    List<LocatedBlock> blks = dfs.getNamenode().getBlockLocations(filename, 0,
        Long.MAX_VALUE).getLocatedBlocks();
    if (blks == null || blks.size() == 0) {
      out.print("Can't locate file blocks");
      dfs.close();
      return;
    }

    boolean needBlockToken = conf.getBoolean(
            DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY,
            DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);

    for (int i = 0; i < blks.size(); i++) {
      if (blks.get(i).getBlock().getBlockId() == blockId) {
        bpid = blks.get(i).getBlock().getBlockPoolId();
        if (needBlockToken) {
          blockToken = blks.get(i).getBlockToken();
        }
        break;
      }
    }

    final Long genStamp = JspHelper.validateLong(req.getParameter("genstamp"));
    if (genStamp == null) {
      out.print("Invalid input (genstamp absent)");
      return;
    }

    long blockSize = 0;
    final String blockSizeStr = req.getParameter("blockSize");
    if (blockSizeStr == null) {
      out.print("Invalid input (blockSize absent)");
      return;
    }
    blockSize = Long.parseLong(blockSizeStr);

    final int chunkSizeToView = JspHelper.string2ChunkSizeToView(req
        .getParameter("chunkSizeToView"), getDefaultChunkSize(conf));

    String startOffsetStr = req.getParameter("startOffset");
    if (startOffsetStr == null || Long.parseLong(startOffsetStr) < 0)
      startOffset = 0;
    else
      startOffset = Long.parseLong(startOffsetStr);

    String datanodePortStr = req.getParameter("datanodePort");
    if (datanodePortStr == null) {
      out.print("Invalid input (datanodePort absent)");
      return;
    }
    datanodePort = Integer.parseInt(datanodePortStr);
    out.print("<h3>File: ");
    JspHelper.printPathWithLinks(filename, out, namenodeInfoPort,
                                 tokenString, nnAddr);
    out.print("</h3><hr>");
    String parent = new File(filename).getParent();
    JspHelper.printGotoForm(out, namenodeInfoPort, tokenString, parent, nnAddr);
    out.print("<hr>");
    out.print("<a href=\"" + HttpConfig.getSchemePrefix()
        + req.getServerName() + ":" + req.getServerPort()
        + "/browseDirectory.jsp?dir=" + URLEncoder.encode(parent, "UTF-8")
        + "&namenodeInfoPort=" + namenodeInfoPort
        + JspHelper.getDelegationTokenUrlParam(tokenString)
        + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr)
        + "\"><i>Go back to dir listing</i></a><br>");
    out.print("<a href=\"#viewOptions\">Advanced view/download options</a><br>");
    out.print("<hr>");

    // Determine the prev & next blocks
    long nextStartOffset = 0;
    long nextBlockSize = 0;
    String nextBlockIdStr = null;
    String nextGenStamp = null;
    String nextHost = req.getServerName();
    int nextPort = req.getServerPort();
    int nextDatanodePort = datanodePort;
    // determine data for the next link
    if (startOffset + chunkSizeToView >= blockSize) {
      // we have to go to the next block from this point onwards
      List<LocatedBlock> blocks = dfs.getNamenode().getBlockLocations(filename, 0,
          Long.MAX_VALUE).getLocatedBlocks();
      for (int i = 0; i < blocks.size(); i++) {
        if (blocks.get(i).getBlock().getBlockId() == blockId) {
          if (i != blocks.size() - 1) {
            LocatedBlock nextBlock = blocks.get(i + 1);
            nextBlockIdStr = Long.toString(nextBlock.getBlock().getBlockId());
            nextGenStamp = Long.toString(nextBlock.getBlock()
                .getGenerationStamp());
            nextStartOffset = 0;
            nextBlockSize = nextBlock.getBlock().getNumBytes();
            DatanodeInfo d = JspHelper.bestNode(nextBlock, conf);
            nextDatanodePort = d.getXferPort();
            nextHost = d.getIpAddr();
            nextPort = d.getInfoPort();
          }
        }
      }
    } else {
      // we are in the same block
      nextBlockIdStr = blockId.toString();
      nextStartOffset = startOffset + chunkSizeToView;
      nextBlockSize = blockSize;
      nextGenStamp = genStamp.toString();
    }
    String nextUrl = null;
    if (nextBlockIdStr != null) {
      nextUrl = HttpConfig.getSchemePrefix() + canonicalize(nextHost) + ":" + nextPort
          + "/browseBlock.jsp?blockId=" + nextBlockIdStr
          + "&blockSize=" + nextBlockSize
          + "&startOffset=" + nextStartOffset
          + "&genstamp=" + nextGenStamp
          + "&filename=" + URLEncoder.encode(filename, "UTF-8")
          + "&chunkSizeToView=" + chunkSizeToView
          + "&datanodePort=" + nextDatanodePort
          + "&namenodeInfoPort=" + namenodeInfoPort
          + JspHelper.getDelegationTokenUrlParam(tokenString)
          + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
      out.print("<a href=\"" + nextUrl + "\">View Next chunk</a>&nbsp;&nbsp;");
    }
    // determine data for the prev link
    String prevBlockIdStr = null;
    String prevGenStamp = null;
    long prevStartOffset = 0;
    long prevBlockSize = 0;
    String prevHost = req.getServerName();
    int prevPort = req.getServerPort();
    int prevDatanodePort = datanodePort;
    if (startOffset == 0) {
      List<LocatedBlock> blocks = dfs.getNamenode().getBlockLocations(filename, 0,
          Long.MAX_VALUE).getLocatedBlocks();
      for (int i = 0; i < blocks.size(); i++) {
        if (blocks.get(i).getBlock().getBlockId() == blockId) {
          if (i != 0) {
            LocatedBlock prevBlock = blocks.get(i - 1);
            prevBlockIdStr = Long.toString(prevBlock.getBlock().getBlockId());
            prevGenStamp = Long.toString(prevBlock.getBlock()
                .getGenerationStamp());
            prevStartOffset = prevBlock.getBlock().getNumBytes()
                - chunkSizeToView;
            if (prevStartOffset < 0)
              prevStartOffset = 0;
            prevBlockSize = prevBlock.getBlock().getNumBytes();
            DatanodeInfo d = JspHelper.bestNode(prevBlock, conf);
            prevDatanodePort = d.getXferPort();
            prevHost = d.getIpAddr();
            prevPort = d.getInfoPort();
          }
        }
      }
    } else {
      // we are in the same block
      prevBlockIdStr = blockId.toString();
      prevStartOffset = startOffset - chunkSizeToView;
      if (prevStartOffset < 0)
        prevStartOffset = 0;
      prevBlockSize = blockSize;
      prevGenStamp = genStamp.toString();
    }

    String prevUrl = null;
    if (prevBlockIdStr != null) {
      prevUrl = HttpConfig.getSchemePrefix() + canonicalize(prevHost) + ":" + prevPort
          + "/browseBlock.jsp?blockId=" + prevBlockIdStr
          + "&blockSize=" + prevBlockSize
          + "&startOffset=" + prevStartOffset
          + "&filename=" + URLEncoder.encode(filename, "UTF-8")
          + "&chunkSizeToView=" + chunkSizeToView
          + "&genstamp=" + prevGenStamp
          + "&datanodePort=" + prevDatanodePort
          + "&namenodeInfoPort=" + namenodeInfoPort
          + JspHelper.getDelegationTokenUrlParam(tokenString)
          + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
      out.print("<a href=\"" + prevUrl + "\">View Prev chunk</a>&nbsp;&nbsp;");
    }
    out.print("<hr>");
    out.print("<textarea cols=\"100\" rows=\"25\" wrap=\"virtual\" style=\"width:100%\" READONLY>");
    try {
      JspHelper.streamBlockInAscii(new InetSocketAddress(req.getServerName(),
          datanodePort), bpid, blockId, blockToken, genStamp, blockSize,
          startOffset, chunkSizeToView, out, conf, dfs.getDataEncryptionKey());
    } catch (Exception e) {
      out.print(e);
    }
    out.print("</textarea>");
    dfs.close();
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.DFSClient

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.