Examples of DatanodeDescriptor

org.apache.cayenne.configuration.DataNodeDescriptor
A descriptor of {@link DataNode} configuration. @since 3.1
org.apache.hadoop.dfs.DatanodeDescriptor
DatanodeDescriptor tracks stats on a given DataNode, such as available storage capacity, last update time, etc., and maintains a set of blocks stored on the datanode. This data structure is a data structure that is internal to the namenode. It is *not* sent over-the-wire to the Client or the Datnodes. Neither is it stored persistently in the fsImage. @author Mike Cafarella @author Konstantin Shvachko
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor
This class extends the DatanodeInfo class with ephemeral information (eg health, capacity, what blocks are associated with the Datanode) that is private to the Namenode, ie this class is not exposed to clients.
org.apache.hadoop.hdfs.server.namenode.DatanodeDescriptor
DatanodeDescriptor tracks stats on a given DataNode, such as available storage capacity, last update time, etc., and maintains a set of blocks stored on the datanode. This data structure is a data structure that is internal to the namenode. It is *not* sent over-the-wire to the Client or the Datnodes. Neither is it stored persistently in the fsImage.

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

      QuotaExceededException, SafeModeException, UnresolvedLinkException,
      IOException {
    checkBlock(previous);
    long fileLength, blockSize;
    int replication;
    DatanodeDescriptor clientNode = null;
    Block newBlock = null;


    if(NameNode.stateChangeLog.isDebugEnabled()) {
      NameNode.stateChangeLog.debug(
          "BLOCK* NameSystem.getAdditionalBlock: file "
          +src+" for "+clientName);
    }


    writeLock();
    try {
      if (isInSafeMode()) {
        throw new SafeModeException("Cannot add block to " + src, safeMode);
      }


      // have we exceeded the configured limit of fs objects.
      checkFsObjectLimit();


      INodeFileUnderConstruction pendingFile  = checkLease(src, clientName);


      // commit the last block and complete it if it has minimum replicas
      commitOrCompleteLastBlock(pendingFile, ExtendedBlock.getLocalBlock(previous));


      //
      // If we fail this, bad things happen!
      //
      if (!checkFileProgress(pendingFile, false)) {
        throw new NotReplicatedYetException("Not replicated yet:" + src);
      }
      fileLength = pendingFile.computeContentSummary().getLength();
      blockSize = pendingFile.getPreferredBlockSize();
      clientNode = pendingFile.getClientNode();
      replication = (int)pendingFile.getReplication();
    } finally {
      writeUnlock();
    }


    // choose targets for the new block to be allocated.
    final DatanodeDescriptor targets[] = blockManager.chooseTarget(
        src, replication, clientNode, excludedNodes, blockSize);


    // Allocate a new block and record it in the INode. 
    writeLock();
    try {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

      final int numAdditionalNodes, final String clientName
      ) throws IOException {
    //check if the feature is enabled
    dtpReplaceDatanodeOnFailure.checkEnabled();


    final DatanodeDescriptor clientnode;
    final long preferredblocksize;
    final List<DatanodeDescriptor> chosen;
    readLock();
    try {
      //check safe mode
      if (isInSafeMode()) {
        throw new SafeModeException("Cannot add datanode; src=" + src
            + ", blk=" + blk, safeMode);
      }


      //check lease
      final INodeFileUnderConstruction file = checkLease(src, clientName);
      clientnode = file.getClientNode();
      preferredblocksize = file.getPreferredBlockSize();


      //find datanode descriptors
      chosen = new ArrayList<DatanodeDescriptor>();
      for(DatanodeInfo d : existings) {
        final DatanodeDescriptor descriptor = blockManager.getDatanodeManager(
            ).getDatanode(d);
        if (descriptor != null) {
          chosen.add(descriptor);
        }
      }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

      LocatedBlocks blocks = client.getNamenode().getBlockLocations(
          fileName.toString(), 0, blockSize);
      DatanodeInfo[] nodes = blocks.get(0).getLocations();
      assertEquals(nodes.length, 3);
      DataNode staleNode = null;
      DatanodeDescriptor staleNodeInfo = null;
      // stop the heartbeat of the first node
      staleNode = this.stopDataNodeHeartbeat(cluster, nodes[0].getHostName());
      assertNotNull(staleNode);
      // set the first node as stale
      staleNodeInfo = cluster.getNameNode().getNamesystem().getBlockManager()
          .getDatanodeManager()
          .getDatanode(staleNode.getDatanodeId());
      staleNodeInfo.setLastUpdate(Time.now() - staleInterval - 1);


      LocatedBlocks blocksAfterStale = client.getNamenode().getBlockLocations(
          fileName.toString(), 0, blockSize);
      DatanodeInfo[] nodesAfterStale = blocksAfterStale.get(0).getLocations();
      assertEquals(nodesAfterStale.length, 3);
      assertEquals(nodesAfterStale[2].getHostName(), nodes[0].getHostName());


      // restart the staleNode's heartbeat
      DataNodeTestUtils.setHeartbeatsDisabledForTests(staleNode, false);
      // reset the first node as non-stale, so as to avoid two stale nodes
      staleNodeInfo.setLastUpdate(Time.now());


      LocatedBlock lastBlock = client.getLocatedBlocks(fileName.toString(), 0,
          Long.MAX_VALUE).getLastLocatedBlock();
      nodes = lastBlock.getLocations();
      assertEquals(nodes.length, 3);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

        UUID.randomUUID().toString(),
        port, port, port, port);
  }


  public static DatanodeDescriptor getLocalDatanodeDescriptor() {
    return new DatanodeDescriptor(getLocalDatanodeID());
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

    return storages;
  }
  public static DatanodeStorageInfo createDatanodeStorageInfo(
      String storageID, String ip, String rack, String hostname) {
    final DatanodeStorage storage = new DatanodeStorage(storageID);
    final DatanodeDescriptor dn = BlockManagerTestUtil.getDatanodeDescriptor(ip, rack, storage, hostname);
    return BlockManagerTestUtil.newDatanodeStorageInfo(dn, storage);
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

    DatanodeID dnId = new DatanodeID(ipAddr, hostname,
        UUID.randomUUID().toString(), port,
        DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
        DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT,
        DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT);
    return new DatanodeDescriptor(dnId, rackLocation);
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

    dataNodes[10].setDecommissioned();
  }
  
  @Test
  public void testContains() throws Exception {
    DatanodeDescriptor nodeNotInMap = 
      DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/d2/r4");
    for (int i=0; i < dataNodes.length; i++) {
      assertTrue(cluster.contains(dataNodes[i]));
    }
    assertFalse(cluster.contains(nodeNotInMap));

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

  }


  @Test
  public void testCreateInvalidTopology() throws Exception {
    NetworkTopology invalCluster = new NetworkTopology();
    DatanodeDescriptor invalDataNodes[] = new DatanodeDescriptor[] {
        DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1"),
        DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1"),
        DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1")
    };
    invalCluster.add(invalDataNodes[0]);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

    assertTrue(testNodes[1] == dataNodes[5]);
    assertTrue(testNodes[2] == dataNodes[3]);


    // Array is just local rack nodes
    // Expect a random first node depending on the seed (normally the block ID).
    DatanodeDescriptor first = null;
    boolean foundRandom = false;
    for (int i=5; i<=7; i++) {
      testNodes[0] = dataNodes[5];
      testNodes[1] = dataNodes[6];
      testNodes[2] = dataNodes[7];

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

    if(LOG.isDebugEnabled()) {
      LOG.debug("Running " + GenericTestUtils.getMethodName());    
    }
    LeaseManager.Lease lm = mock(LeaseManager.Lease.class);
    Path file = spy(new Path("/test.dat"));
    DatanodeDescriptor dnd = mock(DatanodeDescriptor.class);
    PermissionStatus ps =
      new PermissionStatus("test", "test", new FsPermission((short)0777));
    
    fsn.dir.addFile(file.toString(), ps, (short)3, 1l, 
      "test", "test-machine", dnd, 1001l);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.