Package org.apache.hadoop.hdfs.net

Examples of org.apache.hadoop.hdfs.net.Peer

1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
    int cacheTries = 0;
    DomainSocketFactory dsFactory = dfsClient.getDomainSocketFactory();
    BlockReader reader = null;
    final int nCachedConnRetry = dfsClient.getConf().nCachedConnRetry;
    for (; cacheTries < nCachedConnRetry; ++cacheTries) {
      Peer peer = peerCache.get(chosenNode, true);
      if (peer == null) break;
      try {
        boolean allowShortCircuitLocalReads = dfsClient.getConf().
            shortCircuitLocalReads && (!shortCircuitForbidden());
        reader = BlockReaderFactory.newBlockReader(
            dfsClient.getConf(), file, block, blockToken, startOffset,
            len, verifyChecksum, clientName, peer, chosenNode,
            dsFactory, peerCache, fileInputStreamCache,
            allowShortCircuitLocalReads, cachingStrategy);
        return reader;
      } catch (IOException ex) {
        DFSClient.LOG.debug("Error making BlockReader with DomainSocket. " +
            "Closing stale " + peer, ex);
      } finally {
        if (reader == null) {
          IOUtils.closeQuietly(peer);
        }
      }
    }

    // Try to create a DomainPeer.
    DomainSocket domSock = dsFactory.create(dnAddr, this);
    if (domSock != null) {
      Peer peer = new DomainPeer(domSock);
      try {
        boolean allowShortCircuitLocalReads = dfsClient.getConf().
            shortCircuitLocalReads && (!shortCircuitForbidden());
        reader = BlockReaderFactory.newBlockReader(
            dfsClient.getConf(), file, block, blockToken, startOffset,
            len, verifyChecksum, clientName, peer, chosenNode,
            dsFactory, peerCache, fileInputStreamCache,
            allowShortCircuitLocalReads, cachingStrategy);
        return reader;
      } catch (IOException e) {
        DFSClient.LOG.warn("failed to connect to " + domSock, e);
      } finally {
        if (reader == null) {
         // If the Peer that we got the error from was a DomainPeer,
         // mark the socket path as bad, so that newDataSocket will not try
         // to re-open this socket for a while.
         dsFactory.disableDomainSocketPath(domSock.getPath());
         IOUtils.closeQuietly(peer);
        }
      }
    }

    // Look for cached peers.
    for (; cacheTries < nCachedConnRetry; ++cacheTries) {
      Peer peer = peerCache.get(chosenNode, false);
      if (peer == null) break;
      try {
        reader = BlockReaderFactory.newBlockReader(
            dfsClient.getConf(), file, block, blockToken, startOffset,
            len, verifyChecksum, clientName, peer, chosenNode,
            dsFactory, peerCache, fileInputStreamCache, false,
            cachingStrategy);
        return reader;
      } catch (IOException ex) {
        DFSClient.LOG.debug("Error making BlockReader. Closing stale " +
          peer, ex);
      } finally {
        if (reader == null) {
          IOUtils.closeQuietly(peer);
        }
      }
    }
    if (tcpReadsDisabledForTesting) {
      throw new IOException("TCP reads are disabled.");
    }
    // Try to create a new remote peer.
    Peer peer = newTcpPeer(dnAddr);
    return BlockReaderFactory.newBlockReader(
        dfsClient.getConf(), file, block, blockToken, startOffset,
        len, verifyChecksum, clientName, peer, chosenNode,
        dsFactory, peerCache, fileInputStreamCache, false,
        cachingStrategy);
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
226
227
    assertEquals(CAPACITY, cache.size());
    assertSame(null, cache.get(dnIds[0], false));

    // Make sure that the other entries are still there
    for (int i = 1; i < CAPACITY; ++i) {
      Peer peer = cache.get(dnIds[i], false);
      assertSame(peers[i], peer);
      assertTrue(!peer.isClosed());
      peer.close();
    }
    assertEquals(1, cache.size());
    cache.close();
  }
View Full Code Here
240
241
242
243
244
245
246
247
248
249
250
251
252
      cache.put(dnId, peer);
    }
    // Check that all of the peers ended up in the cache
    assertEquals(CAPACITY, cache.size());
    while (!peers.isEmpty()) {
      Peer peer = cache.get(dnId, false);
      assertTrue(peer != null);
      assertTrue(!peer.isClosed());
      peers.remove(peer);
    }
    assertEquals(0, cache.size());
    cache.close();
  }
View Full Code Here
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
    }
    // Check that all of the peers ended up in the cache
    assertEquals(CAPACITY, cache.size());
    // Test that get(requireDomainPeer=true) finds the peer with the
    // domain socket.
    Peer peer = cache.get(dnId, true);
    assertTrue(peer.getDomainSocket() != null);
    peers.remove(peer);
    // Test that get(requireDomainPeer=true) returns null when there are
    // no more peers with domain sockets.
    peer = cache.get(dnId, true);
    assertTrue(peer == null);
    // Check that all of the other peers ended up in the cache.
    while (!peers.isEmpty()) {
      peer = cache.get(dnId, false);
      assertTrue(peer != null);
      assertTrue(!peer.isClosed());
      peers.remove(peer);
    }
    assertEquals(0, cache.size());
    cache.close();
  }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
                   DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_DEFAULT));
  }

  @Override
  public void run() {
    Peer peer = null;
    while (datanode.shouldRun) {
      try {
        peer = peerServer.accept();

        // Make sure the xceiver count is not exceeded
View Full Code Here
938
939
940
941
942
943
944
945
946
947
948
      addToDeadNodes(chosenNode);
    }
  }

  private Peer newTcpPeer(InetSocketAddress addr) throws IOException {
    Peer peer = null;
    boolean success = false;
    Socket sock = null;
    try {
      sock = dfsClient.socketFactory.createSocket();
      NetUtils.connect(sock, addr,
View Full Code Here
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
    int cacheTries = 0;
    DomainSocketFactory dsFactory = dfsClient.getDomainSocketFactory();
    BlockReader reader = null;
    final int nCachedConnRetry = dfsClient.getConf().nCachedConnRetry;
    for (; cacheTries < nCachedConnRetry; ++cacheTries) {
      Peer peer = peerCache.get(chosenNode, true);
      if (peer == null) break;
      try {
        boolean allowShortCircuitLocalReads = dfsClient.getConf().
            shortCircuitLocalReads && (!shortCircuitForbidden());
        reader = BlockReaderFactory.newBlockReader(
            dfsClient.getConf(), file, block, blockToken, startOffset,
            len, verifyChecksum, clientName, peer, chosenNode,
            dsFactory, peerCache, fileInputStreamCache,
            allowShortCircuitLocalReads);
        return reader;
      } catch (IOException ex) {
        DFSClient.LOG.debug("Error making BlockReader with DomainSocket. " +
            "Closing stale " + peer, ex);
      } finally {
        if (reader == null) {
          IOUtils.closeQuietly(peer);
        }
      }
    }

    // Try to create a DomainPeer.
    DomainSocket domSock = dsFactory.create(dnAddr, this);
    if (domSock != null) {
      Peer peer = new DomainPeer(domSock);
      try {
        boolean allowShortCircuitLocalReads = dfsClient.getConf().
            shortCircuitLocalReads && (!shortCircuitForbidden());
        reader = BlockReaderFactory.newBlockReader(
            dfsClient.getConf(), file, block, blockToken, startOffset,
            len, verifyChecksum, clientName, peer, chosenNode,
            dsFactory, peerCache, fileInputStreamCache,
            allowShortCircuitLocalReads);
        return reader;
      } catch (IOException e) {
        DFSClient.LOG.warn("failed to connect to " + domSock, e);
      } finally {
        if (reader == null) {
         // If the Peer that we got the error from was a DomainPeer,
         // mark the socket path as bad, so that newDataSocket will not try
         // to re-open this socket for a while.
         dsFactory.disableDomainSocketPath(domSock.getPath());
         IOUtils.closeQuietly(peer);
        }
      }
    }

    // Look for cached peers.
    for (; cacheTries < nCachedConnRetry; ++cacheTries) {
      Peer peer = peerCache.get(chosenNode, false);
      if (peer == null) break;
      try {
        reader = BlockReaderFactory.newBlockReader(
            dfsClient.getConf(), file, block, blockToken, startOffset,
            len, verifyChecksum, clientName, peer, chosenNode,
            dsFactory, peerCache, fileInputStreamCache, false);
        return reader;
      } catch (IOException ex) {
        DFSClient.LOG.debug("Error making BlockReader. Closing stale " +
          peer, ex);
      } finally {
        if (reader == null) {
          IOUtils.closeQuietly(peer);
        }
      }
    }
    if (tcpReadsDisabledForTesting) {
      throw new IOException("TCP reads are disabled.");
    }
    // Try to create a new remote peer.
    Peer peer = newTcpPeer(dnAddr);
    return BlockReaderFactory.newBlockReader(
        dfsClient.getConf(), file, block, blockToken, startOffset,
        len, verifyChecksum, clientName, peer, chosenNode,
        dsFactory, peerCache, fileInputStreamCache, false);
  }
View Full Code Here
217
218
219
220
221
222
223
224
225
226
227
228
229
    assertEquals(CAPACITY, cache.size());
    assertSame(null, cache.get(dnIds[0], false));

    // Make sure that the other entries are still there
    for (int i = 1; i < CAPACITY; ++i) {
      Peer peer = cache.get(dnIds[i], false);
      assertSame(peers[i], peer);
      assertTrue(!peer.isClosed());
      peer.close();
    }
    assertEquals(1, cache.size());
    cache.close();
  }
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
253
254
      cache.put(dnId, peer);
    }
    // Check that all of the peers ended up in the cache
    assertEquals(CAPACITY, cache.size());
    while (!peers.isEmpty()) {
      Peer peer = cache.get(dnId, false);
      assertTrue(peer != null);
      assertTrue(!peer.isClosed());
      peers.remove(peer);
    }
    assertEquals(0, cache.size());
    cache.close();
  }
View Full Code Here
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
    }
    // Check that all of the peers ended up in the cache
    assertEquals(CAPACITY, cache.size());
    // Test that get(requireDomainPeer=true) finds the peer with the
    // domain socket.
    Peer peer = cache.get(dnId, true);
    assertTrue(peer.getDomainSocket() != null);
    peers.remove(peer);
    // Test that get(requireDomainPeer=true) returns null when there are
    // no more peers with domain sockets.
    peer = cache.get(dnId, true);
    assertTrue(peer == null);
    // Check that all of the other peers ended up in the cache.
    while (!peers.isEmpty()) {
      peer = cache.get(dnId, false);
      assertTrue(peer != null);
      assertTrue(!peer.isClosed());
      peers.remove(peer);
    }
    assertEquals(0, cache.size());
    cache.close();
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.net.Peer

  • org.apache.hadoop.hdfs.BlockReaderFactory
  • org.apache.hadoop.hdfs.BlockReaderTestUtil
  • org.apache.hadoop.hdfs.DFSClient
  • org.apache.hadoop.hdfs.DFSInputStream
  • org.apache.hadoop.hdfs.PeerCache
  • org.apache.hadoop.hdfs.server.blockmanagement.TestBlockTokenWithDFS
  • org.apache.hadoop.hdfs.server.common.JspHelper
  • org.apache.hadoop.hdfs.server.datanode.DataXceiverServer
  • org.apache.hadoop.hdfs.server.datanode.TestDataNodeVolumeFailure
  • org.apache.hadoop.hdfs.server.namenode.NamenodeFsck
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.