Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.BlockLocation

210
211
212
213
214
215
216
217
218
219
220
  public static void waitReplication(FileSystem fs, Path fileName,
      short replFactor)  throws IOException {
    boolean good;
    do {
      good = true;
      BlockLocation locs[] = fs.getFileBlockLocations(
        fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
      for (int j = 0; j < locs.length; j++) {
        String[] loc = locs[j].getHosts();
        if (loc.length != replFactor) {
          System.out.println("File " + fileName + " has replication factor " +
View Full Code Here
480
481
482
483
484
485
486
487
488
489
490
  public static void waitReplication(FileSystem fs, Path fileName,
      short replFactor)  throws IOException {
    boolean good;
    do {
      good = true;
      BlockLocation locs[] = fs.getFileBlockLocations(
        fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
      for (int j = 0; j < locs.length; j++) {
        String[] hostnames = locs[j].getNames();
        if (hostnames.length != replFactor) {
          String hostNameList = "";
View Full Code Here
278
279
280
281
282
283
284
285
286
287
        numLocations++;
      BlockLocation[] blockLocations = new BlockLocation[numLocations];
      for (int i = 0; i < numLocations; i++) {
        String[] names = new String[] { "b" + i };
        String[] hosts = new String[] { "host" + i };
        blockLocations[i] = new BlockLocation(names, hosts, i * splitSize,
            Math.min(splitSize, size - (splitSize * i)));
      }
      return blockLocations;
    }
View Full Code Here
276
277
278
279
280
281
282
283
284
285
286
        xferAddrs[hCnt] = locations[hCnt].getXferAddr();
        NodeBase node = new NodeBase(xferAddrs[hCnt],
                                     locations[hCnt].getNetworkLocation());
        racks[hCnt] = node.toString();
      }
      blkLocations[idx] = new BlockLocation(xferAddrs, hosts, racks,
                                            blk.getStartOffset(),
                                            blk.getBlockSize(),
                                            blk.isCorrupt());
      idx++;
    }
View Full Code Here
407
408
409
410
411
412
413
414
415
416
417
      DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);

      // Decommission one of the hosts with the block, this should cause
      // the block to get replicated to another host on the same rack,
      // otherwise the rack policy is violated.
      BlockLocation locs[] = fs.getFileBlockLocations(
          fs.getFileStatus(filePath), 0, Long.MAX_VALUE);
      String name = locs[0].getNames()[0];
      DFSTestUtil.writeFile(localFileSys, excludeFile, name);
      ns.getBlockManager().getDatanodeManager().refreshNodes(conf);
      DFSTestUtil.waitForDecommission(fs, name);
View Full Code Here
460
461
462
463
464
465
466
467
468
469
470
      fs.setReplication(filePath, REPLICATION_FACTOR);

      // Decommission one of the hosts with the block that is not on
      // the lone host on rack2 (if we decomission that host it would
      // be impossible to respect the rack policy).
      BlockLocation locs[] = fs.getFileBlockLocations(
          fs.getFileStatus(filePath), 0, Long.MAX_VALUE);
      for (String top : locs[0].getTopologyPaths()) {
        if (!top.startsWith("/rack2")) {
          String name = top.substring("/rack1".length()+1);
          DFSTestUtil.writeFile(localFileSys, excludeFile, name);
View Full Code Here
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
        // We can't call directly getBlockLocations, it's not available in HFileSystem
        // We're trying multiple times to be sure, as the order is random

        BlockLocation[] bls = rfs.getFileBlockLocations(fsLog, 0, 1);
        if (bls.length > 0) {
          BlockLocation bl = bls[0];

          LOG.info(bl.getHosts().length + " replicas for block 0 in " + logFile + " ");
          for (int i = 0; i < bl.getHosts().length - 1; i++) {
            LOG.info(bl.getHosts()[i] + "    " + logFile);
            Assert.assertNotSame(bl.getHosts()[i], host4);
          }
          String last = bl.getHosts()[bl.getHosts().length - 1];
          LOG.info(last + "    " + logFile);
          if (host4.equals(last)) {
            nbTest++;
            LOG.info(logFile + " is on the new datanode and is ok");
            if (bl.getHosts().length == 3) {
              // We can test this case from the file system as well
              // Checking the underlying file system. Multiple times as the order is random
              testFromDFS(dfs, logFile, repCount, host4);

              // now from the master
View Full Code Here
239
240
241
242
243
244
245
246
247
248
249
  public static void waitReplication(FileSystem fs, Path fileName,
      short replFactor)  throws IOException {
    boolean good;
    do {
      good = true;
      BlockLocation locs[] = fs.getFileBlockLocations(
        fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
      for (int j = 0; j < locs.length; j++) {
        String[] loc = locs[j].getHosts();
        if (loc.length != replFactor) {
          System.out.println("File " + fileName + " has replication factor " +
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
        names[hCnt] = locations[hCnt].getName();
        NodeBase node = new NodeBase(names[hCnt],
                                     locations[hCnt].getNetworkLocation());
        racks[hCnt] = node.toString();
      }
      blkLocations[idx] = new BlockLocation(names, hosts, racks,
                                            blk.getStartOffset(),
                                            blk.getBlockSize());
      idx++;
    }
    return blkLocations;
View Full Code Here
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
                remainLen > stepSize ? stepSize : remainLen;
            long offsetBegin = fileLen - remainLen;
            long offsetEnd = offsetBegin + splitBytes;
            int indexBegin = getStartBlockIndex(startOffsets, offsetBegin);
            int indexEnd = getEndBlockIndex(startOffsets, offsetEnd);
            BlockLocation firstBlock = locations[indexBegin];
            BlockLocation lastBlock = locations[indexEnd-1];
            long lastBlockOffsetBegin = lastBlock.getOffset();
            long lastBlockOffsetEnd =
                lastBlockOffsetBegin + lastBlock.getLength();
            if ((firstBlock.getOffset() > offsetBegin)
                || (lastBlockOffsetEnd < offsetEnd)) {
              throw new AssertionError(
                  "Block locations returned by getFileBlockLocations do not cover requested range");
            }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.BlockLocation

  • co.cask.cdap.data.stream.StreamDataFileSplitter
  • com.mongodb.hadoop.splitter.BSONSplitter
  • com.quantcast.qfs.hadoop.QuantcastFileSystem
  • org.apache.cassandra.hadoop.fs.CassandraFileSystemThriftStore
  • org.apache.drill.exec.store.TestAffinityCalculator
  • org.apache.hadoop.dfs.DFSTestUtil
  • org.apache.hadoop.fs.ceph.CephFileSystem
  • org.apache.hadoop.fs.glusterfs.GlusterFSXattr
  • org.apache.hadoop.fs.kfs.KosmosFileSystem
  • org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.