Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.BlockLocation

72
73
74
75
76
77
78
79
80
81
82
83
      hosts[i] = "host" + i;
      names[i] = "host:" + port;
    }

    BlockLocation[] blockLocations = new BlockLocation[4];
    blockLocations[0] = new BlockLocation(new String[]{names[0]}, new String[]{hosts[0]}, 0, blockSize);
    blockLocations[1] = new BlockLocation(new String[]{names[1]}, new String[]{hosts[1]}, blockSize, blockSize);
    blockLocations[3] = new BlockLocation(new String[]{names[3]}, new String[]{hosts[3]}, blockSize*2, blockSize);
    blockLocations[2] = new BlockLocation(new String[]{names[2]}, new String[]{hosts[2]}, blockSize*3, blockSize);

    return blockLocations;
  }
View Full Code Here
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
                if (diff < 0) return -1;
                if (diff > 0) return 1;
                return 0;
              }
            });
            BlockLocation firstBlock = locations[0];
            BlockLocation lastBlock = locations[locations.length - 1];
            long lastBlockOffsetBegin = lastBlock.getOffset();
            long lastBlockOffsetEnd =
                lastBlockOffsetBegin + lastBlock.getLength();
            if ((firstBlock.getOffset() > offsetBegin)
                || (lastBlockOffsetEnd < offsetEnd)) {
              throw new AssertionError(
                  "Block locations returned by getFileBlockLocations do not cover requested range");
            }
View Full Code Here
497
498
499
500
501
502
503
504
505
506
507
      if (locations == null) {
        blocks = new OneBlockInfo[0];
      } else {

        if(locations.length == 0) {
          locations = new BlockLocation[] { new BlockLocation() };
        }

        if (!isSplitable) {
          // if the file is not splitable, just create the one block with
          // full file length
View Full Code Here
503
504
505
506
507
508
509
510
511
512
513
  public static void waitReplication(FileSystem fs, Path fileName,
      short replFactor)  throws IOException {
    boolean good;
    do {
      good = true;
      BlockLocation locs[] = fs.getFileBlockLocations(
        fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
      for (int j = 0; j < locs.length; j++) {
        String[] hostnames = locs[j].getNames();
        if (hostnames.length != replFactor) {
          String hostNameList = "";
View Full Code Here
510
511
512
513
514
515
516
517
518
519
520
      if (locations == null) {
        blocks = new OneBlockInfo[0];
      } else {

        if(locations.length == 0) {
          locations = new BlockLocation[] { new BlockLocation() };
        }

        if (!isSplitable) {
          // if the file is not splitable, just create the one block with
          // full file length
View Full Code Here
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
        // We can't call directly getBlockLocations, it's not available in HFileSystem
        // We're trying multiple times to be sure, as the order is random

        BlockLocation[] bls = rfs.getFileBlockLocations(fsLog, 0, 1);
        if (bls.length > 0) {
          BlockLocation bl = bls[0];

          LOG.info(bl.getHosts().length + " replicas for block 0 in " + logFile + " ");
          for (int i = 0; i < bl.getHosts().length - 1; i++) {
            LOG.info(bl.getHosts()[i] + "    " + logFile);
            Assert.assertNotSame(bl.getHosts()[i], host4);
          }
          String last = bl.getHosts()[bl.getHosts().length - 1];
          LOG.info(last + "    " + logFile);
          if (host4.equals(last)) {
            nbTest++;
            LOG.info(logFile + " is on the new datanode and is ok");
            if (bl.getHosts().length == 3) {
              // We can test this case from the file system as well
              // Checking the underlying file system. Multiple times as the order is random
              testFromDFS(dfs, logFile, repCount, host4);

              // now from the master
View Full Code Here
422
423
424
425
426
427
428
429
430
431
432
        xferAddrs[hCnt] = locations[hCnt].getXferAddr();
        NodeBase node = new NodeBase(xferAddrs[hCnt],
                                     locations[hCnt].getNetworkLocation());
        racks[hCnt] = node.toString();
      }
      blkLocations[idx] = new BlockLocation(xferAddrs, hosts, racks,
                                            blk.getStartOffset(),
                                            blk.getBlockSize(),
                                            blk.isCorrupt());
      idx++;
    }
View Full Code Here
282
283
284
285
286
287
288
289
290
291
        numLocations++;
      BlockLocation[] blockLocations = new BlockLocation[numLocations];
      for (int i = 0; i < numLocations; i++) {
        String[] names = new String[] { "b" + i };
        String[] hosts = new String[] { "host" + i };
        blockLocations[i] = new BlockLocation(names, hosts, i * splitSize,
            Math.min(splitSize, size - (splitSize * i)));
      }
      return blockLocations;
    }
View Full Code Here
183
184
185
186
187
188
189
190
191
192
        numLocations++;
      BlockLocation[] blockLocations = new BlockLocation[numLocations];
      for (int i = 0; i < numLocations; i++) {
        String[] names = new String[] { "b" + i };
        String[] hosts = new String[] { "host" + i };
        blockLocations[i] = new BlockLocation(names, hosts, i * splitSize,
            Math.min(splitSize, size - (splitSize * i)));
      }
      return blockLocations;
    }
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
  public static void waitReplication(FileSystem fs, Path fileName,
      short replFactor)  throws IOException {
    boolean good;
    do {
      good = true;
      BlockLocation locs[] = fs.getFileBlockLocations(
        fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
      for (int j = 0; j < locs.length; j++) {
        String[] loc = locs[j].getHosts();
        if (loc.length != replFactor) {
          System.out.println("File " + fileName + " has replication factor " +
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.BlockLocation

  • co.cask.cdap.data.stream.StreamDataFileSplitter
  • com.mongodb.hadoop.splitter.BSONSplitter
  • com.quantcast.qfs.hadoop.QuantcastFileSystem
  • org.apache.cassandra.hadoop.fs.CassandraFileSystemThriftStore
  • org.apache.drill.exec.store.TestAffinityCalculator
  • org.apache.hadoop.dfs.DFSTestUtil
  • org.apache.hadoop.fs.ceph.CephFileSystem
  • org.apache.hadoop.fs.glusterfs.GlusterFSXattr
  • org.apache.hadoop.fs.kfs.KosmosFileSystem
  • org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.