Examples of org.apache.hadoop.fs.BlockLocation

org.apache.hadoop.fs.BlockLocation

    for (int i = 0; i < blocks.length; i++) {
      if ((blocks[i].getOffset() <= offset) && (offset < blocks[i].getOffset() + blocks[i].getLength())) {
        return i;
      }
    }
    BlockLocation block = blocks[blocks.length - 1];
    long length = block.getOffset() + block.getLength() - 1;
    throw new IllegalArgumentException("Offset " + offset + " is outside of file with length=" + length);
  }

View Full Code Here

  protected static Path mockWithFileSystem(int blockCount, long blockSize, long extraBlockSize) throws Exception {
    final ArrayList<BlockLocation> blocks = new ArrayList<BlockLocation>();
    long offset = 0;
    int i = 0;
    for (; i < blockCount; i++) {
      blocks.add(new BlockLocation(new String[]{"names"+i}, new String[]{"hosts"+i}, offset, blockSize));
      offset += blockSize;
    }


    // extra just means that we add a non full last block
    if (extraBlockSize > 0 && extraBlockSize < blockSize) {
      blocks.add(new BlockLocation(new String[]{"names"+i}, new String[]{"hosts"+i}, offset, extraBlockSize));
      offset += extraBlockSize;
    }


    FileStatus mStatus = mock(FileStatus.class);
    Path mPath = mock(Path.class);

View Full Code Here

      if ((blkLocations[i].getOffset() <= offset) &&
          (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
        return i;
      }
    }
    BlockLocation last = blkLocations[blkLocations.length -1];
    long fileLength = last.getOffset() + last.getLength() -1;
    throw new IllegalArgumentException("Offset " + offset + 
                                       " is outside of file (0.." +
                                       fileLength + ")");
  }

View Full Code Here

      if ((blkLocations[i].getOffset() <= offset) &&
          (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
        return i;
      }
    }
    BlockLocation last = blkLocations[blkLocations.length -1];
    long fileLength = last.getOffset() + last.getLength() -1;
    throw new IllegalArgumentException("Offset " + offset + 
                                       " is outside of file (0.." +
                                       fileLength + ")");
  }

View Full Code Here

  public static void waitReplication(FileSystem fs, Path fileName, 
      short replFactor)  throws IOException {
    boolean good;
    do {
      good = true;
      BlockLocation locs[] = fs.getFileBlockLocations(
        fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
      for (int j = 0; j < locs.length; j++) {
        String[] loc = locs[j].getHosts();
        if (loc.length != replFactor) {
          System.out.println("File " + fileName + " has replication factor " +

View Full Code Here

                if (diff < 0) return -1;
                if (diff > 0) return 1;
                return 0;
              }
            });
            BlockLocation firstBlock = locations[0];
            BlockLocation lastBlock = locations[locations.length - 1];
            long lastBlockOffsetBegin = lastBlock.getOffset();
            long lastBlockOffsetEnd =
                lastBlockOffsetBegin + lastBlock.getLength();
            if ((firstBlock.getOffset() > offsetBegin)
                || (lastBlockOffsetEnd < offsetEnd)) {
              throw new AssertionError(
                  "Block locations returned by getFileBlockLocations do not cover requested range");
            }

View Full Code Here

      if (locations == null) {
        blocks = new OneBlockInfo[0];
      } else {


        if(locations.length == 0) {
          locations = new BlockLocation[] { new BlockLocation() };
        }


        if (!isSplitable) {
          // if the file is not splitable, just create the one block with
          // full file length

View Full Code Here

        xferAddrs[hCnt] = locations[hCnt].getXferAddr();
        NodeBase node = new NodeBase(xferAddrs[hCnt], 
                                     locations[hCnt].getNetworkLocation());
        racks[hCnt] = node.toString();
      }
      blkLocations[idx] = new BlockLocation(xferAddrs, hosts, racks,
                                            blk.getStartOffset(),
                                            blk.getBlockSize(),
                                            blk.isCorrupt());
      idx++;
    }

View Full Code Here

          && (offset < blkLocations[i].getOffset()
              + blkLocations[i].getLength())) {
        return i;
      }
    }
    BlockLocation last = blkLocations[blkLocations.length - 1];
    long fileLength = last.getOffset() + last.getLength() - 1;
    throw new IllegalArgumentException("Offset " + offset
        + " is outside of file (0.." + fileLength + ")");
  }

View Full Code Here

                remainLen > stepSize ? stepSize : remainLen;
            long offsetBegin = fileLen - remainLen;
            long offsetEnd = offsetBegin + splitBytes;
            int indexBegin = getStartBlockIndex(startOffsets, offsetBegin);
            int indexEnd = getEndBlockIndex(startOffsets, offsetEnd);
            BlockLocation firstBlock = locations[indexBegin];
            BlockLocation lastBlock = locations[indexEnd-1];
            long lastBlockOffsetBegin = lastBlock.getOffset();
            long lastBlockOffsetEnd =
                lastBlockOffsetBegin + lastBlock.getLength();
            if ((firstBlock.getOffset() > offsetBegin)
                || (lastBlockOffsetEnd < offsetEnd)) {
              throw new AssertionError(
                  "Block locations returned by getFileBlockLocations do not cover requested range");
            }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.fs.BlockLocation

co.cask.cdap.data.stream.StreamDataFileSplitter

com.mongodb.hadoop.splitter.BSONSplitter

com.quantcast.qfs.hadoop.QuantcastFileSystem

org.apache.cassandra.hadoop.fs.CassandraFileSystemThriftStore

org.apache.drill.exec.store.TestAffinityCalculator

org.apache.hadoop.dfs.DFSTestUtil

org.apache.hadoop.fs.ceph.CephFileSystem

org.apache.hadoop.fs.glusterfs.GlusterFSXattr

org.apache.hadoop.fs.kfs.KosmosFileSystem

org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.