Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.BlockLocation


        numLocations++;
      BlockLocation[] blockLocations = new BlockLocation[numLocations];
      for (int i = 0; i < numLocations; i++) {
        String[] names = new String[] { "b" + i };
        String[] hosts = new String[] { "host" + i };
        blockLocations[i] = new BlockLocation(names, hosts, i * splitSize,
            Math.min(splitSize, size - (splitSize * i)));
      }
      return blockLocations;
    }
View Full Code Here


    @Override
    public BlockLocation[] getFileBlockLocations(Path p, long start, long len)
        throws IOException {
      return new BlockLocation[] {
          new BlockLocation(new String[] { "localhost:50010" },
              new String[] { "localhost" }, 0, len) };
    }
View Full Code Here

                        key = "DISTRIBUTE-" + dcount;
                        brick = vol.get(key).get(0);

                        if (hnts == null) {
                                result = new BlockLocation[1];
                                result[0] = new BlockLocation(null, new String[] {brick2host(brick)}, start, len);
                        } else
                                hnts.put(0, new GlusterFSBrickClass(brick, start, len, false, -1, -1, -1));
                        break;

                case R:
                case DR:
                        /* just the name says it's striped - the volume isn't */
                        stripedBricks = new ArrayList<String>();

                        for (i = 1; i <= rcount; i++) {
                                key = "REPLICATE-" + i;
                                brickList = vol.get(key);
                                it = brickList.iterator();
                                while (it.hasNext()) {
                                        stripedBricks.add(it.next());
                                }
                        }

                        nrAllocs = stripedBricks.size();
                        if (hnts == null) {
                                result = new BlockLocation[1];
                                blks = new String[nrAllocs];
                        }

                        for (i = 0; i < nrAllocs; i++) {
                                if (hnts == null)
                                        blks[i] = brick2host(stripedBricks.get(i));
                                else
                                        hnts.put(i, new GlusterFSBrickClass(stripedBricks.get(i), start, len, false, -1, -1, -1));
                        }

                        if (hnts == null)
                                result[0] = new BlockLocation(null, blks, start, len);

                        break;

                case SR:
                case DSR:
                        int rsize = 0;
                        ArrayList<ArrayList<String>> replicas = new ArrayList<ArrayList<String>>();

                        stripedBricks = new ArrayList<String>();

                        if (rcount == 0)
                                throw new IOException("got replicated volume with replication count 0");

                        for (i = 1; i <= rcount; i++) {
                                key = "REPLICATE-" + i;
                                brickList = vol.get(key);
                                it = brickList.iterator();
                                replicas.add(i - 1, new ArrayList<String>());
                                while (it.hasNext()) {
                                        replicas.get(i - 1).add(it.next());
                                }
                        }

                        stripeSize = meta.get("block-size");

                        nrAllocs = (int) (((len - start) / stripeSize) + 1);
                        if (hnts == null) {
                                result = new BlockLocation[nrAllocs];
                                repl = new GlusterFSBrickRepl[nrAllocs];
                        }

                        // starting stripe position
                        counter = (int) ((start / stripeSize) % rcount);
                        stripeStart = start;

                        key = null;
                        int currAlloc = 0;
                        boolean hntsDone = false;
                        while ((stripeStart < len) && !done) {
                                stripeEnd = (stripeStart - (stripeStart % stripeSize)) + stripeSize - 1;
                                if (stripeEnd > start + len) {
                                        stripeEnd = start + len - 1;
                                        done = true;
                                }

                                rsize = replicas.get(counter).size();

                                if (hnts == null)
                                        repl[allocCtr] = new GlusterFSBrickRepl(rsize, stripeStart, (stripeEnd - stripeStart));

                                for (i = 0; i < rsize; i++) {
                                        brick = replicas.get(counter).get(i);
                                        currAlloc = (allocCtr * rsize) + i;

                                        if (hnts == null)
                                                repl[allocCtr].addHost(brick2host(brick));
                                        else
                                                if (currAlloc <= (rsize * rcount) - 1) {
                                                        hnts.put(currAlloc, new GlusterFSBrickClass(brick, stripeStart,
                                                                                                    (stripeEnd - stripeStart),
                                                                                                    true, stripeSize, rcount, rsize));
                                                } else
                                                        hntsDone = true;
                                }

                                if (hntsDone)
                                        break;

                                stripeStart = stripeEnd + 1;

                                allocCtr++;
                                counter++;

                                if (counter >= replicas.size())
                                        counter = 0;
                        }

                        if (hnts == null)
                                for (int k = 0; k < nrAllocs; k++)
                                        result[k] = new BlockLocation(null, repl[k].getReplHosts(), repl[k].getStartLen(), repl[k].getOffLen());

                        break;

                case S:
                case DS:
                        if (scount == 0)
                                throw new IOException("got striped volume with stripe count 0");

                        stripedBricks = new ArrayList<String>();
                        stripeSize = meta.get("block-size");

                        key = "STRIPE-" + scount;
                        brickList = vol.get(key);
                        it = brickList.iterator();
                        while (it.hasNext()) {
                                stripedBricks.add(it.next());
                        }

                        nrAllocs = (int) ((len - start) / stripeSize) + 1;
                        if (hnts == null)
                                result = new BlockLocation[nrAllocs];

                        // starting stripe position
                        counter = (int) ((start / stripeSize) % stripedBricks.size());
                        stripeStart = start;

                        key = null;
                        while ((stripeStart < len) && !done) {
                                brick = stripedBricks.get(counter);

                                stripeEnd = (stripeStart - (stripeStart % stripeSize)) + stripeSize - 1;
                                if (stripeEnd > start + len) {
                                        stripeEnd = start + len - 1;
                                        done = true;
                                }

                                if (hnts == null)
                                        result[allocCtr] = new BlockLocation(null, new String[] {brick2host(brick)},
                                                                             stripeStart, (stripeEnd - stripeStart));
                                else
                                        if (allocCtr <= stripedBricks.size()) {
                                                hnts.put(allocCtr, new GlusterFSBrickClass(brick, stripeStart, (stripeEnd - stripeStart),
                                                                                           true, stripeSize, stripedBricks.size(), -1));
View Full Code Here

      hosts[i] = "host" + i;
      names[i] = "host:" + port;
    }

    BlockLocation[] blockLocations = new BlockLocation[3];
    blockLocations[0] = new BlockLocation(new String[]{names[0], names[1], names[2]}, new String[]{hosts[0], hosts[1], hosts[2]}, 0, blockSize);
    blockLocations[1] = new BlockLocation(new String[]{names[0], names[2], names[3]}, new String[]{hosts[0], hosts[2], hosts[3]}, blockSize, blockSize);
    blockLocations[2] = new BlockLocation(new String[]{names[0], names[1], names[3]}, new String[]{hosts[0], hosts[1], hosts[3]}, blockSize*2, blockSize);

    return blockLocations;
  }
View Full Code Here

      hosts[i] = "host" + i;
      names[i] = "host:" + port;
    }

    BlockLocation[] blockLocations = new BlockLocation[4];
    blockLocations[0] = new BlockLocation(new String[]{names[0]}, new String[]{hosts[0]}, 0, blockSize);
    blockLocations[1] = new BlockLocation(new String[]{names[1]}, new String[]{hosts[1]}, blockSize, blockSize);
    blockLocations[3] = new BlockLocation(new String[]{names[3]}, new String[]{hosts[3]}, blockSize*2, blockSize);
    blockLocations[2] = new BlockLocation(new String[]{names[2]}, new String[]{hosts[2]}, blockSize*3, blockSize);

    return blockLocations;
  }
View Full Code Here

      DatanodeInfo[] cachedLocations = blk.getCachedLocations();
      String[] cachedHosts = new String[cachedLocations.length];
      for (int i=0; i<cachedLocations.length; i++) {
        cachedHosts[i] = cachedLocations[i].getHostName();
      }
      blkLocations[idx] = new BlockLocation(xferAddrs, hosts, cachedHosts,
                                            racks,
                                            blk.getStartOffset(),
                                            blk.getBlockSize(),
                                            blk.isCorrupt());
      idx++;
View Full Code Here

        names[hCnt] = locations[hCnt].getName();
        NodeBase node = new NodeBase(names[hCnt],
                                     locations[hCnt].getNetworkLocation());
        racks[hCnt] = node.toString();
      }
      blkLocations[idx] = new BlockLocation(names, hosts, racks,
                                            blk.getStartOffset(),
                                            blk.getBlockSize(),
                                            blk.isCorrupt());
      idx++;
    }
View Full Code Here

        // We can't call directly getBlockLocations, it's not available in HFileSystem
        // We're trying multiple times to be sure, as the order is random

        BlockLocation[] bls = rfs.getFileBlockLocations(fsLog, 0, 1);
        if (bls.length > 0) {
          BlockLocation bl = bls[0];

          LOG.info(bl.getHosts().length + " replicas for block 0 in " + logFile + " ");
          for (int i = 0; i < bl.getHosts().length - 1; i++) {
            LOG.info(bl.getHosts()[i] + "    " + logFile);
            Assert.assertNotSame(bl.getHosts()[i], host4);
          }
          String last = bl.getHosts()[bl.getHosts().length - 1];
          LOG.info(last + "    " + logFile);
          if (host4.equals(last)) {
            nbTest++;
            LOG.info(logFile + " is on the new datanode and is ok");
            if (bl.getHosts().length == 3) {
              // We can test this case from the file system as well
              // Checking the underlying file system. Multiple times as the order is random
              testFromDFS(dfs, logFile, repCount, host4);

              // now from the master
View Full Code Here

  private int getBlockIndex(BlockLocation[] blockLocations, long offset, int startIdx) {
    if (blockLocations == null) {
      return -1;
    }
    for (int i = startIdx; i < blockLocations.length; i++) {
      BlockLocation blockLocation = blockLocations[i];
      long endOffset = blockLocation.getOffset() + blockLocation.getLength();

      if (blockLocation.getOffset() <= offset && offset < endOffset) {
        return i;
      }
    }
    return -1;
  }
View Full Code Here

        xferAddrs[hCnt] = locations[hCnt].getXferAddr();
        NodeBase node = new NodeBase(xferAddrs[hCnt],
                                     locations[hCnt].getNetworkLocation());
        racks[hCnt] = node.toString();
      }
      blkLocations[idx] = new BlockLocation(xferAddrs, hosts, racks,
                                            blk.getStartOffset(),
                                            blk.getBlockSize(),
                                            blk.isCorrupt());
      idx++;
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.BlockLocation

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.