Examples of org.apache.hadoop.fs.DF

org.apache.hadoop.fs.DF
Filesystem disk space usage statistics. Uses the unix 'df' program to get mount points, and java.io.File for space utilization. Tested on Linux, FreeBSD, Windows.

    FSVolume(File currentDir, Configuration conf) throws IOException {
      this.reserved = conf.getLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY,
                                   DFSConfigKeys.DFS_DATANODE_DU_RESERVED_DEFAULT);
      this.currentDir = currentDir; 
      File parent = currentDir.getParentFile();
      this.usage = new DF(parent, conf);
    }

View Full Code Here

    for (URI directoryUri : directoriesToCheck) {
      File dir = new File(directoryUri.getPath());
      if (!dir.exists()) {
        throw new IOException("Missing directory "+dir.getAbsolutePath());
      }
      DF df = new DF(dir, conf);
      volumes.put(df.getFilesystem(), df);
    }
  }

View Full Code Here

            configCapacity));
        assertTrue(percentBpUsed == DFSUtil.getPercentUsed(bpUsed,
            configCapacity));
      }   
      
      DF df = new DF(new File(cluster.getDataDirectory()), conf);
     
      //
      // Currently two data directories are created by the data node
      // in the MiniDFSCluster. This results in each data directory having
      // capacity equals to the disk capacity of the data directory.
      // Hence the capacity reported by the data node is twice the disk space
      // the disk capacity
      //
      // So multiply the disk capacity and reserved space by two 
      // for accommodating it
      //
      int numOfDataDirs = 2;
      
      long diskCapacity = numOfDataDirs * df.getCapacity();
      reserved *= numOfDataDirs;
      
      configCapacity = namesystem.getCapacityTotal();
      used = namesystem.getCapacityUsed();
      nonDFSUsed = namesystem.getNonDfsUsedSpace();

View Full Code Here

    NameNodeResourceChecker nnrc = new NameNodeResourceChecker(conf);
    
    // For the purpose of this test, we need to force the name dirs to appear to
    // be on different volumes.
    Map<String, DF> volumes = new HashMap<String, DF>();
    volumes.put("volume1", new DF(nameDir1, conf));
    volumes.put("volume2", new DF(nameDir2, conf));
    nnrc.setVolumes(volumes);
    
    NameNodeResourceChecker spyNnrc = Mockito.spy(nnrc);
    
    Mockito.when(spyNnrc.getVolumesLowOnSpace()).thenReturn(

View Full Code Here

    if (minSpace == 0) {
      return true;
    }
    String[] localDirs = fConf.getLocalDirs();
    for (int i = 0; i < localDirs.length; i++) {
      DF df = null;
      if (localDirsDf.containsKey(localDirs[i])) {
        df = localDirsDf.get(localDirs[i]);
      } else {
        df = new DF(new File(localDirs[i]), fConf);
        localDirsDf.put(localDirs[i], df);
      }


      if (df.getAvailable() > minSpace)
        return true;
    }


    return false;
  }

View Full Code Here

      try {
        // canonicalize f   
        File f = pathToFile(p).getCanonicalFile();
      
        // find highest writable parent dir of f on the same device
        String device = new DF(f.toString(), getConf()).getMount();
        File parent = f.getParentFile();
        File dir;
        do {
          dir = parent;
          parent = parent.getParentFile();

View Full Code Here

      if (!detachDir.mkdirs()) {
        if (!detachDir.isDirectory()) {
          throw new IOException("Mkdirs failed to create " + detachDir.toString());
        }
      }
      this.usage = new DF(parent, conf);
      this.dfsUsage = new DU(parent, conf);
      this.dfsUsage.start();
    }

View Full Code Here

    FSVolume(File currentDir, Configuration conf) throws IOException {
      this.reserved = conf.getLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY,
                                   DFSConfigKeys.DFS_DATANODE_DU_RESERVED_DEFAULT);
      this.currentDir = currentDir; 
      File parent = currentDir.getParentFile();
      this.usage = new DF(parent, conf);
    }

View Full Code Here

  
  private long getFreeSpace() throws IOException {
    long biggestSeenSoFar = 0;
    String[] localDirs = fConf.getLocalDirs();
    for (int i = 0; i < localDirs.length; i++) {
      DF df = null;
      if (localDirsDf.containsKey(localDirs[i])) {
        df = localDirsDf.get(localDirs[i]);
      } else {
        df = new DF(new File(localDirs[i]), fConf);
        localDirsDf.put(localDirs[i], df);
      }


      long availOnThisVol = df.getAvailable();
      if (availOnThisVol > biggestSeenSoFar) {
        biggestSeenSoFar = availOnThisVol;
      }
    }

View Full Code Here

  
  private long getFreeSpace() throws IOException {
    long biggestSeenSoFar = 0;
    String[] localDirs = localStorage.getDirs();
    for (int i = 0; i < localDirs.length; i++) {
      DF df = null;
      if (localDirsDf.containsKey(localDirs[i])) {
        df = localDirsDf.get(localDirs[i]);
      } else {
        df = new DF(new File(localDirs[i]), fConf);
        localDirsDf.put(localDirs[i], df);
      }


      long availOnThisVol = df.getAvailable();
      if (availOnThisVol > biggestSeenSoFar) {
        biggestSeenSoFar = availOnThisVol;
      }
    }

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.hadoop.fs.DF

org.apache.hadoop.fs.LocalFileSystem

org.apache.hadoop.hdfs.server.datanode.FSDataset$FSVolume

org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl

org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker

org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume

org.apache.hadoop.hdfs.server.namenode.TestNamenodeCapacityReport

org.apache.hadoop.hdfs.server.namenode.TestNameNodeResourceChecker

org.apache.hadoop.mapred.TaskTracker

java.io.FileNotFoundException

java.io.BufferedReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.