Examples of org.apache.hadoop.hdfs.protocol.PolicyInfo

org.apache.hadoop.hdfs.protocol.PolicyInfo
Maintains information about one policy

  /**
   * Fix a specific file
   */
  private void fixFile(Path badFile) throws IOException {


    PolicyInfo pinfo = null;
    String filename = badFile.toString();


    LOG.info("File  = file to fix:" + badFile);


    // Find the policy that maps this file
    for (PathToPolicy pp: pathToPolicy) {
      if (filename.startsWith(pp.spath)) {
        pinfo = pp.pinfo;
        break;
      }
    }
    if (pinfo == null) {
      throw new IOException("Unable to find matching policy for " +
                            badFile);
    }


    // process the file and fix it. 
    Path src;
    HighTideNode.getMetrics().fixAttempt.inc();
    
    if (filename.startsWith(pinfo.getSrcPath().toString())) {
      // srcPath is corrupted, pick the first destPath as source of truth.
      String[] splits = filename.split(pinfo.getSrcPath().toString());
      src = new Path(pinfo.getDestPaths().get(0).rpath.toString() + splits[1]);
    } else {
      // dest file is corrupted, copy from source to destination
      String[] splits = filename.split(pinfo.getDestPaths().get(0).rpath.toString());
      src = new Path(pinfo.getSrcPath().toString() + splits[1]);
    }
    DistributedFileSystem srcFs = (DistributedFileSystem) src.getFileSystem(pinfo.getConf());
    DistributedFileSystem destFs = (DistributedFileSystem) badFile.getFileSystem(pinfo.getConf());


    FileStatus sstat = srcFs.getFileStatus(src);
    FileStatus dstat = destFs.getFileStatus(badFile);


    // assert that modtime of the two files are same

View Full Code Here


        if (srcPathPrefix == null || srcPathPrefix.length() == 0) {
          throw new HighTideConfigurationException("Bad configuration file: " + 
            "srcPath node does not have a path.");
        }
        PolicyInfo policyInfo = new PolicyInfo(srcPathPrefix, conf);
        policyName = srcPathPrefix;
        Properties policyProperties;


        // loop through all elements of this policy
        NodeList policies = element.getChildNodes();
        for (int j = 0; j < policies.getLength(); j++) {
          Node node1 = policies.item(j);
          if (!(node1 instanceof Element)) {
            continue;
          }
          Element policy = (Element)node1;
          if ((!"property".equalsIgnoreCase(policy.getTagName())) &&
              (!"destPath".equalsIgnoreCase(policy.getTagName()))) {
            throw new HighTideConfigurationException("Bad configuration file: " + 
              "Expecting <property> or <destPath> for srcPath " + srcPathPrefix +
              " but found " + policy.getTagName());
          }


          // parse the <destPath> items
          if ("destPath".equalsIgnoreCase(policy.getTagName())) {
            String destPath = policy.getAttribute("name");
            if (destPath == null) {
              throw new HighTideConfigurationException("Bad configuration file: " + 
                "<destPath> tag should have an attribute named 'name'.");
            }
            NodeList properties = policy.getChildNodes();
            Properties destProperties = new Properties();
            for (int k = 0; k < properties.getLength(); k++) {
              Node node2 = properties.item(k);
              if (!(node2 instanceof Element)) {
                continue;
              }
              Element property = (Element)node2;
              String propertyName = property.getTagName();
              if (!("property".equalsIgnoreCase(propertyName))) {
                throw new HighTideConfigurationException("Bad configuration file: " + 
                  "<destPath> can have only <property> children." +
                  " but found " + propertyName);
              }
              NodeList nl = property.getChildNodes();
              String pname=null,pvalue=null;
              for (int l = 0; l < nl.getLength(); l++){
                Node node3 = nl.item(l);
                if (!(node3 instanceof Element)) {
                  continue;
                }
                Element item = (Element) node3;
                String itemName = item.getTagName();
                if ("name".equalsIgnoreCase(itemName)){
                  pname = ((Text)item.getFirstChild()).getData().trim();
                } else if ("value".equalsIgnoreCase(itemName)){
                  pvalue = ((Text)item.getFirstChild()).getData().trim();
                }
              }
              if (pname == null || pvalue == null) {
                throw new HighTideConfigurationException("Bad configuration file: " + 
                  "All property for destPath " +  destPath + 
                  "  must have name and value ");
              }
              LOG.info(policyName + "." + pname + " = " + pvalue);
              destProperties.setProperty(pname, pvalue);
            }
            policyInfo.addDestPath(destPath, destProperties); 


          } else if ("property".equalsIgnoreCase(policy.getTagName())) {
            Element property = (Element)node1;
            NodeList nl = property.getChildNodes();
            String pname=null,pvalue=null;
            for (int l = 0; l < nl.getLength(); l++){
              Node node3 = nl.item(l);
              if (!(node3 instanceof Element)) {
               continue;
              }
              Element item = (Element) node3;
              String itemName = item.getTagName();
              if ("name".equalsIgnoreCase(itemName)){
                pname = ((Text)item.getFirstChild()).getData().trim();
              } else if ("value".equalsIgnoreCase(itemName)){
                pvalue = ((Text)item.getFirstChild()).getData().trim();
              }
            }
            if (pname == null || pvalue == null) {
              throw new HighTideConfigurationException("Bad configuration file: " + 
                "All property for srcPath " + srcPathPrefix +
                " must have name and value ");
            }
            LOG.info(policyName + "." + pname + " = " + pvalue);
            policyInfo.setProperty(pname,pvalue);
          }
        }
        existingPolicies.add(policyInfo);
      } else {
        throw new HighTideConfigurationException("Bad configuration file: " +

View Full Code Here

  /**
   * Fix a specific file
   */
  private void fixFile(Path badFile) throws IOException {


    PolicyInfo pinfo = null;
    String filename = badFile.toString();


    LOG.info("File  = file to fix:" + badFile);


    // Find the policy that maps this file
    for (PathToPolicy pp: pathToPolicy) {
      if (filename.startsWith(pp.spath)) {
        pinfo = pp.pinfo;
        break;
      }
    }
    if (pinfo == null) {
      throw new IOException("Unable to find matching policy for " +
                            badFile);
    }


    // process the file and fix it. 
    Path src;
    HighTideNode.getMetrics().fixAttempt.inc();
    
    if (filename.startsWith(pinfo.getSrcPath().toString())) {
      // srcPath is corrupted, pick the first destPath as source of truth.
      String[] splits = filename.split(pinfo.getSrcPath().toString());
      src = new Path(pinfo.getDestPaths().get(0).rpath.toString() + splits[1]);
    } else {
      // dest file is corrupted, copy from source to destination
      String[] splits = filename.split(pinfo.getDestPaths().get(0).rpath.toString());
      src = new Path(pinfo.getSrcPath().toString() + splits[1]);
    }
    DistributedFileSystem srcFs = (DistributedFileSystem) src.getFileSystem(pinfo.getConf());
    DistributedFileSystem destFs = (DistributedFileSystem) badFile.getFileSystem(pinfo.getConf());


    FileStatus sstat = srcFs.getFileStatus(src);
    FileStatus dstat = destFs.getFileStatus(badFile);


    // assert that modtime of the two files are same

View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.protocol.PolicyInfo

org.apache.hadoop.fs.Path

org.apache.hadoop.hdfs.server.hightidenode.ConfigManager

org.apache.hadoop.hdfs.server.hightidenode.FileFixer

java.io.IOException

java.util.Properties

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.