Package org.apache.hadoop.raid.protocol

Examples of org.apache.hadoop.raid.protocol.PolicyInfo


          if (!"policy".equalsIgnoreCase(policy.getTagName())) {
            throw new RaidConfigurationException("Bad configuration file: " +
              "Expecting <policy> for srcPath " + srcPathPrefix);
          }
          String policyName = policy.getAttribute("name");
          PolicyInfo curr = new PolicyInfo(policyName, conf);
          if (srcPathPrefix != null && srcPathPrefix.length() > 0) {
            curr.setSrcPath(srcPathPrefix);
          }
          // loop through all the properties of this policy
          NodeList properties = policy.getChildNodes();
          PolicyInfo parent = null;
          for (int k = 0; k < properties.getLength(); k++) {
            Node node2 = properties.item(k);
            if (!(node2 instanceof Element)) {
              continue;
            }
            Element property = (Element)node2;
            String propertyName = property.getTagName();
            if ("erasureCode".equalsIgnoreCase(propertyName)) {
              String text = ((Text)property.getFirstChild()).getData().trim();
              LOG.info(policyName + ".erasureCode = " + text);
              curr.setErasureCode(text);
            } else if ("description".equalsIgnoreCase(propertyName)) {
              String text = ((Text)property.getFirstChild()).getData().trim();
              curr.setDescription(text);
            } else if ("parentPolicy".equalsIgnoreCase(propertyName)) {
              String text = ((Text)property.getFirstChild()).getData().trim();
              parent = existingPolicies.get(text);
            } else if ("property".equalsIgnoreCase(propertyName)) {
              NodeList nl = property.getChildNodes();
              String pname=null,pvalue=null;
              for (int l = 0; l < nl.getLength(); l++){
                Node node3 = nl.item(l);
                if (!(node3 instanceof Element)) {
                  continue;
                }
                Element item = (Element) node3;
                String itemName = item.getTagName();
                if ("name".equalsIgnoreCase(itemName)){
                  pname = ((Text)item.getFirstChild()).getData().trim();
                } else if ("value".equalsIgnoreCase(itemName)){
                  pvalue = ((Text)item.getFirstChild()).getData().trim();
                }
              }
              if (pname != null && pvalue != null) {
                LOG.info(policyName + "." + pname + " = " + pvalue);
                curr.setProperty(pname,pvalue);
              }
            } else {
              LOG.warn("Found bad property " + propertyName +
                       " for srcPath" + srcPathPrefix +
                       " policy name " + policyName +
                       ". Ignoring.");
            }
          }  // done with all properties of this policy

          PolicyInfo pinfo;
          if (parent != null) {
            pinfo = new PolicyInfo(policyName, conf);
            pinfo.copyFrom(parent);
            pinfo.copyFrom(curr);
          } else {
            pinfo = curr;
          }
          if (policyList != null) {
            policyList.add(pinfo);
View Full Code Here


        new SequenceFile.Reader(conf, Reader.file(inputFile));
      long prev = 0L;
      final int opsPerTask = conf.getInt(OPS_PER_TASK, DEFAULT_OPS_PER_TASK);
      try {
        Text key = new Text();
        PolicyInfo value = new PolicyInfo();
        int count = 0; // count src
        while (in.next(key, value)) {
          long curr = in.getPosition();
          long delta = curr - prev;
          if (++count > opsPerTask) {
View Full Code Here

      Path topDir = new Path(TEST_DIR + "/testFileFilter");
      int targetRepl = 1;
      createTestTree(topDir);
      Path file = new Path(topDir.toString() + "/a/f1");
      FileStatus stat = fs.getFileStatus(file);
      PolicyInfo info = new PolicyInfo("testFileFilter", conf);
      info.setSrcPath(topDir.toString());
      info.setErasureCode("rs");
      info.setDescription("test policy");
      info.setProperty("targetReplication", "1");
      info.setProperty("metaReplication", "1");

      DirectoryTraversal.FileFilter timeBasedXORFilter =
        new RaidFilter.TimeBasedFilter(conf,
          RaidNode.xorDestinationPath(conf), targetRepl,
            System.currentTimeMillis(), 0);
View Full Code Here

  public void testLayeredPolicies() throws Exception {
    mySetup();
    Path src1 = new Path("/user/foo");
    Path src2 = new Path("/user/foo/bar");

    PolicyInfo info1 = new PolicyInfo("p1", conf);
    info1.setSrcPath(src1.toString());
    info1.setErasureCode("xor");
    info1.setDescription("test policy");
    info1.setProperty("targetReplication", "1");
    info1.setProperty("metaReplication", "1");
    info1.setProperty("modTimePeriod", "0");

    PolicyInfo info2 = new PolicyInfo("p2", conf);
    info2.setSrcPath(src2.toString());
    info2.setErasureCode("xor");
    info2.setDescription("test policy");
    info2.setProperty("targetReplication", "1");
    info2.setProperty("metaReplication", "1");
    info2.setProperty("modTimePeriod", "0");

    ArrayList<PolicyInfo> all = new ArrayList<PolicyInfo>();
    all.add(info1);
    all.add(info2);
View Full Code Here

  public void testPurgePreference() throws Exception {
    createClusters(true);
    Path dir = new Path("/user/test/raidtest/");
    Path file1 = new Path(dir + "/file1");

    PolicyInfo infoXor = new PolicyInfo("testPurgePreference", conf);
    infoXor.setSrcPath("/user/test/raidtest");
    infoXor.setErasureCode("xor");
    infoXor.setDescription("test policy");
    infoXor.setProperty("targetReplication", "2");
    infoXor.setProperty("metaReplication", "2");

    PolicyInfo infoRs = new PolicyInfo("testPurgePreference", conf);
    infoRs.setSrcPath("/user/test/raidtest");
    infoRs.setErasureCode("rs");
    infoRs.setDescription("test policy");
    infoRs.setProperty("targetReplication", "1");
    infoRs.setProperty("metaReplication", "1");
    try {
      TestRaidNode.createOldFile(fileSys, file1, 1, 9, 8192L);
      FileStatus stat = fileSys.getFileStatus(file1);

      // Create the parity files.
      RaidNode.doRaid(
        conf, infoXor, stat, new RaidNode.Statistics(), Reporter.NULL);
      RaidNode.doRaid(
        conf, infoRs, stat, new RaidNode.Statistics(), Reporter.NULL);
      Path xorParity =
        new Path(RaidNode.DEFAULT_RAID_LOCATION, "user/test/raidtest/file1");
      Path rsParity =
        new Path(RaidNode.DEFAULT_RAIDRS_LOCATION, "user/test/raidtest/file1");
      assertTrue(fileSys.exists(xorParity));
      assertTrue(fileSys.exists(rsParity));

      // Check purge of a single parity file.
      RaidNode cnode = RaidNode.createRaidNode(conf);
      FileStatus raidRsStat =
        fileSys.getFileStatus(new Path(RaidNode.DEFAULT_RAIDRS_LOCATION));
      cnode.purgeMonitor.recursePurge(infoRs.getErasureCode(), fileSys, fileSys,
         RaidNode.DEFAULT_RAIDRS_LOCATION, raidRsStat);

      // Calling purge under the RS path has no effect.
      assertTrue(fileSys.exists(xorParity));
      assertTrue(fileSys.exists(rsParity));
View Full Code Here

  public void testPurgePreference() throws Exception {
    createClusters(true);
    Path dir = new Path("/user/test/raidtest/");
    Path file1 = new Path(dir + "/file1");

    PolicyInfo infoXor = new PolicyInfo("testPurgePreference", conf);
    infoXor.setSrcPath("/user/test/raidtest");
    infoXor.setErasureCode("xor");
    infoXor.setDescription("test policy");
    infoXor.setProperty("targetReplication", "2");
    infoXor.setProperty("metaReplication", "2");

    PolicyInfo infoRs = new PolicyInfo("testPurgePreference", conf);
    infoRs.setSrcPath("/user/test/raidtest");
    infoRs.setErasureCode("rs");
    infoRs.setDescription("test policy");
    infoRs.setProperty("targetReplication", "1");
    infoRs.setProperty("metaReplication", "1");
    try {
      TestRaidNode.createOldFile(fileSys, file1, 1, 9, 8192L);
      FileStatus stat = fileSys.getFileStatus(file1);

      // Create the parity files.
      RaidNode.doRaid(
        conf, infoXor, stat, new RaidNode.Statistics(), Reporter.NULL);
      RaidNode.doRaid(
        conf, infoRs, stat, new RaidNode.Statistics(), Reporter.NULL);
      Path xorParity =
        new Path(RaidNode.DEFAULT_RAID_LOCATION, "user/test/raidtest/file1");
      Path rsParity =
        new Path(RaidNode.DEFAULT_RAIDRS_LOCATION, "user/test/raidtest/file1");
      assertTrue(fileSys.exists(xorParity));
      assertTrue(fileSys.exists(rsParity));

      // Check purge of a single parity file.
      RaidNode cnode = RaidNode.createRaidNode(conf);
      FileStatus raidRsStat =
        fileSys.getFileStatus(new Path(RaidNode.DEFAULT_RAIDRS_LOCATION));
      cnode.purgeMonitor.recursePurge(infoRs.getErasureCode(), fileSys, fileSys,
         RaidNode.DEFAULT_RAIDRS_LOCATION, raidRsStat);

      // Calling purge under the RS path has no effect.
      assertTrue(fileSys.exists(xorParity));
      assertTrue(fileSys.exists(rsParity));
View Full Code Here

  public String recoverFile(String inStr, long corruptOffset) throws IOException {

    LOG.info("Recover File for " + inStr + " for corrupt offset " + corruptOffset);
    Path inputPath = new Path(inStr);
    Path srcPath = inputPath.makeQualified(inputPath.getFileSystem(conf));
    PolicyInfo info = findMatchingPolicy(srcPath);
    if (info != null) {

      // find stripe length from config
      int stripeLength = getStripeLength(conf, info);
View Full Code Here

    PolicyList[] sorted = all.toArray(new PolicyList[all.size()]);
    Arrays.sort(sorted, lexi);

    // loop through all categories of policies.
    for (PolicyList category : sorted) {
      PolicyInfo first = category.getAll().iterator().next();
      if (first != null) {
        Path[] srcPaths = first.getSrcPathExpanded(); // input src paths unglobbed
        if (srcPaths == null) {
          continue;
        }

        for (Path src: srcPaths) {
View Full Code Here

      FileSystem fs = srcs.getFileSystem(job);

      List<FileSplit> splits = new ArrayList<FileSplit>(numSplits);

      Text key = new Text();
      PolicyInfo value = new PolicyInfo();
      SequenceFile.Reader in = null;
      long prev = 0L;
      int count = 0; // count src
      try {
        for (in = new SequenceFile.Reader(fs, srcs, job); in.next(key, value);) {
View Full Code Here

      // activate all categories
      Collection<PolicyList> all = configMgr.getAllPolicies();  
      PolicyList[] sorted = all.toArray(new PolicyList[all.size()]);
      Iterator<PolicyInfo> pi = sorted[0].getAll().iterator();
      PolicyInfo p = pi.next();
      List<FileStatus> ll = new ArrayList<FileStatus>();

      for(int i = 0 ; i < 10; i++){
        Path file = new Path("/user/dhruba/raidtest/file"+i);
        createOldFile(fileSys, file, 1, 7, blockSize);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.raid.protocol.PolicyInfo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.