Examples of StorageDirectory

Examples of com.alimama.mdrill.editlog.defined.StorageDirectory

    {
      long t1=System.currentTimeMillis();
    List<StorageDirectory> editsDirs = new ArrayList<StorageDirectory>();
    if("hdfs".equals(SolrCore.getBinglogType()))
    {
      editsDirs.add(new StorageDirectory(FileSystem.get(conf), new Path(params.hdfsPath, "editlogs_v9")));
    }else{
      editsDirs.add(new StorageDirectory(FileSystem.getLocal(conf), new Path(params.baseDir, "editlogs_v9")));
    }
    LOG.info("recoverFromEditlog begin:"+this.params.getLogStr());
    editlog = new FSEditLog(conf, editsDirs);
    editlog.initJournalsForWrite();
    editlog.recoverUnclosedStreams();

View Full Code Here

Examples of com.alimama.mdrill.editlog.defined.StorageDirectory

    {
      long t1=System.currentTimeMillis();
    List<StorageDirectory> editsDirs = new ArrayList<StorageDirectory>();
    if("hdfs".equals(SolrCore.getBinglogType()))
    {
      editsDirs.add(new StorageDirectory(FileSystem.get(conf), new Path(params.hdfsPath, "editlogs_v9")));
    }else{
      editsDirs.add(new StorageDirectory(FileSystem.getLocal(conf), new Path(params.baseDir, "editlogs_v9")));
    }
    LOG.info("recoverFromEditlog begin:"+this.params.getLogStr());
    editlog = new FSEditLog(conf, editsDirs);
    editlog.initJournalsForWrite();
    editlog.recoverUnclosedStreams();

View Full Code Here

Examples of org.apache.hadoop.dfs.Storage.StorageDirectory

        break;
      case DATA_NODE:
        storage = new DataStorage(version, "doNotCare");
        break;
      }
      StorageDirectory sd = storage.new StorageDirectory(parent[i].getParentFile());
      sd.write(versionFile);
      versionFiles[i] = versionFile;
    }
    return versionFiles;
  }

View Full Code Here

Examples of org.apache.hadoop.dfs.Storage.StorageDirectory

        break;
      case DATA_NODE:
        storage = new DataStorage(version, "doNotCare");
        break;
      }
      StorageDirectory sd = storage.new StorageDirectory(parent[i].getParentFile());
      sd.write(versionFile);
      versionFiles[i] = versionFile;
    }
    return versionFiles;
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

   * test
   */
  public void printStorages(FSImage fs) {
    LOG.info("current storages and corresoponding sizes:");
    for(Iterator<StorageDirectory> it = fs.dirIterator(); it.hasNext(); ) {
      StorageDirectory sd = it.next();
      
      if(sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) {
        File imf = FSImage.getImageFile(sd, NameNodeFile.IMAGE);
        LOG.info("  image file " + imf.getAbsolutePath() + "; len = " + imf.length());  
      }
      if(sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
        File edf = FSImage.getImageFile(sd, NameNodeFile.EDITS);
        LOG.info("  edits file " + edf.getAbsolutePath() + "; len = " + edf.length()); 
      }
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

    numTransactions = totalTimeTransactions = numTransactionsBatchedInSync = 0;
    if (editStreams == null)
      editStreams = new ArrayList<EditLogOutputStream>();
    for (Iterator<StorageDirectory> it = 
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      StorageDirectory sd = it.next();
      File eFile = getEditFile(sd);
      try {
        EditLogOutputStream eStream = new EditLogFileOutputStream(eFile);
        editStreams.add(eStream);
      } catch (IOException e) {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

    // Open edits.new
    //
    boolean failedSd = false;
    for (Iterator<StorageDirectory> it = 
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      StorageDirectory sd = it.next();
      try {
        EditLogFileOutputStream eStream = 
             new EditLogFileOutputStream(getEditNewFile(sd));
        eStream.create();
        editStreams.add(eStream);
      } catch (IOException e) {
        failedSd = true;
        // remove stream and this storage directory from list
        FSImage.LOG.warn("rollEdidLog: removing storage " + sd.getRoot().getPath());
        sd.unlock();
        fsimage.removedStorageDirs.add(sd);
        it.remove();
      }
    }
    if(failedSd)

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

    //
    // Delete edits and rename edits.new to edits.
    //
    for (Iterator<StorageDirectory> it = 
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      StorageDirectory sd = it.next();
      if (!getEditNewFile(sd).renameTo(getEditFile(sd))) {
        //
        // renameTo() fails on Windows if the destination
        // file exists.
        //
        getEditFile(sd).delete();
        if (!getEditNewFile(sd).renameTo(getEditFile(sd))) {
          // Should we also remove from edits
          NameNode.LOG.warn("purgeEditLog: removing failed storage " + sd.getRoot().getPath());
          fsimage.removedStorageDirs.add(sd);
          it.remove(); 
        }
      }
    }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory


  /**
   * Return the name of the edit file
   */
  synchronized File getFsEditName() throws IOException {
    StorageDirectory sd = null;
    for (Iterator<StorageDirectory> it = 
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();)
      sd = it.next();
    return getEditFile(sd);
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

    // Also check that the edits file is empty here
    // and that temporary checkpoint files are gone.
    FSImage image = cluster.getNameNode().getFSImage();
    for (Iterator<StorageDirectory> it = 
             image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
      StorageDirectory sd = it.next();
      assertFalse(FSImage.getImageFile(sd, NameNodeFile.IMAGE_NEW).exists());
    }
    for (Iterator<StorageDirectory> it = 
            image.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      StorageDirectory sd = it.next();
      assertFalse(image.getEditNewFile(sd).exists());
      File edits = image.getEditFile(sd);
      assertTrue(edits.exists()); // edits should exist and be empty
      long editsLen = edits.length();
      assertTrue(editsLen == Integer.SIZE/Byte.SIZE);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.