Examples of EditLogFileInputStream

com.alimama.mdrill.editlog.read.EditLogFileInputStream
org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream
An implementation of the abstract class {@link EditLogInputStream}, which reads edits from a file. That file may be either on the local disk or accessible via a URL.
org.apache.hadoop.hdfs.server.namenode.FSEditLog.EditLogFileInputStream

Examples of com.alimama.mdrill.editlog.read.EditLogFileInputStream

        LOG.debug("passing over " + elf + " because it ends at " +
            elf.lastTxId + ", but we only care about transactions " +
            "as new as " + fromTxId);
        continue;
      }
      EditLogFileInputStream elfis = new EditLogFileInputStream(elf.getFs(),elf.getFile(),
            elf.getFirstTxId(), elf.getLastTxId(), elf.isInProgress());
      LOG.debug("selecting edit log stream " + elf);
      streams.add(elfis);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

   * @return number of edits loaded
   * @throws IOException
   */
  int loadFSEdits(StorageDirectory sd) throws IOException {
    int numEdits = 0;
    EditLogFileInputStream edits = 
      new EditLogFileInputStream(getImageFile(sd, NameNodeFile.EDITS));
    
    FSEditLogLoader loader = new FSEditLogLoader(namesystem);
    numEdits += loader.loadFSEdits(edits, editLog.getCurrentTxId());
    edits.close();
    File editsNew = getImageFile(sd, NameNodeFile.EDITS_NEW);
    if (editsNew.exists() && editsNew.length() > 0) {
      edits = new EditLogFileInputStream(editsNew);
      numEdits += loader.loadFSEdits(edits, loader.getCurrentTxId());
      edits.close();
    }
    editLog.setStartTransactionId(loader.getCurrentTxId());
    // update the counts.
    getFSNamesystem().dir.updateCountForINodeWithQuota();    
    return numEdits;

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

            3 + expectedTxns - 1);
        assertTrue("Expect " + editFile + " exists", editFile.exists());
        
        System.out.println("Verifying file: " + editFile);
        long numEdits = loader.loadFSEdits(
            new EditLogFileInputStream(editFile), 3, null);
        int numLeases = namesystem.leaseManager.countLease();
        System.out.println("Number of outstanding leases " + numLeases);
        assertEquals(0, numLeases);
        assertTrue("Verification for " + editFile + " failed. " +
                   "Expected " + expectedTxns + " transactions. "+

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

        File editFile = NNStorage.getFinalizedEditsFile(sd, 1, 1 + expectedTransactions - 1);
        System.out.println("Verifying file: " + editFile);
        
        FSEditLogLoader loader = new FSEditLogLoader(namesystem);        
        int numEdits = loader.loadFSEdits(
            new EditLogFileInputStream(editFile), 1);
        assertEquals("Verification for " + editFile, expectedTransactions, numEdits);
      }
    } finally {
      if(fileSys != null) fileSys.close();
      if(cluster != null) cluster.shutdown();

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

        File file = null;
        EditLogInputStream elis = null;
        OfflineEditsLoader loader = null;
        try {
          file = new File(inputFileName);
          elis = new EditLogFileInputStream(file, -1, -1, false);
          loader = new OfflineEditsBinaryLoader(visitor, elis);
        } finally {
          if ((loader == null) && (elis != null)) {
            elis.close();
          }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream


      File editFile = new File(sd.getCurrentDir(), logFileName);
        
      System.out.println("Verifying file: " + editFile);
      FSEditLogLoader loader = new FSEditLogLoader(namesystem, startTxId);
      long numEditsThisLog = loader.loadFSEdits(new EditLogFileInputStream(editFile), 
          startTxId, null);
      
      System.out.println("Number of edits: " + numEditsThisLog);
      assertTrue(numEdits == -1 || numEditsThisLog == numEdits);
      numEdits = numEditsThisLog;

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

            3 + expectedTxns - 1);
        assertTrue("Expect " + editFile + " exists", editFile.exists());
        
        System.out.println("Verifying file: " + editFile);
        int numEdits = loader.loadFSEdits(
            new EditLogFileInputStream(editFile), 3);
        int numLeases = namesystem.leaseManager.countLease();
        System.out.println("Number of outstanding leases " + numLeases);
        assertEquals(0, numLeases);
        assertTrue("Verification for " + editFile + " failed. " +
                   "Expected " + expectedTxns + " transactions. "+

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

    for (Iterator<StorageDirectory> it = 
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      File editFile = FSImage.getImageFile(it.next(), NameNodeFile.EDITS);
      System.out.println("Verifying file: " + editFile);
      int numEdits = namesystem.getEditLog().loadFSEdits(
        new EditLogFileInputStream(editFile));
      System.out.println("Number of edits: " + numEdits);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

    int numEdits = 0;
    File jSpoolFile = getJSpoolFile(sdEdits);
    long startTime = FSNamesystem.now();
    if(jSpoolFile.exists()) {
      // load edits.new
      EditLogFileInputStream edits = new EditLogFileInputStream(jSpoolFile);
      DataInputStream in = edits.getDataInputStream();
      numEdits += editLog.loadFSEdits(in, false);
  
      // first time reached the end of spool
      jsState = JSpoolState.WAIT;
      numEdits += editLog.loadEditRecords(getLayoutVersion(), in, true);
      getFSNamesystem().dir.updateCountForINodeWithQuota();
      edits.close();
    }


    FSImage.LOG.info("Edits file " + jSpoolFile.getCanonicalPath() 
        + " of size " + jSpoolFile.length() + " edits # " + numEdits 
        + " loaded in " + (FSNamesystem.now()-startTime)/1000 + " seconds.");

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

      for (Iterator<StorageDirectory> it = 
              fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
        File editFile = FSImage.getImageFile(it.next(), NameNodeFile.EDITS);
        System.out.println("Verifying file: " + editFile);
        int numEdits = namesystem.getEditLog().loadFSEdits(
                                  new EditLogFileInputStream(editFile));
        int numLeases = namesystem.leaseManager.countLease();
        System.out.println("Number of outstanding leases " + numLeases);
        assertEquals(0, numLeases);
        assertTrue("Verification for " + editFile + " failed. " +
                   "Expected " + (NUM_THREADS * 2 * NUM_TRANSACTIONS) + " transactions. "+

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.