Examples of FileJournalManager

com.alimama.mdrill.editlog.write.FileJournalManager
org.apache.hadoop.hdfs.server.namenode.FileJournalManager
Journal manager for the common case of edits files being written to a storage directory. Note: this class is not thread-safe and should be externally synchronized.

Examples of org.apache.hadoop.hdfs.server.namenode.FileJournalManager

    Callable<Void> producerThread = new Callable<Void>() {
      @Override
      public Void call() throws Exception {
        int currentTxId = 0;
        try {
          FileJournalManager jm = new FileJournalManager(storageDirectory);
          for (int segment = 0; segment < numSegments; segment++) {
            // start new segment
            int firstTxId = currentTxId;
            EditLogOutputStream out = jm.startLogSegment(currentTxId);


            // starting transaction
            FSEditLogOp startOp = new LogSegmentOp(
                FSEditLogOpCodes.OP_START_LOG_SEGMENT);
            startOp.setTransactionId(firstTxId);
            FSEditLogTestUtil.writeToStreams(startOp, out);
            LOG.info("Written op: " + startOp);
            writtenTransactions.add(startOp);


            currentTxId++;


            // other transactions
            List<FSEditLogOp> transactions = FSEditLogTestUtil
                .getContiguousLogSegment(currentTxId, currentTxId
                    + editsPerFile);


            for (int i = 0; i < editsPerFile; i++) {
              FSEditLogOp op = transactions.get(i);
              op.setTransactionId(currentTxId);
              FSEditLogTestUtil.writeToStreams(op, out);
              writtenTransactions.add(op);
              currentTxId++;
              LOG.info("Written op: " + op);
              if (i % 100 == 0) {
                Thread.sleep(10);
                FSEditLogTestUtil.flushStreams(out);
              }
            }


            // write ending transactions if needed
            if (endLogSegment || (segment == numSegments - 1)) {
              int lastTxId = currentTxId;
              FSEditLogOp endOp = new LogSegmentOp(
                  FSEditLogOpCodes.OP_END_LOG_SEGMENT);
              endOp.setTransactionId(lastTxId);
              FSEditLogTestUtil.writeToStreams(endOp, out);
              LOG.info("Written op: " + endOp);
              writtenTransactions.add(endOp);
              currentTxId++;
            }


            FSEditLogTestUtil.flushStreams(out);
            FSEditLogTestUtil.closeStreams(out);
            jm.finalizeLogSegment(firstTxId, currentTxId - 1);


            // simulate NFS cache delay (reader won't see
            // the new file for 1 second
            if (r.nextBoolean())
              ;

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FileJournalManager

      
      long segmentTxId = ServletUtil.parseLongParam(request,
          SEGMENT_TXID_PARAM);
      long position = ServletUtil.parseLongParam(request, POSITION_PARAM);


      FileJournalManager fjm = journal.getJournalManager();
      File editFile;
      InputStream fStream;
      
      long lengthToSend;
      synchronized (journal) {
        // Synchronize on the journal so that the file doesn't get finalized
        // out from underneath us while we're in the process of opening
        // it up.


        // get all log segments
        List<EditLogFile> logFiles = fjm.getLogFiles(segmentTxId);
        
        // no files found
        if (logFiles.size() == 0) {
          response.sendError(HttpServletResponse.SC_NOT_FOUND,
              "No edit log found starting at txid " + segmentTxId);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FileJournalManager

          response)) {
        return;
      }


      long startTxId = ServletUtil.parseLongParam(request, START_TXID_PARAM);
      FileJournalManager fjm = journal.getJournalManager();


      LOG.info("getJournalManifest request: journalId " + journalId
          + ", start txid: " + startTxId + ", storage: "
          + storage.toColonSeparatedString());


      String output = JSON.toString(new ArrayList<String>());


      synchronized (journal) {
        // Synchronize on the journal so that the files do not change
        // get all log segments
        List<EditLogFile> logFiles = fjm.getLogFiles(startTxId, false);
        List<String> manifest = new ArrayList<String>();
        for (EditLogFile elf : logFiles) {
          manifest.add(elf.toColonSeparatedString());
        }
        output = JSON.toString(manifest);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FileJournalManager

    // initialize storage directories
    journalStorage = new JNStorage(logDir, errorReporter, false, journalNode.getConf());
    imageStorage = new JNStorage(imageDir, errorReporter, true, journalNode.getConf());


    // initialize journal and image managers
    this.fjm = new FileJournalManager(journalStorage.getSingularStorageDir(),
        null, errorReporter);


    this.imageManager = new FileImageManager(
        imageStorage.getStorageDirectory(), imageStorage);

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.