Package org.apache.hadoop.hdfs.server.protocol

Examples of org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest

646
647
648
649
650
651
652
653
654
655
656
      RemoteEditLog bestLog = Collections.max(logGroup);
      logs.add(bestLog);
      // And then start looking from after that point
      curStartTxId = bestLog.getEndTxId() + 1;
    }
    RemoteEditLogManifest ret = new RemoteEditLogManifest(logs);
   
    if (LOG.isDebugEnabled()) {
      LOG.debug("Generated manifest for logs since " + fromTxId + ":"
          + ret);     
    }
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
    // Make sure we're talking to the same NN!
    sig.validateStorageInfo(bnImage);

    long lastApplied = bnImage.getLastAppliedTxId();
    LOG.debug("Doing checkpoint. Last applied: " + lastApplied);
    RemoteEditLogManifest manifest =
      getRemoteNamenodeProxy().getEditLogManifest(bnImage.getLastAppliedTxId() + 1);

    boolean needReloadImage = false;
    if (!manifest.getLogs().isEmpty()) {
      RemoteEditLog firstRemoteLog = manifest.getLogs().get(0);
      // we don't have enough logs to roll forward using only logs. Need
      // to download and load the image.
      if (firstRemoteLog.getStartTxId() > lastApplied + 1) {
        LOG.info("Unable to roll forward using only logs. Downloading " +
            "image with txid " + sig.mostRecentCheckpointTxId);
        MD5Hash downloadedHash = TransferFsImage.downloadImageToStorage(
            backupNode.nnHttpAddress, sig.mostRecentCheckpointTxId, bnStorage,
            true);
        bnImage.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE,
            sig.mostRecentCheckpointTxId, downloadedHash);
        lastApplied = sig.mostRecentCheckpointTxId;
        needReloadImage = true;
      }

      if (firstRemoteLog.getStartTxId() > lastApplied + 1) {
        throw new IOException("No logs to roll forward from " + lastApplied);
      }
 
      // get edits files
      for (RemoteEditLog log : manifest.getLogs()) {
        TransferFsImage.downloadEditsToStorage(
            backupNode.nnHttpAddress, log, bnStorage);
      }

      if(needReloadImage) {
View Full Code Here
356
357
358
359
360
361
362
    List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(manifest
        .getLogsList().size());
    for (RemoteEditLogProto l : manifest.getLogsList()) {
      logs.add(convert(l));
    }
    return new RemoteEditLogManifest(logs);
  }
View Full Code Here
522
523
524
525
526
527
528
529
530
531
532
    if (ErrorSimulator.getErrorSimulation(0)) {
      throw new IOException("Simulating error0 " +
                            "after creating edits.new");
    }

    RemoteEditLogManifest manifest =
      namenode.getEditLogManifest(sig.mostRecentCheckpointTxId + 1);

    boolean loadImage = downloadCheckpointFiles(
        fsName, checkpointImage, sig, manifest);   // Fetch fsimage and edits
    doMerge(sig, manifest, loadImage, checkpointImage);
View Full Code Here
773
774
775
776
777
778
779
780
781
782
783
      RemoteEditLog bestLog = Collections.max(logGroup);
      logs.add(bestLog);
      // And then start looking from after that point
      curStartTxId = bestLog.getEndTxId() + 1;
    }
    RemoteEditLogManifest ret = new RemoteEditLogManifest(logs);
   
    if (LOG.isDebugEnabled()) {
      LOG.debug("Generated manifest for logs since " + fromTxId + ":"
          + ret);     
    }
View Full Code Here
782
783
784
785
786
787
788
789
790
791
792
      RemoteEditLog bestLog = Collections.max(logGroup);
      logs.add(bestLog);
      // And then start looking from after that point
      curStartTxId = bestLog.getEndTxId() + 1;
    }
    RemoteEditLogManifest ret = new RemoteEditLogManifest(logs);
   
    if (LOG.isDebugEnabled()) {
      LOG.debug("Generated manifest for logs since " + fromTxId + ":"
          + ret);     
    }
View Full Code Here
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
  @Test
  public void testConvertRemoteEditLogManifest() {
    List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>();
    logs.add(new RemoteEditLog(1, 10));
    logs.add(new RemoteEditLog(11, 20));
    RemoteEditLogManifest m = new RemoteEditLogManifest(logs);
    RemoteEditLogManifestProto mProto = PBHelper.convert(m);
    RemoteEditLogManifest m1 = PBHelper.convert(mProto);
   
    List<RemoteEditLog> logs1 = m1.getLogs();
    assertEquals(logs.size(), logs1.size());
    for (int i = 0; i < logs.size(); i++) {
      compare(logs.get(i), logs1.get(i));
    }
  }
View Full Code Here
440
441
442
443
444
445
446
447
448
449
450
   * Get all available log segments present in the underlying storage directory.
   * This function will never return null, or empty list of segments - it will
   * throw exception in this case.
   */
  List<RemoteEditLog> getManifest() throws IOException {
    RemoteEditLogManifest rm = remoteJournalManager.getEditLogManifest(-1);
    if (rm == null || rm.getLogs().size() == 0) {
      throw new IOException("Cannot obtain the list of log segments");
    }
    return rm.getLogs();
  }
View Full Code Here
775
776
777
778
779
780
781
            " which is in the middle of ledger " + ledger);
      }
    }

    Collections.sort(ret);
    return new RemoteEditLogManifest(ret, false);
  }
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
  @Override
  public GetEditLogManifestResponseProto getEditLogManifest(byte[] jid,
      long sinceTxId) throws IOException {
   
    RemoteEditLogManifest manifest = jn.getOrCreateJournal(jid)
        .getEditLogManifest(sinceTxId);
   
    GetEditLogManifestResponseProto ret = new GetEditLogManifestResponseProto();
    ret.setLogs(manifest.getLogs());
    ret.setHttpPort(jn.getBoundHttpAddress().getPort());
   
    if (JournalNode.LOG.isDebugEnabled()) {
      JournalNode.LOG.info("Returning manifest " + manifest.toString());
    }

    return ret;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest

  • org.apache.hadoop.hdfs.notifier.server.ServerLogReaderTransactional
  • org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolServerSideTranslatorPB
  • org.apache.hadoop.hdfs.protocolPB.PBHelper
  • org.apache.hadoop.hdfs.protocolPB.TestPBHelper
  • org.apache.hadoop.hdfs.qjournal.client.IPCLoggerChannel
  • org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager
  • org.apache.hadoop.hdfs.qjournal.server.Journal
  • org.apache.hadoop.hdfs.qjournal.server.JournalNodeRpcServer
  • org.apache.hadoop.hdfs.server.namenode.bookkeeper.BookKeeperJournalManager
  • org.apache.hadoop.hdfs.server.namenode.Checkpointer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.