Package org.apache.hadoop.hdfs.server.namenode

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream

471
472
473
474
475
476
477
478
479
480
481
      RemoteEditLogManifest manifest = e.getValue();
     
      for (RemoteEditLog remoteLog : manifest.getLogs()) {
        URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());

        EditLogInputStream elis = EditLogFileInputStream.fromUrl(
            connectionFactory, url, remoteLog.getStartTxId(),
            remoteLog.getEndTxId(), remoteLog.isInProgress());
        allStreams.add(elis);
      }
    }
View Full Code Here
319
320
321
322
323
324
325
326
327
328
329
      if (!throwExceptionOnRead) {
        return streams;
      } else {
        Collection<EditLogInputStream> ret = new LinkedList<EditLogInputStream>();
        for (EditLogInputStream stream : streams) {
          EditLogInputStream spyStream = spy(stream);
          doAnswer(new Answer<FSEditLogOp>() {

            @Override
            public FSEditLogOp answer(InvocationOnMock invocation)
                throws Throwable {
View Full Code Here
274
275
276
277
278
279
280
281
282
283
284
285
286
    }
    out.close();
    bkjm.finalizeLogSegment(1, numTransactions);

    
    EditLogInputStream in = bkjm.getInputStream(1, true);
    try {
      assertEquals(numTransactions,
                   FSEditLogTestUtil.countTransactionsInStream(in));
    } finally {
      in.close();
    }
  }
View Full Code Here
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
  @Override
  public void selectInputStreams(Collection<EditLogInputStream> streams,
      long fromTxId, boolean inProgressOk) {
    // NOTE: could probably be rewritten more efficiently
    while (true) {
      EditLogInputStream elis;
      try {
        elis = getInputStream(fromTxId, inProgressOk);
      } catch (IOException e) {
        LOG.error(e);
        return;
      }
      if (elis == null) {
        return;
      }
      streams.add(elis);
      if (elis.getLastTxId() == HdfsConstants.INVALID_TXID) {
        return;
      }
      fromTxId = elis.getLastTxId() + 1;
    }
  }
View Full Code Here
472
473
474
475
476
477
478
479
480
481
482
      RemoteEditLogManifest manifest = e.getValue();
     
      for (RemoteEditLog remoteLog : manifest.getLogs()) {
        URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());

        EditLogInputStream elis = EditLogFileInputStream.fromUrl(
            url, remoteLog.getStartTxId(), remoteLog.getEndTxId(),
            remoteLog.isInProgress());
        allStreams.add(elis);
      }
    }
View Full Code Here
472
473
474
475
476
477
478
479
480
481
482
      RemoteEditLogManifest manifest = e.getValue();
     
      for (RemoteEditLog remoteLog : manifest.getLogs()) {
        URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());

        EditLogInputStream elis = EditLogFileInputStream.fromUrl(
            url, remoteLog.getStartTxId(), remoteLog.getEndTxId(),
            remoteLog.isInProgress());
        allStreams.add(elis);
      }
    }
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
    readerQjm.selectInputStreams(streams, 0, false);
    try {
      assertEquals(1, streams.size());
      // Validate the actual stream contents.
      EditLogInputStream stream = streams.get(0);
      assertEquals(1, stream.getFirstTxId());
      assertEquals(3, stream.getLastTxId());
     
      verifyEdits(streams, 1, 3);
      assertNull(stream.readOp());
    } finally {
      IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
      streams.clear();
    }
   
    // Ensure correct results when there is a stream in-progress, but we don't
    // ask for in-progress.
    writeSegment(cluster, qjm, 4, 3, false);
    readerQjm.selectInputStreams(streams, 0, false);
    try {
      assertEquals(1, streams.size());
      EditLogInputStream stream = streams.get(0);
      assertEquals(1, stream.getFirstTxId());
      assertEquals(3, stream.getLastTxId());
      verifyEdits(streams, 1, 3);
    } finally {
      IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
      streams.clear();
    }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
        String inputFileName, boolean xmlInput) throws IOException {
      if (xmlInput) {
        return new OfflineEditsXmlLoader(visitor, new File(inputFileName));
      } else {
        File file = null;
        EditLogInputStream elis = null;
        OfflineEditsLoader loader = null;
        try {
          file = new File(inputFileName);
          elis = new EditLogFileInputStream(file, -1, -1, false);
          loader = new OfflineEditsBinaryLoader(visitor, elis);
        } finally {
          if ((loader == null) && (elis != null)) {
            elis.close();
          }
        }
        return loader;
      }
    }
View Full Code Here
355
356
357
358
359
360
361
362
363
364
365
366
367
    }
    out.close();
    bkjm.finalizeLogSegment(1, numTransactions);

    
    EditLogInputStream in = bkjm.getInputStream(1, true);
    try {
      assertEquals(numTransactions,
                   FSEditLogTestUtil.countTransactionsInStream(in));
    } finally {
      in.close();
    }
  }
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
    readerQjm.selectInputStreams(streams, 0, false);
    try {
      assertEquals(1, streams.size());
      // Validate the actual stream contents.
      EditLogInputStream stream = streams.get(0);
      assertEquals(1, stream.getFirstTxId());
      assertEquals(3, stream.getLastTxId());
     
      verifyEdits(streams, 1, 3);
      assertNull(stream.readOp());
    } finally {
      IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
      streams.clear();
    }
   
    // Ensure correct results when there is a stream in-progress, but we don't
    // ask for in-progress.
    writeSegment(cluster, qjm, 4, 3, false);
    readerQjm.selectInputStreams(streams, 0, false);
    try {
      assertEquals(1, streams.size());
      EditLogInputStream stream = streams.get(0);
      assertEquals(1, stream.getFirstTxId());
      assertEquals(3, stream.getLastTxId());
      verifyEdits(streams, 1, 3);
    } finally {
      IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
      streams.clear();
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream

  • org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager
  • org.apache.hadoop.contrib.bkjournal.TestBookKeeperJournalManager
  • org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager
  • org.apache.hadoop.hdfs.qjournal.client.TestQJMWriteRead
  • org.apache.hadoop.hdfs.qjournal.client.TestQuorumJournalManager
  • org.apache.hadoop.hdfs.qjournal.client.TestQuorumJournalManagerInputStream
  • org.apache.hadoop.hdfs.qjournal.QJMTestUtil
  • org.apache.hadoop.hdfs.server.namenode.bookkeeper.TestBookKeeperJournalManager
  • org.apache.hadoop.hdfs.server.namenode.ha.TestFailureToReadEdits$LimitedEditLogAnswer
  • org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader$OfflineEditsLoaderFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.