Package org.apache.hadoop.hdfs.server.namenode

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
  }
 
  public static EditLogOutputStream writeSegment(MiniJournalCluster cluster,
      QuorumJournalManager qjm, long startTxId, int numTxns,
      boolean finalize) throws IOException {
    EditLogOutputStream stm = qjm.startLogSegment(startTxId);
    // Should create in-progress
    assertExistsInQuorum(cluster,
        NNStorage.getInProgressEditsFileName(startTxId));
   
    writeTxns(stm, startTxId, numTxns);
    if (finalize) {
      stm.close();
      qjm.finalizeLogSegment(startTxId, startTxId + numTxns - 1);
      return null;
    } else {
      return stm;
    }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
  }
 
  public static EditLogOutputStream writeRandomSegment(MiniJournalCluster cluster,
      QuorumJournalManager qjm, long startTxId, int numTxns,
      boolean finalize, List<FSEditLogOp> writtenTxns) throws IOException {
    EditLogOutputStream stm = qjm.startLogSegment(startTxId);
    // Should create in-progress
    assertExistsInQuorum(cluster,
        NNStorage.getInProgressEditsFileName(startTxId));
   
    List<FSEditLogOp> txns = FSEditLogTestUtil.getContiguousLogSegment(
        (int) startTxId, ((int) startTxId + numTxns - 1));
    for (FSEditLogOp op : txns) {
      stm.write(op);
      stm.setReadyToFlush();
      stm.flush();
      if (writtenTxns != null) {
        writtenTxns.add(op);
      }
    }
   
    if (finalize) {
      stm.close();
      qjm.finalizeLogSegment(startTxId, startTxId + numTxns - 1);
      return null;
    } else {
      return stm;
    }
View Full Code Here
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
      QuorumJournalManager qjm, long txid, int numTxns, Holder<Throwable> thrown) {
   
    long firstTxId = txid;
    long lastAcked = txid - 1;
    try {
      EditLogOutputStream stm = qjm.startLogSegment(txid);
     
      for (int i = 0; i < numTxns; i++) {
        QJMTestUtil.writeTxns(stm, txid++, 1);
        lastAcked++;
      }
     
      stm.close();
      qjm.finalizeLogSegment(firstTxId, lastAcked);
    } catch (Throwable t) {
      thrown.held = t;
    }
    return lastAcked;
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
  public void testTailing() throws Exception {
    // Unlike the other unit test, numEdits here is constant as this is
    // a longer running test
    final int numEdits = 1000;
    final AtomicBoolean finishedProducing = new AtomicBoolean(false);
    final EditLogOutputStream out = qjm.startLogSegment(0);

    Callable<Void> producerThread = new Callable<Void>() {
      @Override
      public Void call() throws Exception {
        try {
View Full Code Here
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
  @Test
  public void testCrashAtBeginningOfSegment() throws Exception {
    writeSegment(cluster, qjm, 1, 3, true);
    waitForAllPendingCalls(qjm.getLoggerSetForTests());
   
    EditLogOutputStream stm = qjm.startLogSegment(4);
    try {
      waitForAllPendingCalls(qjm.getLoggerSetForTests());
    } finally {
      stm.abort();
    }
   
   
    // Make a new QJM
    qjm = new QuorumJournalManager(
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    writeSegment(cluster, qjm, 1, 3, true);
    waitForAllPendingCalls(qjm.getLoggerSetForTests());
    cluster.getJournalNode(nodeMissingSegment).stopAndJoin(0);
   
    // Open segment on 2/3 nodes
    EditLogOutputStream stm = qjm.startLogSegment(4);
    try {
      waitForAllPendingCalls(qjm.getLoggerSetForTests());
     
      // Write transactions to only 1/3 nodes
      failLoggerAtTxn(spies.get(nodeWithEmptySegment), 4);
      try {
        writeTxns(stm, 4, 1);
        fail("Did not fail even though 2/3 failed");
      } catch (QuorumException qe) {
        GenericTestUtils.assertExceptionContains("mock failure", qe);
      }
    } finally {
      stm.abort();
    }
   
    // Bring back the down JN.
    cluster.restartJournalNode(nodeMissingSegment);
   
View Full Code Here
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
   
    // Logger 1: fail at txn id 4
    failLoggerAtTxn(spies.get(1), 4L);
   
    writeSegment(cluster, qjm, 1, 3, true);
    EditLogOutputStream stm = qjm.startLogSegment(4);
    try {
      writeTxns(stm, 4, 1);
      fail("Did not fail to write");
    } catch (QuorumException qe) {
      // Should fail, because logger 1 had an injected fault and
      // logger 0 should detect writer out of sync
      GenericTestUtils.assertExceptionContains("Writer out of sync",
          qe);
    } finally {
      stm.abort();
      qjm.close();
    }
   
    // State:
    // Logger 0: 1-3 in-progress (since it missed finalize)
View Full Code Here
551
552
553
554
555
556
557
558
559
560
561
   * - JN2: edits 1-5 in progress
   *
   * None of the loggers have any associated paxos info.
   */
  private void setupLoggers345() throws Exception {
    EditLogOutputStream stm = qjm.startLogSegment(1);
   
    failLoggerAtTxn(spies.get(0), 4);
    failLoggerAtTxn(spies.get(1), 5);
   
    writeTxns(stm, 1, 3);
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
   * it should be disabled.
   */
  @Test
  public void testRefresh() throws Exception {
    // start new segment
    EditLogOutputStream stm = qjm.startLogSegment(0);
    // write a bunch of transactions
    QJMTestUtil.writeTxns(stm, 0, 10);
   
    // get input stream
    List<EditLogInputStream> streams = Lists.newArrayList();
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
   * startup)
   */
  @Test
  public void testRefreshOnlyForInprogress() throws Exception {
    // start new segment
    EditLogOutputStream stm = qjm.startLogSegment(0);
    // write a bunch of transactions
    QJMTestUtil.writeTxns(stm, 0, 10);
    qjm.finalizeLogSegment(0, 9);
   
    // get input stream
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

  • org.apache.hadoop.contrib.bkjournal.TestBookKeeperJournalManager
  • org.apache.hadoop.hdfs.notifier.server.TestTransactionalEditLogReader
  • org.apache.hadoop.hdfs.qjournal.client.TestQJMRecovery
  • org.apache.hadoop.hdfs.qjournal.client.TestQJMWithFaults
  • org.apache.hadoop.hdfs.qjournal.client.TestQJMWriteRead
  • org.apache.hadoop.hdfs.qjournal.client.TestQuorumJournalManager
  • org.apache.hadoop.hdfs.qjournal.client.TestQuorumJournalManagerInputStream
  • org.apache.hadoop.hdfs.qjournal.client.TestQuorumJournalManagerUnit
  • org.apache.hadoop.hdfs.qjournal.QJMTestUtil
  • org.apache.hadoop.hdfs.server.namenode.bookkeeper.TestBookKeeperJournalManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.