Examples of EditLogInputStream

com.alimama.mdrill.editlog.read.EditLogInputStream
org.apache.hadoop.hdfs.server.namenode.EditLogInputStream
A generic abstract class to support reading edits log data from persistent storage. It should stream bytes from the storage exactly as they were written into the # {@link EditLogOutputStream}.

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream

    }
    out.close();
    bkjm.finalizeLogSegment(1, numTransactions);


     
    EditLogInputStream in = bkjm.getInputStream(1, true);
    try {
      assertEquals(numTransactions, 
                   FSEditLogTestUtil.countTransactionsInStream(in));
    } finally {
      in.close();
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream


    readerQjm.selectInputStreams(streams, 0, false);
    try {
      assertEquals(1, streams.size());
      // Validate the actual stream contents.
      EditLogInputStream stream = streams.get(0);
      assertEquals(1, stream.getFirstTxId());
      assertEquals(3, stream.getLastTxId());
      
      verifyEdits(streams, 1, 3);
      assertNull(stream.readOp());
    } finally {
      IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
      streams.clear();
    }
    
    // Ensure correct results when there is a stream in-progress, but we don't
    // ask for in-progress.
    writeSegment(cluster, qjm, 4, 3, false);
    readerQjm.selectInputStreams(streams, 0, false);
    try {
      assertEquals(1, streams.size());
      EditLogInputStream stream = streams.get(0);
      assertEquals(1, stream.getFirstTxId());
      assertEquals(3, stream.getLastTxId());
      verifyEdits(streams, 1, 3);
    } finally {
      IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
      streams.clear();
    }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream

      if (!throwExceptionOnRead) {
        return streams;
      } else {
        Collection<EditLogInputStream> ret = new LinkedList<EditLogInputStream>();
        for (EditLogInputStream stream : streams) {
          EditLogInputStream spyStream = spy(stream);
          doAnswer(new Answer<FSEditLogOp>() {


            @Override
            public FSEditLogOp answer(InvocationOnMock invocation)
                throws Throwable {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream

      RemoteEditLogManifest manifest = e.getValue();
      
      for (RemoteEditLog remoteLog : manifest.getLogs()) {
        URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());


        EditLogInputStream elis = EditLogFileInputStream.fromUrl(
            url, remoteLog.getStartTxId(), remoteLog.getEndTxId(),
            remoteLog.isInProgress());
        allStreams.add(elis);
      }
    }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream

      if (!throwExceptionOnRead) {
        return streams;
      } else {
        Collection<EditLogInputStream> ret = new LinkedList<EditLogInputStream>();
        for (EditLogInputStream stream : streams) {
          EditLogInputStream spyStream = spy(stream);
          doAnswer(new Answer<FSEditLogOp>() {


            @Override
            public FSEditLogOp answer(InvocationOnMock invocation)
                throws Throwable {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream

      RemoteEditLogManifest manifest = e.getValue();
      
      for (RemoteEditLog remoteLog : manifest.getLogs()) {
        URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());


        EditLogInputStream elis = EditLogFileInputStream.fromUrl(
            connectionFactory, url, remoteLog.getStartTxId(),
            remoteLog.getEndTxId(), remoteLog.isInProgress());
        allStreams.add(elis);
      }
    }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream

      if (!throwExceptionOnRead) {
        return streams;
      } else {
        Collection<EditLogInputStream> ret = new LinkedList<EditLogInputStream>();
        for (EditLogInputStream stream : streams) {
          EditLogInputStream spyStream = spy(stream);
          doAnswer(new Answer<FSEditLogOp>() {


            @Override
            public FSEditLogOp answer(InvocationOnMock invocation)
                throws Throwable {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream

        OfflineEditsViewer.Flags flags) throws IOException {
      if (xmlInput) {
        return new OfflineEditsXmlLoader(visitor, new File(inputFileName), flags);
      } else {
        File file = null;
        EditLogInputStream elis = null;
        OfflineEditsLoader loader = null;
        try {
          file = new File(inputFileName);
          elis = new EditLogFileInputStream(file, HdfsConstants.INVALID_TXID,
              HdfsConstants.INVALID_TXID, false);
          loader = new OfflineEditsBinaryLoader(visitor, elis, flags);
        } finally {
          if ((loader == null) && (elis != null)) {
            elis.close();
          }
        }
        return loader;
      }
    }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream

  public static void verifyEdits(List<EditLogInputStream> streams,
      int firstTxnId, int lastTxnId) throws IOException {
    
    Iterator<EditLogInputStream> iter = streams.iterator();
    assertTrue(iter.hasNext());
    EditLogInputStream stream = iter.next();
    
    for (int expected = firstTxnId;
        expected <= lastTxnId;
        expected++) {
      
      FSEditLogOp op = stream.readOp();
      while (op == null) {
        assertTrue("Expected to find txid " + expected + ", " +
            "but no more streams available to read from",
            iter.hasNext());
        stream = iter.next();
        op = stream.readOp();
      }
      
      assertEquals(FSEditLogOpCodes.OP_MKDIR, op.opCode);
      assertEquals(expected, op.getTransactionId());
    }
    
    assertNull(stream.readOp());
    assertFalse("Expected no more txns after " + lastTxnId +
        " but more streams are available", iter.hasNext());
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream


    readerQjm.selectInputStreams(streams, 0, false);
    try {
      assertEquals(1, streams.size());
      // Validate the actual stream contents.
      EditLogInputStream stream = streams.get(0);
      assertEquals(1, stream.getFirstTxId());
      assertEquals(3, stream.getLastTxId());
      
      verifyEdits(streams, 1, 3);
      assertNull(stream.readOp());
    } finally {
      IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
      streams.clear();
    }
    
    // Ensure correct results when there is a stream in-progress, but we don't
    // ask for in-progress.
    writeSegment(cluster, qjm, 4, 3, false);
    readerQjm.selectInputStreams(streams, 0, false);
    try {
      assertEquals(1, streams.size());
      EditLogInputStream stream = streams.get(0);
      assertEquals(1, stream.getFirstTxId());
      assertEquals(3, stream.getLastTxId());
      verifyEdits(streams, 1, 3);
    } finally {
      IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
      streams.clear();
    }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.