Package com.cloudera.flume.agent.diskfailover

Source Code of com.cloudera.flume.agent.diskfailover.TestDiskFailoverManager

/**
* Licensed to Cloudera, Inc. under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  Cloudera, Inc. licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloudera.flume.agent.diskfailover;

import static org.junit.Assert.assertEquals;

import java.io.File;
import java.io.IOException;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Before;
import org.junit.Test;

import com.cloudera.flume.conf.FlumeSpecException;
import com.cloudera.flume.core.Event;
import com.cloudera.flume.core.EventImpl;
import com.cloudera.flume.core.EventSink;
import com.cloudera.flume.core.EventSource;
import com.cloudera.flume.handlers.debug.ConsoleEventSink;
import com.cloudera.flume.handlers.rolling.ProcessTagger;
import com.cloudera.flume.handlers.rolling.Tagger;
import com.cloudera.util.BenchmarkHarness;
import com.cloudera.util.FileUtil;

/**
* This tests the disk failover manager. Values append to it should be written
* to disk and then eventually resent. AFter data reaches the SENT state, it can
* be deleted.
*/
public class TestDiskFailoverManager {

  static Logger LOG = Logger
      .getLogger(TestDiskFailoverManager.class.getName());
  // has 5 good entries.
  final static String WAL_OK = "src/data/hadoop_logs_5.hdfs";

  @Before
  public void setUp() {
    LOG.setLevel(Level.DEBUG);
    LOG.info("====================================================");
  }

  /**
   * Tests import to make sure it gets into the logged state properly.
   */
  @Test
  public void testImport() throws IOException {

    File dir = FileUtil.mktempdir();
    // putting in large ridiculous constant
    NaiveFileFailoverManager wal = new NaiveFileFailoverManager(dir);
    wal.open();

    File logdir = new File(dir, NaiveFileFailoverManager.IMPORTDIR);
    File src = new File(WAL_OK);
    File dest = new File(logdir, "ok.0000000.20091104-101213997-0800.seq");
    FileUtil.dumbfilecopy(src, dest);
    dest.deleteOnExit();

    assertEquals(0, wal.getWritingTags().size());
    assertEquals(0, wal.getLoggedTags().size());
    assertEquals(0, wal.getSendingTags().size());

    wal.importData();

    assertEquals(0, wal.getWritingTags().size());
    assertEquals(1, wal.getLoggedTags().size());
    assertEquals(0, wal.getSendingTags().size());

    wal.close();

  }

  @Test
  public void testTransitions() throws IOException {
    File dir = FileUtil.mktempdir();
    // putting in large ridiculous constant
    NaiveFileFailoverManager wal = new NaiveFileFailoverManager(dir);
    wal.open();

    // get a sink write to it and then be done with it.
    Tagger t = new ProcessTagger();
    EventSink sink = wal.newWritingSink(t);
    sink.open();
    sink.append(new EventImpl("foo".getBytes()));
    assertEquals(1, wal.getWritingTags().size());

    // close moves to logged state.
    sink.close();
    assertEquals(0, wal.getWritingTags().size());
    assertEquals(1, wal.getLoggedTags().size());

    // logged values can transition to the sending state by getting a sender.
    EventSource curSource = wal.getUnsentSource();
    // no state change
    assertEquals(0, wal.getWritingTags().size());
    assertEquals(0, wal.getLoggedTags().size());
    assertEquals(1, wal.getSendingTags().size());

    // open changes state
    curSource.open();
    assertEquals(0, wal.getLoggedTags().size());
    assertEquals(1, wal.getSendingTags().size());

    // read next event
    Event e = null;
    ConsoleEventSink console = new ConsoleEventSink();
    while ((e = curSource.next()) != null) {
      console.append(e);
    }
    curSource.close();
    assertEquals(0, wal.getSendingTags().size());
  }

  /**
   * This test puts a file in each log dir and makes sure they are all
   * recovered.
   */
  @Test
  public void testRecovers() throws IOException, FlumeSpecException {
    BenchmarkHarness.setupLocalWriteDir();
    File tmp = BenchmarkHarness.tmpdir;

    // putting in large ridiculous constant
    NaiveFileFailoverManager dfo = new NaiveFileFailoverManager(tmp);
    dfo.open(); // create dirs

    File acked = new File(WAL_OK);

    // copy files and then recover them.
    FileUtil.dumbfilecopy(acked, new File(dfo.writingDir,
        "writing.00000000.20100204-015814F430-0800.seq"));
    FileUtil.dumbfilecopy(acked, new File(dfo.loggedDir,
        "logged.00000000.20100204-015814F430-0800.seq"));
    FileUtil.dumbfilecopy(acked, new File(dfo.sendingDir,
        "sending.00000000.20100204-015814F430-0800.seq"));
    FileUtil.dumbfilecopy(acked, new File(dfo.importDir,
        "import.00000000.20100204-015814F430-0800.seq"));
    FileUtil.dumbfilecopy(acked, new File(dfo.errorDir,
        "error.00000000.20100204-015814F430-0800.seq"));
    dfo.recover();

    // check to make sure wal file is gone
    assertEquals(1,
        new File(tmp, NaiveFileFailoverManager.ERRORDIR).list().length);
    assertEquals(1,
        new File(tmp, NaiveFileFailoverManager.IMPORTDIR).list().length);
    assertEquals(3,
        new File(tmp, NaiveFileFailoverManager.LOGGEDDIR).list().length);
    assertEquals(0,
        new File(tmp, NaiveFileFailoverManager.SENDINGDIR).list().length);
    assertEquals(0,
        new File(tmp, NaiveFileFailoverManager.WRITINGDIR).list().length);

    BenchmarkHarness.cleanupLocalWriteDir();
  }

  /**
   * Tests import to make sure it gets into the logged state properly.
   */

  public void doTestBadOpen(String conflict) throws IOException {
    File src = new File(WAL_OK);
    File tmpdir = FileUtil.mktempdir();

    FileUtil.dumbfilecopy(src, new File(tmpdir, conflict));

    // putting in large ridiculous constant
    DiskFailoverManager wal = new NaiveFileFailoverManager(tmpdir);

    try {
      wal.open();
    } catch (IOException ioe) {
      throw ioe;
    } finally {
      FileUtil.rmr(tmpdir);
    }
  }

  /**
   * Tests import to make sure we handle errors with dir problems
   */
  @Test(expected = IOException.class)
  public void testBadOpenImport() throws IOException {
    doTestBadOpen(NaiveFileFailoverManager.IMPORTDIR);
  }

  /**
   * Tests import to make sure we handle errors with dir problems
   */
  @Test(expected = IOException.class)
  public void testBadOpenWriting() throws IOException {
    doTestBadOpen(NaiveFileFailoverManager.WRITINGDIR);
  }

  /**
   * Tests import to make sure we handle errors with dir problems
   */
  @Test(expected = IOException.class)
  public void testBadOpenLogged() throws IOException {
    doTestBadOpen(NaiveFileFailoverManager.LOGGEDDIR);
  }

  /**
   * Tests import to make sure we handle errors with dir problems
   */
  @Test(expected = IOException.class)
  public void testBadOpenSending() throws IOException {
    doTestBadOpen(NaiveFileFailoverManager.SENDINGDIR);
  }

  /**
   * Tests import to make sure we handle errors with dir problems
   */
  @Test(expected = IOException.class)
  public void testBadOpenError() throws IOException {
    doTestBadOpen(NaiveFileFailoverManager.ERRORDIR);
  }

  /**
   * Tests import to make sure it gets into the logged state properly.
   */
  public void doTestBadRecover(String conflict) throws IOException {
    File tmpdir = FileUtil.mktempdir();

    File dir = new File(new File(tmpdir, conflict), "foo");
    dir.mkdirs();

    // putting in large ridiculous constant
    NaiveFileFailoverManager wal = new NaiveFileFailoverManager(tmpdir);

    wal.open();
    try {
      wal.recover();
    } catch (IOException ioe) {
      throw ioe;
    } finally {
      FileUtil.rmr(tmpdir);
    }
  }

  @Test(expected = IOException.class)
  public void testBadRecoverSending() throws IOException {
    doTestBadRecover(NaiveFileFailoverManager.SENDINGDIR);
  }

  @Test(expected = IOException.class)
  public void testBadRecoverWriting() throws IOException {
    doTestBadRecover(NaiveFileFailoverManager.WRITINGDIR);
  }

}
TOP

Related Classes of com.cloudera.flume.agent.diskfailover.TestDiskFailoverManager

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.