* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

  private HDFSEventSink sink;

  private String testPath;

  private static final Logger LOG = LoggerFactory

      .getLogger(HDFSEventSink.class);

  static {

    System.setProperty("java.security.krb5.realm", "flume");

    System.setProperty("java.security.krb5.kdc", "blah");

  }

  private void dirCleanup() {

    Configuration conf = new Configuration();

    try {

      FileSystem fs = FileSystem.get(conf);

      Path dirPath = new Path(testPath);

      if (fs.exists(dirPath)) {

        fs.delete(dirPath, true);

      }

    } catch (IOException eIO) {

      LOG.warn("IO Error in test cleanup", eIO);

    }

  }

  // TODO: use System.getProperty("file.separator") instead of hardcoded '/'

  @Before

  public void setUp() {

    LOG.debug("Starting...");

    /*

     * FIXME: Use a dynamic path to support concurrent test execution. Also,

     * beware of the case where this path is used for something or when the

     * Hadoop config points at file:/// rather than hdfs://. We need to find a

     * better way of testing HDFS related functionality.

     */

    testPath = "file:///tmp/flume-test."

        + Calendar.getInstance().getTimeInMillis() + "."

        + Thread.currentThread().getId();

    sink = new HDFSEventSink();

    sink.setName("HDFSEventSink-" + UUID.randomUUID().toString());

    dirCleanup();

  }

  @After

  public void tearDown() {

    if (System.getenv("hdfs_keepFiles") == null)

      dirCleanup();

  }

  @Test

  public void testTextBatchAppend() throws Exception {

    doTestTextBatchAppend(false);

  }

  @Test

  public void testTextBatchAppendRawFS() throws Exception {

    doTestTextBatchAppend(true);

  }

  public void doTestTextBatchAppend(boolean useRawLocalFileSystem)

      throws Exception {

    LOG.debug("Starting...");

    final long rollCount = 10;

    final long batchSize = 2;

    final String fileName = "FlumeData";

    String newPath = testPath + "/singleTextBucket";

    int totalEvents = 0;

    int i = 1, j = 1;

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(newPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    Context context = new Context();

    // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(rollCount));

    context.put("hdfs.rollInterval", "0");

    context.put("hdfs.rollSize", "0");

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    context.put("hdfs.writeFormat", "Text");

    context.put("hdfs.useRawLocalFileSystem",

        Boolean.toString(useRawLocalFileSystem));

    context.put("hdfs.fileType", "DataStream");

    Configurables.configure(sink, context);

    Channel channel = new MemoryChannel();

    Configurables.configure(channel, context);

    sink.setChannel(channel);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    List<String> bodies = Lists.newArrayList();

    // push the event batches into channel to roll twice

    for (i = 1; i <= (rollCount*10)/batchSize; i++) {

      Transaction txn = channel.getTransaction();

      txn.begin();

      for (j = 1; j <= batchSize; j++) {

        Event event = new SimpleEvent();

        eventDate.clear();

        eventDate.set(2011, i, i, i, 0); // yy mm dd

        String body = "Test." + i + "." + j;

        event.setBody(body.getBytes());

        bodies.add(body);

        channel.put(event);

        totalEvents++;

      }

      txn.commit();

      txn.close();

      // execute sink to process the events

      sink.process();

    }

    sink.stop();

    // loop through all the files generated and check their contains

    FileStatus[] dirStat = fs.listStatus(dirPath);

    Path fList[] = FileUtil.stat2Paths(dirStat);

    // check that the roll happened correctly for the given data

    long expectedFiles = totalEvents / rollCount;

    if (totalEvents % rollCount > 0) expectedFiles++;

    Assert.assertEquals("num files wrong, found: " +

        Lists.newArrayList(fList), expectedFiles, fList.length);

    // check the contents of the all files

    verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);

  }

  @Test

  public void testLifecycle() throws InterruptedException, LifecycleException {

    LOG.debug("Starting...");

    Context context = new Context();

    context.put("hdfs.path", testPath);

    /*

     * context.put("hdfs.rollInterval", String.class);

     * context.get("hdfs.rollSize", String.class); context.get("hdfs.rollCount",

     * String.class);

     */

    Configurables.configure(sink, context);

    sink.setChannel(new MemoryChannel());

    sink.start();

    sink.stop();

  }

  @Test

  public void testEmptyChannelResultsInStatusBackoff()

      throws InterruptedException, LifecycleException, EventDeliveryException {

    LOG.debug("Starting...");

    Context context = new Context();

    Channel channel = new MemoryChannel();

    context.put("hdfs.path", testPath);

    context.put("keep-alive", "0");

    Configurables.configure(sink, context);

    Configurables.configure(channel, context);

    sink.setChannel(channel);

    sink.start();

    Assert.assertEquals(Status.BACKOFF, sink.process());

    sink.stop();

  }

  @Test

  public void testKerbFileAccess() throws InterruptedException,

      LifecycleException, EventDeliveryException, IOException {

    LOG.debug("Starting testKerbFileAccess() ...");

    final String fileName = "FlumeData";

    final long rollCount = 5;

    final long batchSize = 2;

    String newPath = testPath + "/singleBucket";

    String kerbConfPrincipal = "user1/localhost@EXAMPLE.COM";

    String kerbKeytab = "/usr/lib/flume/nonexistkeytabfile";

    //turn security on

    Configuration conf = new Configuration();

    conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,

        "kerberos");

    UserGroupInformation.setConfiguration(conf);

    Context context = new Context();

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(rollCount));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    context.put("hdfs.kerberosPrincipal", kerbConfPrincipal);

    context.put("hdfs.kerberosKeytab", kerbKeytab);

    try {

      Configurables.configure(sink, context);

      Assert.fail("no exception thrown");

    } catch (IllegalArgumentException expected) {

      Assert.assertTrue(expected.getMessage().contains(

          "is nonexistent or can't read."));

    } finally {

      //turn security off

      conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,

          "simple");

      UserGroupInformation.setConfiguration(conf);

    }

  }

  @Test

  public void testTextAppend() throws InterruptedException, LifecycleException,

      EventDeliveryException, IOException {

    LOG.debug("Starting...");

    final long rollCount = 3;

    final long batchSize = 2;

    final String fileName = "FlumeData";

    String newPath = testPath + "/singleTextBucket";

    int totalEvents = 0;

    int i = 1, j = 1;

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(newPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    Context context = new Context();

    // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(rollCount));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    context.put("hdfs.writeFormat", "Text");

    context.put("hdfs.fileType", "DataStream");

    Configurables.configure(sink, context);

    Channel channel = new MemoryChannel();

    Configurables.configure(channel, context);

    sink.setChannel(channel);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    List<String> bodies = Lists.newArrayList();

    // push the event batches into channel

    for (i = 1; i < 4; i++) {

      Transaction txn = channel.getTransaction();

      txn.begin();

      for (j = 1; j <= batchSize; j++) {

        Event event = new SimpleEvent();

        eventDate.clear();

        eventDate.set(2011, i, i, i, 0); // yy mm dd

        event.getHeaders().put("timestamp",

            String.valueOf(eventDate.getTimeInMillis()));

        event.getHeaders().put("hostname", "Host" + i);

        String body = "Test." + i + "." + j;

        event.setBody(body.getBytes());

        bodies.add(body);

        channel.put(event);

        totalEvents++;

      }

      txn.commit();

      txn.close();

      // execute sink to process the events

      sink.process();

    }

    sink.stop();

    // loop through all the files generated and check their contains

    FileStatus[] dirStat = fs.listStatus(dirPath);

    Path fList[] = FileUtil.stat2Paths(dirStat);

    // check that the roll happened correctly for the given data

    long expectedFiles = totalEvents / rollCount;

    if (totalEvents % rollCount > 0) expectedFiles++;

    Assert.assertEquals("num files wrong, found: " +

        Lists.newArrayList(fList), expectedFiles, fList.length);

    verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);

  }

  @Test

  public void testAvroAppend() throws InterruptedException, LifecycleException,

      EventDeliveryException, IOException {

    LOG.debug("Starting...");

    final long rollCount = 3;

    final long batchSize = 2;

    final String fileName = "FlumeData";

    String newPath = testPath + "/singleTextBucket";

    int totalEvents = 0;

    int i = 1, j = 1;

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(newPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    Context context = new Context();

    // context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(rollCount));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    context.put("hdfs.writeFormat", "Text");

    context.put("hdfs.fileType", "DataStream");

    context.put("serializer", "AVRO_EVENT");

    Configurables.configure(sink, context);

    Channel channel = new MemoryChannel();

    Configurables.configure(channel, context);

    sink.setChannel(channel);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    List<String> bodies = Lists.newArrayList();

    // push the event batches into channel

    for (i = 1; i < 4; i++) {

      Transaction txn = channel.getTransaction();

      txn.begin();

      for (j = 1; j <= batchSize; j++) {

        Event event = new SimpleEvent();

        eventDate.clear();

        eventDate.set(2011, i, i, i, 0); // yy mm dd

        event.getHeaders().put("timestamp",

            String.valueOf(eventDate.getTimeInMillis()));

        event.getHeaders().put("hostname", "Host" + i);

        String body = "Test." + i + "." + j;

        event.setBody(body.getBytes());

        bodies.add(body);

        channel.put(event);

        totalEvents++;

      }

      txn.commit();

      txn.close();

      // execute sink to process the events

      sink.process();

    }

    sink.stop();

    // loop through all the files generated and check their contains

    FileStatus[] dirStat = fs.listStatus(dirPath);

    Path fList[] = FileUtil.stat2Paths(dirStat);

    // check that the roll happened correctly for the given data

    long expectedFiles = totalEvents / rollCount;

    if (totalEvents % rollCount > 0) expectedFiles++;

    Assert.assertEquals("num files wrong, found: " +

        Lists.newArrayList(fList), expectedFiles, fList.length);

    verifyOutputAvroFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);

  }

  @Test

  public void testSimpleAppend() throws InterruptedException,

      LifecycleException, EventDeliveryException, IOException {

    LOG.debug("Starting...");

    final String fileName = "FlumeData";

    final long rollCount = 5;

    final long batchSize = 2;

    final int numBatches = 4;

    String newPath = testPath + "/singleBucket";

    int totalEvents = 0;

    int i = 1, j = 1;

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(newPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    Context context = new Context();

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(rollCount));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    Configurables.configure(sink, context);

    Channel channel = new MemoryChannel();

    Configurables.configure(channel, context);

    sink.setChannel(channel);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    List<String> bodies = Lists.newArrayList();

    // push the event batches into channel

    for (i = 1; i < numBatches; i++) {

      Transaction txn = channel.getTransaction();

      txn.begin();

      for (j = 1; j <= batchSize; j++) {

        Event event = new SimpleEvent();

        eventDate.clear();

        eventDate.set(2011, i, i, i, 0); // yy mm dd

        event.getHeaders().put("timestamp",

            String.valueOf(eventDate.getTimeInMillis()));

        event.getHeaders().put("hostname", "Host" + i);

        String body = "Test." + i + "." + j;

        event.setBody(body.getBytes());

        bodies.add(body);

        channel.put(event);

        totalEvents++;

      }

      txn.commit();

      txn.close();

      // execute sink to process the events

      sink.process();

    }

    sink.stop();

    // loop through all the files generated and check their contains

    FileStatus[] dirStat = fs.listStatus(dirPath);

    Path fList[] = FileUtil.stat2Paths(dirStat);

    // check that the roll happened correctly for the given data

    long expectedFiles = totalEvents / rollCount;

    if (totalEvents % rollCount > 0) expectedFiles++;

    Assert.assertEquals("num files wrong, found: " +

        Lists.newArrayList(fList), expectedFiles, fList.length);

    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);

  }

  @Test

  public void testSimpleAppendLocalTime() throws InterruptedException,

    LifecycleException, EventDeliveryException, IOException {

    final long currentTime = System.currentTimeMillis();

    Clock clk = new Clock() {

      @Override

      public long currentTimeMillis() {

        return currentTime;

      }

    };

    LOG.debug("Starting...");

    final String fileName = "FlumeData";

    final long rollCount = 5;

    final long batchSize = 2;

    final int numBatches = 4;

    String newPath = testPath + "/singleBucket/%s" ;

    String expectedPath = testPath + "/singleBucket/" +

      String.valueOf(currentTime/1000);

    int totalEvents = 0;

    int i = 1, j = 1;

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(expectedPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    Context context = new Context();

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(rollCount));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    context.put("hdfs.useLocalTimeStamp", String.valueOf(true));

    Configurables.configure(sink, context);

    Channel channel = new MemoryChannel();

    Configurables.configure(channel, context);

    sink.setChannel(channel);

    sink.setBucketClock(clk);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    List<String> bodies = Lists.newArrayList();

    // push the event batches into channel

    for (i = 1; i < numBatches; i++) {

      Transaction txn = channel.getTransaction();

      txn.begin();

      for (j = 1; j <= batchSize; j++) {

        Event event = new SimpleEvent();

        eventDate.clear();

        eventDate.set(2011, i, i, i, 0); // yy mm dd

        event.getHeaders().put("timestamp",

          String.valueOf(eventDate.getTimeInMillis()));

        event.getHeaders().put("hostname", "Host" + i);

        String body = "Test." + i + "." + j;

        event.setBody(body.getBytes());

        bodies.add(body);

        channel.put(event);

        totalEvents++;

      }

      txn.commit();

      txn.close();

      // execute sink to process the events

      sink.process();

    }

    sink.stop();

    // loop through all the files generated and check their contains

    FileStatus[] dirStat = fs.listStatus(dirPath);

    Path fList[] = FileUtil.stat2Paths(dirStat);

    // check that the roll happened correctly for the given data

    long expectedFiles = totalEvents / rollCount;

    if (totalEvents % rollCount > 0) expectedFiles++;

    Assert.assertEquals("num files wrong, found: " +

      Lists.newArrayList(fList), expectedFiles, fList.length);

    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);

    // The clock in bucketpath is static, so restore the real clock

    sink.setBucketClock(new SystemClock());

  }

  @Test

  public void testAppend() throws InterruptedException, LifecycleException,

      EventDeliveryException, IOException {

    LOG.debug("Starting...");

    final long rollCount = 3;

    final long batchSize = 2;

    final String fileName = "FlumeData";

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(testPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    Context context = new Context();

    context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");

    context.put("hdfs.timeZone", "UTC");

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(rollCount));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    Configurables.configure(sink, context);

    Channel channel = new MemoryChannel();

    Configurables.configure(channel, context);

    sink.setChannel(channel);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    List<String> bodies = Lists.newArrayList();

    // push the event batches into channel

    for (int i = 1; i < 4; i++) {

      Transaction txn = channel.getTransaction();

      txn.begin();

      for (int j = 1; j <= batchSize; j++) {

        Event event = new SimpleEvent();

        eventDate.clear();

        eventDate.set(2011, i, i, i, 0); // yy mm dd

        event.getHeaders().put("timestamp",

            String.valueOf(eventDate.getTimeInMillis()));

        event.getHeaders().put("hostname", "Host" + i);

        String body = "Test." + i + "." + j;

        event.setBody(body.getBytes());

        bodies.add(body);

        channel.put(event);

      }

      txn.commit();

      txn.close();

      // execute sink to process the events

      sink.process();

    }

    sink.stop();

    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);

  }

  // inject fault and make sure that the txn is rolled back and retried

  @Test

  public void testBadSimpleAppend() throws InterruptedException,

      LifecycleException, EventDeliveryException, IOException {

    LOG.debug("Starting...");

    final String fileName = "FlumeData";

    final long rollCount = 5;

    final long batchSize = 2;

    final int numBatches = 4;

    String newPath = testPath + "/singleBucket";

    int totalEvents = 0;

    int i = 1, j = 1;

    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();

    sink = new HDFSEventSink(badWriterFactory);

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(newPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    Context context = new Context();

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(rollCount));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);

    Configurables.configure(sink, context);

    Channel channel = new MemoryChannel();

    Configurables.configure(channel, context);

    sink.setChannel(channel);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    List<String> bodies = Lists.newArrayList();

    // push the event batches into channel

    for (i = 1; i < numBatches; i++) {

      Transaction txn = channel.getTransaction();

      txn.begin();

      for (j = 1; j <= batchSize; j++) {

        Event event = new SimpleEvent();

        eventDate.clear();

        eventDate.set(2011, i, i, i, 0); // yy mm dd

        event.getHeaders().put("timestamp",

            String.valueOf(eventDate.getTimeInMillis()));

        event.getHeaders().put("hostname", "Host" + i);

        String body = "Test." + i + "." + j;

        event.setBody(body.getBytes());

        bodies.add(body);

        // inject fault

        if ((totalEvents % 30) == 1) {

          event.getHeaders().put("fault-once", "");

        }

        channel.put(event);

        totalEvents++;

      }

      txn.commit();

      txn.close();

      LOG.info("Process events: " + sink.process());

    }

    LOG.info("Process events to end of transaction max: " + sink.process());

    LOG.info("Process events to injected fault: " + sink.process());

    LOG.info("Process events remaining events: " + sink.process());

    sink.stop();

    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);

  }

  private List<String> getAllFiles(String input) {

    List<String> output = Lists.newArrayList();

    File dir = new File(input);

    if(dir.isFile()) {

      output.add(dir.getAbsolutePath());

    } else if(dir.isDirectory()) {

      for(String file : dir.list()) {

        File subDir = new File(dir, file);

        output.addAll(getAllFiles(subDir.getAbsolutePath()));

      }

    }

    return output;

  }

  private void verifyOutputSequenceFiles(FileSystem fs, Configuration conf, String dir, String prefix, List<String> bodies) throws IOException {

    int found = 0;

    int expected = bodies.size();

    for(String outputFile : getAllFiles(dir)) {

      String name = (new File(outputFile)).getName();

      if(name.startsWith(prefix)) {

        SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFile), conf);

        LongWritable key = new LongWritable();

        BytesWritable value = new BytesWritable();

        while(reader.next(key, value)) {

          String body = new String(value.getBytes(), 0, value.getLength());

          if (bodies.contains(body)) {

            LOG.debug("Found event body: {}", body);

            bodies.remove(body);

            found++;

          }

        }

        reader.close();

      }

    }

    if (!bodies.isEmpty()) {

      for (String body : bodies) {

        LOG.error("Never found event body: {}", body);

      }

    }

    Assert.assertTrue("Found = " + found + ", Expected = "  +

        expected + ", Left = " + bodies.size() + " " + bodies,

          bodies.size() == 0);

  }

  private void verifyOutputTextFiles(FileSystem fs, Configuration conf, String dir, String prefix, List<String> bodies) throws IOException {

    int found = 0;

    int expected = bodies.size();

    for(String outputFile : getAllFiles(dir)) {

      String name = (new File(outputFile)).getName();

      if(name.startsWith(prefix)) {

        FSDataInputStream input = fs.open(new Path(outputFile));

        BufferedReader reader = new BufferedReader(new InputStreamReader(input));

        String body = null;

        while((body = reader.readLine()) != null) {

          bodies.remove(body);

          found++;

        }

        reader.close();

      }

    }

    Assert.assertTrue("Found = " + found + ", Expected = "  +

        expected + ", Left = " + bodies.size() + " " + bodies,

          bodies.size() == 0);

  }

  private void verifyOutputAvroFiles(FileSystem fs, Configuration conf, String dir, String prefix, List<String> bodies) throws IOException {

    int found = 0;

    int expected = bodies.size();

    for(String outputFile : getAllFiles(dir)) {

      String name = (new File(outputFile)).getName();

      if(name.startsWith(prefix)) {

        FSDataInputStream input = fs.open(new Path(outputFile));

        DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();

        DataFileStream<GenericRecord> avroStream =

            new DataFileStream<GenericRecord>(input, reader);

        GenericRecord record = new GenericData.Record(avroStream.getSchema());

        while (avroStream.hasNext()) {

          avroStream.next(record);

          ByteBuffer body = (ByteBuffer) record.get("body");

          CharsetDecoder decoder = Charsets.UTF_8.newDecoder();

          String bodyStr = decoder.decode(body).toString();

          LOG.debug("Removing event: {}", bodyStr);

          bodies.remove(bodyStr);

          found++;

        }

        avroStream.close();

        input.close();

      }

    }

    Assert.assertTrue("Found = " + found + ", Expected = "  +

        expected + ", Left = " + bodies.size() + " " + bodies,

          bodies.size() == 0);

  }

  /**

   * Ensure that when a write throws an IOException we are

   * able to continue to progress in the next process() call.

   * This relies on Transactional rollback semantics for durability and

   * the behavior of the BucketWriter class of close()ing upon IOException.

   */

@Test

  public void testCloseReopen() throws InterruptedException,

      LifecycleException, EventDeliveryException, IOException {

    LOG.debug("Starting...");

    final int numBatches = 4;

    final String fileName = "FlumeData";

    final long rollCount = 5;

    final long batchSize = 2;

    String newPath = testPath + "/singleBucket";

    int i = 1, j = 1;

    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();

    sink = new HDFSEventSink(badWriterFactory);

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(newPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    Context context = new Context();

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(rollCount));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);

    Configurables.configure(sink, context);

    MemoryChannel channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    sink.setChannel(channel);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    List<String> bodies = Lists.newArrayList();

    // push the event batches into channel

    for (i = 1; i < numBatches; i++) {

      channel.getTransaction().begin();

      try {

        for (j = 1; j <= batchSize; j++) {

          Event event = new SimpleEvent();

          eventDate.clear();

          eventDate.set(2011, i, i, i, 0); // yy mm dd

          event.getHeaders().put("timestamp",

              String.valueOf(eventDate.getTimeInMillis()));

          event.getHeaders().put("hostname", "Host" + i);

          String body = "Test." + i + "." + j;

          event.setBody(body.getBytes());

          bodies.add(body);

          // inject fault

          event.getHeaders().put("fault-until-reopen", "");

          channel.put(event);

        }

        channel.getTransaction().commit();

      } finally {

        channel.getTransaction().close();

      }

      LOG.info("execute sink to process the events: " + sink.process());

    }

    LOG.info("clear any events pending due to errors: " + sink.process());

    sink.stop();

    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);

  }

  /**

   * Test that the old bucket writer is closed at the end of rollInterval and

   * a new one is used for the next set of events.

   */

  @Test

  public void testCloseReopenOnRollTime() throws InterruptedException,

    LifecycleException, EventDeliveryException, IOException {

    LOG.debug("Starting...");

    final int numBatches = 4;

    final String fileName = "FlumeData";

    final long batchSize = 2;

    String newPath = testPath + "/singleBucket";

    int i = 1, j = 1;

    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();

    sink = new HDFSEventSink(badWriterFactory);

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(newPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    Context context = new Context();

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(0));

    context.put("hdfs.rollSize", String.valueOf(0));

    context.put("hdfs.rollInterval", String.valueOf(2));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);

    Configurables.configure(sink, context);

    MemoryChannel channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    sink.setChannel(channel);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    List<String> bodies = Lists.newArrayList();

    // push the event batches into channel

    for (i = 1; i < numBatches; i++) {

      channel.getTransaction().begin();

      try {

        for (j = 1; j <= batchSize; j++) {

          Event event = new SimpleEvent();

          eventDate.clear();

          eventDate.set(2011, i, i, i, 0); // yy mm dd

          event.getHeaders().put("timestamp",

            String.valueOf(eventDate.getTimeInMillis()));

          event.getHeaders().put("hostname", "Host" + i);

          String body = "Test." + i + "." + j;

          event.setBody(body.getBytes());

          bodies.add(body);

          // inject fault

          event.getHeaders().put("count-check", "");

          channel.put(event);

        }

        channel.getTransaction().commit();

      } finally {

        channel.getTransaction().close();

      }

      LOG.info("execute sink to process the events: " + sink.process());

      // Make sure the first file gets rolled due to rollTimeout.

      if (i == 1) {

        Thread.sleep(2001);

      }

    }

    LOG.info("clear any events pending due to errors: " + sink.process());

    sink.stop();

    Assert.assertTrue(badWriterFactory.openCount.get() >= 2);

    LOG.info("Total number of bucket writers opened: {}",

      badWriterFactory.openCount.get());

    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName,

      bodies);

  }

  /**

   * Test that a close due to roll interval removes the bucketwriter from

   * sfWriters map.

   */

  @Test

  public void testCloseRemovesFromSFWriters() throws InterruptedException,

    LifecycleException, EventDeliveryException, IOException {

    LOG.debug("Starting...");

    final String fileName = "FlumeData";

    final long batchSize = 2;

    String newPath = testPath + "/singleBucket";

    int i = 1, j = 1;

    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();

    sink = new HDFSEventSink(badWriterFactory);

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(newPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    Context context = new Context();

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(0));

    context.put("hdfs.rollSize", String.valueOf(0));

    context.put("hdfs.rollInterval", String.valueOf(1));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);

    String expectedLookupPath = newPath + "/FlumeData";

    Configurables.configure(sink, context);

    MemoryChannel channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    sink.setChannel(channel);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    List<String> bodies = Lists.newArrayList();

    // push the event batches into channel

    channel.getTransaction().begin();

    try {

      for (j = 1; j <= 2 * batchSize; j++) {

        Event event = new SimpleEvent();

        eventDate.clear();

        eventDate.set(2011, i, i, i, 0); // yy mm dd

        event.getHeaders().put("timestamp",

          String.valueOf(eventDate.getTimeInMillis()));

        event.getHeaders().put("hostname", "Host" + i);

        String body = "Test." + i + "." + j;

        event.setBody(body.getBytes());

        bodies.add(body);

        // inject fault

        event.getHeaders().put("count-check", "");

        channel.put(event);

      }

      channel.getTransaction().commit();

    } finally {

      channel.getTransaction().close();

    }

    LOG.info("execute sink to process the events: " + sink.process());

    Assert.assertTrue(sink.getSfWriters().containsKey(expectedLookupPath));

    // Make sure the first file gets rolled due to rollTimeout.

    Thread.sleep(2001);

    Assert.assertFalse(sink.getSfWriters().containsKey(expectedLookupPath));

    LOG.info("execute sink to process the events: " + sink.process());

    // A new bucket writer should have been created for this bucket. So

    // sfWriters map should not have the same key again.

    Assert.assertTrue(sink.getSfWriters().containsKey(expectedLookupPath));

    sink.stop();

    LOG.info("Total number of bucket writers opened: {}",

      badWriterFactory.openCount.get());

    verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName,

      bodies);

  }

  /*

   * append using slow sink writer.

   * verify that the process returns backoff due to timeout

   */

  @Test

  public void testSlowAppendFailure() throws InterruptedException,

      LifecycleException, EventDeliveryException, IOException {

    LOG.debug("Starting...");

    final String fileName = "FlumeData";

    final long rollCount = 5;

    final long batchSize = 2;

    final int numBatches = 2;

    String newPath = testPath + "/singleBucket";

    int i = 1, j = 1;

    // clear the test directory

    Configuration conf = new Configuration();

    FileSystem fs = FileSystem.get(conf);

    Path dirPath = new Path(newPath);

    fs.delete(dirPath, true);

    fs.mkdirs(dirPath);

    // create HDFS sink with slow writer

    HDFSTestWriterFactory badWriterFactory = new HDFSTestWriterFactory();

    sink = new HDFSEventSink(badWriterFactory);

    Context context = new Context();

    context.put("hdfs.path", newPath);

    context.put("hdfs.filePrefix", fileName);

    context.put("hdfs.rollCount", String.valueOf(rollCount));

    context.put("hdfs.batchSize", String.valueOf(batchSize));

    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);

    context.put("hdfs.callTimeout", Long.toString(1000));

    Configurables.configure(sink, context);

    Channel channel = new MemoryChannel();

    Configurables.configure(channel, context);

    sink.setChannel(channel);

    sink.start();

    Calendar eventDate = Calendar.getInstance();

    // push the event batches into channel

    for (i = 0; i < numBatches; i++) {

      Transaction txn = channel.getTransaction();

      txn.begin();

      for (j = 1; j <= batchSize; j++) {

        Event event = new SimpleEvent();

        eventDate.clear();

        eventDate.set(2011, i, i, i, 0); // yy mm dd

        event.getHeaders().put("timestamp",

            String.valueOf(eventDate.getTimeInMillis()));

        event.getHeaders().put("hostname", "Host" + i);

        event.getHeaders().put("slow", "1500");

        event.setBody(("Test." + i + "." + j).getBytes());

        channel.put(event);

      }

      txn.commit();

      txn.close();

      // execute sink to process the events

      Status satus = sink.process();

      // verify that the append returned backoff due to timeotu

      Assert.assertEquals(satus, Status.BACKOFF);

    }

    sink.stop();

  }

  /*

   * append using slow sink writer with specified append timeout

   * verify that the data is written correctly to files

   */

  private void slowAppendTestHelper (long appendTimeout)  throws InterruptedException, IOException,

  LifecycleException, EventDeliveryException, IOException {

    final String fileName = "FlumeData";

    final long rollCount = 5;

    final long batchSize = 2;

    final int numBatches = 2;

    String newPath = testPath + "/singleBucket";

    int totalEvents = 0;

    int i = 1, j = 1;