/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.flume.channel.file;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.flume.Context;
import org.apache.flume.conf.Configurables;
import org.apache.flume.sink.NullSink;
import org.apache.flume.source.SequenceGeneratorSource;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
public class TestIntegration {
private static final Logger LOG = LoggerFactory
.getLogger(TestIntegration.class);
private FileChannel channel;
private File baseDir;
private File checkpointDir;
private File[] dataDirs;
private String dataDir;
@Before
public void setup() {
baseDir = Files.createTempDir();
checkpointDir = new File(baseDir, "chkpt");
Assert.assertTrue(checkpointDir.mkdirs() || checkpointDir.isDirectory());
dataDirs = new File[3];
dataDir = "";
for (int i = 0; i < dataDirs.length; i++) {
dataDirs[i] = new File(baseDir, "data" + (i+1));
Assert.assertTrue(dataDirs[i].mkdirs() || dataDirs[i].isDirectory());
dataDir += dataDirs[i].getAbsolutePath() + ",";
}
dataDir = dataDir.substring(0, dataDir.length() - 1);
}
@After
public void teardown() {
if(channel != null && channel.isOpen()) {
channel.stop();
}
FileUtils.deleteQuietly(baseDir);
}
@Test
public void testIntegration() throws IOException, InterruptedException {
// set shorter checkpoint and filesize to ensure
// checkpoints and rolls occur during the test
Context context = new Context();
context.put(FileChannelConfiguration.CHECKPOINT_DIR,
checkpointDir.getAbsolutePath());
context.put(FileChannelConfiguration.DATA_DIRS, dataDir);
context.put(FileChannelConfiguration.CAPACITY, String.valueOf(10000));
// Set checkpoint for 5 seconds otherwise test will run out of memory
context.put(FileChannelConfiguration.CHECKPOINT_INTERVAL, "5000");
context.put(FileChannelConfiguration.MAX_FILE_SIZE,
String.valueOf(1024 * 1024 * 5));
// do reconfiguration
channel = new FileChannel();
channel.setName("FileChannel-" + UUID.randomUUID());
Configurables.configure(channel, context);
channel.start();
Assert.assertTrue(channel.isOpen());
SequenceGeneratorSource source = new SequenceGeneratorSource();
CountingSourceRunner sourceRunner = new CountingSourceRunner(source, channel);
NullSink sink = new NullSink();
sink.setChannel(channel);
CountingSinkRunner sinkRunner = new CountingSinkRunner(sink);
sinkRunner.start();
sourceRunner.start();
TimeUnit.SECONDS.sleep(30);
// shutdown source
sourceRunner.shutdown();
while(sourceRunner.isAlive()) {
Thread.sleep(10L);
}
// wait for queue to clear
while(channel.getDepth() > 0) {
Thread.sleep(10L);
}
// shutdown size
sinkRunner.shutdown();
// wait a few seconds
TimeUnit.SECONDS.sleep(5);
List<File> logs = Lists.newArrayList();
for (int i = 0; i < dataDirs.length; i++) {
logs.addAll(LogUtils.getLogs(dataDirs[i]));
}
LOG.info("Total Number of Logs = " + logs.size());
for(File logFile : logs) {
LOG.info("LogFile = " + logFile);
}
LOG.info("Source processed " + sinkRunner.getCount());
LOG.info("Sink processed " + sourceRunner.getCount());
for(Exception ex : sourceRunner.getErrors()) {
LOG.warn("Source had error", ex);
}
for(Exception ex : sinkRunner.getErrors()) {
LOG.warn("Sink had error", ex);
}
Assert.assertEquals(sinkRunner.getCount(), sinkRunner.getCount());
Assert.assertEquals(Collections.EMPTY_LIST, sinkRunner.getErrors());
Assert.assertEquals(Collections.EMPTY_LIST, sourceRunner.getErrors());
}
}