/*
* Copyright 2011 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.data.hadoop.batch;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.junit.Before;
import org.junit.Test;
import org.springframework.batch.core.scope.context.StepSynchronizationManager;
import org.springframework.context.support.GenericXmlApplicationContext;
import org.springframework.core.io.Resource;
import org.springframework.data.hadoop.TestUtils;
import org.springframework.data.hadoop.fs.HdfsResourceLoader;
import static org.junit.Assert.assertTrue;
/**
* Basic data workflow test writing data into HDFS, executing a basic job and then reading data out of HDFS.
*
* @author Costin Leau
*/
public class ReadWriteHdfsTest {
{
TestUtils.hackHadoopStagingOnWin();
}
@Before
public void before() {
StepSynchronizationManager.release();
StepSynchronizationManager.close();
}
@Test
public void testWorkflow() throws Exception {
GenericXmlApplicationContext ctx = new GenericXmlApplicationContext(
"/org/springframework/data/hadoop/batch/in-do-out.xml");
ctx.registerShutdownHook();
FileSystem fs = FileSystem.get(ctx.getBean(Configuration.class));
System.out.println("FS is " + fs.getClass().getName());
HdfsResourceLoader hrl = ctx.getBean(HdfsResourceLoader.class);
Resource resource = hrl.getResource("/ide-test/output/word/");
assertTrue(ctx.isPrototype("script-tasklet"));
fs.delete(new Path(resource.getURI().toString()), true);
JobsTrigger.startJobs(ctx);
Path p = new Path("/ide-test/output/word/");
Job job = (Job) ctx.getBean("mr-job");
Configuration c = job.getConfiguration();
FileSystem fs2 = p.getFileSystem(c);
System.out.println("FS is " + fs2.getClass().getName());
fs2.exists(p);
ctx.close();
}
@Test
public void testWorkflowNS() throws Exception {
GenericXmlApplicationContext ctx = new GenericXmlApplicationContext(
"/org/springframework/data/hadoop/batch/in-do-out-ns.xml");
ctx.registerShutdownHook();
FileSystem fs = FileSystem.get(ctx.getBean(Configuration.class));
fs.delete(new Path("/ide-test/output/word/"), true);
assertTrue(ctx.isPrototype("hadoop-tasklet"));
JobsTrigger.startJobs(ctx);
ctx.close();
}
}