/*
* Copyright 2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.yarn.batch.config;
import java.util.HashSet;
import java.util.Set;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.step.StepLocator;
import org.springframework.batch.support.transaction.ResourcelessTransactionManager;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.support.ConversionServiceFactoryBean;
import org.springframework.core.convert.ConversionService;
import org.springframework.core.convert.converter.Converter;
import org.springframework.core.convert.support.GenericConversionService;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.channel.QueueChannel;
import org.springframework.integration.config.ConsumerEndpointFactoryBean;
import org.springframework.integration.ip.tcp.TcpOutboundGateway;
import org.springframework.integration.ip.tcp.connection.TcpNetClientConnectionFactory;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.yarn.YarnSystemConstants;
import org.springframework.yarn.batch.repository.RemoteJobExplorer;
import org.springframework.yarn.batch.repository.RemoteJobRepository;
import org.springframework.yarn.batch.support.BeanFactoryStepLocator;
import org.springframework.yarn.integration.convert.ConverterRegistrar;
import org.springframework.yarn.integration.convert.MindHolderToObjectConverter;
import org.springframework.yarn.integration.convert.MindObjectToHolderConverter;
import org.springframework.yarn.integration.ip.mind.AppmasterMindScOperations;
import org.springframework.yarn.integration.ip.mind.DefaultMindAppmasterServiceClient;
import org.springframework.yarn.integration.ip.mind.MindRpcSerializer;
import org.springframework.yarn.integration.support.IntegrationContextUtils;
import org.springframework.yarn.integration.support.Jackson2ObjectMapperFactoryBean;
import com.fasterxml.jackson.databind.ObjectMapper;
@Configuration
public class SimpleYarnRemoteBatchConfiguration {
@Value("${SHDP_AMSERVICE_HOST}")
private String host;
@Value("${SHDP_AMSERVICE_PORT}")
private int port;
@Bean
public JobBuilderFactory jobBuilders() throws Exception {
return new JobBuilderFactory(jobRepository());
}
@Bean
public StepBuilderFactory stepBuilders() throws Exception {
return new StepBuilderFactory(jobRepository(), transactionManager());
}
@Bean
public PlatformTransactionManager transactionManager() throws Exception {
return new ResourcelessTransactionManager();
}
@Bean
public StepLocator stepLocator() {
return new BeanFactoryStepLocator();
}
@Bean
public JobRepository jobRepository() {
return new RemoteJobRepository(appmasterServiceClient());
}
@Bean
public JobExplorer jobExplorer() {
return new RemoteJobExplorer(appmasterServiceClient());
}
@Bean
public DirectChannel directChannel() {
return new DirectChannel();
}
@Bean
public QueueChannel queueChannel() {
return new QueueChannel();
}
@Bean
public MindRpcSerializer mindRpcSerializer() {
return new MindRpcSerializer();
}
@Bean
public TcpNetClientConnectionFactory tcpNetClientConnectionFactory() {
TcpNetClientConnectionFactory factory = new TcpNetClientConnectionFactory(host, port);
factory.setSerializer(mindRpcSerializer());
factory.setDeserializer(mindRpcSerializer());
return factory;
}
@Bean
public TcpOutboundGateway tcpOutboundGateway() {
TcpOutboundGateway gateway = new TcpOutboundGateway();
gateway.setConnectionFactory(tcpNetClientConnectionFactory());
gateway.setOutputChannel(directChannel());
gateway.setReplyChannel(queueChannel());
gateway.setRequestTimeout(60000);
gateway.setRemoteTimeout(60000);
return gateway;
}
@Bean
public ConsumerEndpointFactoryBean consumerEndpointFactoryBean() {
ConsumerEndpointFactoryBean endpointFactoryBean = new ConsumerEndpointFactoryBean();
endpointFactoryBean.setHandler(tcpOutboundGateway());
endpointFactoryBean.setInputChannel(directChannel());
return endpointFactoryBean;
}
@Bean(name=YarnSystemConstants.DEFAULT_ID_AMSERVICE_CLIENT)
public AppmasterMindScOperations appmasterServiceClient() {
DefaultMindAppmasterServiceClient client = new DefaultMindAppmasterServiceClient();
client.setRequestChannel(directChannel());
client.setResponseChannel(queueChannel());
return client;
}
@Bean
public ObjectMapper objectMapper() {
Jackson2ObjectMapperFactoryBean factory = new Jackson2ObjectMapperFactoryBean();
factory.afterPropertiesSet();
return factory.getObject();
}
@Bean(name=IntegrationContextUtils.YARN_INTEGRATION_CONVERSION_SERVICE_BEAN_NAME)
public ConversionService conversionService() {
CustomConversionServiceFactoryBean factory = new CustomConversionServiceFactoryBean();
factory.afterPropertiesSet();
return factory.getObject();
}
@Bean
public ConverterRegistrar converterRegistrar() {
Set<Converter<?, ?>> converters = new HashSet<Converter<?,?>>();
converters.add(new MindObjectToHolderConverter(objectMapper()));
String[] packages = new String[]{
"org.springframework.yarn.batch.repository.bindings",
"org.springframework.yarn.batch.repository.bindings.exp",
"org.springframework.yarn.batch.repository.bindings.repo"};
converters.add(new MindHolderToObjectConverter(objectMapper(), packages));
return new ConverterRegistrar(converters);
}
static class CustomConversionServiceFactoryBean extends ConversionServiceFactoryBean {
@Override
public ConversionService getObject() {
ConversionService service = super.getObject();
if (service instanceof GenericConversionService) {
((GenericConversionService) service).removeConvertible(Object.class, Object.class);
}
return service;
}
}
}