package com.lingbobu.flashdb.transfer;
import java.io.Closeable;
import java.io.IOException;
import java.lang.reflect.Array;
import java.math.BigDecimal;
import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.lingbobu.flashdb.common.ColumnInfo;
import com.lingbobu.flashdb.common.DataTypes;
import com.lingbobu.flashdb.execsql.ExecutorUtil;
import com.lingbobu.flashdb.transfer.TransferInput.PartableInput;
import com.lingbobu.flashdb.transfer.TransferInput.RowIterator;
import com.lingbobu.flashdb.transfer.TransferInput.RowIteratorCompletor;
import static com.lingbobu.flashdb.common.DataTypes.*;
/**
* 数据迁移工具
*/
public class Transfer {
private static final Logger LOG = LoggerFactory.getLogger(Transfer.class);
/**
* 执行迁移
*/
public static int doTransfer(TransferInput input, TransferOutput output) {
return doTransfer(input, output, null);
}
public static int doTransfer(Iterable<Map<String, Object>> iterable, TransferOutput output) {
return doTransfer(iterable, output, null);
}
public static int doTransfer(final Iterable<Map<String, Object>> iterable, TransferOutput output, TransferConverter converter) {
TransferInput input = new TransferInput() {
public RowIterator iterator() {
return new IteratorRowIterator(iterable.iterator());
}
};
return doTransfer(input, output, converter);
}
public static int paralDoTransfer(int threadCount, PartableInput input, final TransferOutput output) {
return paralDoTransfer(threadCount, input, output, null);
}
public static int paralDoTransfer(int threadCount, PartableInput input, final TransferOutput output, final TransferConverter converter) {
long beginTime = System.currentTimeMillis();
final TransferInput[] partInputs = input.getPartInputs();
output.beginOutput();
final ColumnInfo[] columnInfos = output.getOutputColumns();
int outputRows;
if (partInputs.length > 1) {
final AtomicInteger effectRowCount = new AtomicInteger(0);
ExecutorUtil.execute(threadCount, partInputs.length, new ExecutorUtil.TaskRunner() {
@Override
public void run(int taskIndex) {
int outputRows = runTransfer(1+taskIndex, columnInfos, partInputs[taskIndex], output, converter);
effectRowCount.addAndGet(outputRows);
}
});
outputRows = effectRowCount.get();
}
else if (partInputs.length == 1)
outputRows = runTransfer(0, columnInfos, partInputs[0], output, converter);
else
outputRows = 0;
output.endOutput();
long endTime = System.currentTimeMillis();
LOG.info("Transfer completed. Output " + outputRows +" rows. Used time "+(endTime - beginTime)+"ms.");
return outputRows;
}
public static int doTransfer(TransferInput input, final TransferOutput output, final TransferConverter converter) {
long beginTime = System.currentTimeMillis();
output.beginOutput();
ColumnInfo[] columnInfos = output.getOutputColumns();
int outputRows = runTransfer(0, columnInfos, input, output, converter);
output.endOutput();
long endTime = System.currentTimeMillis();
LOG.info("Transfer completed. Output " + outputRows +" rows. Used time "+(endTime - beginTime)+"ms.");
return outputRows;
}
private static int runTransfer(int threadId, ColumnInfo[] columnInfos,
TransferInput input, TransferOutput output, TransferConverter converter) {
boolean isOneManyConv = false;
TransferConverter.OneMany oneManyConv = null;
TransferConverter.OneOne oneOneConv = null;
if (converter != null) {
if (converter instanceof TransferConverter.OneMany) {
isOneManyConv = true;
oneManyConv = (TransferConverter.OneMany)converter;
}
else if (converter instanceof TransferConverter.OneOne) {
isOneManyConv = false;
oneOneConv = (TransferConverter.OneOne)converter;
}
else
throw new RuntimeException("TransferConverter must OneOne or OneMany");
}
String[] columnNames = new String[columnInfos.length];
int[] columnDataTypes = new int[columnInfos.length];
for (int i=0; i < columnInfos.length; i++) {
columnNames[i] = columnInfos[i].getName();
columnDataTypes[i] = columnInfos[i].getDataType();
}
RowIterator iterator = input.iterator();
int inputRows = 0;
int outputRows = 0;
try {
for (Map<String, Object> rowValues = iterator.next(); rowValues != null; rowValues = iterator.next()) {
if (inputRows % 1000 == 0) LOG.info("Transfering ... " + inputRows + (threadId > 0 ? (" @ Part " + threadId) : "") + " output " + outputRows);
inputRows++;
if (converter == null) {
ensureDataType(rowValues, columnInfos, false);
outputRows++;
output.writeOutput(rowValues);
}
else {
ensureDataType(rowValues, columnInfos, true);
if (isOneManyConv) {
List<Map<String, Object>> rows = oneManyConv.convert(rowValues);
if (rows == null) continue;
for (Map<String, Object> row : rows)
ensureDataType(row, columnInfos, false);
outputRows += rows.size();
for (Map<String, Object> row : rows)
output.writeOutput(row);
}
else {
rowValues = oneOneConv.convert(rowValues);
if (rowValues == null) continue;
ensureDataType(rowValues, columnInfos, false);
outputRows++;
output.writeOutput(rowValues);
}
}
}
} catch (Exception e) {
LOG.error("doTransfer error ocurred at line "+inputRows + (threadId > 0 ? (" @ Part " + threadId) : ""), e);
throw new RuntimeException("doTransfer error ocurred at line "+inputRows + (threadId > 0 ? (" @ Part " + threadId) : "") +" BECAUSE " + e.getMessage(), e);
} finally {
if (iterator instanceof Closeable) {
try {
((Closeable)iterator).close();
} catch (IOException e) {
LOG.error("", e);
}
}
}
if (iterator instanceof RowIteratorCompletor) {
((RowIteratorCompletor)iterator).completed();
}
return outputRows;
}
/*
private static final Object[] TYPES_NULL_DEFAULT = new Object[]{null,
null, Boolean.FALSE, Integer.valueOf(0), Long.valueOf(0), Float.valueOf(0),
Double.valueOf(0), new BigDecimal(0), new Date(0), new Timestamp(0), null
}; */
private static final Pattern[] PAT_TIMES = new Pattern[]{
Pattern.compile("\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d \\d\\d:\\d\\d:\\d\\d")
,Pattern.compile("\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d \\d\\d:\\d\\d:\\d\\d\\.\\d")
,Pattern.compile("\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d \\d\\d:\\d\\d:\\d\\d\\.\\d\\d\\d")
,Pattern.compile("\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d \\d\\d:\\d\\d")
,Pattern.compile("\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d")
};
private static final SimpleDateFormat[] SDF_TIMES = {
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
,new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S")
,new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS")
,new SimpleDateFormat("yyyy-MM-dd HH:mm")
,new SimpleDateFormat("yyyy-MM-dd")
};
private static Date parseDatetime(String value) {
for (int i=0; i < PAT_TIMES.length; i++) {
Matcher matcher = PAT_TIMES[i].matcher(value);
if (! matcher.matches()) continue;
try {
return SDF_TIMES[i].parse(value);
} catch (ParseException e) {
throw new IllegalArgumentException("Cann't convert data '"+value+"' to type Timestamp");
}
}
throw new IllegalArgumentException("Cann't convert data '"+value+"' to type Timestamp");
}
private static void ensureDataType(Map<String, Object> rowValues, ColumnInfo[] columnInfos, boolean skipError) {
for (int n=0; n < columnInfos.length; n++) {
String columnName = columnInfos[n].getName();
Object columnValue = rowValues.get(columnName);
if (columnValue == null) continue;
try {
columnValue = ensureDataType(columnValue, columnInfos[n].getDataType());
} catch (IllegalArgumentException e) {
if (skipError) continue;
else throw new RuntimeException("ensureDataType Error: " + rowValues.toString(), e);
}
rowValues.put(columnName, columnValue);
}
}
@SuppressWarnings("unchecked")
public static Object ensureDataType(Object fieldVal, int dataType) {
if (fieldVal == null) return null;
if ((dataType & TYPE_ARRAY) > 0) {
int dataType1 = dataType - TYPE_ARRAY;
assert (dataType1 >= DataTypes.TYPE_String) && (dataType1 <= DataTypes.TYPE_Binary);
Class<?> valueClass = DataTypes.TYPE_VALUE_CLASS[dataType1];
Object[] resultValues;
if (fieldVal instanceof Collection) {
Collection<?> fieldValues = (Collection<?>)fieldVal;
Set<Object> valuesSet;
if (fieldValues instanceof Set)
valuesSet = (Set<Object>)fieldValues;
else {
valuesSet = new HashSet<Object>();
valuesSet.addAll(fieldValues);
}
resultValues = valuesSet.toArray();
}
else if (fieldVal instanceof Object[]) {
resultValues = (Object[])fieldVal;
}
else if (fieldVal.getClass().isArray()) {
if (fieldVal.getClass().getComponentType() == valueClass) return fieldVal;
int length = Array.getLength(fieldVal);
resultValues = new Object[length];
for (int i=0; i < length; i++) {
resultValues[i] = Array.get(fieldVal, i);
}
}
else throw new IllegalArgumentException("Cann't convert '"+ fieldVal.getClass().getName() +"' to '"+DataTypes.typeName(dataType)+"'");
int length = resultValues.length;
Object resultArray = Array.newInstance(valueClass, length);
for (int i=0; i < length; i++) {
Object fieldValue = resultValues[i];
if (fieldValue == null) continue;
fieldValue = ensureDataType1(fieldValue, dataType1);
Array.set(resultArray, i, fieldValue);
}
return resultArray;
}
else return ensureDataType1(fieldVal, dataType);
}
private static Object ensureDataType1(Object value, int dataType) {
if (dataType == TYPE_String) {
if (value instanceof String)
return value;
else if ((value instanceof Closeable /*TokenStream*/) && (value.getClass().getName().endsWith("TokenStream")))
return value;
else if (value instanceof Collection) {
JSONArray json = JSONArray.fromObject(value);
return json.toString();
}
else if (value instanceof Map) {
JSONObject json = JSONObject.fromObject(value);
return json.toString();
}
else
return value.toString();
}
else if (dataType == TYPE_Boolean) {
if (value instanceof Boolean)
return value;
else if (value instanceof Number) {
int n = ((Number)value).intValue();
if (n == 0) return Boolean.FALSE;
else if (n == 1) return Boolean.TRUE;
}
else if (value instanceof String) {
String strValue = (String)value;
if ("1".equals(value))
return Boolean.TRUE;
else if ("True".equalsIgnoreCase(strValue))
return Boolean.TRUE;
else if ("0".equals(strValue))
return Boolean.FALSE;
else if ("False".equalsIgnoreCase(strValue))
return Boolean.FALSE;
}
}
else if (dataType == TYPE_Integer) {
if (value instanceof Integer)
return value;
else if (value instanceof Number)
return Integer.valueOf(((Number)value).intValue());
else if (value instanceof String) {
String strValue = (String)value;
try {
return Integer.valueOf(Integer.parseInt(strValue));
} catch (NumberFormatException e) {
}
}
}
else if (dataType == TYPE_Long) {
if (value instanceof Long)
return value;
else if (value instanceof Number)
return Long.valueOf(((Number)value).longValue());
else if (value instanceof String) {
String strValue = (String)value;
try {
return Long.valueOf(Long.parseLong(strValue));
} catch (NumberFormatException e) {
}
}
}
else if (dataType == TYPE_Float) {
if (value instanceof Float)
return value;
else if (value instanceof Number)
return Float.valueOf(((Number)value).floatValue());
else if (value instanceof String) {
String strValue = (String)value;
try {
return Float.valueOf(Float.parseFloat(strValue));
} catch (NumberFormatException e) {
}
}
}
else if (dataType == TYPE_Double) {
if (value instanceof Double)
return value;
else if (value instanceof Number)
return Double.valueOf(((Number)value).doubleValue());
else if (value instanceof String) {
String strValue = (String)value;
try {
return Double.valueOf(Double.parseDouble(strValue));
} catch (NumberFormatException e) {
}
}
}
else if (dataType == TYPE_Currency) {
if (value instanceof BigDecimal)
return value;
else if (value instanceof Number)
return new BigDecimal(((Number)value).doubleValue());
else if (value instanceof String) {
String strValue = (String)value;
try {
return new BigDecimal(strValue);
} catch (NumberFormatException e) {
}
}
}
else if (dataType == TYPE_Date) {
if (value instanceof Date)
return value;
else if (value instanceof String) {
String strValue = (String)value;
return parseDatetime(strValue);
}
}
else if (dataType == TYPE_DateTime) {
if (value instanceof Timestamp)
return value;
else if (value instanceof Number) {
return new Timestamp(((Number)value).longValue());
}
else if (value instanceof Date) {
return new Timestamp(((Date)value).getTime());
}
else if (value instanceof String) {
String strValue = (String)value;
return new Timestamp(parseDatetime(strValue).getTime());
}
}
else if (dataType == TYPE_Binary) {
if (value instanceof byte[])
return value;
}
else if (dataType == TYPE_OBJECT) {
return value;
}
throw new IllegalArgumentException("Cann't convert '"+value+"' to data type " + DataTypes.typeName(dataType));
}
/**
* 打印输出. 用于调试输入
*/
public static final TransferOutput PRINT_OUTPUT = new TransferOutput(){
public ColumnInfo[] getOutputColumns() {
return new ColumnInfo[0];
}
public void beginOutput() {
}
public void writeOutput(Map<String, Object> rowValues) {
System.out.println(rowValues.toString());
}
public void endOutput() {
}
};
/**
* 空输出. 用于测试输入性能
*/
public static final TransferOutput EMPTY_OUTPUT = new TransferOutput(){
public ColumnInfo[] getOutputColumns() {
return new ColumnInfo[0];
}
public void beginOutput() {
}
public void writeOutput(Map<String, Object> rowValues) {
}
public void endOutput() {
}
};
}