private void readFromHdfs(ILineSender lineSender) {
FSDataInputStream in = null;
CompressionCodecFactory factory;
CompressionCodec codec;
CompressionInputStream cin = null;
LineIterator itr = null;
try {
conf = DFSUtils.getConf(filePath, null);
fs = DFSUtils.createFileSystem(new URI(filePath), conf);
in = fs.open(new Path(filePath));
factory = new CompressionCodecFactory(conf);
codec = factory.getCodec(new Path(filePath));
if (codec == null) {
LOG.info("codec not found, using text file reader");
itr = new LineIterator(new BufferedReader(
new InputStreamReader(in)));
} else {
LOG.info("found code " + codec.getClass());
cin = codec.createInputStream(in);
itr = new LineIterator(new BufferedReader(
new InputStreamReader(cin)));
}
while (itr.hasNext()) {
ILine oneLine = lineSender.createNewLine();
String line = itr.nextLine();
String[] parts = StringUtils
.splitByWholeSeparatorPreserveAllTokens(line,
FIELD_SEPARATOR);
for (int i = 0; i < parts.length; i++) {
if (HIVE_COLUMN_NULL_VALUE.equals(parts[i])) {
oneLine.addField(null, i);
} else {
oneLine.addField(parts[i], i);
}
}
boolean flag = lineSender.send(oneLine);
if (flag) {
getMonitor().increaseSuccessLines();
} else {
getMonitor().increaseFailedLines();
LOG.debug("failed to send line: " + oneLine.toString('\t'));
}
}
lineSender.flush();
} catch (Exception e) {
LOG.error(e.getCause());
throw new WormholeException(e,
JobStatus.READ_DATA_EXCEPTION.getStatus());
} finally {
if (itr != null) {
itr.close();
}
try {
if (cin != null) {
cin.close();
}