@Override
public RecordWriter<String, Tuple>
getRecordWriter(TaskAttemptContext context
) throws IOException, InterruptedException {
final TaskAttemptContext ctx = context;
return new RecordWriter<String, Tuple>() {
private Map<String, MyLineRecordWriter> storeMap =
new HashMap<String, MyLineRecordWriter>();
private static final int BUFFER_SIZE = 1024;
private ByteArrayOutputStream mOut =
new ByteArrayOutputStream(BUFFER_SIZE);
@Override
public void write(String key, Tuple val) throws IOException {
int sz = val.size();
for (int i = 0; i < sz; i++) {
Object field;
try {
field = val.get(i);
} catch (ExecException ee) {
throw ee;
}
StorageUtil.putField(mOut, field);
if (i != sz - 1) {
mOut.write(fieldDel);
}
}
getStore(key).write(null, new Text(mOut.toByteArray()));
mOut.reset();
}
@Override
public void close(TaskAttemptContext context) throws IOException {
for (MyLineRecordWriter out : storeMap.values()) {
out.close(context);
}
}
private MyLineRecordWriter getStore(String fieldValue) throws IOException {
MyLineRecordWriter store = storeMap.get(fieldValue);
if (store == null) {
DataOutputStream os = createOutputStream(fieldValue);
store = new MyLineRecordWriter(os, keyValueSeparator);
storeMap.put(fieldValue, store);
}
return store;
}
private DataOutputStream createOutputStream(String fieldValue) throws IOException {
Configuration conf = ctx.getConfiguration();
TaskID taskId = ctx.getTaskAttemptID().getTaskID();
Path path = new Path(fieldValue, fieldValue + '-'
+ NumberFormat.getInstance().format(taskId.getId()));
Path workOutputPath = ((FileOutputCommitter)getOutputCommitter(ctx)).getWorkPath();
Path file = new Path(workOutputPath, path);
FileSystem fs = file.getFileSystem(conf);