BatchSchema schema) throws SchemaChangeException, UnsupportedEncodingException {
// TODO - this does not handle schema changes
Map<String, List> combinedVectors = new HashMap();
long totalRecords = 0;
QueryResultBatch batch;
int size = records.size();
for (int i = 0; i < size; i++) {
batch = records.get(0);
loader.load(batch.getHeader().getDef(), batch.getData());
if (schema == null) {
schema = loader.getSchema();
for (MaterializedField mf : schema) {
combinedVectors.put(mf.getPath().toExpr(), new ArrayList());
}
}
logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
totalRecords += loader.getRecordCount();
for (VectorWrapper w : loader) {
String field = w.getField().toExpr();
for (int j = 0; j < loader.getRecordCount(); j++) {
if (totalRecords - loader.getRecordCount() + j > 5000000) {
continue;
}
Object obj = w.getValueVector().getAccessor().getObject(j);
if (obj != null) {
if (obj instanceof Text) {
obj = obj.toString();
if (obj.equals("")) {
System.out.println(w.getField());
}
}
else if (obj instanceof byte[]) {
obj = new String((byte[]) obj, "UTF-8");
}
}
combinedVectors.get(field).add(obj);
}
}
records.remove(0);
batch.release();
loader.clear();
}
return combinedVectors;
}