}
int inputCount = analyst.getScript().getNormalize().calculateInputColumns();
int idealCount = analyst.getScript().getNormalize().calculateOutputColumns();
BasicMLData inputData = new BasicMLData(inputCount);
BasicMLData idealData = new BasicMLData(idealCount);
ReadCSV csv = null;
BufferedMLDataSet buffer = new BufferedMLDataSet(file);
buffer.beginLoad(inputCount,idealCount);
try {
csv = new ReadCSV(getInputFilename().toString(),
isExpectInputHeaders(), getFormat());
resetStatus();
final int outputLength = this.analyst.determineTotalColumns();
// write file contents
while (csv.next() && !shouldStop()) {
updateStatus(false);
double[] output = AnalystNormalizeCSV.extractFields(
this.analyst, this.analystHeaders, csv, outputLength,
false);
if (this.series.getTotalDepth() > 1) {
output = this.series.process(output);
}
// copy the input
int idx = 0;
for(int i=0;i<inputData.size();i++) {
inputData.setData(i, output[idx++]);
}
for(int i=0;i<idealData.size();i++) {
idealData.setData(i, output[idx++]);
}
if (output != null) {
buffer.add(inputData,idealData);
}