* -> collect document and partition by values
* -> exclude partitioned by columns from document
* -> insert into es index (partition determined by partition by value)
*/
TableInfo table = analysis.table();
int clusteredByPrimaryKeyIdx = table.primaryKey().indexOf(analysis.table().clusteredBy());
List<String> partitionedByNames;
List<ColumnIdent> partitionByColumns;
String tableName;
if (analysis.partitionIdent() == null) {
tableName = table.ident().name();
if (table.isPartitioned()) {
partitionedByNames = Lists.newArrayList(
Lists.transform(table.partitionedBy(), ColumnIdent.GET_FQN_NAME_FUNCTION));
partitionByColumns = table.partitionedBy();
} else {
partitionedByNames = Collections.emptyList();
partitionByColumns = Collections.emptyList();
}
} else {
assert table.isPartitioned() : "table must be partitioned if partitionIdent is set";
// partitionIdent is present -> possible to index raw source into concrete es index
tableName = PartitionName.fromPartitionIdent(table.ident().name(), analysis.partitionIdent()).stringValue();
partitionedByNames = Collections.emptyList();
partitionByColumns = Collections.emptyList();
}
SourceIndexWriterProjection sourceIndexWriterProjection = new SourceIndexWriterProjection(
tableName,
table.primaryKey(),
partitionByColumns,
table.clusteredBy(),
clusteredByPrimaryKeyIdx,
analysis.settings(),
null,
partitionedByNames.size() > 0 ? partitionedByNames.toArray(new String[partitionedByNames.size()]) : null,
table.isPartitioned() // autoCreateIndices
);
List<Projection> projections = Arrays.<Projection>asList(sourceIndexWriterProjection);
partitionedByNames.removeAll(Lists.transform(table.primaryKey(), ColumnIdent.GET_FQN_NAME_FUNCTION));
int referencesSize = table.primaryKey().size() + partitionedByNames.size() + 1;
referencesSize = clusteredByPrimaryKeyIdx == -1 ? referencesSize + 1 : referencesSize;
List<Symbol> toCollect = new ArrayList<>(referencesSize);
// add primaryKey columns
for (ColumnIdent primaryKey : table.primaryKey()) {
toCollect.add(new Reference(table.getReferenceInfo(primaryKey)));
}
// add partitioned columns (if not part of primaryKey)
for (String partitionedColumn : partitionedByNames) {
toCollect.add(
new Reference(table.getReferenceInfo(ColumnIdent.fromPath(partitionedColumn)))
);
}
// add clusteredBy column (if not part of primaryKey)
if (clusteredByPrimaryKeyIdx == -1) {
toCollect.add(
new Reference(table.getReferenceInfo(table.clusteredBy())));
}
// finally add _raw or _doc
if (table.isPartitioned() && analysis.partitionIdent() == null) {
toCollect.add(new Reference(table.getReferenceInfo(DocSysColumns.DOC)));
} else {
toCollect.add(new Reference(table.getReferenceInfo(DocSysColumns.RAW)));
}
DiscoveryNodes allNodes = clusterService.state().nodes();
FileUriCollectNode collectNode = new FileUriCollectNode(
"copyFrom",