existsIndices[i] = false;
//logger.warn("No reduce "+i+" output : no output index ["+index_path+","+index_prefix+ "]");
}
}
//2. the target index is the first source index
Index dest = srcIndices[0] != null ? srcIndices[0] : Index.createIndex(index_path, ApplicationSetup.TERRIER_INDEX_PREFIX+"-"+0);
if (dest == null)
{
throw new IllegalArgumentException("No index found at " + index_path + ","+ ApplicationSetup.TERRIER_INDEX_PREFIX+"-"+0);
}
//3. create the new lexicon
LexiconOutputStream<String> lexOut = new FSOMapFileLexiconOutputStream(
dest, tmpLexiconStructure,
(FixedSizeWriteableFactory<Text>) dest.getIndexStructure(lexiconStructure + "-keyfactory"),
(Class<? extends FixedSizeWriteableFactory<LexiconEntry>>) dest.getIndexStructure(lexiconStructure + "-valuefactory").getClass());
//4. append each source lexicon on to the new lexicon, amending the filenumber as we go
int termId = 0;
for(int i=0;i<numberOfReducers;i++)
{
//the partition did not have any stuff
if (! existsIndices[i])
{
//touch an empty inverted index file for this segment, as BitPostingIndex requires that all of the files exist
Files.writeFileStream(BitPostingIndexInputStream.getFilename(
dest, invertedStructure, (byte)numberOfReducers, (byte)i)).close();
continue;
}
//else, append the lexicon
Iterator<Map.Entry<String,LexiconEntry>> lexIn = (Iterator<Map.Entry<String, LexiconEntry>>) srcIndices[i].getIndexStructureInputStream("lexicon");
while(lexIn.hasNext())
{
Map.Entry<String,LexiconEntry> e = lexIn.next();
e.getValue().setTermId(termId);
((BitIndexPointer)e.getValue()).setFileNumber((byte)i);
lexOut.writeNextEntry(e.getKey(), e.getValue());
termId++;
}
IndexUtil.close(lexIn);
//rename the inverted file to be part of the destination index
Files.rename(
BitPostingIndexInputStream.getFilename(srcIndices[i], invertedStructure, (byte)1, (byte)1),
BitPostingIndexInputStream.getFilename(dest, invertedStructure, (byte)numberOfReducers, (byte)i));
}
lexOut.close();
//5. change over lexicon structures
final String[] structureSuffices = new String[]{"", "-entry-inputstream"};
//remove old lexicon structures
for (String suffix : structureSuffices)
{
// if (! IndexUtil.deleteStructure(dest, lexiconStructure + suffix))
//logger.warn("Structure " + lexiconStructure + suffix + " not found when removing");
}
//rename new lexicon structures
for (String suffix : structureSuffices)
{
// if (! IndexUtil.renameIndexStructure(dest, tmpLexiconStructure + suffix, lexiconStructure + suffix))
//logger.warn("Structure " + tmpLexiconStructure + suffix + " not found when renaming");
}
//6. update destimation index
if (FieldScore.FIELDS_COUNT > 0)
dest.addIndexStructure("lexicon-valuefactory", FieldLexiconEntry.Factory.class.getName(), "java.lang.String", "${index.inverted.fields.count}");
dest.setIndexProperty("index."+invertedStructure+".data-files", ""+numberOfReducers);
LexiconBuilder.optimise(dest, lexiconStructure);
dest.flush();
//7. close source and dest indices
for(Index src: srcIndices) //dest is also closed
{
if (src != null)