package org.apache.maven.index.packer;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.TimeZone;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.maven.index.ArtifactInfo;
import org.apache.maven.index.context.IndexCreator;
import org.apache.maven.index.context.IndexUtils;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.context.NexusIndexWriter;
import org.apache.maven.index.context.NexusLegacyAnalyzer;
import org.apache.maven.index.creator.LegacyDocumentUpdater;
import org.apache.maven.index.incremental.IncrementalHandler;
import org.apache.maven.index.updater.IndexDataWriter;
import org.codehaus.plexus.component.annotations.Component;
import org.codehaus.plexus.component.annotations.Requirement;
import org.codehaus.plexus.logging.AbstractLogEnabled;
import org.codehaus.plexus.util.FileUtils;
import org.codehaus.plexus.util.IOUtil;
/**
* A default {@link IndexPacker} implementation. Creates the properties, legacy index zip and new gz files.
*
* @author Tamas Cservenak
* @author Eugene Kuleshov
*/
@Component( role = IndexPacker.class )
public class DefaultIndexPacker
extends AbstractLogEnabled
implements IndexPacker
{
@Requirement( role = IncrementalHandler.class )
IncrementalHandler incrementalHandler;
public void packIndex( IndexPackingRequest request )
throws IOException, IllegalArgumentException
{
if ( request.getTargetDir() == null )
{
throw new IllegalArgumentException( "The target dir is null" );
}
if ( request.getTargetDir().exists() )
{
if ( !request.getTargetDir().isDirectory() )
{
throw new IllegalArgumentException( //
String.format( "Specified target path %s is not a directory",
request.getTargetDir().getAbsolutePath() ) );
}
if ( !request.getTargetDir().canWrite() )
{
throw new IllegalArgumentException( String.format( "Specified target path %s is not writtable",
request.getTargetDir().getAbsolutePath() ) );
}
}
else
{
if ( !request.getTargetDir().mkdirs() )
{
throw new IllegalArgumentException( "Can't create " + request.getTargetDir().getAbsolutePath() );
}
}
// These are all of the files we'll be dealing with (except for the incremental chunks of course)
File legacyFile = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".zip" );
File v1File = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".gz" );
Properties info = null;
try
{
// Note that for incremental indexes to work properly, a valid index.properties file
// must be present
info = readIndexProperties( request );
if ( request.isCreateIncrementalChunks() )
{
List<Integer> chunk = incrementalHandler.getIncrementalUpdates( request, info );
if ( chunk == null )
{
getLogger().debug( "Problem with Chunks, forcing regeneration of whole index" );
incrementalHandler.initializeProperties( info );
}
else if ( chunk.isEmpty() )
{
getLogger().debug( "No incremental changes, not writing new incremental chunk" );
}
else
{
File file =
new File( request.getTargetDir(), //
IndexingContext.INDEX_FILE_PREFIX + "."
+ info.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ) + ".gz" );
writeIndexData( request.getContext(), //
chunk, file );
if ( request.isCreateChecksumFiles() )
{
FileUtils.fileWrite(
new File( file.getParentFile(), file.getName() + ".sha1" ).getAbsolutePath(),
DigesterUtils.getSha1Digest( file ) );
FileUtils.fileWrite(
new File( file.getParentFile(), file.getName() + ".md5" ).getAbsolutePath(),
DigesterUtils.getMd5Digest( file ) );
}
}
}
}
catch ( IOException e )
{
getLogger().info( "Unable to read properties file, will force index regeneration" );
info = new Properties();
incrementalHandler.initializeProperties( info );
}
Date timestamp = request.getContext().getTimestamp();
if ( timestamp == null )
{
timestamp = new Date( 0 ); // never updated
}
if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_LEGACY ) )
{
info.setProperty( IndexingContext.INDEX_LEGACY_TIMESTAMP, format( timestamp ) );
writeIndexArchive( request.getContext(), legacyFile );
if ( request.isCreateChecksumFiles() )
{
FileUtils.fileWrite(
new File( legacyFile.getParentFile(), legacyFile.getName() + ".sha1" ).getAbsolutePath(),
DigesterUtils.getSha1Digest( legacyFile ) );
FileUtils.fileWrite(
new File( legacyFile.getParentFile(), legacyFile.getName() + ".md5" ).getAbsolutePath(),
DigesterUtils.getMd5Digest( legacyFile ) );
}
}
if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_V1 ) )
{
info.setProperty( IndexingContext.INDEX_TIMESTAMP, format( timestamp ) );
writeIndexData( request.getContext(), null, v1File );
if ( request.isCreateChecksumFiles() )
{
FileUtils.fileWrite( new File( v1File.getParentFile(), v1File.getName() + ".sha1" ).getAbsolutePath(),
DigesterUtils.getSha1Digest( v1File ) );
FileUtils.fileWrite( new File( v1File.getParentFile(), v1File.getName() + ".md5" ).getAbsolutePath(),
DigesterUtils.getMd5Digest( v1File ) );
}
}
writeIndexProperties( request, info );
}
private Properties readIndexProperties( IndexPackingRequest request )
throws IOException
{
File file = null;
if ( request.isUseTargetProperties() )
{
file = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
}
else
{
file =
new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE );
}
Properties properties = new Properties();
FileInputStream fos = null;
try
{
fos = new FileInputStream( file );
properties.load( fos );
}
finally
{
if ( fos != null )
{
fos.close();
}
}
return properties;
}
void writeIndexArchive( IndexingContext context, File targetArchive )
throws IOException
{
if ( targetArchive.exists() )
{
targetArchive.delete();
}
OutputStream os = null;
try
{
os = new BufferedOutputStream( new FileOutputStream( targetArchive ), 4096 );
packIndexArchive( context, os );
}
finally
{
IOUtil.close( os );
}
}
/**
* Pack legacy index archive into a specified output stream
*/
public static void packIndexArchive( IndexingContext context, OutputStream os )
throws IOException
{
File indexArchive = File.createTempFile( "nexus-index", "" );
File indexDir = new File( indexArchive.getAbsoluteFile().getParentFile(), indexArchive.getName() + ".dir" );
indexDir.mkdirs();
FSDirectory fdir = FSDirectory.open( indexDir );
try
{
// force the timestamp update
IndexUtils.updateTimestamp( context.getIndexDirectory(), context.getTimestamp() );
IndexUtils.updateTimestamp( fdir, context.getTimestamp() );
final IndexSearcher indexSearcher = context.acquireIndexSearcher();
try
{
copyLegacyDocuments( indexSearcher.getIndexReader(), fdir, context );
}
finally
{
context.releaseIndexSearcher( indexSearcher );
}
packDirectory( fdir, os );
}
finally
{
IndexUtils.close( fdir );
indexArchive.delete();
IndexUtils.delete( indexDir );
}
}
static void copyLegacyDocuments( IndexReader r, Directory targetdir, IndexingContext context )
throws CorruptIndexException, LockObtainFailedException, IOException
{
IndexWriter w = null;
try
{
w = new NexusIndexWriter( targetdir, new NexusLegacyAnalyzer(), true );
for ( int i = 0; i < r.maxDoc(); i++ )
{
if ( !r.isDeleted( i ) )
{
w.addDocument( updateLegacyDocument( r.document( i ), context ) );
}
}
w.optimize();
w.commit();
}
finally
{
IndexUtils.close( w );
}
}
static Document updateLegacyDocument( Document doc, IndexingContext context )
{
ArtifactInfo ai = IndexUtils.constructArtifactInfo( doc, context );
if ( ai == null )
{
return doc;
}
Document document = new Document();
document.add( new Field( ArtifactInfo.UINFO, ai.getUinfo(), Field.Store.YES, Field.Index.NOT_ANALYZED ) );
for ( IndexCreator ic : context.getIndexCreators() )
{
if ( ic instanceof LegacyDocumentUpdater )
{
( (LegacyDocumentUpdater) ic ).updateLegacyDocument( ai, document );
}
}
return document;
}
static void packDirectory( Directory directory, OutputStream os )
throws IOException
{
ZipOutputStream zos = null;
try
{
zos = new ZipOutputStream( os );
zos.setLevel( 9 );
String[] names = directory.listAll();
boolean savedTimestamp = false;
byte[] buf = new byte[8192];
for ( int i = 0; i < names.length; i++ )
{
String name = names[i];
writeFile( name, zos, directory, buf );
if ( name.equals( IndexUtils.TIMESTAMP_FILE ) )
{
savedTimestamp = true;
}
}
// FSDirectory filter out the foreign files
if ( !savedTimestamp && directory.fileExists( IndexUtils.TIMESTAMP_FILE ) )
{
writeFile( IndexUtils.TIMESTAMP_FILE, zos, directory, buf );
}
}
finally
{
IndexUtils.close( zos );
}
}
static void writeFile( String name, ZipOutputStream zos, Directory directory, byte[] buf )
throws IOException
{
ZipEntry e = new ZipEntry( name );
zos.putNextEntry( e );
IndexInput in = directory.openInput( name );
try
{
int toRead = 0;
int bytesLeft = (int) in.length();
while ( bytesLeft > 0 )
{
toRead = ( bytesLeft >= buf.length ) ? buf.length : bytesLeft;
bytesLeft -= toRead;
in.readBytes( buf, 0, toRead, false );
zos.write( buf, 0, toRead );
}
}
finally
{
IndexUtils.close( in );
}
zos.flush();
zos.closeEntry();
}
void writeIndexData( IndexingContext context, List<Integer> docIndexes, File targetArchive )
throws IOException
{
if ( targetArchive.exists() )
{
targetArchive.delete();
}
OutputStream os = null;
try
{
os = new FileOutputStream( targetArchive );
IndexDataWriter dw = new IndexDataWriter( os );
dw.write( context, docIndexes );
os.flush();
}
finally
{
IOUtil.close( os );
}
}
void writeIndexProperties( IndexPackingRequest request, Properties info )
throws IOException
{
File propertyFile =
new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE );
File targetPropertyFile = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
info.setProperty( IndexingContext.INDEX_ID, request.getContext().getId() );
OutputStream os = null;
try
{
os = new FileOutputStream( propertyFile );
info.store( os, null );
}
finally
{
IOUtil.close( os );
}
try
{
os = new FileOutputStream( targetPropertyFile );
info.store( os, null );
}
finally
{
IOUtil.close( os );
}
if ( request.isCreateChecksumFiles() )
{
FileUtils.fileWrite(
new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".sha1" ).getAbsolutePath(),
DigesterUtils.getSha1Digest( targetPropertyFile ) );
FileUtils.fileWrite(
new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".md5" ).getAbsolutePath(),
DigesterUtils.getMd5Digest( targetPropertyFile ) );
}
}
private String format( Date d )
{
SimpleDateFormat df = new SimpleDateFormat( IndexingContext.INDEX_TIME_FORMAT );
df.setTimeZone( TimeZone.getTimeZone( "GMT" ) );
return df.format( d );
}
}