/*
* Created on May 6, 2008
* Created by Paul Gardner
*
* Copyright 2008 Vuze, Inc. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 of the License only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
*/
package com.aelitis.azureus.core.metasearch.impl;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.*;
import org.gudy.azureus2.core3.internat.MessageText;
import org.gudy.azureus2.core3.util.AERunnable;
import org.gudy.azureus2.core3.util.AsyncDispatcher;
import org.gudy.azureus2.core3.util.BDecoder;
import org.gudy.azureus2.core3.util.Base32;
import org.gudy.azureus2.core3.util.Constants;
import org.gudy.azureus2.core3.util.Debug;
import org.gudy.azureus2.core3.util.DelayedEvent;
import org.gudy.azureus2.core3.util.FileUtil;
import org.gudy.azureus2.core3.util.IndentWriter;
import org.gudy.azureus2.core3.util.SimpleTimer;
import org.gudy.azureus2.core3.util.SystemTime;
import org.gudy.azureus2.core3.util.TimerEvent;
import org.gudy.azureus2.core3.util.TimerEventPerformer;
import org.gudy.azureus2.core3.util.TimerEventPeriodic;
import org.gudy.azureus2.plugins.utils.StaticUtilities;
import org.gudy.azureus2.plugins.utils.resourcedownloader.ResourceDownloader;
import org.gudy.azureus2.plugins.utils.resourcedownloader.ResourceDownloaderFactory;
import org.gudy.azureus2.plugins.utils.search.SearchProvider;
import com.aelitis.azureus.core.messenger.config.PlatformMetaSearchMessenger;
import com.aelitis.azureus.core.metasearch.*;
import com.aelitis.azureus.core.metasearch.impl.plugin.PluginEngine;
import com.aelitis.azureus.core.metasearch.impl.web.WebEngine;
import com.aelitis.azureus.core.metasearch.impl.web.rss.RSSEngine;
import com.aelitis.azureus.core.util.CopyOnWriteList;
import com.aelitis.azureus.core.vuzefile.VuzeFile;
import com.aelitis.azureus.core.vuzefile.VuzeFileComponent;
import com.aelitis.azureus.core.vuzefile.VuzeFileHandler;
public class
MetaSearchImpl
implements MetaSearch
{
private static final String CONFIG_FILE = "metasearch.config";
private MetaSearchManagerImpl manager;
private CopyOnWriteList<EngineImpl> engines = new CopyOnWriteList<EngineImpl>();
private Map<String,Long> plugin_map = new HashMap<String,Long>();
private boolean config_dirty;
private CopyOnWriteList<MetaSearchListener> listeners = new CopyOnWriteList<MetaSearchListener>();
private TimerEventPeriodic update_check_timer;
private static final int UPDATE_CHECK_PERIOD = 15*60*1000;
private static final int MIN_UPDATE_CHECK_SECS = 10*60;
private Object MS_UPDATE_CONSEC_FAIL_KEY = new Object();
protected
MetaSearchImpl(
MetaSearchManagerImpl _manager )
{
manager = _manager;
loadConfig();
}
protected MetaSearchManagerImpl
getManager()
{
return( manager );
}
public Engine
importFromBEncodedMap(
Map<String,Object> map )
throws IOException
{
return( EngineImpl.importFromBEncodedMap( this, map ));
}
public Engine
importFromJSONString(
int type,
long id,
long last_updated,
float rank_bias,
String name,
String content )
throws IOException
{
return( EngineImpl.importFromJSONString( this, type, id, last_updated, rank_bias, name, content ));
}
public EngineImpl
importFromPlugin(
String _pid,
SearchProvider provider )
throws IOException
{
synchronized( this ){
// unfortunately pid can be internationalised and thus musn't be used as a key to
// a bencoded-map as it can lead of nastyness. Delete any existing entries that have
// got out of control
Iterator<String> it = plugin_map.keySet().iterator();
while( it.hasNext()){
if ( it.next().length() > 1024 ){
Debug.out( "plugin_map corrupted, resetting" );
plugin_map.clear();
break;
}
}
String pid = Base32.encode( _pid.getBytes( "UTF-8" ));
Long l_id = plugin_map.get( pid );
long id;
if ( l_id == null ){
id = manager.getLocalTemplateID();
plugin_map.put( pid, new Long( id ));
configDirty();
}else{
id = l_id.longValue();
}
EngineImpl engine = (EngineImpl)getEngine( id );
if ( engine == null ){
engine = new PluginEngine( this, id, provider );
engine.setSource( Engine.ENGINE_SOURCE_LOCAL );
engine.setSelectionState( Engine.SEL_STATE_MANUAL_SELECTED );
addEngine( engine );
}else{
if ( engine instanceof PluginEngine ){
((PluginEngine)engine).setProvider( provider );
}else{
Debug.out( "Inconsistent: plugin must be a PluginEngine!" );
plugin_map.remove( pid );
removeEngine( engine );
throw( new IOException( "Inconsistent" ));
}
}
return( engine );
}
}
public Engine
createRSSEngine(
String name,
URL url )
throws MetaSearchException
{
EngineImpl engine =
new RSSEngine(
this,
manager.getLocalTemplateID(),
SystemTime.getCurrentTime(),
1.0f,
name,
url.toExternalForm(),
false,
WebEngine.AM_TRANSPARENT,
null,
new String[0] );
engine.setSource( Engine.ENGINE_SOURCE_RSS );
addEngine( engine, false );
log( "Created RSS engine '" + url + "'" );
return( engine );
}
protected void
enableUpdateChecks()
{
synchronized( this ){
if ( update_check_timer == null ){
update_check_timer = SimpleTimer.addPeriodicEvent(
"MS:updater",
UPDATE_CHECK_PERIOD,
new TimerEventPerformer()
{
public void
perform(
TimerEvent event)
{
checkUpdates();
}
});
}
}
}
protected void
checkUpdates()
{
Iterator<EngineImpl> it = engines.iterator();
while( it.hasNext()){
EngineImpl engine = (EngineImpl)it.next();
String update_url = engine.getUpdateURL();
if ( update_url != null ){
long now = SystemTime.getCurrentTime();
long last_check = engine.getLastUpdateCheck();
if ( last_check > now ){
last_check = now;
engine.setLastUpdateCheck( now );
}
long check_secs = engine.getUpdateCheckSecs();
if ( check_secs < MIN_UPDATE_CHECK_SECS ){
log( "Engine '" + engine.getName() + "': Update check period too small (" + check_secs + " secs) adjusting to " + MIN_UPDATE_CHECK_SECS + ": " + engine.getName());
check_secs = MIN_UPDATE_CHECK_SECS;
}
long check_millis = check_secs*1000;
long next_check = last_check + check_millis;
Object consec_fails_o = engine.getUserData( MS_UPDATE_CONSEC_FAIL_KEY );
int consec_fails = consec_fails_o==null?0:((Integer)consec_fails_o).intValue();
if ( consec_fails > 0 ){
next_check += ( UPDATE_CHECK_PERIOD << consec_fails );
}
if ( next_check < now ){
if ( updateEngine( engine )){
consec_fails = 0;
engine.setLastUpdateCheck( now );
}else{
consec_fails++;
if ( consec_fails > 3 ){
consec_fails = 0;
// skip to next scheduled update time
engine.setLastUpdateCheck( now );
}
}
engine.setUserData( MS_UPDATE_CONSEC_FAIL_KEY, consec_fails==0?null:new Integer( consec_fails ));
}
}
}
}
protected boolean
updateEngine(
EngineImpl engine )
{
String update_url = engine.getUpdateURL();
int pos = update_url.indexOf('?');
if ( pos == -1 ){
update_url += "?";
}else{
update_url += "&";
}
update_url += "az_template_uid=" + engine.getUID() +
"&az_template_version=" + engine.getVersion() +
"&az_version=" + Constants.AZUREUS_VERSION +
"&az_locale=" + MessageText.getCurrentLocale().toString() +
"&az_rand=" + Math.abs( new Random().nextLong());
log( "Engine " + engine.getName() + ": auto-update check via " + update_url );
try{
ResourceDownloaderFactory rdf = StaticUtilities.getResourceDownloaderFactory();
ResourceDownloader url_rd = rdf.create( new URL( update_url ));
ResourceDownloader rd = rdf.getMetaRefreshDownloader( url_rd );
InputStream is = rd.download();
try{
Map<String,Object> map = BDecoder.decode( new BufferedInputStream( is ));
log( " update check reply: " + map );
// reply is either "response" meaning "no update" and giving possibly changed update secs
// or Vuze file with updated template
Map<String,Object> response = (Map<String,Object>)map.get( "response" );
if ( response != null ){
Long update_secs = (Long)response.get( "update_url_check_secs" );
if ( update_secs == null ){
engine.setLocalUpdateCheckSecs( 0 );
}else{
int check_secs = update_secs.intValue();
if ( check_secs < MIN_UPDATE_CHECK_SECS ){
log( " update check secs for to small, min is " + MIN_UPDATE_CHECK_SECS);
check_secs = MIN_UPDATE_CHECK_SECS;
}
engine.setLocalUpdateCheckSecs( check_secs );
}
return( true );
}else{
VuzeFile vf = VuzeFileHandler.getSingleton().loadVuzeFile( map );
if ( vf == null ){
log( " failed to decode vuze file" );
return( false );
}
Engine[] updated_engines = manager.loadFromVuzeFile( vf );
if ( updated_engines.length > 0 ){
String existing_uid = engine.getUID();
boolean found = false;
String engine_str = "";
for (int i=0;i<updated_engines.length;i++){
Engine updated_engine = updated_engines[i];
engine_str += (i==0?"":",") + updated_engine.getName() + ": uid=" + updated_engine.getUID() + ",version=" + updated_engine.getVersion();
if ( updated_engine.getUID().equals( existing_uid )){
found = true;
}
}
if ( !found ){
log( " existing engine not found in updated set, deleting" );
engine.delete();
}
log( " update complete: new engines=" + engine_str );
}else{
log( " no engines found in vuze file" );
}
return( true );
}
}finally{
is.close();
}
}catch( Throwable e ){
log( " update check failed", e );
return( false );
}
}
public void
addEngine(
Engine engine )
{
addEngine( (EngineImpl)engine, false );
}
public Engine
addEngine(
long id )
throws MetaSearchException
{
try{
PlatformMetaSearchMessenger.templateDetails details = PlatformMetaSearchMessenger.getTemplate( manager.getExtensionKey(), id );
log( "Downloading definition of template " + id );
log( details.getValue());
if ( details.isVisible()){
Engine engine =
importFromJSONString(
details.getType()==PlatformMetaSearchMessenger.templateDetails.ENGINE_TYPE_JSON?Engine.ENGINE_TYPE_JSON:Engine.ENGINE_TYPE_REGEX,
details.getId(),
details.getModifiedDate(),
details.getRankBias(),
details.getName(),
details.getValue());
engine.setSource( Engine.ENGINE_SOURCE_VUZE );
engine.setSelectionState( Engine.SEL_STATE_DESELECTED );
addEngine( engine );
return( engine );
}else{
throw( new MetaSearchException( "Search template is not visible" ));
}
}catch( MetaSearchException e ){
throw( e );
}catch( Throwable e ){
throw( new MetaSearchException( "Template load failed", e ));
}
}
public void
addEngine(
EngineImpl new_engine,
boolean loading )
{
boolean add_op = true;
synchronized( this ){
Iterator<EngineImpl> it = engines.iterator();
while( it.hasNext()){
Engine existing_engine = it.next();
if ( existing_engine.getId() == new_engine.getId()){
log( "Updating engine with same ID " + existing_engine.getId() + ": " + existing_engine.getName() + "/" + existing_engine.getUID());
it.remove();
new_engine.setUID( existing_engine.getUID());
if ( existing_engine.sameLogicAs( new_engine )){
new_engine.setVersion( existing_engine.getVersion());
}else{
new_engine.setVersion( existing_engine.getVersion() + 1 );
log( " new version=" + new_engine.getVersion());
}
add_op = false;
}else if ( existing_engine.getUID().equals( new_engine.getUID())){
log( "Removing engine with same UID " + existing_engine.getUID() + "(" + existing_engine.getName() + ")" );
it.remove();
}
}
engines.add( new_engine );
}
if ( new_engine.getUpdateURL() != null ){
enableUpdateChecks();
}
if ( !loading ){
log( "Engine '" + new_engine.getName() + "' added" );
saveConfig();
Iterator<MetaSearchListener> it = listeners.iterator();
while( it.hasNext()){
MetaSearchListener listener = it.next();
try{
if ( add_op ){
listener.engineAdded( new_engine );
}else{
listener.engineUpdated( new_engine );
}
}catch( Throwable e ){
Debug.printStackTrace(e);
}
}
}
}
public void
removeEngine(
Engine engine )
{
if ( engines.remove((EngineImpl)engine )){
log( "Engine '" + engine.getName() + "' removed" );
saveConfig();
Iterator<MetaSearchListener> it = listeners.iterator();
while( it.hasNext()){
try{
it.next().engineRemoved( engine );
}catch( Throwable e ){
Debug.printStackTrace(e);
}
}
}
}
public String
getFUD()
{
List<EngineImpl> l = engines.getList();
List<Long> ids = new ArrayList<Long>();
for ( EngineImpl engine: l ){
if ( engine.getSource() == Engine.ENGINE_SOURCE_VUZE ){
ids.add( engine.getId());
}
}
Collections.sort( ids );
String fud = "";
for ( Long id: ids ){
fud += (fud.length()==0?"":",") +id;
}
return( fud );
}
protected void
addPotentialAssociation(
EngineImpl engine,
String key )
{
manager.addPotentialAssociation(engine, key);
}
public Engine[]
getEngines(
boolean active_only,
boolean ensure_up_to_date )
{
if ( ensure_up_to_date ){
manager.ensureEnginesUpToDate();
}
List<EngineImpl> l = engines.getList();
List<EngineImpl> result;
if ( active_only ){
result = new ArrayList<EngineImpl>();
for (int i=0;i<l.size();i++){
EngineImpl e = l.get(i);
if ( e.isActive()){
result.add( e );
}
}
}else{
result = l;
}
return( (Engine[])result.toArray( new Engine[ result.size() ]));
}
public Engine
getEngine(
long id )
{
List<EngineImpl> l = engines.getList();
for( int i=0;i<l.size();i++){
Engine e = l.get(i);
if ( e.getId() == id ){
return( e );
}
}
return( null );
}
public Engine
getEngineByUID(
String uid )
{
List<EngineImpl> l = engines.getList();
for( int i=0;i<l.size();i++){
Engine e = l.get(i);
if ( e.getUID().equals( uid )){
return( e );
}
}
return( null );
}
public int
getEngineCount()
{
return( engines.size());
}
public Engine[]
search(
final ResultListener original_listener,
SearchParameter[] searchParameters,
String headers,
int max_results_per_engine )
{
return( search( original_listener, searchParameters, headers, new HashMap<String,String>(), max_results_per_engine ));
}
public Engine[]
search(
final ResultListener original_listener,
SearchParameter[] searchParameters,
String headers,
Map<String,String> context,
int max_results_per_engine )
{
return( search( null, original_listener, searchParameters, headers, context, max_results_per_engine ));
}
public Engine[]
search(
Engine[] engines,
final ResultListener listener,
SearchParameter[] search_parameters,
String headers,
final int max_results_per_engine )
{
return( search( engines, listener, search_parameters, headers, new HashMap<String,String>(), max_results_per_engine ));
}
public void
enginePreferred(
Engine engine )
{
Engine[] engines = getEngines( true, false );
int num_other_preferred = 0;
for ( Engine e: engines ){
if ( e.getId() == engine.getId()){
e.setPreferredDelta( +1 );
}else{
if ( e.getPreferredWeighting() > 0 ){
num_other_preferred++;
}
}
}
if ( num_other_preferred > 0 ){
float negative_weighting = -1 / num_other_preferred;
for ( Engine e: engines ){
if ( e.getId() != engine.getId() && e.getPreferredWeighting() > 0 ){
e.setPreferredDelta( negative_weighting );
}
}
}
}
public Engine[]
search(
Engine[] engines,
final ResultListener original_listener,
SearchParameter[] searchParameters,
String headers,
Map<String,String> context,
final int max_results_per_engine )
{
String batch_millis_str = context.get( Engine.SC_BATCH_PERIOD );
final long batch_millis = batch_millis_str==null?0:Long.parseLong( batch_millis_str );
String rem_dups_str = context.get( Engine.SC_REMOVE_DUP_HASH );
final boolean rem_dups = rem_dups_str==null?false:rem_dups_str.equalsIgnoreCase( "true" );
ResultListener listener =
new ResultListener()
{
// single thread listener calls
private AsyncDispatcher dispatcher = new AsyncDispatcher( 5000 );
final private Map<Engine,List<Result[]>> pending_results = new HashMap<Engine,List<Result[]>>();
final private Map<Engine,Set<String>> result_hashes = new HashMap<Engine, Set<String>>();
public void
contentReceived(
final Engine engine,
final String content )
{
dispatcher.dispatch(
new AERunnable()
{
public void
runSupport()
{
original_listener.contentReceived( engine, content );
}
});
}
public void
matchFound(
final Engine engine,
final String[] fields )
{
dispatcher.dispatch(
new AERunnable()
{
public void
runSupport()
{
original_listener.matchFound( engine, fields );
}
});
}
public void
resultsReceived(
final Engine engine,
final Result[] results )
{
dispatcher.dispatch(
new AERunnable()
{
public void
runSupport()
{
Result[] results_to_return = null;
if ( batch_millis > 0 ){
List<Result[]> list = pending_results.get( engine );
if ( list == null ){
results_to_return = results;
pending_results.put( engine, new ArrayList<Result[]>());
new DelayedEvent(
"SearchBatcher",
batch_millis,
new AERunnable()
{
public void
runSupport()
{
dispatcher.dispatch(
new AERunnable()
{
public void
runSupport()
{
batchResultsComplete( engine );
}
});
}
});
}else{
list.add( results );
}
}else{
results_to_return = results;
}
if ( results_to_return != null ){
results_to_return = truncateResults( engine, results_to_return, max_results_per_engine );
original_listener.resultsReceived( engine, results_to_return );
}
}
});
}
public void
resultsComplete(
final Engine engine )
{
dispatcher.dispatch(
new AERunnable()
{
public void
runSupport()
{
if ( batch_millis > 0 ){
batchResultsComplete( engine );
}
original_listener.resultsComplete( engine );
}
});
}
protected void
batchResultsComplete(
Engine engine )
{
List<Result[]> list = pending_results.remove( engine );
if ( list != null ){
List<Result> x = new ArrayList<Result>();
for ( Result[] y: list ){
x.addAll( Arrays.asList( y ));
}
Result[] results = x.toArray( new Result[ x.size()]);
results = truncateResults( engine, results, max_results_per_engine );
original_listener.resultsReceived( engine, results );
}
}
protected Result[]
truncateResults(
Engine engine,
Result[] a_results,
int max )
{
Set<String> hash_set = result_hashes.get( engine );
if ( hash_set == null ){
hash_set = new HashSet<String>();
result_hashes.put( engine, hash_set );
}
List<Result> results = new ArrayList<Result>( a_results.length );
for ( Result r: a_results ){
String name = r.getName();
if ( name == null || name.trim().length() == 0 ){
continue;
}
if ( rem_dups ){
String hash = r.getHash();
if ( hash == null ||
hash.length() == 0 ){
results.add( r );
}else{
if ( !hash_set.contains( hash )){
results.add( r );
hash_set.add( hash );
}
}
}else{
results.add( r );
}
}
if ( max < results.size() ){
log( "Truncating search results for " + engine.getName() + " from " + results.size() + " to " + max );
Collections.sort(
results,
new Comparator<Result>()
{
Map<Result,Float> ranks = new HashMap<Result, Float>();
public int
compare(
Result r1,
Result r2)
{
Float rank1 = (Float)ranks.get(r1);
if ( rank1 == null ){
rank1 = new Float(r1.getRank());
ranks.put( r1, rank1 );
}
Float rank2 = (Float)ranks.get(r2);
if ( rank2 == null ){
rank2 = new Float(r2.getRank());
ranks.put( r2, rank2 );
}
return( -rank1.compareTo( rank2 ));
}
});
Result[] x = new Result[max];
int pos = 0;
while( pos < max ){
x[pos] = results.get( pos );
pos++;
}
return( x );
}else{
return( results.toArray( new Result[ results.size()] ));
}
}
public void
engineFailed(
final Engine engine,
final Throwable e )
{
dispatcher.dispatch(
new AERunnable()
{
public void
runSupport()
{
original_listener.engineFailed( engine, e );
}
});
}
public void
engineRequiresLogin(
final Engine engine,
final Throwable e )
{
dispatcher.dispatch(
new AERunnable()
{
public void
runSupport()
{
original_listener.engineRequiresLogin( engine, e );
}
});
}
};
SearchExecuter se = new SearchExecuter( context, listener );
if ( engines == null ){
engines = getEngines( true, true );
}
String engines_str = "";
for (int i=0;i<engines.length;i++){
engines_str += (i==0?"":",") + engines[i].getId();
}
log( "Search: engines=" + engines_str );
for (int i=0;i<engines.length;i++){
se.search( engines[i], searchParameters, headers, max_results_per_engine );
}
return( engines );
}
public void
exportEngines(
File target )
throws MetaSearchException
{
Engine[] engines = getEngines( true, false );
VuzeFile vf = VuzeFileHandler.getSingleton().create();
for ( Engine engine: engines ){
try{
vf.addComponent(
VuzeFileComponent.COMP_TYPE_METASEARCH_TEMPLATE,
engine.exportToBencodedMap());
}catch( IOException e ){
Debug.out( e );
}
}
try{
vf.write( target );
}catch( IOException e ){
throw( new MetaSearchException( "Failed to write file", e ));
}
}
public void
addListener(
MetaSearchListener listener )
{
listeners.add( listener );
}
public void
removeListener(
MetaSearchListener listener )
{
listeners.remove( listener );
}
protected void
loadConfig()
{
log( "Loading configuration" );
synchronized( this ){
Map<String,Object> map = FileUtil.readResilientConfigFile( CONFIG_FILE );
List<Map<String,Object>> l_engines = (List<Map<String,Object>>)map.get( "engines" );
if( l_engines != null ){
for (int i=0;i<l_engines.size();i++){
Map<String,Object> m = (Map<String,Object>)l_engines.get(i);
try{
Engine e = importFromBEncodedMap( m );
addEngine( (EngineImpl)e, true );
log( " loaded " + e.getString());
}catch( Throwable e ){
log( "Failed to import engine from " + m, e );
}
}
}
Map<String,Long> p_map = (Map<String,Long>)map.get( "plugin_map" );
if ( p_map != null ){
plugin_map = p_map;
}
}
if ( update_check_timer != null ){
new AsyncDispatcher().dispatch(
new AERunnable()
{
public void
runSupport()
{
checkUpdates();
}
});
}
}
protected void
configDirty()
{
synchronized( this ){
if ( config_dirty ){
return;
}
config_dirty = true;
new DelayedEvent(
"MetaSearch:save", 5000,
new AERunnable()
{
public void
runSupport()
{
synchronized( this ){
if ( !config_dirty ){
return;
}
saveConfig();
}
}
});
}
}
protected void
saveConfig()
{
log( "Saving configuration" );
synchronized( this ){
config_dirty = false;
Map<String,Object> map = new HashMap<String, Object>();
List<Map<String,Object>> l_engines = new ArrayList<Map<String,Object>>();
map.put( "engines", l_engines );
Iterator<EngineImpl> it = engines.iterator();
while( it.hasNext()){
Engine e = it.next();
try{
l_engines.add( e.exportToBencodedMap());
}catch( Throwable f ){
log( "Failed to export engine " + e.getName(), f );
}
}
if ( plugin_map != null ){
map.put( "plugin_map", plugin_map );
}
FileUtil.writeResilientConfigFile( CONFIG_FILE, map );
}
}
protected void
log(
String str )
{
manager.log( "search :" + str );
}
protected void
log(
String str,
Throwable e )
{
manager.log( "search :" + str, e );
}
protected void
generate(
IndentWriter writer )
{
Iterator<EngineImpl> it = engines.iterator();
while( it.hasNext()){
EngineImpl e = it.next();
writer.println( e.getString( true ));
}
}
}