Package com.aelitis.azureus.core.subs.impl

Source Code of com.aelitis.azureus.core.subs.impl.SubscriptionManagerImpl$searchMatcher

/*
* Created on Jul 11, 2008
* Created by Paul Gardner
*
* Copyright 2008 Vuze, Inc.  All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 of the License only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
*/


package com.aelitis.azureus.core.subs.impl;

import java.io.*;
import java.net.InetSocketAddress;
import java.net.URL;
import java.security.KeyPair;
import java.util.*;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import org.bouncycastle.util.encoders.Base64;
import org.gudy.azureus2.core3.config.COConfigurationManager;
import org.gudy.azureus2.core3.config.ParameterListener;
import org.gudy.azureus2.core3.internat.MessageText;
import org.gudy.azureus2.core3.torrent.TOTorrent;
import org.gudy.azureus2.core3.torrent.TOTorrentFactory;
import org.gudy.azureus2.core3.util.*;
import org.gudy.azureus2.plugins.PluginInterface;
import org.gudy.azureus2.plugins.download.*;
import org.gudy.azureus2.plugins.peers.PeerManager;
import org.gudy.azureus2.plugins.torrent.Torrent;
import org.gudy.azureus2.plugins.torrent.TorrentAttribute;
import org.gudy.azureus2.plugins.torrent.TorrentManager;
import org.gudy.azureus2.plugins.ui.UIManager;
import org.gudy.azureus2.plugins.ui.UIManagerEvent;
import org.gudy.azureus2.plugins.utils.DelayedTask;
import org.gudy.azureus2.plugins.utils.StaticUtilities;
import org.gudy.azureus2.plugins.utils.search.SearchException;
import org.gudy.azureus2.plugins.utils.search.SearchInstance;
import org.gudy.azureus2.plugins.utils.search.SearchObserver;
import org.gudy.azureus2.plugins.utils.search.SearchProvider;
import org.gudy.azureus2.plugins.utils.search.SearchResult;
import org.gudy.azureus2.pluginsimpl.local.PluginInitializer;
import org.gudy.azureus2.pluginsimpl.local.torrent.TorrentImpl;
import org.gudy.azureus2.pluginsimpl.local.utils.UtilitiesImpl;

import com.aelitis.azureus.core.AzureusCore;
import com.aelitis.azureus.core.AzureusCoreRunningListener;
import com.aelitis.azureus.core.AzureusCoreFactory;
import com.aelitis.azureus.core.custom.Customization;
import com.aelitis.azureus.core.custom.CustomizationManager;
import com.aelitis.azureus.core.custom.CustomizationManagerFactory;
import com.aelitis.azureus.core.dht.DHT;
import com.aelitis.azureus.core.lws.LightWeightSeed;
import com.aelitis.azureus.core.lws.LightWeightSeedManager;
import com.aelitis.azureus.core.messenger.config.PlatformSubscriptionsMessenger;
import com.aelitis.azureus.core.metasearch.Engine;
import com.aelitis.azureus.core.metasearch.MetaSearchListener;
import com.aelitis.azureus.core.metasearch.MetaSearchManagerFactory;
import com.aelitis.azureus.core.metasearch.impl.web.WebEngine;
import com.aelitis.azureus.core.metasearch.impl.web.rss.RSSEngine;
import com.aelitis.azureus.core.security.CryptoECCUtils;
import com.aelitis.azureus.core.subs.*;
import com.aelitis.azureus.core.subs.SubscriptionUtils.SubscriptionDownloadDetails;
import com.aelitis.azureus.core.torrent.PlatformTorrentUtils;
import com.aelitis.azureus.core.util.CopyOnWriteList;
import com.aelitis.azureus.core.vuzefile.*;
import com.aelitis.azureus.plugins.dht.*;
import com.aelitis.azureus.plugins.magnet.MagnetPlugin;
import com.aelitis.azureus.plugins.magnet.MagnetPluginProgressListener;
import com.aelitis.azureus.util.ImportExportUtils;
import com.aelitis.azureus.util.UrlFilter;
import com.aelitis.net.magneturi.MagnetURIHandler;


public class
SubscriptionManagerImpl
  implements SubscriptionManager, AEDiagnosticsEvidenceGenerator
  private static final String  CONFIG_FILE = "subscriptions.config";
  private static final String  LOGGER_NAME = "Subscriptions";

  private static final String CONFIG_MAX_RESULTS       = "subscriptions.max.non.deleted.results";
  private static final String CONFIG_AUTO_START_DLS     = "subscriptions.auto.start.downloads";
  private static final String CONFIG_AUTO_START_MIN_MB   = "subscriptions.auto.start.min.mb";
  private static final String CONFIG_AUTO_START_MAX_MB   = "subscriptions.auto.start.max.mb";
 
  private static final String  CONFIG_RSS_ENABLE      = "subscriptions.config.rss_enable";

  private static final String  CONFIG_ENABLE_SEARCH      = "subscriptions.config.search_enable";
 
  private static final String  CONFIG_HIDE_SEARCH_TEMPLATES  = "subscriptions.config.hide_search_templates";
 
  private static final String  CONFIG_DL_SUBS_ENABLE    = "subscriptions.config.dl_subs_enable";

  private static final int DELETE_UNUSED_AFTER_MILLIS = 2*7*24*60*60*1000;
 
 
  private static SubscriptionManagerImpl    singleton;
  private static boolean            pre_initialised;
 
  private static final int random_seed = RandomUtils.nextInt( 256 );
 
  public static void
  preInitialise()
  {
    synchronized( SubscriptionManagerImpl.class ){
     
      if ( pre_initialised ){
       
        return;
      }
     
      pre_initialised = true;
    }
   
    VuzeFileHandler.getSingleton().addProcessor(
      new VuzeFileProcessor()
      {
        public void
        process(
          VuzeFile[]    files,
          int        expected_types )
        {
          for (int i=0;i<files.length;i++){
           
            VuzeFile  vf = files[i];
           
            VuzeFileComponent[] comps = vf.getComponents();
           
            for (int j=0;j<comps.length;j++){
             
              VuzeFileComponent comp = comps[j];
             
              int  type = comp.getType();
             
              if (   type == VuzeFileComponent.COMP_TYPE_SUBSCRIPTION ||
                  type == VuzeFileComponent.COMP_TYPE_SUBSCRIPTION_SINGLETON ){
               
                try{
                  ((SubscriptionManagerImpl)getSingleton( false )).importSubscription(
                      type,
                      comp.getContent(),
                      ( expected_types &
                        ( VuzeFileComponent.COMP_TYPE_SUBSCRIPTION | VuzeFileComponent.COMP_TYPE_SUBSCRIPTION_SINGLETON )) == 0 );
                 
                  comp.setProcessed();
                 
                }catch( Throwable e ){
                 
                  Debug.printStackTrace(e);
                }
              }
            }
          }
        }
      });   
  }
   
  public static SubscriptionManager
  getSingleton(
    boolean    stand_alone )
  {
    preInitialise();
   
    synchronized( SubscriptionManagerImpl.class ){
     
      if ( singleton != null ){
     
        return( singleton );
      }
     
      singleton = new SubscriptionManagerImpl( stand_alone );
    }
   
      // saw deadlock here when adding core listener while synced on class - rework
      // to avoid
   
    if ( !stand_alone ){
     
      singleton.initialise();
    }
   
    return( singleton );
  }
 
 
  private boolean    started;
   
  private static final int  TIMER_PERIOD    = 30*1000;
 
  private static final int  ASSOC_CHECK_PERIOD  = 5*60*1000;
  private static final int  ASSOC_CHECK_TICKS  = ASSOC_CHECK_PERIOD/TIMER_PERIOD;
 
  private static final int  SERVER_PUB_CHECK_PERIOD  = 10*60*1000;
  private static final int  SERVER_PUB_CHECK_TICKS  = SERVER_PUB_CHECK_PERIOD/TIMER_PERIOD;
 
  private static final int  TIDY_POT_ASSOC_PERIOD  = 30*60*1000;
  private static final int  TIDY_POT_ASSOC_TICKS  = TIDY_POT_ASSOC_PERIOD/TIMER_PERIOD;

  private static final int  SET_SELECTED_PERIOD    = 23*60*60*1000;
  private static final int  SET_SELECTED_FIRST_TICK  = 3*60*1000 /TIMER_PERIOD;
  private static final int  SET_SELECTED_TICKS    = SET_SELECTED_PERIOD/TIMER_PERIOD;

  private static final Object  SP_LAST_ATTEMPTED  = new Object();
  private static final Object  SP_CONSEC_FAIL    = new Object();
 
  private AzureusCore    azureus_core;
 
  private volatile DHTPlugin  dht_plugin;
 
  private List<SubscriptionImpl>    subscriptions  = new ArrayList<SubscriptionImpl>();
 
  private boolean  config_dirty;
 
  private static final int PUB_ASSOC_CONC_MAX  = 3;
 
  private int    publish_associations_active;
 
  private boolean publish_subscription_active;
 
  private TorrentAttribute    ta_subs_download;
  private TorrentAttribute    ta_subs_download_rd;
  private TorrentAttribute    ta_subscription_info;
  private TorrentAttribute    ta_category;
 
  private boolean          periodic_lookup_in_progress;
  private int            priority_lookup_pending;
 
  private CopyOnWriteList<SubscriptionManagerListener>      listeners = new CopyOnWriteList<SubscriptionManagerListener>();
 
  private SubscriptionSchedulerImpl  scheduler;
 
  private List<Object[]>        potential_associations  = new ArrayList<Object[]>();
  private Map<HashWrapper,Object[]>  potential_associations2  = new HashMap<HashWrapper,Object[]>();
 
  private boolean          meta_search_listener_added;
 
  private Pattern          exclusion_pattern = Pattern.compile( "azdev[0-9]+\\.azureus\\.com" );
 
  private SubscriptionRSSFeed    rss_publisher;
 
  private AEDiagnosticsLogger    logger;
 
 
  protected
  SubscriptionManagerImpl(
    boolean  stand_alone )
  {
    if ( !stand_alone ){
     
      loadConfig();
 
      AEDiagnostics.addEvidenceGenerator( this );
     
      CustomizationManager cust_man = CustomizationManagerFactory.getSingleton();
     
      Customization cust = cust_man.getActiveCustomization();
     
      if ( cust != null ){
       
        String cust_name   = COConfigurationManager.getStringParameter( "subscriptions.custom.name", "" );
        String cust_version = COConfigurationManager.getStringParameter( "subscriptions.custom.version", "0" );
       
        boolean  new_name   = !cust_name.equals( cust.getName());
        boolean  new_version = org.gudy.azureus2.core3.util.Constants.compareVersions( cust_version, cust.getVersion() ) < 0;
       
        if ( new_name || new_version ){

          log( "Customization: checking templates for " + cust.getName() + "/" + cust.getVersion());
         
          try{
            InputStream[] streams = cust.getResources( Customization.RT_SUBSCRIPTIONS );
           
            for (int i=0;i<streams.length;i++){
             
              InputStream is = streams[i];
             
              try{
                VuzeFile vf = VuzeFileHandler.getSingleton().loadVuzeFile(is);
               
                if ( vf != null ){
                 
                  VuzeFileComponent[] comps = vf.getComponents();
                 
                  for (int j=0;j<comps.length;j++){
                   
                    VuzeFileComponent comp = comps[j];
                   
                    int type = comp.getType();
                   
                    if (   type == VuzeFileComponent.COMP_TYPE_SUBSCRIPTION ||
                        type == VuzeFileComponent.COMP_TYPE_SUBSCRIPTION_SINGLETON  ){
                     
                      try{
                        importSubscription(
                            type,
                            comp.getContent(),
                            false );
                       
                        comp.setProcessed();
                       
                      }catch( Throwable e ){
                       
                        Debug.printStackTrace(e);
                      }
                    }
                  }
                }
              }finally{
               
                try{
                  is.close();
                 
                }catch( Throwable e ){
                }
              }
            }
          }finally{
           
            COConfigurationManager.setParameter( "subscriptions.custom.name", cust.getName());
            COConfigurationManager.setParameter( "subscriptions.custom.version", cust.getVersion());
          }
        }
      }
     
      scheduler = new SubscriptionSchedulerImpl( this );
    }
  }
 
  protected void
  initialise()
  {
    AzureusCoreFactory.addCoreRunningListener(new AzureusCoreRunningListener() {
      public void azureusCoreRunning(final AzureusCore core) {
        initWithCore(core);
      }
    });
  }

  protected void
  initWithCore(
    AzureusCore   _core )
  {
    synchronized( this ){
     
      if ( started ){
       
        return;
      }
     
      started  = true;
    }

    azureus_core = _core;
   
    final PluginInterface default_pi = PluginInitializer.getDefaultInterface();

    rss_publisher = new SubscriptionRSSFeed( this, default_pi );
   
    TorrentManager  tm = default_pi.getTorrentManager();
   
    ta_subs_download     = tm.getPluginAttribute( "azsubs.subs_dl" );
    ta_subs_download_rd   = tm.getPluginAttribute( "azsubs.subs_dl_rd" );
    ta_subscription_info   = tm.getPluginAttribute( "azsubs.subs_info" );
    ta_category        = tm.getAttribute( TorrentAttribute.TA_CATEGORY );
   
    PluginInterface  dht_plugin_pi  = AzureusCoreFactory.getSingleton().getPluginManager().getPluginInterfaceByClass( DHTPlugin.class );
       
    if ( dht_plugin_pi != null ){
     
      dht_plugin = (DHTPlugin)dht_plugin_pi.getPlugin();

      /*
      if ( Constants.isCVSVersion()){
       
        addListener(
            new SubscriptionManagerListener()
            {
              public void
              subscriptionAdded(
                Subscription subscription )
              {
              }
       
              public void
              subscriptionChanged(
                Subscription    subscription )
              {
              }
             
              public void
              subscriptionRemoved(
                Subscription subscription )
              {
              }
             
              public void
              associationsChanged(
                byte[] hash )
              {
                System.out.println( "Subscriptions changed: " + ByteFormatter.encodeString( hash ));
               
                Subscription[] subs = getKnownSubscriptions( hash );
             
                for (int i=0;i<subs.length;i++){
                 
                  System.out.println( "    " + subs[i].getString());
                }
              }
            }); 
      }
      */
     
      default_pi.getDownloadManager().addListener(
        new DownloadManagerListener()
        {
          public void
          downloadAdded(
            Download  download )
          {
            Torrent  torrent = download.getTorrent();
           
            if ( torrent != null ){
             
              byte[]  hash = torrent.getHash();
             
              Object[] entry;
             
              synchronized( potential_associations2 ){
               
                entry = (Object[])potential_associations2.remove( new HashWrapper( hash ));
              }
             
              if ( entry != null ){
               
                SubscriptionImpl[] subs = (SubscriptionImpl[])entry[0];
               
                String  subs_str = "";
                for (int i=0;i<subs.length;i++){
                  subs_str += (i==0?"":",") + subs[i].getName();
                }
               
                log( "Applying deferred asocciation for " + ByteFormatter.encodeString( hash ) + " -> " + subs_str );
               
                recordAssociationsSupport(
                  hash,
                  subs,
                  ((Boolean)entry[1]).booleanValue());
              }
            }
          }
         
          public void
          downloadRemoved(
            Download  download )
          { 
          }
        },
        false );
     
      TorrentUtils.addTorrentAttributeListener(
        new TorrentUtils.torrentAttributeListener()
        {
          public void
          attributeSet(
            TOTorrent   torrent,
            String     attribute,
            Object     value )
          {
            if ( attribute == TorrentUtils.TORRENT_AZ_PROP_OBTAINED_FROM ){
             
              try{
                checkPotentialAssociations( torrent.getHash(), (String)value );
               
              }catch( Throwable e ){
               
                Debug.printStackTrace(e);
              }
            }
          }
        });
       
      DelayedTask delayed_task = UtilitiesImpl.addDelayedTask( "Subscriptions",
          new Runnable()
          {
            public void
            run()
            {
              new AEThread2( "Subscriptions:delayInit", true )
              {
                public void
                run()
                {
                  asyncInit();
                }
              }.start();
             
            }
           
            protected void
            asyncInit()
            {
              Download[] downloads = default_pi.getDownloadManager().getDownloads();
                 
              for (int i=0;i<downloads.length;i++){
               
                Download download = downloads[i];
               
                if ( download.getBooleanAttribute( ta_subs_download )){
                 
                  Map  rd = download.getMapAttribute( ta_subs_download_rd );
                 
                  boolean  delete_it;
                 
                  if ( rd == null ){
                   
                    delete_it = true;
                   
                  }else{
                   
                    delete_it = !recoverSubscriptionUpdate( download, rd );
                  }
                 
                  if ( delete_it ){
                   
                    removeDownload( download, true );
                  }
                }
              }
               
              default_pi.getDownloadManager().addListener(
                new DownloadManagerListener()
                {
                  public void
                  downloadAdded(
                    final Download  download )
                  {
                      // if ever changed to handle non-persistent then you need to fix init deadlock
                      // potential with share-hoster plugin
                   
                    if ( download.isPersistent()){
                     
                      if ( !dht_plugin.isInitialising()){
               
                          // if new download then we want to check out its subscription status
                       
                        lookupAssociations( download.getMapAttribute( ta_subscription_info ) == null );
                       
                      }else{
                       
                        new AEThread2( "Subscriptions:delayInit", true )
                        {
                          public void
                          run()
                          {
                            lookupAssociations( download.getMapAttribute( ta_subscription_info ) == null );
                          }
                        }.start();
                      }
                    }
                  }
                 
                  public void
                  downloadRemoved(
                    Download  download )
                  {
                  }
                },
                false );
             
              for (int i=0;i<PUB_ASSOC_CONC_MAX;i++){
             
                if ( publishAssociations()){
                 
                  break;
                }
              }
             
              publishSubscriptions();
             
              COConfigurationManager.addParameterListener(
                  CONFIG_MAX_RESULTS,
                  new ParameterListener()
                  {
                    public void
                    parameterChanged(
                      String   name )
                    {
                      final int  max_results = COConfigurationManager.getIntParameter( CONFIG_MAX_RESULTS );
                     
                      new AEThread2( "Subs:max results changer", true )
                      {
                        public void
                        run()
                        {
                          checkMaxResults( max_results );
                        }
                      }.start();
                    }
                  });
             
              SimpleTimer.addPeriodicEvent(
                  "SubscriptionChecker",
                  TIMER_PERIOD,
                  new TimerEventPerformer()
                  {
                    private int  ticks;
                   
                    public void
                    perform(
                      TimerEvent event )
                    {
                      ticks++;
                     
                      checkStuff( ticks );
                    }
                  });
            }
          });
   
      delayed_task.queue();
    }
   
    if ( isSearchEnabled()){
     
      try{
        default_pi.getUtilities().registerSearchProvider(
          new SearchProvider()
          {
            private Map<Integer,Object>  properties = new HashMap<Integer, Object>();
           
            {
              properties.put( PR_NAME, MessageText.getString( "ConfigView.section.Subscriptions" ));
             
              try{
                URL url =
                  MagnetURIHandler.getSingleton().registerResource(
                    new MagnetURIHandler.ResourceProvider()
                    {
                      public String
                      getUID()
                      {
                        return( SubscriptionManager.class.getName() + ".2" );
                      }
                     
                      public String
                      getFileType()
                      {
                        return( "png" );
                      }
                         
                      public byte[]
                      getData()
                      {
                        InputStream is = getClass().getClassLoader().getResourceAsStream( "com/aelitis/azureus/ui/images/subscription_icon_1616.png" );

                        if ( is == null ){
                         
                          return( null );
                        }
                       
                        try{
                          ByteArrayOutputStream  baos = new ByteArrayOutputStream();
                         
                          try{
                            byte[]  buffer = new byte[8192];
                           
                            while( true ){
               
                              int  len = is.read( buffer );
                     
                              if ( len <= 0 ){
                               
                                break;
                              }
                 
                              baos.write( buffer, 0, len );
                            }
                          }finally{
                           
                            is.close();
                          }
                         
                          return( baos.toByteArray());
                         
                        }catch( Throwable e ){
                         
                          return( null );
                        }
                      }
                    });
                                   
                properties.put( PR_ICON_URL, url.toExternalForm());
               
              }catch( Throwable e ){
               
                Debug.out( e );
              }
            }
           
            public SearchInstance
            search(
              Map<String,Object>  search_parameters,
              SearchObserver    observer )
           
              throws SearchException
            {   
              try{
                return( searchSubscriptions( search_parameters, observer ));
               
              }catch( Throwable e ){
               
                throw( new SearchException( "Search failed", e ));
              }
            }
           
            public Object
            getProperty(
              int      property )
            {
              return( properties.get( property ));
            }
           
            public void
            setProperty(
              int      property,
              Object    value )
            {
              properties.put( property, value );
            }
          });
       
      }catch( Throwable e ){
       
        Debug.out( "Failed to register search provider" );
      }
    }
  }

  public SearchInstance
  searchSubscriptions(
    Map<String,Object>    search_parameters,
    final SearchObserver  observer )
 
    throws SearchException
  {
    final String  term = (String)search_parameters.get( SearchProvider.SP_SEARCH_TERM );
   
    final SearchInstance si =
      new SearchInstance()
      {
        public void
        cancel()
        {
          Debug.out( "Cancelled" );
        }
      };
     
    if ( term == null ){
   
      try{
        observer.complete();
       
      }catch( Throwable e ){
       
        Debug.out( e );
      }
    }else{
   
      new AEThread2( "Subscriptions:search", true )
      {
        public void
        run()
        {
          final Set<String>  hashes = new HashSet<String>();
         
          searchMatcher  matcher = new searchMatcher( term );

          try{       
            List<SubscriptionResult>  matches = matchSubscriptionResults( matcher );
             
            for ( final SubscriptionResult result: matches ){
             
              final Map result_properties = result.toPropertyMap();
             
              byte[] hash = (byte[])result_properties.get( SearchResult.PR_HASH );
                           
              if ( hash != null ){
                             
                String hash_str = Base32.encode( hash );
             
                if ( hashes.contains( hash_str )){
               
                  continue;
                }
             
                hashes.add( hash_str );
              }
             
              SearchResult search_result =
                new SearchResult()
                {
                  public Object
                  getProperty(
                    int    property_name )
                  {
                    return( result_properties.get( property_name ));
                  }
                };
           
              try{
                observer.resultReceived( si, search_result );
               
              }catch( Throwable e ){
               
                Debug.out( e );
              }
            }
                       
            Map<String,Object[]> template_matches = new HashMap<String, Object[]>();
               
            Engine[] engines = MetaSearchManagerFactory.getSingleton().getMetaSearch().getEngines( false, false );
           
            Map<Subscription,List<String>>  sub_dl_name_map = null;
           
            for ( Subscription sub: getSubscriptions( false )){
             
              if ( !sub.isSearchTemplate()){
               
                continue;
              }
             
              String  sub_name = sub.getName();
                           
              Engine sub_engine = sub.getEngine();
             
              if ( sub_engine.isActive() || !(sub_engine instanceof RSSEngine )){
               
                continue;
              }
             
              int  pos = sub_name.indexOf( ":" );
             
              String t_name = sub_name.substring( pos+1 );
             
              pos  = t_name.indexOf( "(v" );
             
              int t_ver;
             
              if ( pos == -1 ){
               
                t_ver = 1;
               
              }else{
               
                String s = t_name.substring( pos+2, t_name.length()-1);
               
                t_name = t_name.substring( 0, pos );
               
                try{
             
                  t_ver = Integer.parseInt(s);
                 
                }catch( Throwable e ){
                 
                  t_ver = 1;
                }
              }
             
              t_name = t_name.trim();
             
              boolean skip = false;
             
              for ( Engine e: engines ){
               
                if ( e != sub_engine && e.sameLogicAs( sub_engine )){
                 
                  skip = true;
                 
                  break;
                }
               
                if ( e.getName().equalsIgnoreCase( t_name )){
                 
                  if ( e.getVersion() >= t_ver ){
                   
                    skip = true;
                  }
                }
              }
             
              if ( skip ){
               
                continue;
              }
             
              if ( sub_dl_name_map == null ){
               
                sub_dl_name_map = new HashMap<Subscription, List<String>>();
               
                SubscriptionDownloadDetails[] sdds = SubscriptionUtils.getAllCachedDownloadDetails( azureus_core );
               
                for ( SubscriptionDownloadDetails sdd: sdds ){
                 
                  String name = sdd.getDownload().getDisplayName();
                 
                  if ( matcher.matches( name )){
                   
                    Subscription[] x = sdd.getSubscriptions();
                   
                    for ( Subscription s: x ){
                     
                      List<String> g = sub_dl_name_map.get( s );
                     
                      if ( g == null ){
                       
                        g = new ArrayList<String>();
                       
                        sub_dl_name_map.put( s, g );
                      }
                     
                      g.add( name );
                    }
                  }
                }
              }
             
              List<String> names = sub_dl_name_map.get( sub );
             
              if ( names == null ){
               
                continue;
              }
                         
              String key = t_name.toLowerCase();
             
              Object[] entry = template_matches.get( key );
             
              if ( entry == null ){
               
                entry = new Object[]{ sub, t_ver };
               
                template_matches.put( key, entry );
               
              }else{
               
                if ( t_ver > (Integer)entry[1]){
                 
                  entry[0= sub;
                  entry[1= t_ver;
                }
              }
            }
           
            List<Subscription>  interesting = new ArrayList<Subscription>();
           
            for ( Object[] entry: template_matches.values()){
           
              interesting.add((Subscription)entry[0]);
            }
           
            Collections.sort(
              interesting,
              new Comparator<Subscription>()
              {
                public int
                compare(
                  Subscription o1,
                  Subscription o2)
                {
                  long res = o2.getCachedPopularity() - o1.getCachedPopularity();
                 
                  if ( res < 0 ){
                    return( -1 );
                  }else if ( res > 0 ){
                    return( 1 );
                  }else{
                    return( 0 );
                  }
                }
              });
           
            int  added = 0;
           
            for ( final Subscription sub: interesting ){
               
              if ( added >= 3 ){
               
                break;
              }
             
              try{
                String subs_url_str = ((RSSEngine)sub.getEngine()).getSearchUrl( true );
               
                URL subs_url = new URL( subs_url_str );
               
                final byte[] vf_bytes = FileUtil.readInputStreamAsByteArray(subs_url.openConnection().getInputStream());
 
                VuzeFile vf = VuzeFileHandler.getSingleton().loadVuzeFile( vf_bytes );
               
                if ( MetaSearchManagerFactory.getSingleton().isImportable( vf )){
                 
                  final URL url =
                    MagnetURIHandler.getSingleton().registerResource(
                      new MagnetURIHandler.ResourceProvider()
                      {
                        public String
                        getUID()
                        {
                          return( SubscriptionManager.class.getName() + ".sid." + sub.getID() );
                        }
                       
                        public String
                        getFileType()
                        {
                          return( "vuze" );
                        }
                           
                        public byte[]
                        getData()
                        {
                          return( vf_bytes );
                        }
                      });
                     
                  SearchResult search_result =
                    new SearchResult()
                    {
                      public Object
                      getProperty(
                        int    property_name )
                      {
                        if ( property_name == SearchResult.PR_NAME ){
                         
                          return( sub.getName());
                         
                        }else if (   property_name == SearchResult.PR_DOWNLOAD_LINK ||
                              property_name == SearchResult.PR_DOWNLOAD_BUTTON_LINK ){
                         
                          return( url.toExternalForm());
                         
                        }else if ( property_name == SearchResult.PR_PUB_DATE ){
                         
                          return( new Date(sub.getAddTime()));
                         
                        }else if ( property_name == SearchResult.PR_SIZE ){
                         
                          return( 1024L );
                         
                        }else if (   property_name == SearchResult.PR_SEED_COUNT ||
                              property_name == SearchResult.PR_VOTES ){
                                             
                          return((long)sub.getCachedPopularity());
                         
                        }else if ( property_name == SearchResult.PR_RANK ){
                       
                          return( 100L );
                        }
                       
                        return( null );
                      }
                    };
             
                  added++;
                   
                  try{
                    observer.resultReceived( si, search_result );
                   
                  }catch( Throwable e ){
                   
                    Debug.out( e );
                  }
                }
              }catch( Throwable e ){
               
                Debug.out( e );
              }
            }
          }catch( Throwable e ){
           
            Debug.out( e );
           
          }finally{
           
            observer.complete();
          }
        }
      }.start();
    }
   
    return( si );
  }
 
  private List<SubscriptionResult>
  matchSubscriptionResults(
    searchMatcher  matcher  )
  {
    List<SubscriptionResult> result = new ArrayList<SubscriptionResult>();
       
    for ( Subscription sub: getSubscriptions( true )){
     
      SubscriptionResult[] results = sub.getResults( false );
     
      for ( SubscriptionResult r: results ){
       
        Map properties = r.toPropertyMap();
       
        String name = (String)properties.get( SearchResult.PR_NAME );
       
        if ( name == null ){
         
          continue;
        }
       
        if ( matcher.matches( name )){
         
          result.add( r );
        }
      }
    }
   
    return( result );
  }
 
  protected void
  checkMaxResults(
    int    max )
  {
    Subscription[] subs = getSubscriptions();
   
    for (int i=0;i<subs.length;i++){
     
      ((SubscriptionHistoryImpl)subs[i].getHistory()).checkMaxResults( max );
    }
  }
 
  public SubscriptionScheduler
  getScheduler()
  {
    return( scheduler );
  }
 
  public boolean
  isRSSPublishEnabled()
  {
    return( COConfigurationManager.getBooleanParameter( CONFIG_RSS_ENABLE, false ));
  }
 
  public void
  setRSSPublishEnabled(
    boolean    enabled )
  {
    COConfigurationManager.setParameter( CONFIG_RSS_ENABLE, enabled );
  }
 
  public boolean
  isSearchEnabled()
  {
    return( COConfigurationManager.getBooleanParameter( CONFIG_ENABLE_SEARCH, true ));
  }
 
  public void
  setSearchEnabled(
    boolean    enabled )
  {
    COConfigurationManager.setParameter( CONFIG_ENABLE_SEARCH, enabled );
  }
 
  public boolean
  hideSearchTemplates()
  {
    return( COConfigurationManager.getBooleanParameter( CONFIG_HIDE_SEARCH_TEMPLATES, true ));
  }
 
  public boolean
  isSubsDownloadEnabled()
  {
    return( COConfigurationManager.getBooleanParameter( CONFIG_DL_SUBS_ENABLE, true ));
  }
 
  public void
  setSubsDownloadEnabled(
    boolean    enabled )
  {
    COConfigurationManager.setParameter( CONFIG_DL_SUBS_ENABLE, enabled );
  }
 
  public String
  getRSSLink()
  {
    return( rss_publisher.getFeedURL());
  }
 
  public Subscription
  create(
    String      name,
    boolean      public_subs,
    String      json )
 
    throws SubscriptionException
  {
    name = getUniqueName( name );
   
    SubscriptionImpl subs = new SubscriptionImpl( this, name, public_subs, null, json, SubscriptionImpl.ADD_TYPE_CREATE );
   
    log( "Created new subscription: " + subs.getString());
   
    if ( subs.isPublic()){
     
      updatePublicSubscription( subs );
    }
   
    return( addSubscription( subs ));
  }
 
  public Subscription
  createSingletonRSS(
    String    name,
    URL      url,
    int      check_interval_mins )
   
    throws SubscriptionException
  {
    return( createSingletonRSSSupport( name, url, true, check_interval_mins, SubscriptionImpl.ADD_TYPE_CREATE, true ));
  }
 
  protected SubscriptionImpl
  lookupSingletonRSS(
    String    name,
    URL      url,
    boolean    is_public,
    int      check_interval_mins )
   
    throws SubscriptionException
  {
    checkURL( url );
   
    Map  singleton_details = getSingletonMap(name, url, is_public, check_interval_mins);
   
    byte[] sid = SubscriptionBodyImpl.deriveSingletonShortID(singleton_details);
   
    return( getSubscriptionFromSID( sid ));
  }
 
     
  protected Subscription
  createSingletonRSSSupport(
    String    name,
    URL      url,
    boolean    is_public,
    int      check_interval_mins,
    int      add_type,
    boolean    subscribe )
   
    throws SubscriptionException
  {
    checkURL( url );
   
    try{
      Subscription existing = lookupSingletonRSS( name, url, is_public, check_interval_mins );
     
      if ( existing != null ){
       
        return( existing );
      }
     
      Engine engine = MetaSearchManagerFactory.getSingleton().getMetaSearch().createRSSEngine( name, url );
     
      String  json = SubscriptionImpl.getSkeletonJSON( engine, check_interval_mins );
     
      Map  singleton_details = getSingletonMap(name, url, is_public, check_interval_mins);
     
      SubscriptionImpl subs = new SubscriptionImpl( this, name, is_public, singleton_details, json, add_type );
     
      subs.setSubscribed( subscribe );
     
      log( "Created new singleton subscription: " + subs.getString());
             
      subs = addSubscription( subs );
     
      return( subs );
     
    }catch( SubscriptionException e ){
     
      throw((SubscriptionException)e);
     
    }catch( Throwable e ){
     
      throw( new SubscriptionException( "Failed to create subscription", e ));
    }
  }
 
  protected String
  getUniqueName(
    String  name )
  {
    for ( int i=0;i<1024;i++){
     
      String  test_name = name + (i==0?"":(" (" + i + ")"));
     
      if ( getSubscriptionFromName( test_name ) == null ){

        return( test_name );
      }
    }
   
    return( name );
  }
 
  protected Map
  getSingletonMap(
    String    name,
    URL      url,
    boolean    is_public,
    int      check_interval_mins )
 
    throws SubscriptionException
  {
    try{
      Map  singleton_details = new HashMap();
     
      if ( url.getProtocol().equalsIgnoreCase( "vuze" )){
       
          // hack to minimise encoded url length for our own urls
       
        singleton_details.put( "key", url.toExternalForm().getBytes( Constants.BYTE_ENCODING ));

      }else{
        singleton_details.put( "key", url.toExternalForm().getBytes( "UTF-8" ));
      }
           
      String  name2 = name.length() > 64?name.substring(0,64):name;
     
      singleton_details.put( "name", name2 );
     
      if ( check_interval_mins != SubscriptionHistoryImpl.DEFAULT_CHECK_INTERVAL_MINS ){
       
        singleton_details.put( "ci", new Long( check_interval_mins ));
      }
     
      return( singleton_details );
   
    }catch( Throwable e ){
     
      throw( new SubscriptionException( "Failed to create subscription", e ));
    }
  }
 
  protected SubscriptionImpl
  createSingletonSubscription(
    Map      singleton_details,
    int      add_type,
    boolean    subscribe )
 
    throws SubscriptionException
  {
    try{
      String name = ImportExportUtils.importString( singleton_details, "name", "(Anonymous)" );
     
      URL  url = new URL( ImportExportUtils.importString( singleton_details, "key" ));
     
      int  check_interval_mins = (int)ImportExportUtils.importLong( singleton_details, "ci", SubscriptionHistoryImpl.DEFAULT_CHECK_INTERVAL_MINS );
     
        // only defined type is singleton rss
     
      SubscriptionImpl s = (SubscriptionImpl)createSingletonRSSSupport( name, url, true, check_interval_mins, add_type, subscribe );
     
      return( s );
     
    }catch( Throwable e ){
     
      log( "Creation of singleton from " + singleton_details + " failed", e );
     
      throw( new SubscriptionException( "Creation of singleton from " + singleton_details + " failed", e ));
    }
  }
 
  public Subscription
  createRSS(
    String    name,
    URL      url,
    int      check_interval_mins,
    Map      user_data )
 
    throws SubscriptionException
  {
    checkURL( url );
   
    try{
      name = getUniqueName(name);
   
      Engine engine = MetaSearchManagerFactory.getSingleton().getMetaSearch().createRSSEngine( name, url );
     
      String  json = SubscriptionImpl.getSkeletonJSON( engine, check_interval_mins );
     
        // engine name may have been modified so re-read it for subscription default
     
      SubscriptionImpl subs = new SubscriptionImpl( this, engine.getName(), engine.isPublic(), null, json, SubscriptionImpl.ADD_TYPE_CREATE );
     
      if ( user_data != null ){
       
        Iterator it = user_data.entrySet().iterator();
       
        while( it.hasNext()){
         
          Map.Entry entry = (Map.Entry)it.next();
         
          subs.setUserData( entry.getKey(), entry.getValue());
        }
      }
     
      log( "Created new subscription: " + subs.getString());
         
      subs = addSubscription( subs );
     
      if ( subs.isPublic()){
     
        updatePublicSubscription( subs );
      }
     
      return( subs );
     
    }catch( Throwable e ){
     
      throw( new SubscriptionException( "Failed to create subscription", e ));
    }
  }

  protected void
  checkURL(
    URL    url )
 
    throws SubscriptionException
  {
    if ( url.getHost().trim().length() == 0 ){
     
      String protocol = url.getProtocol().toLowerCase();
     
      if ( ! ( protocol.equals( "azplug" ) || protocol.equals( "file" ) || protocol.equals( "vuze" ))){
     
        throw( new SubscriptionException( "Invalid URL '" + url + "'" ));
      }
    }
  }
 
  protected SubscriptionImpl
  addSubscription(
    SubscriptionImpl    subs )
  {
    SubscriptionImpl existing;
   
    synchronized( this ){

      int index = Collections.binarySearch(subscriptions, subs, new Comparator<Subscription>() {
        public int compare(Subscription arg0, Subscription arg1) {
          return arg0.getID().compareTo(arg1.getID());
        }
      });
      if (index < 0) {
        existing = null;
        index = -1 * index - 1; // best guess

        subscriptions.add( index, subs );
     
        saveConfig();
      } else {
        existing = (SubscriptionImpl) subscriptions.get(index);
      }
    }
   
    if ( existing != null ){
     
      log( "Attempted to add subscription when already present: " + subs.getString());
     
      subs.destroy();
     
      return( existing );
    }
   
    if ( subs.isMine()){
     
      addMetaSearchListener();
    }
   
    if ( subs.getCachedPopularity() == -1 ){
     
      try{
        subs.getPopularity(
          new SubscriptionPopularityListener()
          {
            public void
            gotPopularity(
              long            popularity )
            {
            }
           
            public void
            failed(
              SubscriptionException    error )
            {
            }
          });
       
      }catch( Throwable e ){
       
        log( "", e );
      }
    }
   
    Iterator it = listeners.iterator();
   
    while( it.hasNext()){
     
      try{
        ((SubscriptionManagerListener)it.next()).subscriptionAdded( subs );
       
      }catch( Throwable e ){
       
        Debug.printStackTrace(e);
      }
    }
   
    if ( subs.isSubscribed() && subs.isPublic()){
     
      setSelected( subs );
    }
   
    if ( dht_plugin != null ){
       
      new AEThread2( "Publish check", true )
      {
        public void
        run()
        {
          publishSubscriptions();
        }
      }.start();
    }
   
    return( subs );
  }
 
  protected void
  addMetaSearchListener()
  {
    synchronized( this ){
     
      if ( meta_search_listener_added ){
       
        return;
      }
     
      meta_search_listener_added = true;
    }
   
    MetaSearchManagerFactory.getSingleton().getMetaSearch().addListener(
      new MetaSearchListener()
      {
        public void
        engineAdded(
          Engine    engine )
        {
         
        }
       
        public void
        engineUpdated(
          Engine    engine )
        {
          synchronized( this ){
           
            for (int i=0;i<subscriptions.size();i++){
             
              SubscriptionImpl  subs = (SubscriptionImpl)subscriptions.get(i);
             
              if ( subs.isMine()){
               
                subs.engineUpdated( engine );
              }
            }
          }
        }
       
        public void
        engineRemoved(
          Engine    engine )
        {
         
        }
      });
  }
 
  protected void
  changeSubscription(
    SubscriptionImpl    subs )
  {
    if ( !subs.isRemoved()){
     
      Iterator it = listeners.iterator();
     
      while( it.hasNext()){
       
        try{
          ((SubscriptionManagerListener)it.next()).subscriptionChanged( subs );
         
        }catch( Throwable e ){
         
          Debug.printStackTrace(e);
        }
      }
    }
  }
 
  protected void
  selectSubscription(
    SubscriptionImpl    subs )
  {
    if ( !subs.isRemoved()){

      Iterator it = listeners.iterator();
     
      while( it.hasNext()){
       
        try{
          ((SubscriptionManagerListener)it.next()).subscriptionSelected( subs );
         
        }catch( Throwable e ){
         
          Debug.printStackTrace(e);
        }
      }
    }
  }
 
  protected void
  removeSubscription(
    SubscriptionImpl    subs )
  {
    synchronized( this ){
     
      if ( subscriptions.remove( subs )){
     
        saveConfig();
       
      }else{
     
        return;
      }
    }
   
    try{
      Engine engine = subs.getEngine( true );
     
      if ( engine.getType() == Engine.ENGINE_TYPE_RSS ){
       
        engine.delete();
       
        log( "Removed engine " + engine.getName() + " due to subscription removal" );
      }
     
    }catch( Throwable e ){
     
      log( "Failed to check for engine deletion", e );
    }
   
    Iterator<SubscriptionManagerListener> it = listeners.iterator();
   
    while( it.hasNext()){
     
      try{
        it.next().subscriptionRemoved( subs );
       
      }catch( Throwable e ){
       
        Debug.printStackTrace(e);
      }
    }
   
    try{
      FileUtil.deleteResilientFile( getResultsFile( subs ));
     
      File vuze_file = getVuzeFile( subs );
     
      vuze_file.delete();
     
      new File( vuze_file.getParent(), vuze_file.getName() + ".bak" ).delete();
     
    }catch( Throwable e ){
     
      log( "Failed to delete results/vuze file", e );
    }
  }
 
  protected void
  updatePublicSubscription(
    SubscriptionImpl    subs )
  {   
    if ( subs.isSingleton()){
     
        // never update singletons
     
      subs.setServerPublished();
     
    }else{
     
      Long  l_last_pub   = (Long)subs.getUserData( SP_LAST_ATTEMPTED );
      Long  l_consec_fail = (Long)subs.getUserData( SP_CONSEC_FAIL );
     
      if ( l_last_pub != null && l_consec_fail != null ){
       
        long  delay = SERVER_PUB_CHECK_PERIOD;
       
        for (int i=0;i<l_consec_fail.longValue();i++){
         
          delay <<= 1;
         
          if ( delay > 24*60*60*1000 ){
           
            break;
          }
        }
       
        if ( l_last_pub.longValue() + delay > SystemTime.getMonotonousTime()){
         
          return;
        }
      }
     
      try{   
        File vf = getVuzeFile( subs );
 
        byte[] bytes = FileUtil.readFileAsByteArray( vf );
       
        byte[]  encoded_subs = Base64.encode( bytes );
 
        PlatformSubscriptionsMessenger.updateSubscription(
            !subs.getServerPublished(),
            subs.getName(),
            subs.getPublicKey(),
            subs.getPrivateKey(),
            subs.getShortID(),
            subs.getVersion(),
            new String( encoded_subs ));
       
        subs.setUserData( SP_LAST_ATTEMPTED, null );
        subs.setUserData( SP_CONSEC_FAIL, null );

        subs.setServerPublished();
       
        log( "    Updated public subscription " + subs.getString());
       
      }catch( Throwable e ){
       
        log( "    Failed to update public subscription " + subs.getString(), e );
       
        subs.setUserData( SP_LAST_ATTEMPTED, new Long( SystemTime.getMonotonousTime()));
       
        subs.setUserData( SP_CONSEC_FAIL, new Long( l_consec_fail==null?1:(l_consec_fail.longValue()+1)));

        subs.setServerPublicationOutstanding();
      }
    }
  }
 
  protected void
  checkSingletonPublish(
    SubscriptionImpl    subs )
 
    throws SubscriptionException
  {
    if ( subs.getSingletonPublishAttempted()){
     
      throw( new SubscriptionException( "Singleton publish already attempted" ));
    }
   
    subs.setSingletonPublishAttempted();
   
    try{
      File vf = getVuzeFile( subs );
 
      byte[] bytes = FileUtil.readFileAsByteArray( vf );
     
      byte[]  encoded_subs = Base64.encode( bytes );
     
        // use a transient key-pair as we won't have the private key in general
     
      KeyPair  kp = CryptoECCUtils.createKeys();
     
      byte[] public_key     = CryptoECCUtils.keyToRawdata( kp.getPublic());
      byte[] private_key     = CryptoECCUtils.keyToRawdata( kp.getPrivate());
 
      PlatformSubscriptionsMessenger.updateSubscription(
          true,
          subs.getName(),
          public_key,
          private_key,
          subs.getShortID(),
          1,
          new String( encoded_subs ));
     
      log( "    created singleton public subscription " + subs.getString());
     
    }catch( Throwable e ){
     
      throw( new SubscriptionException( "Failed to publish singleton", e ));
    }
  }
 
  protected void
  checkServerPublications(
    List    subs )
  {
    for (int i=0;i<subs.size();i++){
     
      SubscriptionImpl  sub = (SubscriptionImpl)subs.get(i);
     
      if ( sub.getServerPublicationOutstanding()){
       
        updatePublicSubscription( sub );
      }
    }
  }
 
  protected void
  checkStuff(
    int    ticks )
  {
    long now = SystemTime.getCurrentTime();
   
    List<SubscriptionImpl> subs;
   
    synchronized( this ){
     
      subs = new ArrayList<SubscriptionImpl>( subscriptions );
    }
   
    SubscriptionImpl  expired_subs = null;
   
    for (int i=0;i<subs.size();i++){
     
      SubscriptionImpl sub = subs.get( i );
     
      if ( !( sub.isMine() || sub.isSubscribed())){
       
        long  age = now - sub.getAddTime();
       
        if ( age > DELETE_UNUSED_AFTER_MILLIS ){
         
          if (   expired_subs == null ||
              ( sub.getAddTime() < expired_subs.getAddTime())){
           
            expired_subs = sub;
          }
         
          continue;
        }
      }
     
     
      sub.checkPublish();
    }
   
    if ( expired_subs != null ){
     
      log( "Removing unsubscribed subscription '" + expired_subs.getName() + "' as expired" );
         
      expired_subs.remove();
    }
   
    if ( ticks % ASSOC_CHECK_TICKS == 0 ){
     
      lookupAssociations( false );
    }
   
    if ( ticks % SERVER_PUB_CHECK_TICKS == 0 ){
     
      checkServerPublications( subs );
    }
   
    if ( ticks % TIDY_POT_ASSOC_TICKS == 0 ){
     
      tidyPotentialAssociations();
    }
   
    if (   ticks == SET_SELECTED_FIRST_TICK ||
        ticks % SET_SELECTED_TICKS == 0 ){
     
      setSelected( subs );
    }
  }
 
  public Subscription
  importSubscription(
    int      type,
    Map      map,
    boolean    warn_user )
 
    throws SubscriptionException
  {
    boolean  log_errors = true;
   
    try{
      try{
        if ( type == VuzeFileComponent.COMP_TYPE_SUBSCRIPTION_SINGLETON ){
         
          String  name = new String((byte[])map.get( "name" ), "UTF-8" );
         
          URL  url = new URL( new String((byte[])map.get( "url" ), "UTF-8" ));
         
          Long  l_interval = (Long)map.get( "check_interval_mins" );
         
          int  check_interval_mins = l_interval==null?SubscriptionHistoryImpl.DEFAULT_CHECK_INTERVAL_MINS:l_interval.intValue();
         
          Long  l_public = (Long)map.get( "public" );
         
          boolean is_public = l_public==null?true:l_public.longValue()==1;
         
          SubscriptionImpl existing = lookupSingletonRSS(name, url, is_public, check_interval_mins );
         
          if ( UrlFilter.getInstance().urlCanRPC( url.toExternalForm())){
           
            warn_user = false;
          }
         
          if ( existing != null && existing.isSubscribed()){
           
            if ( warn_user ){
             
              UIManager ui_manager = StaticUtilities.getUIManager( 120*1000 );
             
              String details = MessageText.getString(
                  "subscript.add.dup.desc",
                  new String[]{ existing.getName()});
             
              ui_manager.showMessageBox(
                  "subscript.add.dup.title",
                  "!" + details + "!",
                  UIManagerEvent.MT_OK );
            }
           
            selectSubscription( existing );
           
            return( existing );
           
          }else{
           
            if ( warn_user ){
             
              UIManager ui_manager = StaticUtilities.getUIManager( 120*1000 );
       
              String details = MessageText.getString(
                  "subscript.add.desc",
                  new String[]{ name });
             
              long res = ui_manager.showMessageBox(
                  "subscript.add.title",
                  "!" + details + "!",
                  UIManagerEvent.MT_YES | UIManagerEvent.MT_NO );
             
              if ( res != UIManagerEvent.MT_YES ){ 
             
                log_errors = false;
               
                throw( new SubscriptionException( "User declined addition" ));
              }
            }
           
            if ( existing == null ){
         
              SubscriptionImpl new_subs = (SubscriptionImpl)createSingletonRSSSupport( name, url, is_public, check_interval_mins, SubscriptionImpl.ADD_TYPE_IMPORT, true );
                                 
              log( "Imported new singleton subscription: " + new_subs.getString());
                 
              return( new_subs );
             
            }else{
             
              existing.setSubscribed( true );
             
              selectSubscription( existing );
             
              return( existing );
            }
          }
        }else{
         
          SubscriptionBodyImpl body = new SubscriptionBodyImpl( this, map );
             
          SubscriptionImpl existing = getSubscriptionFromSID( body.getShortID());
         
          if ( existing != null && existing.isSubscribed()){
         
            if ( existing.getVersion() >= body.getVersion()){
             
              log( "Not upgrading subscription: " + existing.getString() + " as supplied (" +  body.getVersion() + ") is not more recent than existing (" + existing.getVersion() + ")");
             
              if ( warn_user ){
               
                UIManager ui_manager = StaticUtilities.getUIManager( 120*1000 );
               
                String details = MessageText.getString(
                    "subscript.add.dup.desc",
                    new String[]{ existing.getName()});
               
                ui_manager.showMessageBox(
                    "subscript.add.dup.title",
                    "!" + details + "!",
                    UIManagerEvent.MT_OK );
              }
                // we have a newer one, ignore
             
              selectSubscription( existing );
             
              return( existing );
             
            }else{
             
              if ( warn_user ){
               
                UIManager ui_manager = StaticUtilities.getUIManager( 120*1000 );
       
                String details = MessageText.getString(
                    "subscript.add.upgrade.desc",
                    new String[]{ existing.getName()});
               
                long res = ui_manager.showMessageBox(
                    "subscript.add.upgrade.title",
                    "!" + details + "!",
                    UIManagerEvent.MT_YES | UIManagerEvent.MT_NO );
               
                if ( res != UIManagerEvent.MT_YES ){ 
               
                  throw( new SubscriptionException( "User declined upgrade" ));
                }
              }
             
              log( "Upgrading subscription: " + existing.getString());
     
              existing.upgrade( body );
             
              saveConfig();
             
              subscriptionUpdated();
             
              return( existing );
            }
          }else{
           
            SubscriptionImpl new_subs = new SubscriptionImpl( this, body, SubscriptionImpl.ADD_TYPE_IMPORT, true );
     
            if ( warn_user ){
             
              UIManager ui_manager = StaticUtilities.getUIManager( 120*1000 );
       
              String details = MessageText.getString(
                  "subscript.add.desc",
                  new String[]{ new_subs.getName()});
             
              long res = ui_manager.showMessageBox(
                  "subscript.add.title",
                  "!" + details + "!",
                  UIManagerEvent.MT_YES | UIManagerEvent.MT_NO );
             
              if ( res != UIManagerEvent.MT_YES ){ 
             
                throw( new SubscriptionException( "User declined addition" ));
              }
            }
           
            log( "Imported new subscription: " + new_subs.getString());
           
            if ( existing != null ){
             
              existing.remove();
            }
           
            new_subs = addSubscription( new_subs );
           
            return( new_subs );
          }
        }
      }catch( Throwable e ){
       
        throw( new SubscriptionException( "Subscription import failed", e ));
      }
    }catch( SubscriptionException e ){
     
      if ( warn_user && log_errors ){
       
        UIManager ui_manager = StaticUtilities.getUIManager( 120*1000 );
       
        String details = MessageText.getString(
            "subscript.import.fail.desc",
            new String[]{ Debug.getNestedExceptionMessage(e)});
       
        ui_manager.showMessageBox(
            "subscript.import.fail.title",
            "!" + details + "!",
            UIManagerEvent.MT_OK );
      }
     
      throw( e );
    }
  }
 
  public Subscription[]
  getSubscriptions()
  {
    synchronized( this ){
     
      return((SubscriptionImpl[])subscriptions.toArray( new SubscriptionImpl[subscriptions.size()]));
    }
  }
 
  public Subscription[]
  getSubscriptions(
    boolean  subscribed_only )
  {
    if ( !subscribed_only ){
     
      return( getSubscriptions());
    }
   
    List  result = new ArrayList();
       
    synchronized( this ){
     
      for (int i=0;i<subscriptions.size();i++){
       
        SubscriptionImpl subs = (SubscriptionImpl)subscriptions.get(i);
       
        if ( subs.isSubscribed()){
         
          result.add( subs );
        }
      }
    }
   
    return((SubscriptionImpl[])result.toArray( new SubscriptionImpl[result.size()]));
  }
 
  public int
  getSubscriptionCount(
    boolean  subscribed_only )
  {
    if ( subscribed_only ){
 
      int total = 0;
     
      synchronized( this ){
       
        for ( Subscription subs: subscriptions ){
         
          if ( subs.isSubscribed()){
           
            total++;
          }
        }
      }
     
      return( total );
     
    }else{
      return( subscriptions.size());
    }
  }

  protected SubscriptionImpl
  getSubscriptionFromName(
    String    name )
  {
    synchronized( this ){
     
      for (int i=0;i<subscriptions.size();i++){
       
        SubscriptionImpl s = (SubscriptionImpl)subscriptions.get(i);
       
        if ( s.getName().equalsIgnoreCase( name )){
         
          return( s );
        }
      }
    }
   
    return( null );
  }
 
  public Subscription
  getSubscriptionByID(
    String    id )
  {
    synchronized( this ){
     
      int index = Collections.binarySearch(subscriptions, id, new Comparator() {
        public int compare(Object o1, Object o2) {
          String id1 = (o1 instanceof Subscription) ? ((Subscription) o1).getID() : o1.toString();
          String id2 = (o2 instanceof Subscription) ? ((Subscription) o2).getID() : o2.toString();
          return id1.compareTo(id2);
        }
      });
     
      if (index >= 0) {
        return subscriptions.get(index);
      }
    }
   
    return null;
  }
 
  protected SubscriptionImpl
  getSubscriptionFromSID(
    byte[]    sid )
  {
    return (SubscriptionImpl) getSubscriptionByID( Base32.encode(sid));
  }
 
  protected File
  getSubsDir()
 
    throws IOException
  {
    File dir = new File(SystemProperties.getUserPath());

    dir = new File( dir, "subs" );
    
     if ( !dir.exists()){
      
       if ( !dir.mkdirs()){
        
         throw( new IOException( "Failed to create '" + dir + "'" ));
       }
     } 
    
     return( dir );
  }
 
  protected File
  getVuzeFile(
    SubscriptionImpl     subs )
 
    throws IOException
  {
     File dir = getSubsDir();
    
     return( new File( dir, ByteFormatter.encodeString( subs.getShortID()) + ".vuze" ));
  }
 
  protected File
  getResultsFile(
    SubscriptionImpl     subs )
 
    throws IOException
  {
     File dir = getSubsDir();
    
     return( new File( dir, ByteFormatter.encodeString( subs.getShortID()) + ".results" ));
  }
 
  public int
  getKnownSubscriptionCount()
  {
    PluginInterface pi = PluginInitializer.getDefaultInterface();

    ByteArrayHashMap<String> results = new ByteArrayHashMap<String>();
   
    try{
      Download[] downloads = pi.getDownloadManager().getDownloads();
     
      for ( Download download: downloads ){
       
        Map  m = download.getMapAttribute( ta_subscription_info );
       
        if ( m != null ){
         
          List s = (List)m.get("s");
         
          if ( s != null && s.size() > 0 ){
           
            List  result = new ArrayList( s.size());
           
            for (int i=0;i<s.size();i++){
             
              byte[]  sid = (byte[])s.get(i);
             
              results.put( sid, "" );
            }
          }
        }
      }
    }catch( Throwable e ){
     
      log( "Failed to get known subscriptions", e );
    }
   
    return( results.size());
  }
 
  public Subscription[]
  getKnownSubscriptions(
    byte[]            hash )
  {
    PluginInterface pi = PluginInitializer.getDefaultInterface();

    try{
      Download download = pi.getDownloadManager().getDownload( hash );
     
      if ( download != null ){
       
        Map  m = download.getMapAttribute( ta_subscription_info );
       
        if ( m != null ){
         
          List s = (List)m.get("s");
         
          if ( s != null && s.size() > 0 ){
           
            List  result = new ArrayList( s.size());
           
            boolean hide_search = hideSearchTemplates();
           
            for (int i=0;i<s.size();i++){
             
              byte[]  sid = (byte[])s.get(i);
             
              SubscriptionImpl subs = getSubscriptionFromSID(sid);
             
              if ( subs != null ){
               
                if ( isVisible( subs )){
               
                  if ( hide_search && subs.isSearchTemplate()){
                 
                  }else{
                   
                    result.add( subs );
                  }
                }
              }
            }
           
            return((Subscription[])result.toArray( new Subscription[result.size()]));
          }
        }
      }
    }catch( Throwable e ){
     
      log( "Failed to get known subscriptions", e );
    }
   
    return( new Subscription[0] );
  }
 
  protected boolean
  subscriptionExists(
    Download      download,
    SubscriptionImpl  subs )
  {
    byte[]  sid = subs.getShortID();
 
    Map  m = download.getMapAttribute( ta_subscription_info );
   
    if ( m != null ){
     
      List s = (List)m.get("s");
     
      if ( s != null && s.size() > 0 ){
               
        for (int i=0;i<s.size();i++){
         
          byte[]  x = (byte[])s.get(i);
         
          if ( Arrays.equals( x, sid )){
           
            return( true );
          }
        }
      }
    }
   
    return( false );
  }
 
  protected boolean
  isVisible(
    SubscriptionImpl    subs )
  {
      // to avoid development links polluting production we filter out such subscriptions
   
    if ( Constants.isCVSVersion() || subs.isSubscribed()){
     
      return( true );
    }
   
    try{
      Engine engine = subs.getEngine( true );
     
      if ( engine instanceof WebEngine ){
       
        String url = ((WebEngine)engine).getSearchUrl();
       
        try{
          String host = new URL( url ).getHost();
         
          return( !exclusion_pattern.matcher( host ).matches());
         
        }catch( Throwable e ){
        }
      }
     
      return( true );
     
    }catch( Throwable e ){
     
      log( "isVisible failed for " + subs.getString(), e );
     
      return( false );
    }
  }
   
  public Subscription[]
  getLinkedSubscriptions(
    byte[]            hash )
  {
    PluginInterface pi = PluginInitializer.getDefaultInterface();

    try{
      Download download = pi.getDownloadManager().getDownload( hash );
     
      if ( download != null ){
       
        Map  m = download.getMapAttribute( ta_subscription_info );
       
        if ( m != null ){
         
          List s = (List)m.get("s");
         
          if ( s != null && s.size() > 0 ){
           
            List  result = new ArrayList( s.size());
           
            for (int i=0;i<s.size();i++){
             
              byte[]  sid = (byte[])s.get(i);
             
              SubscriptionImpl subs = getSubscriptionFromSID(sid);
             
              if ( subs != null ){
               
                if ( subs.hasAssociation( hash )){
               
                  result.add( subs );
                }
              }
            }
           
            return((Subscription[])result.toArray( new Subscription[result.size()]));
          }
        }
      }
    }catch( Throwable e ){
     
      log( "Failed to get known subscriptions", e );
    }
   
    return( new Subscription[0] );
  }
 
  protected void
  lookupAssociations(
    boolean    high_priority )
  {
    synchronized( this ){
     
      if ( periodic_lookup_in_progress ){
       
        if ( high_priority ){
       
          priority_lookup_pending++;
        }
       
        return;
      }
     
      periodic_lookup_in_progress  = true;
    }
   
    try{
      PluginInterface pi = PluginInitializer.getDefaultInterface();
     
      Download[] downloads = pi.getDownloadManager().getDownloads();
         
      long  now = SystemTime.getCurrentTime();
     
      long    newest_time    = 0;
      Download  newest_download  = null;
     
     
      for( int i=0;i<downloads.length;i++){
       
        Download  download = downloads[i];
       
        if ( download.getTorrent() == null || !download.isPersistent()){
         
          continue;
        }
       
        Map  map = download.getMapAttribute( ta_subscription_info );
       
        if ( map == null ){
         
          map = new LightHashMap();
         
        }else{
         
          map = new LightHashMap( map );
       
       
        Long  l_last_check = (Long)map.get( "lc" );
       
        long  last_check = l_last_check==null?0:l_last_check.longValue();
       
        if ( last_check > now ){
         
          last_check = now;
         
          map.put( "lc", new Long( last_check ));
         
          download.setMapAttribute( ta_subscription_info, map );
        }
       
        List  subs = (List)map.get( "s" );
       
        int  sub_count = subs==null?0:subs.size();
       
        if ( sub_count > 8 ){
         
          continue;
        }
       
        long  create_time = download.getCreationTime();
       
        int  time_between_checks = (sub_count + 1) * 24*60*60*1000 + (int)(create_time%4*60*60*1000);
       
        if ( now - last_check >= time_between_checks ){
         
          if ( create_time > newest_time ){
           
            newest_time    = create_time;
            newest_download  = download;
          }
        }
      }
     
      if ( newest_download != null ){
     
        byte[] hash = newest_download.getTorrent().getHash();
       
        log( "Association lookup starts for " + newest_download.getName() + "/" + ByteFormatter.encodeString( hash ));

        lookupAssociationsSupport(
          hash,
          new  SubscriptionLookupListener()
          {
            public void
            found(
              byte[]          hash,
              Subscription      subscription )
            {             
            }
           
            public void
            failed(
              byte[]          hash,
              SubscriptionException  error )
            {
              log( "Association lookup failed for " + ByteFormatter.encodeString( hash ), error );

              associationLookupComplete();
            }
           
            public void
            complete(
              byte[]       hash,
              Subscription[]  subs )
            {
              log( "Association lookup complete for " + ByteFormatter.encodeString( hash ));
             
              associationLookupComplete();
            }
          });
           
      }else{
       
        associationLookupComplete();
      }
    }catch( Throwable e ){
     
      log( "Association lookup check failed", e );

      associationLookupComplete();     
    }
  }
 
  protected void
  associationLookupComplete()
  {
    boolean  recheck;
   
    synchronized( SubscriptionManagerImpl.this ){
     
      periodic_lookup_in_progress = false;
     
      recheck = priority_lookup_pending > 0;
       
      if ( recheck ){
       
        priority_lookup_pending--;
      }
    }
   
    if ( recheck ){
     
      new AEThread2( "SM:priAssLookup", true )
      {
        public void run()
        {
          lookupAssociations( false );
        }
      }.start();
    }
  }
 
  protected void
  setSelected(
    List    subs )
  {
    List  sids     = new ArrayList();
    List  used_subs  = new ArrayList();
   
    for (int i=0;i<subs.size();i++){
     
      SubscriptionImpl  sub = (SubscriptionImpl)subs.get(i);
     
      if ( sub.isSubscribed()){
       
        if ( sub.isPublic()){
       
          used_subs.add( sub );
       
          sids.add( sub.getShortID());
         
        }else{
         
          checkInitialDownload( sub );
        }
      }
    }
   
    if ( sids.size() > 0 ){
     
      try{
        List[] result = PlatformSubscriptionsMessenger.setSelected( sids );
       
        List  versions     = result[0];
        List  popularities  = result[1];
       
        log( "Popularity update: updated " + sids.size());
       
        final List dht_pops = new ArrayList();
       
        for (int i=0;i<sids.size();i++){
         
          SubscriptionImpl sub = (SubscriptionImpl)used_subs.get(i);
         
          int  latest_version = ((Long)versions.get(i)).intValue();
         
          if ( latest_version > sub.getVersion()){
           
            updateSubscription( sub, latest_version );
           
          }else{
           
            checkInitialDownload( sub );
          }
         
          if ( latest_version > 0 ){
           
            try{
              long  pop = ((Long)popularities.get(i)).longValue();
             
              if ( pop >= 0 && pop != sub.getCachedPopularity()){
               
                sub.setCachedPopularity( pop );
              }
            }catch( Throwable e ){
             
              log( "Popularity update: Failed to extract popularity", e );
            }
          }else{
           
            dht_pops.add( sub );
          }
        }
       
        if ( dht_pops.size() <= 3 ){
         
          for (int i=0;i<dht_pops.size();i++){
             
            updatePopularityFromDHT((SubscriptionImpl)dht_pops.get(i), false );
          }
        }else{
         
          new AEThread2( "SM:asyncPop", true )
          {
            public void
            run()
            {
              for (int i=0;i<dht_pops.size();i++){
               
                updatePopularityFromDHT((SubscriptionImpl)dht_pops.get(i), true );
              }
            }
          }.start();
        }
      }catch( Throwable e ){
       
        log( "Popularity update: Failed to record selected subscriptions", e );
      }
    }else{
     
      log( "Popularity update: No selected, public subscriptions" );
    }
  }
 
  protected void
  checkUpgrade(
    SubscriptionImpl    sub )
  {
    setSelected( sub );
  }
 
  protected void
  setSelected(
    final SubscriptionImpl  sub )
  {
    if ( sub.isSubscribed()){
     
      if ( sub.isPublic()){
     
        new DelayedEvent(
          "SM:setSelected",
          0,
          new AERunnable()
          {
            public void
            runSupport()
            {
              try{
                List  sids = new ArrayList();
               
                sids.add( sub.getShortID());
             
                List[] result = PlatformSubscriptionsMessenger.setSelected( sids );
               
                log( "setSelected: " + sub.getName());
               
                int  latest_version = ((Long)result[0].get(0)).intValue();
               
                if ( latest_version == 0 ){
                 
                  if ( sub.isSingleton()){
                 
                    checkSingletonPublish( sub );
                  }
                }else if ( latest_version > sub.getVersion()){
                 
                  updateSubscription( sub, latest_version );
                 
                }else{
                 
                  checkInitialDownload( sub );
                }
               
                if ( latest_version > 0 ){
                 
                  try{
                    long  pop = ((Long)result[1].get(0)).longValue();
                   
                    if ( pop >= 0 && pop != sub.getCachedPopularity()){
                     
                      sub.setCachedPopularity( pop );
                    }
                  }catch( Throwable e ){
                   
                    log( "Popularity update: Failed to extract popularity", e );
                  }
                }else{
                 
                  updatePopularityFromDHT( sub, true );
                }
              }catch( Throwable e ){
               
                log( "setSelected: failed for " + sub.getName(), e );
              }
            }
          });
      }else{
       
        checkInitialDownload( sub );
      }
    }
  }
 
  protected void
  checkInitialDownload(
    SubscriptionImpl    subs )
  {
    if ( subs.getHistory().getLastScanTime() == 0 ){
     
      scheduler.download(
        subs,
        true,
        new SubscriptionDownloadListener()
        {
          public void
          complete(
            Subscription    subs )
          {
            log( "Initial download of " + subs.getName() + " complete" );
          }
         
          public void
          failed(
            Subscription      subs,
            SubscriptionException  error )
          {
            log( "Initial download of " + subs.getName() + " failed", error );
          }
        });
    }
  }
 
  public SubscriptionAssociationLookup
  lookupAssociations(
    final byte[]               hash,
    final SubscriptionLookupListener    listener )
 
    throws SubscriptionException
  {
    if ( dht_plugin != null && !dht_plugin.isInitialising()){
     
      return( lookupAssociationsSupport( hash, listener ));
    }
   
    final boolean[]  cancelled = { false };
   
    final SubscriptionAssociationLookup[]  actual_res = { null };
   
    final SubscriptionAssociationLookup res =
      new SubscriptionAssociationLookup()
      {
        public void
        cancel()
        {
          log( "    Association lookup cancelled" );
 
          synchronized( actual_res ){
           
            cancelled[0] = true;
           
            if ( actual_res[0] != null ){
             
              actual_res[0].cancel();
            }
          }
        }
      };
     
    new AEThread2( "SM:initwait", true )
    {
      public void
      run()
      {
        try{
          SubscriptionAssociationLookup x = lookupAssociationsSupport( hash, listener );
         
          synchronized( actual_res ){

            actual_res[0] = x;
           
            if ( cancelled[0] ){
             
              x.cancel();
            }
          }
         
        }catch( SubscriptionException e ){
         
          listener.failed( hash, e );
        }
       
      }
    }.start();
   
    return( res );
  }
 
  protected SubscriptionAssociationLookup
  lookupAssociationsSupport(
    final byte[]               hash,
    final SubscriptionLookupListener    listener )
 
    throws SubscriptionException
  {
    log( "Looking up associations for '" + ByteFormatter.encodeString( hash ));
   
    final String  key = "subscription:assoc:" + ByteFormatter.encodeString( hash );
     
    final boolean[]  cancelled = { false };
   
    dht_plugin.get(
      key.getBytes(),
      "Subscription association read: " + ByteFormatter.encodeString( hash ),
      DHTPlugin.FLAG_SINGLE_VALUE,
      30,
      60*1000,
      true,
      true,
      new DHTPluginOperationListener()
      {
        private Map<HashWrapper,Integer>  hits           = new HashMap<HashWrapper, Integer>();
        private AESemaphore          hits_sem        = new AESemaphore( "Subs:lookup" );
        private List<Subscription>      found_subscriptions   = new ArrayList<Subscription>();
       
        private boolean  complete;
       
        public void
        diversified()
        {
        }
       
        public void
        starts(
          byte[]         key )
        {
        }
       
        public void
        valueRead(
          DHTPluginContact  originator,
          DHTPluginValue    value )
        {
          if ( isCancelled2()){
           
            return;
          }
         
          byte[]  val = value.getValue();
         
          if ( val.length > 4 ){
           
            int  ver = ((val[0]<<16)&0xff0000) | ((val[1]<<8)&0xff00) | (val[2]&0xff);

            byte[]  sid = new byte[ val.length - 4 ];
           
            System.arraycopy( val, 4, sid, 0, sid.length );
                       
            HashWrapper hw = new HashWrapper( sid );
           
            boolean  new_sid = false;
           
            synchronized( this ){
             
              if ( complete ){
               
                return;
              }
             
              Integer v = (Integer)hits.get(hw);
             
              if ( v != null ){
               
                if ( ver > v.intValue()){
                 
                  hits.put( hw, new Integer( ver ));               
                }
              }else{
               
                new_sid = true;
                               
                hits.put( hw, new Integer( ver ));
              }
            }
           
            if ( new_sid ){
             
              log( "    Found subscription " + ByteFormatter.encodeString( sid ) + " version " + ver );

                // check if already subscribed
             
              SubscriptionImpl subs = getSubscriptionFromSID( sid );
             
              if ( subs != null ){
               
                found_subscriptions.add( subs );
               
                try{
                  listener.found( hash, subs );
                 
                }catch( Throwable e ){
                 
                  Debug.printStackTrace(e);
                }
                               
                hits_sem.release();
               
              }else{
               
                lookupSubscription(
                  hash,
                  sid,
                  ver,
                  new subsLookupListener()
                  {
                    private boolean sem_done = false;
                   
                    public void
                    found(
                      byte[]          hash,
                      Subscription      subscription )
                    {
                    }
                   
                    public void
                    complete(
                      byte[]          hash,
                      Subscription[]      subscriptions )
                    {
                      done( subscriptions );
                    }
                   
                    public void
                    failed(
                      byte[]          hash,
                      SubscriptionException  error )
                    {
                      done( new Subscription[0]);
                    }
                   
                    protected void
                    done(
                      Subscription[]      subs )
                    {
                      if ( isCancelled()){
                       
                        return;
                      }
                     
                      synchronized( this ){
                       
                        if ( sem_done ){
                         
                          return;
                        }
                       
                        sem_done = true;
                      }
                     
                      if ( subs.length > 0 ){
                       
                        found_subscriptions.add( subs[0] );
                       
                        try{
                          listener.found( hash, subs[0] );
                         
                        }catch( Throwable e ){
                         
                          Debug.printStackTrace(e);
                        }
                      }
                                           
                      hits_sem.release();
                    }
                   
                    public boolean
                    isCancelled()
                    {
                      return( isCancelled2());
                    }
                  });
              }
            }
          }
        }
       
        public void
        valueWritten(
          DHTPluginContact  target,
          DHTPluginValue    value )
        {
        }
       
        public void
        complete(
          byte[]        original_key,
          boolean        timeout_occurred )
        {
          new AEThread2( "Subs:lookup wait", true )
          {
            public void
            run()
            {
              int  num_hits;
             
              synchronized( this ){
               
                if ( complete ){
                 
                  return;
                }
               
                complete = true;
               
                num_hits = hits.size();
              }
             
             
              for (int i=0;i<num_hits;i++){
               
                if ( isCancelled2()){
                 
                  return;
                }

                hits_sem.reserve();
              }

              if ( isCancelled2()){
               
                return;
              }

              SubscriptionImpl[] s;
             
              synchronized( this ){
               
                s = (SubscriptionImpl[])found_subscriptions.toArray( new SubscriptionImpl[ found_subscriptions.size() ]);
              }
             
              log( "    Association lookup complete - " + s.length + " found" );

              try
                  // record zero assoc here for completeness
               
                recordAssociations( hash, s, true );
               
              }finally{
               
                listener.complete( hash, s );
              }
            }
          }.start();
        }
       
        protected boolean
        isCancelled2()
        {
          synchronized( cancelled ){
           
            return( cancelled[0] );
          }
        }
      });
   
    return(
      new SubscriptionAssociationLookup()
      {
        public void
        cancel()
        {
          log( "    Association lookup cancelled" );

          synchronized( cancelled ){
           
            cancelled[0] = true;
          }
        }
      });
  }
 
  interface
  subsLookupListener
    extends SubscriptionLookupListener
  {
    public boolean
    isCancelled();
  }
 
  protected void
  getPopularity(
    final SubscriptionImpl          subs,
    final SubscriptionPopularityListener  listener )
 
    throws SubscriptionException
  {
    try{
      long pop = PlatformSubscriptionsMessenger.getPopularityBySID( subs.getShortID());

      if ( pop >= 0 ){ 
       
        log( "Got popularity of " + subs.getName() + " from platform: " + pop );
       
        listener.gotPopularity( pop );

        return;
       
      }else{
       
          // unknown sid - if singleton try to register for popularity tracking purposes
       
        if ( subs.isSingleton()){
         
          try{
            checkSingletonPublish( subs );
           
          }catch( Throwable e ){           
          }
         
          listener.gotPopularity( subs.isSubscribed()?1:0 );
           
          return;
        }
      }
     
    }catch( Throwable e ){
     
      log( "Subscription lookup via platform failed", e );
    }
   
    getPopularityFromDHT( subs, listener, true );
  }
 
  protected void
  getPopularityFromDHT(
    final SubscriptionImpl          subs,
    final SubscriptionPopularityListener  listener,
    final boolean              sync )

  {
    if ( dht_plugin != null && !dht_plugin.isInitialising()){

      getPopularitySupport( subs, listener, sync );
     
    }else{
     
      new AEThread2( "SM:popwait", true )
      {
        public void
        run()
        {
          getPopularitySupport( subs, listener, sync );
        }
      }.start();
    }
  }
 
  protected void
  updatePopularityFromDHT(
    final SubscriptionImpl    subs,
    boolean            sync )
  {
    getPopularityFromDHT(
      subs,
      new SubscriptionPopularityListener()
      {
        public void
        gotPopularity(
          long            popularity )
        {
          subs.setCachedPopularity( popularity );
        }
       
        public void
        failed(
          SubscriptionException    error )
        {
          log( "Failed to update subscription popularity from DHT", error );
        }
      },
      sync );
  }
 
  protected void
  getPopularitySupport(
    final SubscriptionImpl          subs,
    final SubscriptionPopularityListener  listener,
    final boolean              sync )
  {
    log( "Getting popularity of " + subs.getName() + " from DHT" );

    byte[]  hash = subs.getPublicationHash();
   
    final AESemaphore sem = new AESemaphore( "SM:pop" );
   
    final long[] result = { -1 };
   
    final int timeout = 15*1000;
   
    dht_plugin.get(
        hash,
        "Popularity lookup for subscription " + subs.getName(),
        DHT.FLAG_STATS,
        5,
        timeout,
        false,
        true,
        new DHTPluginOperationListener()
        {
          private boolean  diversified;
         
          private int  hits = 0;
         
          public void
          diversified()
          {
            diversified = true;
          }
         
          public void
          starts(
            byte[]         key )
          {
          }
         
          public void
          valueRead(
            DHTPluginContact  originator,
            DHTPluginValue    value )
          {
            DHTPluginKeyStats stats = dht_plugin.decodeStats( value );
           
            result[0] = Math.max( result[0], stats.getEntryCount());
           
            hits++;
           
            if ( hits >= 3 ){
             
              done();
            }
          }
         
          public void
          valueWritten(
            DHTPluginContact  target,
            DHTPluginValue    value )
          {
           
          }
         
          public void
          complete(
            byte[]        key,
            boolean        timeout_occurred )
          {
            if ( diversified ){
             
                // TODO: fix?
             
              result[0] *= 11;
             
              if ( result[0] == 0 ){
               
                result[0] = 10;
              }
            }
           
            done();
          }
         
          protected void
          done()
          {           
            if ( sync ){
     
              sem.release();

            }else{
             
              if ( result[0] == -1 ){
               
                log( "Failed to get popularity of " + subs.getName() + " from DHT" );
         
                listener.failed( new SubscriptionException( "Timeout" ));
               
              }else{
             
                log( "Get popularity of " + subs.getName() + " from DHT: " + result[0] );
         
                listener.gotPopularity( result[0] );
              }
            }
          }
        });
   
    if ( sync ){
     
      sem.reserve( timeout );
     
      if ( result[0] == -1 ){
       
        log( "Failed to get popularity of " + subs.getName() + " from DHT" );
 
        listener.failed( new SubscriptionException( "Timeout" ));
       
      }else{
     
        log( "Get popularity of " + subs.getName() + " from DHT: " + result[0] );
 
        listener.gotPopularity( result[0] );
      }
    }
  }
 
  protected void
  lookupSubscription(
    final byte[]            association_hash,
    final byte[]            sid,
    final int              version,
    final subsLookupListener      listener )
  {
    try{
      SubscriptionImpl subs = getSubscriptionFromPlatform( sid, SubscriptionImpl.ADD_TYPE_LOOKUP );

      log( "Added temporary subscription: " + subs.getString());
     
      subs = addSubscription( subs );
     
      listener.complete( association_hash, new Subscription[]{ subs });
     
      return;
     
    }catch( Throwable e ){
     
      if ( listener.isCancelled()){
       
        return;
      }
     
      final String sid_str = ByteFormatter.encodeString( sid );
     
      log( "Subscription lookup via platform for " + sid_str + " failed", e );
     
      if ( getSubscriptionDownloadCount() > 8 ){
       
        log( "Too many existing subscription downloads" );
       
        listener.complete( association_hash, new Subscription[0]);

        return;
      }
     
        // fall back to DHT
     
      log( "Subscription lookup via DHT starts for " + sid_str );

      final String  key = "subscription:publish:" + ByteFormatter.encodeString( sid ) + ":" + version;
     
      dht_plugin.get(
        key.getBytes(),
        "Subscription lookup read: " + ByteFormatter.encodeString( sid ) + ":" + version,
        DHTPlugin.FLAG_SINGLE_VALUE,
        12,
        60*1000,
        false,
        true,
        new DHTPluginOperationListener()
        {
          private boolean listener_handled;
         
          public void
          diversified()
          {
          }
         
          public void
          starts(
            byte[]         key )
          {
          }
         
          public void
          valueRead(
            DHTPluginContact  originator,
            DHTPluginValue    value )
          {
            byte[]  data = value.getValue();
               
            try{
              final Map  details = decodeSubscriptionDetails( data );
             
              if ( SubscriptionImpl.getPublicationVersion( details ) == version ){
                               
                Map  singleton_details = (Map)details.get( "x" );
               
                if ( singleton_details == null ){
                 
                  synchronized( this ){
                   
                    if ( listener_handled  ){
                     
                      return;
                    }
                   
                    listener_handled = true;
                  }
                 
                  log( "    found " + sid_str + ", non-singleton" );

                  new AEThread2( "Subs:lookup download", true )
                  {
                    public void
                    run()
                    {
                      downloadSubscription(
                        association_hash,
                        SubscriptionImpl.getPublicationHash( details ),
                        sid,
                        version,
                        SubscriptionImpl.getPublicationSize( details ),
                        listener );
                    }
                  }.start();
                 
                }else{
                                     
                  synchronized( this ){
                     
                    if ( listener_handled  ){
                       
                      return;
                    }
                     
                    listener_handled = true;
                  }
                 
                  log( "    found " + sid_str + ", singleton" );

                  try{
                    SubscriptionImpl subs = createSingletonSubscription( singleton_details, SubscriptionImpl.ADD_TYPE_LOOKUP, false );
                                     
                    listener.complete( association_hash, new Subscription[]{ subs });
                                     
                  }catch( Throwable e ){
                                         
                    listener.failed( association_hash, new SubscriptionException( "Subscription creation failed", e ));
                  }
                }
              }else{
               
                log( "    found " + sid_str + " but version mismatch" );

              }
            }catch( Throwable e ){
             
              log( "    found " + sid_str + " but verification failed", e );

            }
          }
         
          public void
          valueWritten(
            DHTPluginContact  target,
            DHTPluginValue    value )
          {
          }
         
          public void
          complete(
            byte[]        original_key,
            boolean        timeout_occurred )
          {
            if ( listener.isCancelled()){
             
              return;
            }
           
            log( "    " + sid_str + " complete" );
           
            synchronized( this ){
             
              if ( !listener_handled ){
           
                listener_handled = true;
               
                listener.complete( association_hash, new Subscription[0] );
              }
            }
          }
        });
    }
  }
 
  protected SubscriptionImpl
  getSubscriptionFromPlatform(
    byte[]    sid,
    int      add_type )
 
    throws SubscriptionException
  {
    try{
      PlatformSubscriptionsMessenger.subscriptionDetails details = PlatformSubscriptionsMessenger.getSubscriptionBySID( sid );
     
      SubscriptionImpl res = getSubscriptionFromVuzeFileContent( sid, add_type, details.getContent());
     
      int  pop = details.getPopularity();
     
      if ( pop >= 0 ){
       
        res.setCachedPopularity( pop );
      }
     
      return( res );
     
    }catch( SubscriptionException e ){
     
      throw( e );
     
    }catch( Throwable e ){
     
      throw( new SubscriptionException( "Failed to read subscription from platform", e ));
    }
  }
 
  protected SubscriptionImpl
  getSubscriptionFromVuzeFile(
    byte[]    sid,
    int      add_type,
    File    file )
 
    throws SubscriptionException
  {
    VuzeFileHandler vfh = VuzeFileHandler.getSingleton();
   
    String  file_str = file.getAbsolutePath();
   
    VuzeFile vf = vfh.loadVuzeFile( file_str );

    if ( vf == null ){
     
      log( "Failed to load vuze file from " + file_str );
     
      throw( new SubscriptionException( "Failed to load vuze file from " + file_str ));
    }
   
    return( getSubscriptionFromVuzeFile( sid, add_type, vf ));
  }
 
  protected SubscriptionImpl
  getSubscriptionFromVuzeFileContent(
    byte[]    sid,
    int      add_type,
    String    content )
 
    throws SubscriptionException
  {
    VuzeFileHandler vfh = VuzeFileHandler.getSingleton();
   
    VuzeFile vf = vfh.loadVuzeFile( Base64.decode( content ));

    if ( vf == null ){
     
      log( "Failed to load vuze file from " + content );
     
      throw( new SubscriptionException( "Failed to load vuze file from content" ));
    }
 
    return( getSubscriptionFromVuzeFile( sid, add_type, vf ));
  }
 
  protected SubscriptionImpl
  getSubscriptionFromVuzeFile(
    byte[]    sid,
    int      add_type,
    VuzeFile  vf )
 
    throws SubscriptionException
  {
    VuzeFileComponent[] comps = vf.getComponents();
   
    for (int j=0;j<comps.length;j++){
     
      VuzeFileComponent comp = comps[j];
     
      if ( comp.getType() == VuzeFileComponent.COMP_TYPE_SUBSCRIPTION ){
       
        Map map = comp.getContent();
       
        try{
          SubscriptionBodyImpl body = new SubscriptionBodyImpl( SubscriptionManagerImpl.this, map );
                       
          SubscriptionImpl new_subs = new SubscriptionImpl( SubscriptionManagerImpl.this, body, add_type, false );
             
          if ( Arrays.equals( new_subs.getShortID(), sid )){
                         
            return( new_subs );
          }
        }catch( Throwable e ){
         
          log( "Subscription decode failed", e );
        }
      }
    }
   
    throw( new SubscriptionException( "Subscription not found" ));
  }
 
  protected void
  downloadSubscription(
    final byte[]            association_hash,
    byte[]                torrent_hash,
    final byte[]            sid,
    int                  version,
    int                  size,
    final subsLookupListener       listener )
  {
    try{
      Object[] res = downloadTorrent( torrent_hash, size );
     
      if ( listener.isCancelled()){
       
        return;
      }
     
      if ( res == null ){
       
        listener.complete( association_hash, new Subscription[0] );
       
        return;
      }

      downloadSubscription(
        (TOTorrent)res[0],
        (InetSocketAddress)res[1],
        sid,
        version,
        "Subscription " + ByteFormatter.encodeString( sid ) + " for " + ByteFormatter.encodeString( association_hash ),
        new downloadListener()
        {
          public void
          complete(
            File    data_file )
          {
            boolean  reported = false;
           
            try{
              if ( listener.isCancelled()){
               
                return;
              }
             
              SubscriptionImpl subs = getSubscriptionFromVuzeFile( sid, SubscriptionImpl.ADD_TYPE_LOOKUP, data_file );
           
              log( "Added temporary subscription: " + subs.getString());
             
              subs = addSubscription( subs );
             
              listener.complete( association_hash, new Subscription[]{ subs });
             
              reported = true;
 
            }catch( Throwable e ){
             
              log( "Subscription decode failed", e );
             
            }finally{
                           
              if ( !reported ){
               
                listener.complete( association_hash, new Subscription[0] );
              }
            }
          }
         
          public void
          complete(
            Download  download, 
            File    torrent_file )
          {
            File  data_file = new File( download.getSavePath());
           
            try{
              removeDownload( download, false );

              complete( data_file );
             
            }catch( Throwable e ){
             
              log( "Failed to remove download", e );
             
              listener.complete( association_hash, new Subscription[0] );
             
            }finally{
             
              torrent_file.delete();
             
              data_file.delete();
            }
          }
           
          public void
          failed(
            Throwable  error )
          {
            listener.complete( association_hash, new Subscription[0] );
          }
         
          public Map
          getRecoveryData()
          {
            return( null );
          }
         
          public boolean
          isCancelled()
          {
            return( listener.isCancelled());
          }
        });
       
    }catch( Throwable e ){
     
      log( "Subscription download failed",e );
     
      listener.complete( association_hash, new Subscription[0] );
    }
  }
 
  protected int
  getSubscriptionDownloadCount()
  {
    PluginInterface pi = PluginInitializer.getDefaultInterface();
   
    Download[] downloads = pi.getDownloadManager().getDownloads();
   
    int  res = 0;
   
    for( int i=0;i<downloads.length;i++){
     
      Download  download = downloads[i];
     
      if ( download.getBooleanAttribute( ta_subs_download )){
       
        res++;
      }
    }
   
    return( res );
  }
 
  protected void
  associationAdded(
    SubscriptionImpl      subscription,
    byte[]            association_hash )
  {
    recordAssociations( association_hash, new SubscriptionImpl[]{ subscription }, false );

    if ( dht_plugin != null ){
     
      publishAssociations();
    }
  }
 
  protected void
  addPotentialAssociation(
    SubscriptionImpl      subs,
    String            result_id,
    String            key )
  {
    if ( key == null ){
     
      Debug.out( "Attempt to add null key!" );
     
      return;
    }
   
    log( "Added potential association: " + subs.getName() + "/" + result_id + " -> " + key );
   
    synchronized( potential_associations ){
     
      potential_associations.add( new Object[]{ subs, result_id, key, new Long( System.currentTimeMillis())} );
     
      if ( potential_associations.size() > 512 ){
       
        potential_associations.remove(0);
      }
    }
  }
 
  protected void
  checkPotentialAssociations(
    byte[]        hash,
    String        key )
  {
    log( "Checking potential association: " + key + " -> " + ByteFormatter.encodeString( hash ));
   
    SubscriptionImpl   subs     = null;
    String        result_id  = null;
   
    synchronized( potential_associations ){

      Iterator<Object[]> it = potential_associations.iterator();
     
      while( it.hasNext()){
       
        Object[]  entry = it.next();
       
        String  this_key = (String)entry[2];
       
          // startswith as actual URL may have had additional parameters added such as azid
       
        if ( key.startsWith( this_key )){
         
          subs    = (SubscriptionImpl)entry[0];
          result_id  = (String)entry[1];
         
          log( "    key matched to subscription " + subs.getName() + "/" + result_id);

          it.remove();
         
          break;
        }
      }
    }
   
    if ( subs == null ){
     
      log( "    no potential associations found" );
     
    }else{
     
      SubscriptionResult  result = subs.getHistory().getResult( result_id );
     
      if ( result != null ){
       
        log( "    result found, marking as read" );

        result.setRead( true );
       
      }else{
       
        log( "    result not found" );
      }
     
      log( "    adding association" );
     
      subs.addAssociation( hash );
    }
  }
 
  protected void
  tidyPotentialAssociations()
  {
    long  now = SystemTime.getCurrentTime();
   
    synchronized( potential_associations ){
     
      Iterator it = potential_associations.iterator();
     
      while( it.hasNext() && potential_associations.size() > 16 ){
       
        Object[]  entry = (Object[])it.next();
       
        long  created = ((Long)entry[3]).longValue();
       
        if ( created > now ){
         
          entry[3] = new Long( now );
         
        }else if ( now - created > 60*60*1000 ){
         
          SubscriptionImpl   subs = (SubscriptionImpl)entry[0];

          String  result_id  = (String)entry[1];
          String  key      = (String)entry[2];

          log( "Removing expired potential association: " + subs.getName() + "/" + result_id + " -> " + key );
         
          it.remove();
        }
      }
    }
   
    synchronized( potential_associations2 ){
     
      Iterator it = potential_associations2.entrySet().iterator();
     
      while( it.hasNext() && potential_associations2.size() > 16 ){
       
        Map.Entry  map_entry = (Map.Entry)it.next();
       
        byte[]    hash = ((HashWrapper)map_entry.getKey()).getBytes();
       
        Object[]  entry = (Object[])map_entry.getValue();
       
        long  created = ((Long)entry[2]).longValue();
       
        if ( created > now ){
         
          entry[2] = new Long( now );
         
        }else if ( now - created > 60*60*1000 ){
         
          SubscriptionImpl[]   subs = (SubscriptionImpl[])entry[0];

          String  subs_str = "";
         
          for (int i=0;i<subs.length;i++){
            subs_str += (i==0?"":",") + subs[i].getName();
          }
         
          log( "Removing expired potential association: " + ByteFormatter.encodeString(hash) + " -> " + subs_str );
         
          it.remove();
        }
      }
    }
  }
 
  protected void
  recordAssociations(
    byte[]            association_hash,
    SubscriptionImpl[]      subscriptions,
    boolean            full_lookup )
  {
    HashWrapper  hw = new HashWrapper( association_hash );
   
    synchronized( potential_associations2 ){
     
      potential_associations2.put( hw, new Object[]{ subscriptions, new Boolean( full_lookup ), new Long( SystemTime.getCurrentTime())});
    }
     
    if ( recordAssociationsSupport( association_hash, subscriptions, full_lookup )){
     
      synchronized( potential_associations2 ){

        potential_associations2.remove( hw );
      }
    }else{
     
      log( "Deferring association for " + ByteFormatter.encodeString( association_hash ));
    }
  }
 
  protected boolean
  recordAssociationsSupport(
    byte[]            association_hash,
    SubscriptionImpl[]      subscriptions,
    boolean            full_lookup )
  {
    PluginInterface pi = PluginInitializer.getDefaultInterface();

    boolean  download_found  = false;
    boolean  changed     = false;
   
    try{
      Download download = pi.getDownloadManager().getDownload( association_hash );
     
      if ( download != null ){
         
        if ( subscriptions.length > 0 ){
         
          String  category = subscriptions[0].getCategory();
         
          if ( category != null ){
           
            String existing = download.getAttribute( ta_category );
               
            if ( existing == null ){
                 
              download.setAttribute( ta_category, category );
            }
          }
        }
       
        download_found = true;
       
        Map  map = download.getMapAttribute( ta_subscription_info );
       
        if ( map == null ){
         
          map = new LightHashMap();
         
        }else{
         
          map = new LightHashMap( map );
        }
       
        List  s = (List)map.get( "s" );
       
        for (int i=0;i<subscriptions.length;i++){
       
          byte[]  sid = subscriptions[i].getShortID();
         
          if ( s == null ){
           
            s = new ArrayList();
           
            s.add( sid );
           
            changed  = true;
           
            map.put( "s", s );
           
          }else{
           
            boolean found = false;
           
            for (int j=0;j<s.size();j++){
             
              byte[]  existing = (byte[])s.get(j);
             
              if ( Arrays.equals( sid, existing )){
               
                found = true;
               
                break;
              }
            }
           
            if ( !found ){
           
              s.add( sid );
               
              changed  = true;
            }
          }
        }
       
        if ( full_lookup ){
       
          map.put( "lc", new Long( SystemTime.getCurrentTime()));
         
          changed  = true;
        }
       
        if ( changed ){
       
          download.setMapAttribute( ta_subscription_info, map );
        }
      }
    }catch( Throwable e ){
     
      log( "Failed to record associations", e );
    }
   
    if ( changed ){
     
      Iterator it = listeners.iterator();
     
      while( it.hasNext()){
       
        try{
          ((SubscriptionManagerListener)it.next()).associationsChanged( association_hash );
         
        }catch( Throwable e ){
         
          Debug.printStackTrace(e);
        }
      }
    }
   
    return( download_found );
  }
 
  protected boolean
  publishAssociations()
  {
    SubscriptionImpl         subs_to_publish    = null;
    SubscriptionImpl.association  assoc_to_publish   = null;

    synchronized( this ){
     
      if ( publish_associations_active >= PUB_ASSOC_CONC_MAX ){
       
        return( false );
      }     
     
      publish_associations_active++;
     
      List shuffled_subs = new ArrayList( subscriptions );

      Collections.shuffle( shuffled_subs );
             
      for (int i=0;i<shuffled_subs.size();i++){
         
        SubscriptionImpl sub = (SubscriptionImpl)shuffled_subs.get( i );
         
        if ( sub.isSubscribed() && sub.isPublic()){
         
          assoc_to_publish   = sub.getAssociationForPublish();
           
          if ( assoc_to_publish != null ){
             
            subs_to_publish    = sub;

            break;
          }
        }
      }
    }
   
    if ( assoc_to_publish != null ){
   
      publishAssociation( subs_to_publish, assoc_to_publish );
     
      return( false );
    }else{
         
      log( "Publishing Associations Complete" );
         
      synchronized( this ){

        publish_associations_active--;
      }
     
      return( true );
    }
  }
 
  protected void
  publishAssociation(
    final SubscriptionImpl          subs,
    final SubscriptionImpl.association    assoc )
  {
    log( "Checking association '" + subs.getString() + "' -> '" + assoc.getString() + "'" );
   
    byte[]  sub_id     = subs.getShortID();
    int    sub_version  = subs.getVersion();
   
    byte[]  assoc_hash  = assoc.getHash();
   
    final String  key = "subscription:assoc:" + ByteFormatter.encodeString( assoc_hash );
       
    final byte[]  put_value = new byte[sub_id.length + 4];
   
    System.arraycopy( sub_id, 0, put_value, 4, sub_id.length );
   
    put_value[0= (byte)(sub_version>>16);
    put_value[1= (byte)(sub_version>>8);
    put_value[2= (byte)sub_version;
    put_value[3= (byte)subs.getFixedRandom();
   
    dht_plugin.get(
      key.getBytes(),
      "Subscription association read: " + ByteFormatter.encodeString( assoc_hash ),
      DHTPlugin.FLAG_SINGLE_VALUE,
      30,
      60*1000,
      false,
      false,
      new DHTPluginOperationListener()
      {
        private int      hits;
        private boolean    diversified;
        private int      max_ver;
       
        public void
        diversified()
        {
          diversified = true;
        }
       
        public void
        starts(
          byte[]         key )
        {
        }
       
        public void
        valueRead(
          DHTPluginContact  originator,
          DHTPluginValue    value )
        {
          byte[]  val = value.getValue();
         
          if ( val.length == put_value.length ){
           
            boolean  diff = false;
           
            for (int i=4;i<val.length;i++){
             
              if ( val[i] != put_value[i] ){
               
                diff = true;
               
                break;
              }
            }
           
            if ( !diff ){
             
              hits++;
             
              int  ver = ((val[0]<<16)&0xff0000) | ((val[1]<<8)&0xff00) | (val[2]&0xff);
             
              if ( ver > max_ver ){
               
                max_ver = ver;
              }
            }
          }
        }
       
        public void
        valueWritten(
          DHTPluginContact  target,
          DHTPluginValue    value )
        {
        }
       
        public void
        complete(
          byte[]        original_key,
          boolean        timeout_occurred )
        {
          log( "Checked association '" + subs.getString() + "' -> '" + assoc.getString() + "' - max_ver=" + max_ver + ",hits=" + hits + ",div=" + diversified );

          if ( max_ver > subs.getVersion()){
           
            if ( !subs.isMine()){
           
              updateSubscription( subs, max_ver );
            }
          }
         
          if ( hits < 10 && !diversified ){     
     
            log( "    Publishing association '" + subs.getString() + "' -> '" + assoc.getString() + "', existing=" + hits );

            byte flags = DHTPlugin.FLAG_ANON;
           
            if ( hits < 3 && !diversified ){
             
              flags |= DHTPlugin.FLAG_PRECIOUS;
            }
           
            dht_plugin.put(
              key.getBytes(),
              "Subscription association write: " + ByteFormatter.encodeString( assoc.getHash()) + " -> " + ByteFormatter.encodeString( subs.getShortID() ) + ":" + subs.getVersion(),
              put_value,
              flags,
              new DHTPluginOperationListener()
              {
                public void
                diversified()
                {
                }
               
                public void
                starts(
                  byte[]         key )
                {
                }
               
                public void
                valueRead(
                  DHTPluginContact  originator,
                  DHTPluginValue    value )
                {
                }
               
                public void
                valueWritten(
                  DHTPluginContact  target,
                  DHTPluginValue    value )
                {
                }
               
                public void
                complete(
                  byte[]        key,
                  boolean        timeout_occurred )
                {
                  log( "        completed '" + subs.getString() + "' -> '" + assoc.getString() + "'" );
       
                  publishNext();
                }
              });
          }else{
           
            log( "    Not publishing association '" + subs.getString() + "' -> '" + assoc.getString() + "', existing =" + hits );

            publishNext();
          }
        }
       
        protected void
        publishNext()
        {
          synchronized( this ){
           
            publish_associations_active--;
          }
         
          publishAssociations();
        }
      });
  }
 
  protected void
  subscriptionUpdated()
  {
    if ( dht_plugin != null ){
     
      publishSubscriptions();
    }
  }
 
  protected void
  publishSubscriptions()
  {
    List   shuffled_subs;

    synchronized( this ){
     
      if ( publish_subscription_active ){
       
        return;
      }     
     
      shuffled_subs = new ArrayList( subscriptions );

      publish_subscription_active = true;
    }
 
    boolean  publish_initiated = false;
   
    try{
      Collections.shuffle( shuffled_subs );
           
      for (int i=0;i<shuffled_subs.size();i++){
       
        SubscriptionImpl sub = (SubscriptionImpl)shuffled_subs.get( i );
       
        if ( sub.isSubscribed() && sub.isPublic() && !sub.getPublished()){
                 
          sub.setPublished( true );
         
          publishSubscription( sub );
         
          publish_initiated = true;
         
          break;
        }
      }
    }finally{
     
      if ( !publish_initiated ){
       
        log( "Publishing Subscriptions Complete" );
       
        synchronized( this ){
 
          publish_subscription_active = false;
        }
      }
    }
  }
 
  protected void
  publishSubscription(
    final SubscriptionImpl          subs )
  {
    log( "Checking subscription publication '" + subs.getString() + "'" );
   
    byte[]  sub_id     = subs.getShortID();
    int    sub_version  = subs.getVersion();
       
    final String  key = "subscription:publish:" + ByteFormatter.encodeString( sub_id ) + ":" + sub_version;
           
    dht_plugin.get(
      key.getBytes(),
      "Subscription presence read: " + ByteFormatter.encodeString( sub_id ) + ":" + sub_version,
      DHTPlugin.FLAG_SINGLE_VALUE,
      24,
      60*1000,
      false,
      false,
      new DHTPluginOperationListener()
      {
        private int    hits;
        private boolean  diversified;
       
        public void
        diversified()
        {         
          diversified = true;
        }
       
        public void
        starts(
          byte[]         key )
        {
        }
       
        public void
        valueRead(
          DHTPluginContact  originator,
          DHTPluginValue    value )
        {         
          byte[]  data = value.getValue();
           
          try{
            Map  details = decodeSubscriptionDetails( data );

            if ( subs.getVerifiedPublicationVersion( details ) == subs.getVersion()){
           
              hits++;
            }
          }catch( Throwable e ){
           
          }
        }
       
        public void
        valueWritten(
          DHTPluginContact  target,
          DHTPluginValue    value )
        {
        }
       
        public void
        complete(
          byte[]        original_key,
          boolean        timeout_occurred )
        {
          log( "Checked subscription publication '" + subs.getString() + "' - hits=" + hits + ",div=" + diversified );

          if ( hits < 10 && !diversified ){     
     
            log( "    Publishing subscription '" + subs.getString() + ", existing=" + hits );

            try{
              byte[]  put_value = encodeSubscriptionDetails( subs );           
             
              if ( put_value.length < DHTPlugin.MAX_VALUE_SIZE ){
               
                byte  flags = DHTPlugin.FLAG_SINGLE_VALUE;
               
                if ( hits < 3 && !diversified ){
                 
                  flags |= DHTPlugin.FLAG_PRECIOUS;
                }
               
                dht_plugin.put(
                  key.getBytes(),
                  "Subscription presence write: " + ByteFormatter.encodeString( subs.getShortID() ) + ":" + subs.getVersion(),
                  put_value,
                  flags,
                  new DHTPluginOperationListener()
                  {
                    public void
                    diversified()
                    {
                    }
                   
                    public void
                    starts(
                      byte[]         key )
                    {
                    }
                   
                    public void
                    valueRead(
                      DHTPluginContact  originator,
                      DHTPluginValue    value )
                    {
                    }
                   
                    public void
                    valueWritten(
                      DHTPluginContact  target,
                      DHTPluginValue    value )
                    {
                    }
                   
                    public void
                    complete(
                      byte[]        key,
                      boolean        timeout_occurred )
                    {
                      log( "        completed '" + subs.getString() + "'" );
           
                      publishNext();
                    }
                  });
               
              }else{
               
                publishNext();
              }
            }catch( Throwable e ){
             
              Debug.printStackTrace( e );
             
              publishNext();
            }
           
          }else{
           
            log( "    Not publishing subscription '" + subs.getString() + "', existing =" + hits );

            publishNext();
          }
        }
       
        protected void
        publishNext()
        {
          synchronized( this ){
           
            publish_subscription_active = false;
          }
         
          publishSubscriptions();
        }
      });
  }
 
  protected void
  updateSubscription(
    final SubscriptionImpl    subs,
    final int          new_version )
  {
    log( "Subscription " + subs.getString() + " - higher version found: " + new_version );
   
    if ( !subs.canAutoUpgradeCheck()){
     
      log( "    Checked too recently or not updateable, ignoring" );
     
      return;
    }
   
    if ( subs.getHighestUserPromptedVersion() >= new_version ){
     
      log( "    User has already been prompted for version " + new_version + " so ignoring" );
     
      return;
    }
         
    byte[]  sub_id     = subs.getShortID();
     
    try{
      PlatformSubscriptionsMessenger.subscriptionDetails details = PlatformSubscriptionsMessenger.getSubscriptionBySID( sub_id );
     
      if ( !askIfCanUpgrade( subs, new_version )){
       
        return;
      }
     
      VuzeFileHandler vfh = VuzeFileHandler.getSingleton();
     
      VuzeFile vf = vfh.loadVuzeFile( Base64.decode( details.getContent()));
             
      vfh.handleFiles( new VuzeFile[]{ vf }, VuzeFileComponent.COMP_TYPE_SUBSCRIPTION );
     
      return;
     
    }catch( Throwable e ){
     
      log( "Failed to read subscription from platform, trying DHT" );
    }
   
    log( "Checking subscription '" + subs.getString() + "' upgrade to version " + new_version );

    final String  key = "subscription:publish:" + ByteFormatter.encodeString( sub_id ) + ":" + new_version;
           
    dht_plugin.get(
      key.getBytes(),
      "Subscription update read: " + ByteFormatter.encodeString( sub_id ) + ":" + new_version,
      DHTPlugin.FLAG_SINGLE_VALUE,
      12,
      60*1000,
      false,
      false,
      new DHTPluginOperationListener()
      {
        private byte[]  verified_hash;
        private int    verified_size;
       
        public void
        diversified()
        {
        }
       
        public void
        starts(
          byte[]         key )
        {
        }
       
        public void
        valueRead(
          DHTPluginContact  originator,
          DHTPluginValue    value )
        {
          byte[]  data = value.getValue();
             
          try{
            Map  details = decodeSubscriptionDetails( data );
           
            if (   verified_hash == null &&
                subs.getVerifiedPublicationVersion( details ) == new_version ){
             
              verified_hash   = SubscriptionImpl.getPublicationHash( details );
              verified_size  = SubscriptionImpl.getPublicationSize( details );
            }
           
          }catch( Throwable e ){
           
          }
        }
       
        public void
        valueWritten(
          DHTPluginContact  target,
          DHTPluginValue    value )
        {
        }
       
        public void
        complete(
          byte[]        original_key,
          boolean        timeout_occurred )
        {
          if ( verified_hash != null ){     
     
            log( "    Subscription '" + subs.getString() + " upgrade verified as authentic" );

            updateSubscription( subs, new_version, verified_hash, verified_size );
           
          }else{
           
            log( "    Subscription '" + subs.getString() + " upgrade not verified" );
          }
        }
      });
  }
 
  protected byte[]
  encodeSubscriptionDetails(
    SubscriptionImpl    subs )
 
    throws IOException
  {
    Map    details = subs.getPublicationDetails();         
   
      // inject a random element so we can count occurrences properly (as the DHT logic
      // removes duplicates)
   
    details.put( "!", new Long( random_seed ));
   
    byte[] encoded = BEncoder.encode( details );
       
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
       
    GZIPOutputStream os = new GZIPOutputStream( baos );

    os.write( encoded );
   
    os.close();
   
    byte[] compressed = baos.toByteArray();
   
    byte  header;
    byte[]  data;
   
    if ( compressed.length < encoded.length ){
     
      header   = 1;
      data  = compressed;
    }else{
     
      header  = 0;
      data  = encoded;
    }
       
    byte[] result = new byte[data.length+1];
   
    result[0] = header;
   
    System.arraycopy( data, 0, result, 1, data.length );
   
    return( result );
  }
 
  protected Map
  decodeSubscriptionDetails(
    byte[]      data )
 
    throws IOException
  {
    byte[]  to_decode;
   
    if ( data[0] == 0 ){
     
      to_decode = new byte[ data.length-1 ];
     
      System.arraycopy( data, 1, to_decode, 0, data.length - 1 );
     
    }else{
     
      GZIPInputStream is = new GZIPInputStream(new ByteArrayInputStream( data, 1, data.length - 1 ));
     
      to_decode = FileUtil.readInputStreamAsByteArray( is );
     
      is.close();
    }
   
    Map res = BDecoder.decode( to_decode );
   
      // remove any injected random seed
   
    res.remove( "!" );
   
    return( res );
  }
 
  protected void
  updateSubscription(
    final SubscriptionImpl      subs,
    final int            update_version,
    final byte[]          update_hash,
    final int            update_size )
  {
    log( "Subscription " + subs.getString() + " - update hash=" + ByteFormatter.encodeString( update_hash ) + ", size=" + update_size );

    new AEThread2( "SubsUpdate", true )
    {
      public void
      run()
      {
        try{
          Object[] res = downloadTorrent( update_hash, update_size );
         
          if ( res != null ){
         
            updateSubscription( subs, update_version, (TOTorrent)res[0], (InetSocketAddress)res[1] );
          }
        }catch( Throwable e ){
         
          log( "    update failed", e );
        }
      }
    }.start();
  }
 
  protected Object[]
  downloadTorrent(
    byte[]    hash,
    int      update_size )
  {
    if ( !isSubsDownloadEnabled()){
     
      log( "    Can't download subscription " + Base32.encode( hash ) + " as feature disabled" );
     
      return( null );
    }
   
    final MagnetPlugin  magnet_plugin = getMagnetPlugin();
 
    if ( magnet_plugin == null ){
   
      log( "    Can't download, no magnet plugin" );
   
      return( null );
    }

    try{
      final InetSocketAddress[] sender = { null };
     
      byte[] torrent_data = magnet_plugin.download(
        new MagnetPluginProgressListener()
        {
          public void
          reportSize(
            long  size )
          {
          }
         
          public void
          reportActivity(
            String  str )
          {
            log( "    MagnetDownload: " + str );
          }
         
          public void
          reportCompleteness(
            int    percent )
          {
          }
         
          public void
          reportContributor(
            InetSocketAddress  address )
          {
            synchronized( sender ){
           
              sender[0] = address;
            }
          }

          public boolean
          verbose()
          {
            return( false );
          }
        },
        hash,
        "",
        new InetSocketAddress[0],
        300*1000 );
     
      if ( torrent_data == null ){
       
        log( "    download failed - timeout" );
       
        return( null );
      }
     
      log( "Subscription torrent downloaded" );
     
      TOTorrent torrent = TOTorrentFactory.deserialiseFromBEncodedByteArray( torrent_data );
   
        // update size is just that of signed content, torrent itself is .vuze file
        // so take this into account
     
      if ( torrent.getSize() > update_size + 10*1024 ){
     
        log( "Subscription download abandoned, torrent size is " + torrent.getSize() + ", underlying data size is " + update_size );
       
        return( null );
      }
     
      if ( torrent.getSize() > 4*1024*1024 ){
       
        log( "Subscription download abandoned, torrent size is too large (" + torrent.getSize() + ")" );
       
        return( null );
      }
     
      synchronized( sender ){
     
        return( new Object[]{ torrent, sender[0] });
      }
     
    }catch( Throwable e ){
     
      log( "    download failed", e );
     
      return( null );
    }
  }
 
  protected void
  downloadSubscription(
    final TOTorrent      torrent,
    final InetSocketAddress  peer,
    byte[]          subs_id,
    int            version,
    String          name,
    final downloadListener  listener )
  {
    try{
        // testing purposes, see if local exists
     
      LightWeightSeed lws = LightWeightSeedManager.getSingleton().get( new HashWrapper( torrent.getHash()));
 
      if ( lws != null ){
       
        log( "Light weight seed found" );
       
        listener.complete( lws.getDataLocation());
       
      }else{
        String  sid = ByteFormatter.encodeString( subs_id );
       
        File  dir = getSubsDir();
       
        dir = new File( dir, "temp" );
       
        if ( !dir.exists()){
         
          if ( !dir.mkdirs()){
           
            throw( new IOException( "Failed to create dir '" + dir + "'" ));
          }
        }
       
        final File  torrent_file   = new File( dir, sid + "_" + version + ".torrent" );
        final File  data_file     = new File( dir, sid + "_" + version + ".vuze" );
 
        PluginInterface pi = PluginInitializer.getDefaultInterface();
     
        final DownloadManager dm = pi.getDownloadManager();
       
        Download download = dm.getDownload( torrent.getHash());
       
        if ( download == null ){
         
          log( "Adding download for subscription '" + new String(torrent.getName()) + "'" );
         
          boolean is_update = getSubscriptionFromSID( subs_id ) != null;
         
          PlatformTorrentUtils.setContentTitle(torrent, (is_update?"Update":"Download") + " for subscription '" + name + "'" );
         
            // PlatformTorrentUtils.setContentThumbnail(torrent, thumbnail);
           
          TorrentUtils.setFlag( torrent, TorrentUtils.TORRENT_FLAG_LOW_NOISE, true );
         
          Torrent t = new TorrentImpl( torrent );
         
          t.setDefaultEncoding();
         
          t.writeToFile( torrent_file );
         
          download = dm.addDownload( t, torrent_file, data_file );
         
          download.setFlag( Download.FLAG_DISABLE_AUTO_FILE_MOVE, true );

          download.setBooleanAttribute( ta_subs_download, true );
         
          Map rd = listener.getRecoveryData();
         
          if ( rd != null ){
           
            download.setMapAttribute( ta_subs_download_rd, rd );
          }
        }else{
         
          log( "Existing download found for subscription '" + new String(torrent.getName()) + "'" );
        }
       
        final Download f_download = download;
       
        final TimerEventPeriodic[] event = { null };
       
        event[0] =
          SimpleTimer.addPeriodicEvent(
            "SM:cancelTimer",
            10*1000,
            new TimerEventPerformer()
            {
              private long  start_time = SystemTime.getMonotonousTime();
             
              public void
              perform(
                TimerEvent ev )
              {
                boolean  kill = false;
               
                try
                  Download download = dm.getDownload( torrent.getHash());
                 
                  if ( listener.isCancelled() || download == null ){
                   
                    kill = true;
                   
                  }else{
                   
                    int  state = download.getState();
                   
                    if ( state == Download.ST_ERROR ){
                     
                      log( "Download entered error state, removing" );
                     
                      kill = true;
                     
                    }else{
                     
                      long  now = SystemTime.getMonotonousTime();
                     
                      long  running_for = now - start_time;
                     
                      if ( running_for > 2*60*1000 ){
                       
                        DownloadScrapeResult scrape = download.getLastScrapeResult();
                       
                        if ( scrape == null || scrape.getSeedCount() <= 0 ){
                         
                          log( "Download has no seeds, removing" );
                         
                          kill = true;
                        }
                      }else if ( running_for > 4*60*1000 ){
                       
                        if ( download.getStats().getDownloaded() == 0 ){
                         
                          log( "Download has zero downloaded, removing" );
                         
                          kill = true;
                        }
                      }else if ( running_for > 10*60*1000 ){
                       
                        log( "Download hasn't completed in permitted time, removing" );
                       
                        kill = true;
                      }
                    }
                  }
                }catch( Throwable e ){
                 
                  log( "Download failed", e );

                  kill = true;
                }
               
                if ( kill && event[0] != null ){
                 
                  try{
                    event[0].cancel();
                   
                    if ( !listener.isCancelled()){
                                         
                      listener.failed( new SubscriptionException( "Download abandoned" ));
                    }
                  }finally{
                   
                    removeDownload( f_download, true );
                 
                    torrent_file.delete();
                  }
                }
              }
            });
       
        download.addCompletionListener(
          new DownloadCompletionListener()
          {
            public void
            onCompletion(
              Download d )
            {
              listener.complete( d, torrent_file );
            }
          });
       
        if ( download.isComplete()){
         
          listener.complete( download, torrent_file  );
         
        }else{
               
          download.setForceStart( true );
         
          if ( peer != null ){
         
            download.addPeerListener(
              new DownloadPeerListener()
              {
                public void
                peerManagerAdded(
                  Download    download,
                  PeerManager    peer_manager )
                {                 
                  InetSocketAddress tcp = AddressUtils.adjustTCPAddress( peer, true );
                  InetSocketAddress udp = AddressUtils.adjustUDPAddress( peer, true );
                 
                  log( "    Injecting peer into download: " + tcp );

                  peer_manager.addPeer( tcp.getAddress().getHostAddress(), tcp.getPort(), udp.getPort(), true );
                }
               
                public void
                peerManagerRemoved(
                  Download    download,
                  PeerManager    peer_manager )
                {             
                }
              });
          }
        }
      }
    }catch( Throwable e ){
     
      log( "Failed to add download", e );
     
      listener.failed( e );
    }
  }
 
  protected interface
  downloadListener
  {
    public void
    complete(
      File    data_file );
   
    public void
    complete(
      Download  download, 
      File    torrent_file );
     
    public void
    failed(
      Throwable  error );
   
    public Map
    getRecoveryData();
   
    public boolean
    isCancelled();
  }
 
  protected void
  updateSubscription(
    final SubscriptionImpl    subs,
    final int          new_version,
    TOTorrent          torrent,
    InetSocketAddress      peer )
  {
    log( "Subscription " + subs.getString() + " - update torrent: " + new String( torrent.getName()));

    if ( !askIfCanUpgrade( subs, new_version )){
     
      return;
    }
     
    downloadSubscription(
      torrent,
      peer,
      subs.getShortID(),
      new_version,
      subs.getName(),
      new downloadListener()
      {
        public void
        complete(
          File    data_file )
        {
          updateSubscription( subs, data_file );
        }
       
        public void
        complete(
          Download  download, 
          File    torrent_file )
        {
          updateSubscription( subs, download, torrent_file, new File( download.getSavePath()));
        }
         
        public void
        failed(
          Throwable  error )
        {
          log( "Failed to download subscription", error );
        }
       
        public Map
        getRecoveryData()
        {
          Map  rd = new HashMap();
         
          rd.put( "sid", subs.getShortID());
          rd.put( "ver", new Long( new_version ));
         
          return( rd );
        }
       
        public boolean
        isCancelled()
        {
          return( false );
        }
      });
  }

  protected boolean
  askIfCanUpgrade(
    SubscriptionImpl    subs,
    int            new_version )
  {
    subs.setHighestUserPromptedVersion( new_version );
   
    UIManager ui_manager = StaticUtilities.getUIManager( 120*1000 );
   
    String details = MessageText.getString(
        "subscript.add.upgradeto.desc",
        new String[]{ String.valueOf(new_version), subs.getName()});
   
    long res = ui_manager.showMessageBox(
        "subscript.add.upgrade.title",
        "!" + details + "!",
        UIManagerEvent.MT_YES | UIManagerEvent.MT_NO );
   
    if ( res != UIManagerEvent.MT_YES ){ 
   
      log( "    User declined upgrade" );
     
      return( false );
    }
   
    return( true );
  }
 
  protected boolean
  recoverSubscriptionUpdate(
    Download        download,
    final Map        rd )
  {
    byte[]  sid   = (byte[])rd.get( "sid" );
    int    version = ((Long)rd.get( "ver" )).intValue();
   
    final SubscriptionImpl subs = getSubscriptionFromSID( sid );
   
    if ( subs == null ){
   
      log( "Can't recover '" + download.getName() + "' - subscription " + ByteFormatter.encodeString( sid ) " not found" );
     
      return( false );
    }
   
    downloadSubscription(
        ((TorrentImpl)download.getTorrent()).getTorrent(),
        null,
        subs.getShortID(),
        version,
        subs.getName(),
        new downloadListener()
        {
          public void
          complete(
            File    data_file )
          {
            updateSubscription( subs, data_file );
          }
         
          public void
          complete(
            Download  download, 
            File    torrent_file )
          {
            updateSubscription( subs, download, torrent_file, new File( download.getSavePath()));
          }
           
          public void
          failed(
            Throwable  error )
          {
            log( "Failed to download subscription", error );
          }
         
          public Map
          getRecoveryData()
          {
            return( rd );
          }
         
          public boolean
          isCancelled()
          {
            return( false );
          }
        });
   
    return( true );
  }
 
  protected void
  updateSubscription(
    SubscriptionImpl    subs,
    Download        download,
    File          torrent_file,
    File          data_file )
  {
    try{
      removeDownload( download, false );
   
      try{       
        updateSubscription( subs, data_file );
                     
      }finally{
       
        if ( !data_file.delete()){
         
          log( "Failed to delete update file '" + data_file + "'" );
        }
       
        if ( !torrent_file.delete()){
         
          log( "Failed to delete update torrent '" + torrent_file + "'" );
        }
      }
    }catch( Throwable e ){
     
      log( "Failed to remove update download", e );
    }
  }
 
  protected void
  removeDownload(
    Download    download,
    boolean      remove_data )
  {
    try{
      download.stop();
     
    }catch( Throwable e ){
    }
   
    try{
      download.remove( true, remove_data );
     
      log( "Removed download '" + download.getName() + "'" );
     
    }catch( Throwable e ){
     
      log( "Failed to remove download '" + download.getName() + "'", e );
    }
  }
 
  protected void
  updateSubscription(
    SubscriptionImpl    subs,
    File          data_location )
  {
    log( "Updating subscription '" + subs.getString() + " using '" + data_location + "'" );
   
    VuzeFileHandler vfh = VuzeFileHandler.getSingleton();
     
    VuzeFile vf = vfh.loadVuzeFile( data_location.getAbsolutePath());
           
    vfh.handleFiles( new VuzeFile[]{ vf }, VuzeFileComponent.COMP_TYPE_SUBSCRIPTION );
  }
 
  protected MagnetPlugin
  getMagnetPlugin()
  {
    PluginInterface  pi  = AzureusCoreFactory.getSingleton().getPluginManager().getPluginInterfaceByClass( MagnetPlugin.class );
 
    if ( pi == null ){
     
      return( null );
    }
   
    return((MagnetPlugin)pi.getPlugin());
  }
 
  protected Engine
  getEngine(
    SubscriptionImpl    subs,
    Map            json_map,
    boolean          local_only )
 
    throws SubscriptionException
  {
    long id = ((Long)json_map.get( "engine_id" )).longValue();
   
    Engine engine = MetaSearchManagerFactory.getSingleton().getMetaSearch().getEngine( id );
   
    if ( engine != null ){
     
      return( engine );
    }

    if ( !local_only ){
     
      try{
        if ( id >= 0 && id < Integer.MAX_VALUE ){
               
          log( "Engine " + id + " not present, loading" );
           
            // vuze template but user hasn't yet loaded it
           
          try{
            engine = MetaSearchManagerFactory.getSingleton().getMetaSearch().addEngine( id );
                 
            return( engine );
           
          }catch( Throwable e ){
         
            throw( new SubscriptionException( "Failed to load engine '" + id + "'", e ));
          }
        }
      }catch( Throwable e ){
       
        log( "Failed to load search template", e );
      }
    }
   
    engine = subs.extractEngine( json_map, id );
   
    if ( engine != null ){
     
      return( engine );
    }
   
    throw( new SubscriptionException( "Failed to extract engine id " + id ));
  }
 
  protected SubscriptionResultImpl[]
  loadResults(
    SubscriptionImpl      subs )
  {
    List  results = new ArrayList();
   
    try{
      File  f = getResultsFile( subs );
     
      Map  map = FileUtil.readResilientFile( f );
     
      List  list = (List)map.get( "results" );
     
      if ( list != null ){
     
        SubscriptionHistoryImpl  history = (SubscriptionHistoryImpl)subs.getHistory();
       
        for (int i=0;i<list.size();i++){
         
          Map  result_map =(Map)list.get(i);
         
          try{
            SubscriptionResultImpl result = new SubscriptionResultImpl( history, result_map );
           
            results.add( result );
           
          }catch( Throwable e ){
           
            log( "Failed to decode result '" + result_map + "'", e );
          }
        }
      }
     
    }catch( Throwable e ){
     
      log( "Failed to load results for '" + subs.getName() + "' - continuing with empty result set", e );
    }
   
    return((SubscriptionResultImpl[])results.toArray( new SubscriptionResultImpl[results.size()] ));
  }
 
  protected void
    setCategoryOnExisting(
      SubscriptionImpl  subscription,
      String        old_category,
      String        new_category )
    {
    PluginInterface default_pi = PluginInitializer.getDefaultInterface();

      Download[] downloads   = default_pi.getDownloadManager().getDownloads();
               
      for ( Download d: downloads ){
              
        if ( subscriptionExists( d, subscription )){
           
        String existing = d.getAttribute( ta_category );

        if ( existing == null || existing.equals( old_category )){
         
          d.setAttribute( ta_category, new_category );
        }
        }
      }
    }
 
  public int
  getMaxNonDeletedResults()
  {
    return( COConfigurationManager.getIntParameter( CONFIG_MAX_RESULTS ));
  }
 
  public void
  setMaxNonDeletedResults(
    int    max )
  {
    if ( max != getMaxNonDeletedResults()){
     
      COConfigurationManager.setParameter( CONFIG_MAX_RESULTS, max );
    }
  }
 
  public boolean
  getAutoStartDownloads()
  {
    return( COConfigurationManager.getBooleanParameter( CONFIG_AUTO_START_DLS ));   
  }
 
  public void
  setAutoStartDownloads(
    boolean    auto_start )
  {
    if ( auto_start != getAutoStartDownloads()){
     
      COConfigurationManager.setParameter( CONFIG_AUTO_START_DLS, auto_start );
    }   
  }
 
  public int
  getAutoStartMinMB()
  {
    return( COConfigurationManager.getIntParameter( CONFIG_AUTO_START_MIN_MB ));
  }
 
  public void
  setAutoStartMinMB(
    int      mb )
  {
    if ( mb != getAutoStartMinMB()){
     
      COConfigurationManager.setParameter( CONFIG_AUTO_START_MIN_MB, mb );
    }
  }

  public int
  getAutoStartMaxMB()
  {
    return( COConfigurationManager.getIntParameter( CONFIG_AUTO_START_MAX_MB ));
  }
 
  public void
  setAutoStartMaxMB(
    int      mb )
  {
    if ( mb != getAutoStartMaxMB()){
     
      COConfigurationManager.setParameter( CONFIG_AUTO_START_MAX_MB, mb );
    }
  }
 
  protected boolean
  shouldAutoStart(
    Torrent    torrent )
  {
    if ( getAutoStartDownloads()){
     
      long  min = getAutoStartMinMB()*1024*1024L;
      long  max = getAutoStartMaxMB()*1024*1024L;
     
      if ( min <= 0 && max <= 0 ){
       
        return( true );
      }
     
      long size = torrent.getSize();
     
      if ( min > 0 && size < min ){
       
        return( false );
      }
     
      if ( max > 0 && size > max ){
       
        return( false );
      }
     
      return( true );
     
    }else{
     
      return( false );
    }
  }
 
  protected void
   saveResults(
     SubscriptionImpl      subs,
     SubscriptionResultImpl[]  results )
   {
    try{
      File  f = getResultsFile( subs );
 
      Map  map = new HashMap();
     
      List  list = new ArrayList( results.length );
     
      map.put( "results", list );
     
      for (int i=0;i<results.length;i++){
       
        list.add( results[i].toBEncodedMap());
      }
     
      FileUtil.writeResilientFile( f, map );
     
    }catch( Throwable e ){
     
      log( "Failed to save results for '" + subs.getName(), e );
    }
   }
 
  private void
  loadConfig()
  {
    if ( !FileUtil.resilientConfigFileExists( CONFIG_FILE )){
     
      return;
    }
   
    log( "Loading configuration" );
   
    boolean  some_are_mine = false;
   
    synchronized( this ){
     
      Map map = FileUtil.readResilientConfigFile( CONFIG_FILE );
     
      List  l_subs = (List)map.get( "subs" );
     
      if ( l_subs != null ){
       
        for (int i=0;i<l_subs.size();i++){
         
          Map  m = (Map)l_subs.get(i);
         
          try{
            SubscriptionImpl sub = new SubscriptionImpl( this, m );

            int index = Collections.binarySearch(subscriptions, sub, new Comparator<Subscription>() {
              public int compare(Subscription arg0, Subscription arg1) {
                return arg0.getID().compareTo(arg1.getID());
              }
            });
            if (index < 0) {
              index = -1 * index - 1; // best guess

              subscriptions.add( index, sub );
            }

            if ( sub.isMine()){
             
              some_are_mine = true;
            }
           
            log( "    loaded " + sub.getString());
           
          }catch( Throwable e ){
           
            log( "Failed to import subscription from " + m, e );
          }
        }
      }
    }
   
    if ( some_are_mine ){
             
      addMetaSearchListener();
    }
  }
 
  protected void
  configDirty(
    SubscriptionImpl    subs )
  {
    changeSubscription( subs );
   
    configDirty();
  }
 
  protected void
  configDirty()
  {
    synchronized( this ){
     
      if ( config_dirty ){
       
        return;
      }
     
      config_dirty = true;
   
      new DelayedEvent(
        "Subscriptions:save", 5000,
        new AERunnable()
        {
          public void
          runSupport()
          {
            synchronized( this ){
             
              if ( !config_dirty ){

                return;
              }
             
              saveConfig();
           
          }
        });
    }
  }
 
  protected void
  saveConfig()
  {
    log( "Saving configuration" );
   
    synchronized( this ){
     
      config_dirty = false;
     
      if ( subscriptions.size() == 0 ){
       
        FileUtil.deleteResilientConfigFile( CONFIG_FILE );
       
      }else{
       
        Map map = new HashMap();
       
        List  l_subs = new ArrayList();
       
        map.put( "subs", l_subs );
       
        Iterator  it = subscriptions.iterator();
       
        while( it.hasNext()){
         
          SubscriptionImpl sub = (SubscriptionImpl)it.next();
           
          try{
            l_subs.add( sub.toMap());
           
          }catch( Throwable e ){
           
            log( "Failed to save subscription " + sub.getString(), e );
          }
        }
       
        FileUtil.writeResilientConfigFile( CONFIG_FILE, map );
      }
    }
  }
 
  protected synchronized AEDiagnosticsLogger
  getLogger()
  {
    if ( logger == null ){
     
      logger = AEDiagnostics.getLogger( LOGGER_NAME );
    }
   
    return( logger );
  }
 
  public void
  log(
    String     s,
    Throwable   e )
  {
    AEDiagnosticsLogger diag_logger = getLogger();
   
    diag_logger.log( s );
    diag_logger.log( e );
  }
 
  public void
  log(
    String   s )
  {
    AEDiagnosticsLogger diag_logger = getLogger();
   
    diag_logger.log( s );
  }
 
  public void
  addListener(
    SubscriptionManagerListener  listener )
  {
    listeners.add( listener );
  }
 
  public void
  removeListener(
    SubscriptionManagerListener  listener )
  {
    listeners.remove( listener );
  }
 
  public void
  generate(
    IndentWriter    writer )
  {
    writer.println( "Subscriptions" );
     
    try{
      writer.indent();

      Subscription[] subs = getSubscriptions();
     
      for (int i=0;i<subs.length;i++){
       
        SubscriptionImpl sub = (SubscriptionImpl)subs[i];
       
        sub.generate( writer );
      }
     
    }finally{
     
      writer.exdent();
    }
  }
 
  private class
  searchMatcher
  {
    private String[]  bits;
    private int[]    bit_types;
    private Pattern[]  bit_patterns;

    protected
    searchMatcher(
      String    term )
    {
      bits = Constants.PAT_SPLIT_SPACE.split(term.toLowerCase() );
     
      bit_types     = new int[bits.length];
      bit_patterns   = new Pattern[bits.length];
     
      for (int i=0;i<bits.length;i++){
       
        String bit = bits[i] = bits[i].trim();
       
        if ( bit.length() > 0 ){
         
          char  c = bit.charAt(0);
         
          if ( c == '+' ){
           
            bit_types[i] = 1;
           
            bit = bits[i] = bit.substring(1);
           
          }else if ( c == '-' ){
           
            bit_types[i] = 2;
           
            bit = bits[i] = bit.substring(1);
          }
         
          if ( bit.startsWith( "(" ) && bit.endsWith((")"))){
           
            bit = bit.substring( 1, bit.length()-1 );
           
            try{
              bit_patterns[i] = Pattern.compile( bit, Pattern.CASE_INSENSITIVE );
             
            }catch( Throwable e ){
            }
          }else if ( bit.contains( "|" )){
           
            try{
              bit_patterns[i] = Pattern.compile( bit, Pattern.CASE_INSENSITIVE );
             
            }catch( Throwable e ){
            }
          }
        }
      }
    }
   
    public boolean
    matches(
      String    str )
    {     
      // term is made up of space separated bits - all bits must match
      // each bit can be prefixed by + or -, a leading - means 'bit doesn't match'. + doesn't mean anything
      // each bit (with prefix removed) can be "(" regexp ")"
      // if bit isn't regexp but has "|" in it it is turned into a regexp so a|b means 'a or b'
                 
      str = str.toLowerCase();
     
      boolean  match       = true;
      boolean  at_least_one   = false;
     
      for (int i=0;i<bits.length;i++){
       
        String bit = bits[i];
       
        if ( bit.length() > 0 ){
         
          boolean  hit;
         
          if ( bit_patterns[i] == null ){
         
            hit = str.contains( bit );
           
          }else{
         
            hit = bit_patterns[i].matcher( str ).find();
          }
         
          int  type = bit_types[i];
         
          if ( hit ){
                       
            if ( type == 2 ){
             
              match = false;
             
              break;
             
            }else{
             
              at_least_one = true;

            }
          }else{
           
            if ( type == 2 ){
           
              at_least_one = true;
             
            }else{
             
              match = false;
           
              break;
            }
          }
        }
      }
     
      boolean res = match && at_least_one;
       
      return( res );
    }
  }
 
  public static void
  main(
    String[]  args )
  {
    final String   NAME   = "lalalal";
    final String  URL_STR  = "http://www.vuze.com/feed/publisher/ALL/1";
   
    try{
      //AzureusCoreFactory.create();
      /*
      Subscription subs =
        getSingleton(true).createSingletonRSS(
            NAME,
            new URL( URL_STR ),
            240 );
     
      subs.getVuzeFile().write( new File( "C:\\temp\\srss.vuze" ));
     
      subs.remove();
      */
     
      VuzeFile  vf = VuzeFileHandler.getSingleton().create();
     
      Map  map = new HashMap();
     
      map.put( "name", NAME );
      map.put( "url", URL_STR );
      map.put( "public", new Long( 0 ));
      map.put( "check_interval_mins", new Long( 345 ));
     
      vf.addComponent( VuzeFileComponent.COMP_TYPE_SUBSCRIPTION_SINGLETON, map );
     
      vf.write( new File( "C:\\temp\\srss_2.vuze" ) );

    }catch( Throwable e ){
     
      e.printStackTrace();
    }
  }
}
TOP

Related Classes of com.aelitis.azureus.core.subs.impl.SubscriptionManagerImpl$searchMatcher

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.