Package net.myrrix.client

Source Code of net.myrrix.client.ClientRecommender

/*
* Copyright Myrrix Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package net.myrrix.client;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Reader;
import java.io.Writer;
import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.net.URLConnection;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManagerFactory;

import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.net.HostAndPort;
import com.google.common.net.HttpHeaders;
import com.google.common.net.MediaType;
import org.apache.commons.math3.util.FastMath;
import org.apache.commons.math3.util.Pair;
import org.apache.mahout.cf.taste.common.NoSuchItemException;
import org.apache.mahout.cf.taste.common.NoSuchUserException;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.recommender.GenericRecommendedItem;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.recommender.IDRescorer;
import org.apache.mahout.cf.taste.recommender.RecommendedItem;
import org.apache.mahout.cf.taste.recommender.Rescorer;
import org.apache.mahout.common.LongPair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import net.myrrix.common.collection.FastIDSet;
import net.myrrix.common.io.IOUtils;
import net.myrrix.common.LangUtils;
import net.myrrix.common.MyrrixRecommender;
import net.myrrix.common.NotReadyException;
import net.myrrix.common.random.RandomUtils;

/**
* <p>An implementation of {@link MyrrixRecommender} which accesses a remote Serving Layer instance
* over HTTP or HTTPS. This is like a local "handle" on the remote recommender.</p>
*
* <p>It is useful to note here, again, that the API methods {@link #setPreference(long, long)}
* and {@link #removePreference(long, long)}, retained from Apache Mahout, have a somewhat different meaning
* than in Mahout. They add to an association strength, rather than replace it. See the javadoc.</p>
*
* <p>There are a few advanced, system-wide parameters that can be set to affect how the client works.
* These should not normally be used:</p>
*
* <ul>
*   <li>{@code client.connection.close}: Causes the client to request no HTTP keep-alive. This can avoid
*    running out of local sockets on the client side during load testing, but should not otherwise be set.</li>
*   <li>{@code client.https.ignoreHost}: When using HTTPS, ignore the host specified in the certificate. This
*    can make development easier, but must not be used in production.</li>
* </ul>
*
* @author Sean Owen
* @since 1.0
*/
public final class ClientRecommender implements MyrrixRecommender {

  private static final Logger log = LoggerFactory.getLogger(ClientRecommender.class);

  private static final Splitter COMMA = Splitter.on(',');
  private static final String IGNORE_HOSTNAME_KEY = "client.https.ignoreHost";
  private static final String CONNECTION_CLOSE_KEY = "client.connection.close";

  private static final Map<String,String> INGEST_REQUEST_PROPS;
  static {
    INGEST_REQUEST_PROPS = Maps.newHashMapWithExpectedSize(2);
    INGEST_REQUEST_PROPS.put(HttpHeaders.CONTENT_TYPE, MediaType.PLAIN_TEXT_UTF_8.toString());
    INGEST_REQUEST_PROPS.put(HttpHeaders.CONTENT_ENCODING, "gzip");
  }
  private static final String DESIRED_RESPONSE_CONTENT_TYPE = MediaType.CSV_UTF_8.withoutParameters().toString();

  private final MyrrixClientConfiguration config;
  private final boolean needAuthentication;
  private final boolean closeConnection;
  private final boolean ignoreHTTPSHost;
  private final List<List<HostAndPort>> partitions;

  /**
   * Instantiates a new recommender client with the given configuration
   *
   * @param config configuration to use with this client
   * @throws IOException if the HTTP client encounters an error during configuration
   */
  public ClientRecommender(MyrrixClientConfiguration config) throws IOException {
    Preconditions.checkNotNull(config);
    this.config = config;

    final String userName = config.getUserName();
    final String password = config.getPassword();
    needAuthentication = userName != null && password != null;
    if (needAuthentication) {
      Authenticator.setDefault(new Authenticator() {
        @Override
        protected PasswordAuthentication getPasswordAuthentication() {
          return new PasswordAuthentication(userName, password.toCharArray());
        }
      });
    }

    if (config.getKeystoreFile() != null) {
      log.warn("A keystore file has been specified. " +
               "This should only be done to accept self-signed certificates in development.");
      HttpsURLConnection.setDefaultSSLSocketFactory(buildSSLSocketFactory());
    }

    closeConnection = Boolean.valueOf(System.getProperty(CONNECTION_CLOSE_KEY));
    ignoreHTTPSHost = Boolean.valueOf(System.getProperty(IGNORE_HOSTNAME_KEY));

    partitions = config.getPartitions();
  }

  private SSLSocketFactory buildSSLSocketFactory() throws IOException {

    final HostnameVerifier defaultVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
    HttpsURLConnection.setDefaultHostnameVerifier(
      new HostnameVerifier(){
        @Override
        public boolean verify(String hostname, SSLSession sslSession) {
          return ignoreHTTPSHost
              || "localhost".equals(hostname)
              || "127.0.0.1".equals(hostname)
              || defaultVerifier.verify(hostname, sslSession);
        }
      });

    try {

      KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
      File trustStoreFile = config.getKeystoreFile().getAbsoluteFile();
      String password = config.getKeystorePassword();
      Preconditions.checkNotNull(password);

      InputStream in = new FileInputStream(trustStoreFile);
      try {
        keyStore.load(in, password.toCharArray());
      } finally {
        in.close();
      }

      TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
      tmf.init(keyStore);

      SSLContext ctx;
      try {
        ctx = SSLContext.getInstance("TLSv1.1"); // Java 7 only
      } catch (NoSuchAlgorithmException ignored) {
        log.info("TLSv1.1 unavailable, falling back to TLSv1");
        ctx = SSLContext.getInstance("TLSv1"); // Java 6      
        // This also seems to be necessary:
        if (System.getProperty("https.protocols") == null) {
          System.setProperty("https.protocols", "TLSv1");
        }
      }
      ctx.init(null, tmf.getTrustManagers(), null);
      return ctx.getSocketFactory();

    } catch (NoSuchAlgorithmException nsae) {
      // can't happen?
      throw new IllegalStateException(nsae);
    } catch (KeyStoreException kse) {
      throw new IOException(kse);
    } catch (KeyManagementException kme) {
      throw new IOException(kme);
    } catch (CertificateException ce) {
      throw new IOException(ce);
    }
  }

  /**
   * @param replica host and port of replica to connect to
   * @param path URL to access (relative to context root)
   * @param method HTTP method to use
   */
  private HttpURLConnection buildConnectionToReplica(HostAndPort replica,
                                                     String path,
                                                     String method) throws IOException {
    return buildConnectionToReplica(replica, path, method, false, false, null);
  }

  /**
   * @param replica host and port of replica to connect to
   * @param path URL to access (relative to context root)
   * @param method HTTP method to use
   * @param doOutput if true, will need to write data into the request body
   * @param chunkedStreaming if true, use chunked streaming to accommodate a large upload, if possible
   * @param requestProperties additional request key/value pairs or {@code null} for none
   */
  private HttpURLConnection buildConnectionToReplica(HostAndPort replica,
                                                     String path,
                                                     String method,
                                                     boolean doOutput,
                                                     boolean chunkedStreaming,
                                                     Map<String,String> requestProperties) throws IOException {
    String contextPath = config.getContextPath();
    if (contextPath != null) {
      path = '/' + contextPath + path;
    }
    String protocol = config.isSecure() ? "https" : "http";
    URL url;
    try {
      url = new URL(protocol, replica.getHostText(), replica.getPort(), path);
    } catch (MalformedURLException mue) {
      // can't happen
      throw new IllegalStateException(mue);
    }
    log.debug("{} {}", method, url);

    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
    connection.setRequestMethod(method);
    connection.setDoInput(true);
    connection.setDoOutput(doOutput);
    connection.setUseCaches(false);
    connection.setAllowUserInteraction(false);
    connection.setRequestProperty(HttpHeaders.ACCEPT, DESIRED_RESPONSE_CONTENT_TYPE);
    if (closeConnection) {
      connection.setRequestProperty(HttpHeaders.CONNECTION, "close");
    }
    if (chunkedStreaming) {
      if (needAuthentication) {
        // Must buffer in memory if using authentication since it won't handle the authorization challenge
        log.debug("Authentication is enabled, so ingest data must be buffered in memory");
      } else {
        connection.setChunkedStreamingMode(0); // Use default buffer size
      }
    }
    if (requestProperties != null) {
      for (Map.Entry<String,String> entry : requestProperties.entrySet()) {
        connection.setRequestProperty(entry.getKey(), entry.getValue());
      }
    }
    return connection;
  }

  /**
   * @param unnormalizedID ID value that determines partition
   */
  private Iterable<HostAndPort> choosePartitionAndReplicas(long unnormalizedID) {
    List<HostAndPort> replicas = partitions.get(LangUtils.mod(unnormalizedID, partitions.size()));
    int numReplicas = replicas.size();
    if (numReplicas <= 1) {
      return replicas;
    }
    // Fix first replica; cycle through remainder in order since the remainder doesn't matter
    int currentReplica = LangUtils.mod(RandomUtils.md5HashToLong(unnormalizedID), numReplicas);
    Collection<HostAndPort> rotatedReplicas = Lists.newArrayListWithCapacity(numReplicas);
    for (int i = 0; i < numReplicas; i++) {
      rotatedReplicas.add(replicas.get(currentReplica));
      if (++currentReplica == numReplicas) {
        currentReplica = 0;
      }
    }
    return rotatedReplicas;
  }

  /**
   * Calls {@link #setPreference(long, long, float)} with value 1.0.
   */
  @Override
  public void setPreference(long userID, long itemID) throws TasteException {
    setPreference(userID, itemID, 1.0f);
  }

  @Override
  public void setPreference(long userID, long itemID, float value) throws TasteException {
    doSetOrRemovePreference(userID, itemID, value, true);
  }

  @Override
  public void removePreference(long userID, long itemID) throws TasteException {
    doSetOrRemovePreference(userID, itemID, 1.0f, false); // 1.0 is a dummy value that gets ignored
  }

  private void doSetOrRemovePreference(long userID, long itemID, float value, boolean set) throws TasteException {
    doSetOrRemove("/pref/" + userID + '/' + itemID, userID, value, set);
  }

  private void doSetOrRemove(String path, long unnormalizedID, float value, boolean set) throws TasteException {
    boolean sendValue = value != 1.0f;
    Map<String,String> requestProperties;
    byte[] bytes;
    if (sendValue) {
      requestProperties = Maps.newHashMapWithExpectedSize(2);
      bytes = Float.toString(value).getBytes(Charsets.UTF_8);
      requestProperties.put(HttpHeaders.CONTENT_TYPE, MediaType.PLAIN_TEXT_UTF_8.toString());
      requestProperties.put(HttpHeaders.CONTENT_LENGTH, Integer.toString(bytes.length));
    } else {
      requestProperties = null;
      bytes = null;
    }

    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(unnormalizedID)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica,
                                              path,
                                              set ? "POST" : "DELETE",
                                              sendValue,
                                              false,
                                              requestProperties);
        if (sendValue) {
          OutputStream out = connection.getOutputStream();
          out.write(bytes);
          out.close();
        }
        // Should not be able to return Not Available status
        if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
          throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
        return;
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", path, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", path, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  @Override
  public void setUserTag(long userID, String tag) throws TasteException {
    setUserTag(userID, tag, 1.0f);
  }

  @Override
  public void setUserTag(long userID, String tag, float value) throws TasteException {
    Preconditions.checkNotNull(tag);
    Preconditions.checkArgument(!tag.isEmpty());
    doSetOrRemove("/tag/user/" + userID + '/' + IOUtils.urlEncode(tag), userID, value, true);
  }

  @Override
  public void setItemTag(String tag, long itemID) throws TasteException {
    setItemTag(tag, itemID, 1.0f);
  }

  @Override
  public void setItemTag(String tag, long itemID, float value) throws TasteException {
    setItemTag(tag, itemID, value, null);
  }

  /**
   * Like {@link #setItemTag(String, long, float)}, but allows caller to specify the user for
   * which the request is being made. This information does not directly affect the computation,
   * but affects <em>routing</em> of the request in a distributed context. This is always recommended
   * when there is a user in whose context the request is being made, as it will ensure that the
   * request can take into account all the latest information from the user, including a very new
   * item like {@code itemID}.
   */
  public void setItemTag(String tag, long itemID, float value, Long contextUserID) throws TasteException {
    Preconditions.checkNotNull(tag);
    Preconditions.checkArgument(!tag.isEmpty());
    long idToPartitionOn = contextUserID == null ? itemID : contextUserID;
    doSetOrRemove("/tag/item/" + itemID + '/' + IOUtils.urlEncode(tag), idToPartitionOn, value, true);
  }

  /**
   * @param userID user ID whose preference is to be estimated
   * @param itemID item ID to estimate preference for
   * @return an estimate of the strength of the association between the user and item. These values are the
   *  same as will be returned from {@link #recommend(long, int)}. They are opaque values and have no interpretation
   *  other than that larger means stronger. The values are typically in the range [0,1] but are not guaranteed
   *  to be so. Note that 0 will be returned if the user or item is not known in the data.
   * @throws NotReadyException if the recommender has no model available yet
   * @throws TasteException if another error occurs
   */
  @Override
  public float estimatePreference(long userID, long itemID) throws TasteException {
    return estimatePreferences(userID, itemID)[0];
  }

  @Override
  public float[] estimatePreferences(long userID, long... itemIDs) throws TasteException {
    StringBuilder urlPath = new StringBuilder();
    urlPath.append("/estimate/");
    urlPath.append(userID);
    for (long itemID : itemIDs) {
      urlPath.append('/').append(itemID);
    }

    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(userID)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
            try {
              float[] result = new float[itemIDs.length];
              for (int i = 0; i < itemIDs.length; i++) {
                result[i] = LangUtils.parseFloat(reader.readLine());
              }
              return result;
            } finally {
              reader.close();
            }
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  @Override
  public float estimateForAnonymous(long toItemID, long[] itemIDs) throws TasteException {
    return estimateForAnonymous(toItemID, itemIDs, null);
  }

  @Override
  public float estimateForAnonymous(long toItemID, long[] itemIDs, float[] values) throws TasteException {
    return estimateForAnonymous(toItemID, itemIDs, values, null);
  }

  public float estimateForAnonymous(long toItemID, long[] itemIDs, float[] values, Long contextUserID)
      throws TasteException {
    Preconditions.checkArgument(values == null || values.length == itemIDs.length,
                                "Number of values doesn't match number of items");
    StringBuilder urlPath = new StringBuilder(32);
    urlPath.append("/estimateForAnonymous/");
    urlPath.append(toItemID);
    for (int i = 0; i < itemIDs.length; i++) {
      urlPath.append('/').append(itemIDs[i]);
      if (values != null) {
        urlPath.append('=').append(values[i]);
      }
    }

    // Requests are typically partitioned by user, but this request does not directly depend on a user.
    // If a user ID is supplied anyway, use it for partitioning since it will follow routing for other
    // requests related to that user. Otherwise just partition on the "to" item ID, which is at least
    // deterministic.
    long idToPartitionOn = contextUserID == null ? toItemID : contextUserID;

    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(idToPartitionOn)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
            try {
              return LangUtils.parseFloat(reader.readLine());
            } finally {
              reader.close();
            }
          case HttpURLConnection.HTTP_NOT_FOUND:
            throw new NoSuchItemException(Arrays.toString(itemIDs) + ' ' + toItemID);
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  /**
   * Like {@link #recommend(long, int, boolean, IDRescorer)}, and sets {@code considerKnownItems} to {@code false}
   * and {@code rescorer} to {@code null}.
   */
  @Override
  public List<RecommendedItem> recommend(long userID, int howMany) throws TasteException {
    return recommend(userID, howMany, false, (String[]) null);
  }

  /**
   * @param userID user for which recommendations are to be computed
   * @param howMany desired number of recommendations
   * @param considerKnownItems if true, items that the user is already associated to are candidates
   *  for recommendation. Normally this is {@code false}.
   * @param rescorerParams optional parameters to send to the server's {@code RescorerProvider}
   * @return {@link List} of recommended {@link RecommendedItem}s, ordered from most strongly recommend to least
   * @throws NoSuchUserException if the user is not known in the model
   * @throws NotReadyException if the recommender has no model available yet
   * @throws TasteException if another error occurs
   * @throws UnsupportedOperationException if rescorer is not null
   */
  public List<RecommendedItem> recommend(long userID,
                                         int howMany,
                                         boolean considerKnownItems,
                                         String[] rescorerParams) throws TasteException {

    StringBuilder urlPath = new StringBuilder();
    urlPath.append("/recommend/");
    urlPath.append(userID);
    appendCommonQueryParams(howMany, considerKnownItems, rescorerParams, urlPath);

    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(userID)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            return consumeItems(connection);
          case HttpURLConnection.HTTP_NOT_FOUND:
            throw new NoSuchUserException(userID);
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  private static List<RecommendedItem> consumeItems(URLConnection connection) throws IOException {
    List<RecommendedItem> result = Lists.newArrayList();
    BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
    try {
      CharSequence line;
      while ((line = reader.readLine()) != null) {
        Iterator<String> tokens = COMMA.split(line).iterator();
        long itemID = Long.parseLong(tokens.next());
        float value = LangUtils.parseFloat(tokens.next());
        result.add(new GenericRecommendedItem(itemID, value));
      }
    } finally {
      reader.close();
    }
    return result;
  }

  /**
   * @param userIDs users for which recommendations are to be computed
   * @param howMany desired number of recommendations
   * @param considerKnownItems if true, items that the user is already associated to are candidates
   *  for recommendation. Normally this is {@code false}.
   * @param rescorerParams optional parameters to send to the server's {@code RescorerProvider}
   * @return {@link List} of recommended {@link RecommendedItem}s, ordered from most strongly recommend to least
   * @throws NoSuchUserException if <em>none</em> of {@code userIDs} are known in the model. Otherwise unknown
   *  user IDs are ignored.
   * @throws NotReadyException if the recommender has no model available yet
   * @throws TasteException if another error occurs
   * @throws UnsupportedOperationException if rescorer is not null
   */
  public List<RecommendedItem> recommendToMany(long[] userIDs,
                                               int howMany,
                                               boolean considerKnownItems,
                                               String[] rescorerParams) throws TasteException {

    StringBuilder urlPath = new StringBuilder(32);
    urlPath.append("/recommendToMany");
    for (long userID : userIDs) {
      urlPath.append('/').append(userID);
    }
    appendCommonQueryParams(howMany, considerKnownItems, rescorerParams, urlPath);

    // Note that this assumes that all user IDs are on the same partition. It will fail at request
    // time if not since the partition of the first user doesn't contain the others.
    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(userIDs[0])) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            return consumeItems(connection);
          case HttpURLConnection.HTTP_NOT_FOUND:
            throw new NoSuchUserException(Arrays.toString(userIDs));
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  @Override
  public List<RecommendedItem> recommendToAnonymous(long[] itemIDs, int howMany) throws TasteException {
    return recommendToAnonymous(itemIDs, null, howMany);
  }

  @Override
  public List<RecommendedItem> recommendToAnonymous(long[] itemIDs, float[] values, int howMany)
      throws TasteException {
    return recommendToAnonymous(itemIDs, values, howMany, null, null);
  }

  /**
   * Like {@link #recommendToAnonymous(long[], float[], int)}, but allows caller to specify the user for
   * which the request is being made. This information does not directly affect the computation,
   * but affects <em>routing</em> of the request in a distributed context. This is always recommended
   * when there is a user in whose context the request is being made, as it will ensure that the
   * request can take into account all the latest information from the user, including very new
   * items that may be in {@code itemIDs}.
   */
  public List<RecommendedItem> recommendToAnonymous(long[] itemIDs,
                                                    float[] values,
                                                    int howMany,
                                                    String[] rescorerParams,
                                                    Long contextUserID) throws TasteException {
    return anonymousOrSimilar(itemIDs, values, howMany, "/recommendToAnonymous", rescorerParams, contextUserID);
  }

  @Override
  public List<RecommendedItem> mostPopularItems(int howMany) throws TasteException {
    StringBuilder urlPath = new StringBuilder(32);
    urlPath.append("/mostPopularItems");
    appendCommonQueryParams(howMany, false, null, urlPath);

    // Always send to partition 0 for consistency   
    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(0L)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            return consumeItems(connection);
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  /**
   * Computes items most similar to an item or items. The returned items have the highest average similarity
   * to the given items.
   *
   * @param itemIDs items for which most similar items are required
   * @param howMany maximum number of similar items to return; fewer may be returned
   * @return {@link RecommendedItem}s representing the top recommendations for the user, ordered by quality,
   *  descending. The score associated to it is an opaque value. Larger means more similar, but no further
   *  interpretation may necessarily be applied.
   * @throws NoSuchItemException if <em>none</em> of {@code itemIDs} exist in the model. Otherwise, unknown
   *  items are ignored.
   * @throws NotReadyException if the recommender has no model available yet
   * @throws TasteException if another error occurs
   */
  @Override
  public List<RecommendedItem> mostSimilarItems(long[] itemIDs, int howMany) throws TasteException {
    return mostSimilarItems(itemIDs, howMany, null, null);
  }

  /**
   * Like {@link #mostSimilarItems(long[], int)}, but allows caller to specify the user for which the request
   * is being made. This information does not directly affect the computation, but affects <em>routing</em>
   * of the request in a distributed context. This is always recommended when there is a user in whose context
   * the request is being made, as it will ensure that the request can take into account all the latest information
   * from the user, including very new items that may be in {@code itemIDs}.
   */
  public List<RecommendedItem> mostSimilarItems(long[] itemIDs,
                                                int howMany,
                                                String[] rescorerParams,
                                                Long contextUserID) throws TasteException {
    return anonymousOrSimilar(itemIDs, null, howMany, "/similarity", rescorerParams, contextUserID);
  }

  private List<RecommendedItem> anonymousOrSimilar(long[] itemIDs,
                                                   float[] values,
                                                   int howMany,
                                                   String path,
                                                   String[] rescorerParams,
                                                   Long contextUserID) throws TasteException {
    Preconditions.checkArgument(values == null || values.length == itemIDs.length,
                                "Number of values doesn't match number of items");
    StringBuilder urlPath = new StringBuilder();
    urlPath.append(path);
    for (int i = 0; i < itemIDs.length; i++) {
      urlPath.append('/').append(itemIDs[i]);
      if (values != null) {
        urlPath.append('=').append(values[i]);
      }
    }
    appendCommonQueryParams(howMany, false, rescorerParams, urlPath);

    // Requests are typically partitioned by user, but this request does not directly depend on a user.
    // If a user ID is supplied anyway, use it for partitioning since it will follow routing for other
    // requests related to that user. Otherwise just partition on (first0 item ID, which is at least
    // deterministic.
    long idToPartitionOn = contextUserID == null ? itemIDs[0] : contextUserID;

    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(idToPartitionOn)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            return consumeItems(connection);
          case HttpURLConnection.HTTP_NOT_FOUND:
            throw new NoSuchItemException(Arrays.toString(itemIDs));
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  /**
   * One-argument version of {@link #mostSimilarItems(long[], int)}.
   */
  @Override
  public List<RecommendedItem> mostSimilarItems(long itemID, int howMany) throws TasteException {
    return mostSimilarItems(new long[] { itemID }, howMany);
  }

  @Override
  public float[] similarityToItem(long toItemID, long... itemIDs) throws TasteException {
    return similarityToItem(toItemID, itemIDs, null);
  }

  /**
   * Like {@link #similarityToItem(long, long[])}, but allows caller to specify the user for which the request
   * is being made. This information does not directly affect the computation, but affects <em>routing</em>
   * of the request in a distributed context. This is always recommended when there is a user in whose context
   * the request is being made, as it will ensure that the request can take into account all the latest information
   * from the user, including very new items that may be in {@code itemIDs}.
   */
  public float[] similarityToItem(long toItemID, long[] itemIDs, Long contextUserID) throws TasteException {
    StringBuilder urlPath = new StringBuilder(32);
    urlPath.append("/similarityToItem/");
    urlPath.append(toItemID);
    for (long itemID : itemIDs) {
      urlPath.append('/').append(itemID);
    }

    // Requests are typically partitioned by user, but this request does not directly depend on a user.
    // If a user ID is supplied anyway, use it for partitioning since it will follow routing for other
    // requests related to that user. Otherwise just partition on (first0 item ID, which is at least
    // deterministic.
    long idToPartitionOn = contextUserID == null ? itemIDs[0] : contextUserID;

    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(idToPartitionOn)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
            try {
              float[] result = new float[itemIDs.length];
              for (int i = 0; i < itemIDs.length; i++) {
                result[i] = LangUtils.parseFloat(reader.readLine());
              }
              return result;
            } finally {
              reader.close();
            }
          case HttpURLConnection.HTTP_NOT_FOUND:
            throw new NoSuchItemException(connection.getResponseMessage());
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  /**
   * <p>Lists the items that were most influential in recommending a given item to a given user. Exactly how this
   * is determined is left to the implementation, but, generally this will return items that the user prefers
   * and that are similar to the given item.</p>
   *
   * <p>These values by which the results are ordered are opaque values and have no interpretation
   * other than that larger means stronger.</p>
   *
   * @param userID ID of user who was recommended the item
   * @param itemID ID of item that was recommended
   * @param howMany maximum number of items to return
   * @return {@link List} of {@link RecommendedItem}, ordered from most influential in recommended the given
   *  item to least
   * @throws NoSuchUserException if the user is not known in the model
   * @throws NoSuchItemException if the item is not known in the model
   * @throws NotReadyException if the recommender has no model available yet
   * @throws TasteException if another error occurs
   */
  @Override
  public List<RecommendedItem> recommendedBecause(long userID, long itemID, int howMany) throws TasteException {
    String urlPath = "/because/" + userID + '/' + itemID + "?howMany=" + howMany;

    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(userID)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath, "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            return consumeItems(connection);
          case HttpURLConnection.HTTP_NOT_FOUND:
            String connectionMessage = connection.getResponseMessage();
            if (connectionMessage != null &&
                connectionMessage.contains(NoSuchUserException.class.getSimpleName())) {
              throw new NoSuchUserException(userID);
            } else {
              throw new NoSuchItemException(itemID);
            }
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  @Override
  public void ingest(File file) throws TasteException {
    Reader reader = null;
    try {
      reader = IOUtils.openReaderMaybeDecompressing(file);
      ingest(reader);
    } catch (IOException ioe) {
      throw new TasteException(ioe);
    } finally {
      try {
        reader.close();
      } catch (IOException e) {
        // Can't happen, continue
      }
    }
  }

  @Override
  public void ingest(Reader reader) throws TasteException {
    Map<Integer,Pair<Writer,HttpURLConnection>> writersAndConnections =
        Maps.newHashMapWithExpectedSize(partitions.size());
    BufferedReader buffered = IOUtils.buffer(reader);
    try {
      try {
        String line;
        while ((line = buffered.readLine()) != null) {
          if (line.isEmpty() || line.charAt(0) == '#') {
            continue;
          }
          long userID;
          try {
            userID = Long.parseLong(COMMA.split(line).iterator().next());
          } catch (NoSuchElementException nsee) {
            throw new TasteException(nsee);
          } catch (NumberFormatException nfe) {
            throw new TasteException(nfe);
          }
          int partition = LangUtils.mod(userID, partitions.size());
          Pair<Writer,HttpURLConnection> writerAndConnection = writersAndConnections.get(partition);
          if (writerAndConnection == null) {
            HttpURLConnection connection = buildConnectionToAReplica(partition);
            Writer writer = IOUtils.buildGZIPWriter(connection.getOutputStream());
            writerAndConnection = new Pair<Writer,HttpURLConnection>(writer, connection);
            writersAndConnections.put(partition, writerAndConnection);
          }
          Writer writer = writerAndConnection.getFirst();
          writer.write(line);
          writer.write('\n');
        }
        for (Pair<Writer,HttpURLConnection> writerAndConnection : writersAndConnections.values()) {
          // Want to know of output stream close failed -- maybe failed to write
          writerAndConnection.getFirst().close();
          HttpURLConnection connection = writerAndConnection.getSecond();
          if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
          }
        }
      } finally {
        for (Pair<Writer,HttpURLConnection> writerAndConnection : writersAndConnections.values()) {
          writerAndConnection.getFirst().close();
          writerAndConnection.getSecond().disconnect();
        }
      }
    } catch (IOException ioe) {
      throw new TasteException(ioe);
    }
  }

  private HttpURLConnection buildConnectionToAReplica(int partition) throws TasteException {
    String urlPath = "/ingest";

    TasteException savedException = null;
    for (HostAndPort replica : partitions.get(partition)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath, "POST", true, true, INGEST_REQUEST_PROPS);
        connection.connect();
        return connection;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  @Deprecated
  @Override
  public void refresh(Collection<Refreshable> alreadyRefreshed) {
    if (alreadyRefreshed != null) {
      log.warn("Ignoring argument {}", alreadyRefreshed);
    }
    refresh();
  }

  /**
   * Requests that the Serving Layer recompute its models. This is a request, and may or may not result
   * in an update.
   */
  @Override
  public void refresh() {
    int numPartitions = partitions.size();
    for (int i = 0; i < numPartitions; i++) {
      refreshPartition(i);
    }
  }

  private void refreshPartition(int partition) {
    String urlPath = "/refresh";

    for (HostAndPort replica : partitions.get(partition)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath, "POST");
        if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
          log.warn("Unable to refresh partition {} ({} {}); continuing",
                   partition, connection.getResponseCode(), connection.getResponseMessage());
          // Yes, continue
        }
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
  }

  /**
   * Not available. The client does not directly use any {@link DataModel}.
   *
   * @throws UnsupportedOperationException
   * @deprecated do not call
   */
  @Deprecated
  @Override
  public DataModel getDataModel() {
    throw new UnsupportedOperationException();
  }

  /**
   * {@link Rescorer}s are not available at this time in the model.
   *
   * @return {@link #recommend(long, int)} if rescorer is null
   * @throws UnsupportedOperationException otherwise
   * @deprecated use {@link #recommend(long, int)} instead
   */
  @Deprecated
  @Override
  public List<RecommendedItem> recommend(long userID, int howMany, IDRescorer rescorer)
      throws TasteException {
    if (rescorer != null) {
      throw new UnsupportedOperationException();
    }
    return recommend(userID, howMany);
  }

  /**
   * {@link Rescorer}s are not available at this time in the model.
   *
   * @return {@link #recommend(long, int, boolean, String[])} if rescorer is null
   * @throws UnsupportedOperationException otherwise
   * @deprecated use {@link #recommend(long, int, boolean, String[])} instead
   */
  @Deprecated
  @Override
  public List<RecommendedItem> recommend(long userID,
                                         int howMany,
                                         boolean considerKnownItems,
                                         IDRescorer rescorer) throws TasteException {
    if (rescorer != null) {
      throw new UnsupportedOperationException();
    }
    return recommend(userID, howMany, considerKnownItems, (String[]) null);
  }

  @Deprecated
  @Override
  public List<RecommendedItem> recommendToMany(long[] userIDs,
                                               int howMany,
                                               boolean considerKnownItems,
                                               IDRescorer rescorer) throws TasteException {
    if (rescorer != null) {
      throw new UnsupportedOperationException();
    }
    return recommendToMany(userIDs, howMany, considerKnownItems, (String[]) null);
  }

  /**
   * Note that {@link IDRescorer} is not supported in the client now and must be null.
   *
   * @return {@link #recommendToAnonymous(long[], int)} if rescorer is null
   * @throws UnsupportedOperationException otherwise
   * @deprecated use {@link #recommendToAnonymous(long[], int)} instead
   */
  @Deprecated
  @Override
  public List<RecommendedItem> recommendToAnonymous(long[] itemIDs,
                                                    int howMany,
                                                    IDRescorer rescorer) throws TasteException {
    if (rescorer != null) {
      throw new UnsupportedOperationException();
    }
    return recommendToAnonymous(itemIDs, howMany);
  }

  /**
   * Note that {@link IDRescorer} is not supported in the client now and must be null.
   *
   * @return {@link #recommendToAnonymous(long[], float[], int)} if rescorer is null
   * @throws UnsupportedOperationException otherwise
   * @deprecated use {@link #recommendToAnonymous(long[], float[], int)} instead
   */
  @Deprecated
  @Override
  public List<RecommendedItem> recommendToAnonymous(long[] itemIDs,
                                                    float[] values,
                                                    int howMany,
                                                    IDRescorer rescorer) throws TasteException {
    if (rescorer != null) {
      throw new UnsupportedOperationException();
    }
    return recommendToAnonymous(itemIDs, values, howMany);
  }

  /**
   * Note that {@link IDRescorer} is not supported in the client now and must be null.
   *
   * @return {@link #mostPopularItems(int)} if rescorer is null
   * @throws UnsupportedOperationException otherwise
   * @deprecated use {@link #mostPopularItems(int)} instead
   */
  @Deprecated
  @Override
  public List<RecommendedItem> mostPopularItems(int howMany, IDRescorer rescorer) throws TasteException {
    if (rescorer != null) {
      throw new UnsupportedOperationException();
    }
    return mostPopularItems(howMany);
  }

  /**
   * {@link Rescorer}s are not available at this time in the model.
   *
   * @return {@link #mostSimilarItems(long, int)} if rescorer is null
   * @throws UnsupportedOperationException otherwise
   * @deprecated use {@link #mostSimilarItems(long, int)} instead
   */
  @Deprecated
  @Override
  public List<RecommendedItem> mostSimilarItems(long itemID, int howMany, Rescorer<LongPair> rescorer)
      throws TasteException {
    if (rescorer != null) {
      throw new UnsupportedOperationException();
    }
    return mostSimilarItems(itemID, howMany);
  }

  /**
   * {@link Rescorer}s are not available at this time in the model.
   *
   * @return {@link #mostSimilarItems(long[], int)} if rescorer is null
   * @throws UnsupportedOperationException otherwise
   * @deprecated use {@link #mostSimilarItems(long[], int)} instead
   */
  @Deprecated
  @Override
  public List<RecommendedItem> mostSimilarItems(long[] itemIDs, int howMany, Rescorer<LongPair> rescorer)
      throws TasteException {
    if (rescorer != null) {
      throw new UnsupportedOperationException();
    }
    return mostSimilarItems(itemIDs, howMany);
  }

  /**
   * {@code excludeItemIfNotSimilarToAll} is not applicable in this implementation.
   *
   * @return {@link #mostSimilarItems(long[], int)} if excludeItemIfNotSimilarToAll is false
   * @throws UnsupportedOperationException otherwise
   * @deprecated use {@link #mostSimilarItems(long[], int)} instead
   */
  @Deprecated
  @Override
  public List<RecommendedItem> mostSimilarItems(long[] itemIDs,
                                                int howMany,
                                                boolean excludeItemIfNotSimilarToAll) throws TasteException {
    if (excludeItemIfNotSimilarToAll) {
      throw new UnsupportedOperationException();
    }
    return mostSimilarItems(itemIDs, howMany);
  }

  /**
   * {@link Rescorer}s are not available at this time in the model.
   * {@code excludeItemIfNotSimilarToAll} is not applicable in this implementation.
   *
   * @return {@link #mostSimilarItems(long[], int)} if excludeItemIfNotSimilarToAll is false and rescorer is null
   * @throws UnsupportedOperationException otherwise
   * @deprecated use {@link #mostSimilarItems(long[], int)} instead
   */
  @Deprecated
  @Override
  public List<RecommendedItem> mostSimilarItems(long[] itemIDs,
                                                int howMany,
                                                Rescorer<LongPair> rescorer,
                                                boolean excludeItemIfNotSimilarToAll) throws TasteException {
    if (excludeItemIfNotSimilarToAll || rescorer != null) {
      throw new UnsupportedOperationException();
    }
    return mostSimilarItems(itemIDs, howMany);
  }

  @Override
  public boolean isReady() throws TasteException {
    int numPartitions = partitions.size();
    for (int i = 0; i < numPartitions; i++) {
      if (!isPartitionReady(i)) {
        return false;
      }
    }
    return true;
  }

  private boolean isPartitionReady(int partition) throws TasteException {
    String urlPath = "/ready";

    TasteException savedException = null;
    for (HostAndPort replica : partitions.get(partition)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath, "HEAD");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            return true;
          case HttpURLConnection.HTTP_UNAVAILABLE:
            return false;
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  @Override
  public void await() throws TasteException, InterruptedException {
    while (!isReady()) {
      Thread.sleep(1000L);
    }
  }

  @Override
  public boolean await(long time, TimeUnit unit) throws TasteException, InterruptedException {
    Preconditions.checkArgument(time >= 0L, "time must be positive: {}", time);
    Preconditions.checkNotNull(unit);
    long waitForMS = TimeUnit.MILLISECONDS.convert(time, unit);
    long waitIntervalMS = FastMath.min(1000L, waitForMS);
    Stopwatch stopwatch = new Stopwatch().start();
    while (!isReady()) {
      if (stopwatch.elapsed(TimeUnit.MILLISECONDS) > waitForMS) {
        return false;
      }
      Thread.sleep(waitIntervalMS);
    }
    return true;
  }

  @Override
  public FastIDSet getAllUserIDs() throws TasteException {
    FastIDSet result = new FastIDSet();
    int numPartitions = partitions.size();
    for (int i = 0; i < numPartitions; i++) {
      getAllIDsFromPartition(i, true, result);
    }
    return result;
  }


  @Override
  public FastIDSet getAllItemIDs() throws TasteException {
    // Yes, loop over all partitions. Most item IDs will be returned from all partitions but it's
    // possible for some to exist only on a few.
    FastIDSet result = new FastIDSet();
    int numPartitions = partitions.size();
    for (int i = 0; i < numPartitions; i++) {
      getAllIDsFromPartition(i, false, result);
    }
    return result;
  }

  private void getAllIDsFromPartition(int partition, boolean user, FastIDSet result) throws TasteException {
    String urlPath = '/' + (user ? "user" : "item") + "/allIDs";

    TasteException savedException = null;
    for (HostAndPort replica : partitions.get(partition)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath, "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            consumeIDs(connection, result);
            return;
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  private static void consumeIDs(URLConnection connection, FastIDSet result) throws IOException {
    BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
    try {
      String line;
      while ((line = reader.readLine()) != null) {
        result.add(Long.parseLong(line));
      }
    } finally {
      reader.close();
    }
  }

  @Override
  public int getNumUserClusters() throws TasteException {
    return getNumClusters(true);
  }

  @Override
  public int getNumItemClusters() throws TasteException {
    return getNumClusters(false);
  }

  private int getNumClusters(boolean user) throws TasteException {
    String urlPath = '/' + (user ? "user" : "item") + "/clusters/count";

    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(0L)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath, "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
            try {
              return Integer.parseInt(reader.readLine());
            } finally {
              reader.close();
            }
          case HttpURLConnection.HTTP_NOT_IMPLEMENTED:
            throw new UnsupportedOperationException();
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  @Override
  public FastIDSet getUserCluster(int n) throws TasteException {
    return getCluster(n, true);
  }

  @Override
  public FastIDSet getItemCluster(int n) throws TasteException {
    return getCluster(n, false);
  }

  private FastIDSet getCluster(int n, boolean user) throws TasteException {
    String urlPath = '/' + (user ? "user" : "item") + "/clusters/" + n;

    TasteException savedException = null;
    for (HostAndPort replica : choosePartitionAndReplicas(0L)) {
      HttpURLConnection connection = null;
      try {
        connection = buildConnectionToReplica(replica, urlPath, "GET");
        switch (connection.getResponseCode()) {
          case HttpURLConnection.HTTP_OK:
            FastIDSet members = new FastIDSet();
            consumeIDs(connection, members);
            return members;          case HttpURLConnection.HTTP_NOT_IMPLEMENTED:
            throw new UnsupportedOperationException();
          case HttpURLConnection.HTTP_UNAVAILABLE:
            throw new NotReadyException();
          default:
            throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
        }
      } catch (TasteException te) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
        savedException = te;
      } catch (IOException ioe) {
        log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
        savedException = new TasteException(ioe);
      } finally {
        if (connection != null) {
          connection.disconnect();
        }
      }
    }
    throw savedException;
  }

  private static void appendCommonQueryParams(int howMany,
                                              boolean considerKnownItems,
                                              String[] rescorerParams,
                                              StringBuilder urlPath) {
    urlPath.append("?howMany=").append(howMany);
    if (considerKnownItems) {
      urlPath.append("&considerKnownItems=true");
    }
    if (rescorerParams != null) {
      for (String rescorerParam : rescorerParams) {
        urlPath.append("&rescorerParams=").append(IOUtils.urlEncode(rescorerParam));
      }
    }
  }

}
TOP

Related Classes of net.myrrix.client.ClientRecommender

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.