Examples of org.apache.hama.commons.io.VectorWritable

org.apache.hama.commons.io.VectorWritable
Writable for dense vectors.

      BSPPeer<Text, VectorWritable, Text, VectorWritable, MapWritable> peer,
      HashSet<Text> requiredUserFeatures,
      HashSet<Text> requiredItemFeatures) 
          throws IOException {
    Text key = new Text();
    VectorWritable value = new VectorWritable();
    int counter = 0;


    requiredUserFeatures = new HashSet<Text>();
    requiredItemFeatures = new HashSet<Text>();


    while(peer.readNext(key, value)) {
      // key format: (0, 1..n)
      //  0 - delimiter, for type of key
      //  1..n - actaul key value
      String firstSymbol = key.toString().substring(0, 1);
      String actualId = key.toString().substring(1);


      if (firstSymbol.equals(inputPreferenceDelim)) {
        // parse as <k:userId, v:(itemId, score)>
        String itemId = Long.toString((long)value.getVector().get(0));
        String score = Double.toString(value.getVector().get(1));


        if (usersMatrix.containsKey(actualId) == false) {
          DenseDoubleVector vals = new DenseDoubleVector(MATRIX_RANK);
          for (int i=0; i<MATRIX_RANK; i++) {
            vals.set(i, rnd.nextDouble());
          }
          VectorWritable rndValues = new VectorWritable(vals);
          usersMatrix.put(actualId, rndValues);
        }


        if (itemsMatrix.containsKey(itemId) == false) {
          DenseDoubleVector vals = new DenseDoubleVector(MATRIX_RANK);
          for (int i=0; i<MATRIX_RANK; i++) {
            vals.set(i, rnd.nextDouble());
          }
          VectorWritable rndValues = new VectorWritable(vals);
          itemsMatrix.put(itemId, rndValues);
        }
        preferences.add(new Preference<String, String>(actualId, itemId, Double.parseDouble(score)));
        indexes.add(counter);

View Full Code Here

      if (log.isDebugEnabled()) {
        log.debug("{}: new theta for cost {} is {}", new Object[]{peer.getPeerName(), cost, theta});
      }
      // master writes down the output
      if (master) {
        peer.write(new VectorWritable(theta), new DoubleWritable(cost));
      }


      peer.reopenInput();
      peer.sync();

View Full Code Here


  private double aggregateTotalCost(
      BSPPeer<VectorWritable, DoubleWritable, VectorWritable, DoubleWritable, VectorWritable> peer,
      double localCost) throws IOException {
    double totalCost = localCost;
    VectorWritable costResult;
    while ((costResult = peer.getCurrentMessage()) != null) {
      totalCost += costResult.getVector().get(0);
    }
    return totalCost;
  }

View Full Code Here

  }


  private double[] aggregatePartialDerivatives(
      BSPPeer<VectorWritable, DoubleWritable, VectorWritable, DoubleWritable, VectorWritable> peer,
      double[] thetaDelta) throws IOException {
    VectorWritable thetaDeltaSlice;
    double[] newTheta = Arrays.copyOf(thetaDelta, thetaDelta.length);
    while ((thetaDeltaSlice = peer.getCurrentMessage()) != null) {
      for (int j = 0; j < theta.getLength(); j++) {
        newTheta[j] += thetaDeltaSlice.getVector().get(j);
      }
    }
    return newTheta;
  }

View Full Code Here

  }


  private void aggregateItemsNumber(
      BSPPeer<VectorWritable, DoubleWritable, VectorWritable, DoubleWritable, VectorWritable> peer,
      int itemCount) throws IOException {
    VectorWritable itemsResult;
    while ((itemsResult = peer.getCurrentMessage()) != null) {
      itemCount += itemsResult.getVector().get(0);
    }


    m = itemCount;
  }

View Full Code Here

  private void broadcastVector(
      BSPPeer<VectorWritable, DoubleWritable, VectorWritable, DoubleWritable, VectorWritable> peer,
      double[] vector) throws IOException {
    for (String peerName : peer.getAllPeerNames()) {
      if (!peerName.equals(peer.getPeerName())) { // avoid sending to oneself
        peer.send(peerName, new VectorWritable(new DenseDoubleVector(vector)));
      }
    }
  }

View Full Code Here

  public void cleanup(
      BSPPeer<VectorWritable, DoubleWritable, VectorWritable, DoubleWritable, VectorWritable> peer)
      throws IOException {
    // master writes down the final output
    if (master) {
      peer.write(new VectorWritable(theta), new DoubleWritable(cost));
      if (log.isInfoEnabled()) {
        log.info("{}:computation finished with cost {} and theta {}", new Object[]{peer.getPeerName(), cost, theta});
      }
    }
  }

View Full Code Here

      } else {
        if (log.isDebugEnabled()) {
          log.debug("{}: getting theta", peer.getPeerName());
        }
        peer.sync();
        VectorWritable vectorWritable = peer.getCurrentMessage();
        theta = vectorWritable.getVector();
      }
    }
  }

View Full Code Here

  }


  private int getXSize(
      BSPPeer<VectorWritable, DoubleWritable, VectorWritable, DoubleWritable, VectorWritable> peer)
      throws IOException {
    VectorWritable key = new VectorWritable();
    DoubleWritable value = new DoubleWritable();
    peer.readNext(key, value);
    peer.reopenInput(); // reset input to start
    if (key.getVector() == null) {
      throw new IOException("cannot read input vector size");
    }
    return key.getVector().getDimension();
  }

View Full Code Here

        fs.createNewFile(dataPath);
        SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf,
            dataPath, LongWritable.class, VectorWritable.class);


        for (int i = 0; i < 1000; ++i) {
          VectorWritable vecWritable = new VectorWritable(trainingData[i % 4]);
          writer.append(new LongWritable(i), vecWritable);
        }
        writer.close();
      }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.hama.commons.io.VectorWritable

org.apache.hama.commons.math.DenseDoubleVector

org.apache.hama.commons.math.DoubleVector

org.apache.hama.commons.math.NamedDoubleVector

org.apache.hama.examples.NeuralNetworkTest

org.apache.hama.examples.util.VectorWritableMatrixGen

org.apache.hama.ml.ann.SmallLayeredNeuralNetworkTrainer

org.apache.hama.ml.ann.TestAutoEncoder

org.apache.hama.ml.ann.TestSmallLayeredNeuralNetwork

org.apache.hama.ml.kmeans.KMeansBSP

org.apache.hama.ml.perception.SmallMLPTrainer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.