Examples of ivory.lsh.data.FloatAsBytesWritable

ivory.lsh.data.FloatAsBytesWritable
A data structure to optimize space when storing an array of floats. Floats are quantized into byte values and stored as a BytesWritable. maxNorm is a maxNormalization factor required to re-compute float from byte. byte = quantize(float) = ((float/maxNorm)*128)==128 ? 127 : ((float/maxNorm) float = dequantize(byte) = ((byte/128)*maxNorm) @author ferhanture

      
      //store quantized byte value
//      System.out.println("quantized: "+normalized2one + "=>" + b);
      bytes[i]=b;
    }
    FloatAsBytesWritable vector2 =  new FloatAsBytesWritable(bytes, max, min);
    
    //debugging    
//    float sum = 0;
//    for(int i=0;i<numSamples;i++){
//      float f1 = vector.get(i);

View Full Code Here


    public void map(IntWritable docno, WeightedIntDocVector docvectorIn,
        OutputCollector<IntWritable, NBitSignature> output,
        Reporter reporter) throws IOException {
      HMapIFW docvector = docvectorIn.getWeightedTerms();
      FloatAsBytesWritable value;


      for(int i=0;i<randomUnitVectors.size();i++){
        value = (FloatAsBytesWritable) randomUnitVectors.get(i);
        double dprod = dotProduct(docvector,value);
        boolean sign = (dprod >= (dotProductThresholds==null ? 0 : dotProductThresholds[i])) ? true : false;

View Full Code Here


      // store quantized byte value
      // System.out.println("quantized: "+normalized2one + "=>" + b);
      bytes[i] = b;
    }
    FloatAsBytesWritable vector2 = new FloatAsBytesWritable(bytes, max, min);


    // debugging
    // float sum = 0;
    // for(int i=0;i<numSamples;i++){
    // float f1 = vector.get(i);

View Full Code Here

    }


    public void map(IntWritable docno, WeightedIntDocVector docvectorIn,
        OutputCollector<IntWritable, NBitSignature> output, Reporter reporter) throws IOException {
      HMapIFW docvector = docvectorIn.getWeightedTerms();
      FloatAsBytesWritable value;


      for (int i = 0; i < randomUnitVectors.size(); i++) {
        value = (FloatAsBytesWritable) randomUnitVectors.get(i);
        double dprod = dotProduct(docvector, value);
        boolean sign = (dprod >= (dotProductThresholds == null ? 0 : dotProductThresholds[i])) ? true

View Full Code Here


      // store quantized byte value
      // System.out.println("quantized: "+normalized2one + "=>" + b);
      bytes[i] = b;
    }
    FloatAsBytesWritable vector2 = new FloatAsBytesWritable(bytes, max, min);


    // debugging
    // float sum = 0;
    // for(int i=0;i<numSamples;i++){
    // float f1 = vector.get(i);

View Full Code Here

    }


    public void map(IntWritable docno, WeightedIntDocVector docvectorIn,
        OutputCollector<IntWritable, NBitSignature> output, Reporter reporter) throws IOException {
      HMapIFW docvector = docvectorIn.getWeightedTerms();
      FloatAsBytesWritable value;


      for (int i = 0; i < randomUnitVectors.size(); i++) {
        value = (FloatAsBytesWritable) randomUnitVectors.get(i);
        double dprod = dotProduct(docvector, value);
        boolean sign = (dprod >= (dotProductThresholds == null ? 0 : dotProductThresholds[i])) ? true

View Full Code Here

          new Path(TMP_FILENAME1), IntWritable.class, ArrayListOfFloatsWritable.class);
    SequenceFile.Writer writer2 = SequenceFile.createWriter(fs, conf,
          new Path(TMP_FILENAME2), IntWritable.class, FloatAsBytesWritable.class);


    ArrayListOfFloatsWritable a1 = WriteRandomVectors.generateUnitRandomVector(100);
    FloatAsBytesWritable a2 = WriteRandomVectors.generateUnitRandomVectorAsBytes(100);


    writer1.append(new IntWritable(1), a1);
    writer1.close();


    writer2.append(new IntWritable(1), a2);
    writer2.close();


    List<PairOfWritables<WritableComparable, Writable>> listOfKeysPairs1 =
      SequenceFileUtils.readFile(new Path(TMP_FILENAME1));
    fs.delete(new Path(TMP_FILENAME1), true);


    List<PairOfWritables<WritableComparable, Writable>> listOfKeysPairs2 =
      SequenceFileUtils.readFile(new Path(TMP_FILENAME2));
    fs.delete(new Path(TMP_FILENAME2), true);


    FloatAsBytesWritable b2 = (FloatAsBytesWritable) listOfKeysPairs2.get(0).getRightElement();
    ArrayListOfFloatsWritable b1 = (ArrayListOfFloatsWritable) listOfKeysPairs1.get(0)
        .getRightElement();


    for (int i = 0; i < 100; i++) {
      float f = b1.get(i);
      float g = a1.get(i);
      assertTrue(f == g);
    }
    for (int i = 0; i < 100; i++) {
      float f2 = b2.getAsFloat(i);
      float g2 = a2.getAsFloat(i);
      assertTrue(f2 == g2);
    }
  }

View Full Code Here

    }


    public void map(IntWritable docno, WeightedIntDocVector docvectorIn,
        OutputCollector<IntWritable, NBitSignature> output, Reporter reporter) throws IOException {
      HMapIFW docvector = docvectorIn.getWeightedTerms();
      FloatAsBytesWritable value;


      for (int i = 0; i < randomUnitVectors.size(); i++) {
        value = (FloatAsBytesWritable) randomUnitVectors.get(i);
        double dprod = dotProduct(docvector, value);
        boolean sign = (dprod >= (dotProductThresholds == null ? 0 : dotProductThresholds[i])) ? true

View Full Code Here

TOP

Related Classes of ivory.lsh.data.FloatAsBytesWritable

ivory.lsh.projection.ComputeSignaturesRandom$MyMapper

ivory.lsh.projection.WriteRandomVectors

ivory.lsh.WriteRandomVectorsTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.