Package org.apache.hadoop.util.bloom

Examples of org.apache.hadoop.util.bloom.Key


            // fields, we'll serialize the whole tuple.
            byte[] b;
            if (t.size() == 1) b = DataType.toBytes(t.get(0));
            else b = DataType.toBytes(t, DataType.TUPLE);

            Key k = new Key(b);
            filter = new BloomFilter(vSize, numHash, hType);
            filter.add(k);

            return TupleFactory.getInstance().newTuple(bloomOut());
        }
View Full Code Here


        }
        byte[] b;
        if (input.size() == 1) b = DataType.toBytes(input.get(0));
        else b = DataType.toBytes(input, DataType.TUPLE);

        Key k = new Key(b);
        return filter.membershipTest(k);
    }
View Full Code Here

      keyToBytesFn = getKeyToBytesMapFn(ptype, getConfiguration());
    }
   
    @Override
    public void process(K input, Emitter<BloomFilter> emitter) {
      bloomFilter.add(new Key(keyToBytesFn.map(input)));
    }
View Full Code Here

      }
    }

    @Override
    public boolean accept(Pair<K, V> input) {
      Key key = new Key(keyToBytesFn.map(input.first()));
      return bloomFilter.membershipTest(key);
    }
View Full Code Here

        }
        byte[] b;
        if (input.size() == 1) b = DataType.toBytes(input.get(0));
        else b = DataType.toBytes(input, DataType.TUPLE);

        Key k = new Key(b);
        return filter.membershipTest(k);
    }
View Full Code Here

            // fields, we'll serialize the whole tuple.
            byte[] b;
            if (t.size() == 1) b = DataType.toBytes(t.get(0));
            else b = DataType.toBytes(t, DataType.TUPLE);

            Key k = new Key(b);
            filter = new BloomFilter(vSize, numHash, hType);
            filter.add(k);

            return TupleFactory.getInstance().newTuple(bloomOut());
        }
View Full Code Here

   */
  public BloomTokenFilter(Filter filter, boolean keepMembers, TokenStream in) {
    super(in);
    this.filter = filter;
    this.keepMembers = keepMembers;
    this.key = new Key();
    this.termAtt = addAttribute(CharTermAttribute.class);
    this.encoder = Charsets.UTF_8.newEncoder().
      onMalformedInput(CodingErrorAction.REPORT).
      onUnmappableCharacter(CodingErrorAction.REPORT);
  }
View Full Code Here

 
  /** test standalone filter without tokenfilter wrapping */
  @Test
  public void testFilter() throws IOException {
    Filter filter = getFilter(filterTokens);
    Key k = new Key();
    for (String s: filterTokens) {
      setKey(k,s);
      assertTrue("Key for string " + s + " should be filter member", filter.membershipTest(k));
    }
   
View Full Code Here

    assertEquals("Analyzer produced too few terms", expected.length, pos);
  }

  private static Filter getFilter(String[] tokens) throws IOException {
    Filter filter = new BloomFilter(100,50, Hash.JENKINS_HASH);
    Key k = new Key();
    for (String s: tokens) {
      setKey(k,s);
      filter.add(k);
    }
    return filter;
View Full Code Here

      keyToBytesFn = getKeyToBytesMapFn(ptype, getConfiguration());
    }
   
    @Override
    public void process(K input, Emitter<BloomFilter> emitter) {
      bloomFilter.add(new Key(keyToBytesFn.map(input)));
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.util.bloom.Key

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.