Package com.cloudera.util.bloom

Examples of com.cloudera.util.bloom.BloomSet

81
82
83
84
85
86
87
88
  }

  /** {@inheritDoc} */
  @Override
  public void open() throws IOException, InterruptedException {
    bloom = new BloomSet(size, hashes);
    super.open();
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
  /**
   * {@inheritDoc}
   */
  @Override
  synchronized public void open() throws IOException, InterruptedException {
    bloom = new BloomSet(size, hashes);
    state = BloomCheckState.UNKNOWN;
    successCount = 0;
    failCount = 0;
    super.open();
    reportSink.open();
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
  public void append(Event e) throws IOException, InterruptedException {
    byte[] data = e.get(BloomGeneratorDeco.A_BLOOMSETDATA);
    // if has BloomSet Data is present
    if (data != null) {
      // process and then drop the event.
      BloomSet subset = new BloomSet(data);
      boolean contained = bloom.contains(subset);

      if (LOG.isDebugEnabled()) {
        LOG.debug("received bloom set: " + Arrays.toString(subset.getBytes()));
        LOG.debug("local bloom set:    " + Arrays.toString(bloom.getBytes()));
      }

      synchronized (this) {
        switch (state) {
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
*/
public class TestBloomSetDecos {

  @Test
  public void testBloomSetCompare() {
    BloomSet b1 = new BloomSet(10000, 5);
    BloomSet b2 = new BloomSet(10000, 5);

    for (int i = 0; i < 10; i++) {
      b1.addInt(i);
      if (i % 2 == 0)
        b2.addInt(i);
    }

    assertEquals(b1, b1);
    assertEquals(b2, b2);
    assertTrue(b1.contains(b2));
    assertFalse(b2.contains(b1));
  }
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
   * The parameters of the bloom filters are different so this should always
   * reject.
   */
  @Test(expected = IllegalArgumentException.class)
  public void testBloomBadSizes() {
    BloomSet b1 = new BloomSet(10000, 5);
    BloomSet b2 = new BloomSet(10000, 6);

    for (int i = 0; i < 10; i++) {
      b1.addInt(i);
      b2.addInt(i);
    }

    assertFalse(b1.contains(b2));
    assertFalse(b2.contains(b1));
  }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
   */
  @Test
  @Ignore("Takes too long to run")
  public void testBloomSetCompare100M() {
    // generally we want about 9-10 bits per entry.
    BloomSet b1 = new BloomSet(1000000000, 2); // 1B bits ~= 125MB
    BloomSet b2 = new BloomSet(1000000000, 2);

    // int drop = 543215432; // drop this one..
    // int drop = 543215431; // drop this one..
    int drop = 54323423;
    for (int i = 0; i < 100000000; i++) { // 100M "entries"
      b1.addInt(i);

      if (i != drop)
        // oops, we dropped one.
        b2.addInt(i);
    }

    assertTrue(b1.contains(b2));
    assertFalse(b2.contains(b1));
  }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
  /** Test it with larger number of events and larger number of slots. */
  @Test
  @Ignore("Takes too long to run")
  public void testBloomSetCompare100Mx10M() {
    // generally we want about 9-10 bits per entry.
    BloomSet b1 = new BloomSet(1000000000, 2); // 1B bits ~= 125MB
    BloomSet b2 = new BloomSet(1000000000, 2);

    for (int i = 0; i < 100000000; i++) { // 100M "entries"
      if (i != 234000)
        b1.addInt(i); // drop one that is included in the other set.

      if (i <= 10000000)
        b2.addInt(i);

      // only add the first 10M to the second hash
    }

    assertFalse(b1.contains(b2)); // b1 doesn't have all b2 has!
    assertFalse(b2.contains(b1));
  }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
*/
public class TestBloomSetDecos {

  @Test
  public void testBloomSetCompare() {
    BloomSet b1 = new BloomSet(10000, 5);
    BloomSet b2 = new BloomSet(10000, 5);

    for (int i = 0; i < 10; i++) {
      b1.addInt(i);
      if (i % 2 == 0)
        b2.addInt(i);
    }

    assertEquals(b1, b1);
    assertEquals(b2, b2);
    assertTrue(b1.contains(b2));
    assertFalse(b2.contains(b1));
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
   * The parameters of the bloom filters are different so this should always
   * reject.
   */
  @Test(expected = IllegalArgumentException.class)
  public void testBloomBadSizes() {
    BloomSet b1 = new BloomSet(10000, 5);
    BloomSet b2 = new BloomSet(10000, 6);

    for (int i = 0; i < 10; i++) {
      b1.addInt(i);
      b2.addInt(i);
    }

    assertFalse(b1.contains(b2));
    assertFalse(b2.contains(b1));
  }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
   */
  @Test
  @Ignore("Takes too long to run")
  public void testBloomSetCompare100M() {
    // generally we want about 9-10 bits per entry.
    BloomSet b1 = new BloomSet(1000000000, 2); // 1B bits ~= 125MB
    BloomSet b2 = new BloomSet(1000000000, 2);

    // int drop = 543215432; // drop this one..
    // int drop = 543215431; // drop this one..
    int drop = 54323423;
    for (int i = 0; i < 100000000; i++) { // 100M "entries"
      b1.addInt(i);

      if (i != drop)
        // oops, we dropped one.
        b2.addInt(i);
    }

    assertTrue(b1.contains(b2));
    assertFalse(b2.contains(b1));
  }
View Full Code Here
TOP

Related Classes of com.cloudera.util.bloom.BloomSet

  • com.cloudera.flume.handlers.debug.BloomCheckDecorator
  • com.cloudera.flume.handlers.debug.BloomGeneratorDeco
  • com.cloudera.flume.handlers.debug.TestBloomSetDecos
  • org.apache.hadoop.util.bloom.BloomFilter
  • org.apache.hadoop.util.bloom.Key
  • java.nio.ByteBuffer
  • java.io.DataOutputStream
  • java.io.ByteArrayOutputStream
  • java.io.ByteArrayInputStream
  • java.io.DataInputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.