Examples of org.apache.pig.builtin.Bloom

org.apache.pig.builtin.Bloom
Use a Bloom filter build previously by BuildBloom. You would first build a bloom filter in a group all job. For example: in a group all job. For example: define bb BuildBloom('jenkins', '100', '0.1'); A = load 'foo' as (x, y); B = group A all; C = foreach B generate bb(A.x); store C into 'mybloom'; The bloom filter can be on multiple keys by passing more than one field (or the entire bag) to BuildBloom. The resulting file can then be used in a Bloom filter as: define bloom Bloom(mybloom); A = load 'foo' as (x, y); B = load 'bar' as (z); C = filter B by bloom(z); D = join C by z, A by x; It uses {@link org.apache.hadoop.util.bloom.BloomFilter}.


        BuildBloom.Initial map =
                new BuildBloom.Initial(hashFunc, "fixed", size, numHash);
        t = map.exec(input);


        Bloom bloom = new Bloom("bla");
        bloom.setFilter((DataByteArray)t.get(0));


        // Test that everything we put in passes.
        Tuple t1 = tf.newTuple(1);
        t1.set(0, 1);
        assertTrue(bloom.exec(t1));


        // A few that don't pass
        for (int i = 100; i < 10; i++) {
            Tuple t2 = tf.newTuple(1);
            t2.set(0, i);
            assertFalse(bloom.exec(t2));
        }
    }

View Full Code Here

        t.set(0, combinerBag);
        BuildBloom.Intermediate combiner =
                new BuildBloom.Intermediate(hashFunc, "fixed", size, numHash);
        t = combiner.exec(t);


        Bloom bloom = new Bloom("bla");
        bloom.setFilter((DataByteArray)t.get(0));


        // Test that everything we put in passes.
        for (int j = 0; j < 3; j++) {
            Tuple t1 = tf.newTuple(1);
            t1.set(0, 10 + j);
            assertTrue(bloom.exec(t1));
        }


        // A few that don't pass
        for (int i = 100; i < 10; i++) {
            Tuple t2 = tf.newTuple(1);
            t2.set(0, i);
            assertFalse(bloom.exec(t2));
        }
    }

View Full Code Here

        t.set(0, reducerBag);
        BuildBloom.Final reducer =
                new BuildBloom.Final(hashFunc, "fixed", size, numHash);
        DataByteArray dba = reducer.exec(t);


        Bloom bloom = new Bloom("bla");
        bloom.setFilter(dba);


        // Test that everything we put in passes.
        for (int i = 0; i < 3; i++) {
            for (int j = 0; j < 3; j++) {
                Tuple t1 = tf.newTuple(1);
                t1.set(0, i * 10 + j);
                assertTrue(bloom.exec(t1));
            }
        }


        // A few that don't pass
        for (int i = 100; i < 10; i++) {
            Tuple t1 = tf.newTuple(1);
            t1.set(0, i);
            assertFalse(bloom.exec(t1));
        }
    }

View Full Code Here

        t.set(0, reducerBag);
        BuildBloom.Final reducer =
                new BuildBloom.Final(hashFunc, numElements, falsePositive);
        DataByteArray dba = reducer.exec(t);


        Bloom bloom = new Bloom("bla");
        bloom.setFilter(dba);


        // Test that everything we put in passes.
        for (int i = 0; i < 3; i++) {
            for (int j = 0; j < 3; j++) {
                Tuple t1 = tf.newTuple(2);
                t1.set(0, i * 10 + j);
                t1.set(1, strs[i][j]);
                assertTrue(bloom.exec(t1));
            }
        }


        // A few that don't pass
        for (int i = 100; i < 10; i++) {
            Tuple t1 = tf.newTuple(2);
            t1.set(0, i);
            t1.set(1, "ichabod");
            assertFalse(bloom.exec(t1));
        }
    }

View Full Code Here

        
        BuildBloom.Initial map =
            new BuildBloom.Initial(hashFunc, "fixed", size, numHash);
        t = map.exec(input);


        Bloom bloom = new Bloom("bla");
        bloom.setFilter((DataByteArray)t.get(0));


        // Test that everything we put in passes.
        Tuple t1 = tf.newTuple(1);
        t1.set(0, 1);
        assertTrue(bloom.exec(t1));


        // A few that don't pass
        for (int i = 100; i < 10; i++) {
            Tuple t2 = tf.newTuple(1);
            t2.set(0, i);
            assertFalse(bloom.exec(t2));
        }
    }

View Full Code Here

        t.set(0, combinerBag);
        BuildBloom.Intermediate combiner =
            new BuildBloom.Intermediate(hashFunc, "fixed", size, numHash);
        t = combiner.exec(t);


        Bloom bloom = new Bloom("bla");
        bloom.setFilter((DataByteArray)t.get(0));


        // Test that everything we put in passes.
        for (int j = 0; j < 3; j++) {
            Tuple t1 = tf.newTuple(1);
            t1.set(0, 10 + j);
            assertTrue(bloom.exec(t1));
        }


        // A few that don't pass
        for (int i = 100; i < 10; i++) {
            Tuple t2 = tf.newTuple(1);
            t2.set(0, i);
            assertFalse(bloom.exec(t2));
        }
    }

View Full Code Here

        t.set(0, reducerBag);
        BuildBloom.Final reducer =
            new BuildBloom.Final(hashFunc, "fixed", size, numHash);
        DataByteArray dba = reducer.exec(t);


        Bloom bloom = new Bloom("bla");
        bloom.setFilter(dba);


        // Test that everything we put in passes.
        for (int i = 0; i < 3; i++) {
            for (int j = 0; j < 3; j++) {
                Tuple t1 = tf.newTuple(1);
                t1.set(0, i * 10 + j);
                assertTrue(bloom.exec(t1));
            }
        }


        // A few that don't pass
        for (int i = 100; i < 10; i++) {
            Tuple t1 = tf.newTuple(1);
            t1.set(0, i);
            assertFalse(bloom.exec(t1));
        }
    }

View Full Code Here

        t.set(0, reducerBag);
        BuildBloom.Final reducer =
            new BuildBloom.Final(hashFunc, numElements, falsePositive);
        DataByteArray dba = reducer.exec(t);


        Bloom bloom = new Bloom("bla");
        bloom.setFilter(dba);


        // Test that everything we put in passes.
        for (int i = 0; i < 3; i++) {
            for (int j = 0; j < 3; j++) {
                Tuple t1 = tf.newTuple(2);
                t1.set(0, i * 10 + j);
                t1.set(1, strs[i][j]);
                assertTrue(bloom.exec(t1));
            }
        }


        // A few that don't pass
        for (int i = 100; i < 10; i++) {
            Tuple t1 = tf.newTuple(2);
            t1.set(0, i);
            t1.set(1, "ichabod");
            assertFalse(bloom.exec(t1));
        }
    }}

View Full Code Here

TOP

Related Classes of org.apache.pig.builtin.Bloom

org.apache.hadoop.fs.Path

org.apache.hadoop.util.bloom.BloomFilter

org.apache.hadoop.util.bloom.Key

org.apache.pig.test.TestBloom

java.io.File

java.io.FilenameFilter

java.io.FileInputStream

java.io.ByteArrayInputStream

java.io.DataInputStream

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.