Package org.apache.cassandra.utils

Examples of org.apache.cassandra.utils.BloomFilter

56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
            assert keyInDisk.equals(decoratedKey)
                   : String.format("%s != %s in %s", keyInDisk, decoratedKey, file.getPath());
            file.readInt(); // data size

            /* Read the bloom filter and index summarizing the columns */
            BloomFilter bf = IndexHelper.defreezeBloomFilter(file);
            List<IndexHelper.IndexInfo> indexList = IndexHelper.deserializeIndex(file);

            cf = ColumnFamily.serializer().deserializeFromSSTableNoColumns(ssTable.makeColumnFamily(), file);

            // we can stop early if bloom filter says none of the columns actually exist -- but,
            // we can't stop before initializing the cf above, in case there's a relevant tombstone
            List<byte[]> filteredColumnNames = new ArrayList<byte[]>(columnNames.size());
            for (byte[] name : columnNames)
            {
                if (bf.isPresent(name))
                {
                    filteredColumnNames.add(name);
                }
            }
            if (filteredColumnNames.isEmpty())
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
   * @throws IOException
   */
    public static void serialize(ColumnFamily columnFamily, DataOutput dos)
  {
        Collection<IColumn> columns = columnFamily.getSortedColumns();
        BloomFilter bf = createColumnBloomFilter(columns);                   
        /* Write out the bloom filter. */
        DataOutputBuffer bufOut = new DataOutputBuffer();
        try
        {
            BloomFilter.serializer().serialize(bf, bufOut);
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
        for (IColumn column : columns)
        {
            columnCount += column.getObjectCount();
        }

        BloomFilter bf = BloomFilter.getFilter(columnCount, 4);
        for (IColumn column : columns)
        {
            bf.add(column.name());
            /* If this is SuperColumn type Column Family we need to get the subColumns too. */
            if (column instanceof SuperColumn)
            {
                Collection<IColumn> subColumns = column.getSubColumns();
                for (IColumn subColumn : subColumns)
                {
                    bf.add(subColumn.name());
                }
            }
        }
        return bf;
    }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
        File f = File.createTempFile("sstable", "");
        SSTable ssTable;

        // write test data
        ssTable = new SSTable(f.getParent(), f.getName(), new OrderPreservingPartitioner());
        BloomFilter bf = new BloomFilter(1000, 8);
        Random random = new Random();
        byte[] bytes = new byte[1024];
        random.nextBytes(bytes);

        String key = Integer.toString(1);
        ssTable.append(key, bytes);
        bf.add(key);
        ssTable.close(bf);

        // verify
        SSTable.indexMetadataMap_.clear(); // force reloading the index
        ssTable = new SSTable(f.getPath() + "-Data.db", new OrderPreservingPartitioner());
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
            map.put(Integer.toString(i), ("Avinash Lakshman is a good man: " + i).getBytes());
        }

        // write
        ssTable = new SSTable(f.getParent(), f.getName(), new OrderPreservingPartitioner());
        BloomFilter bf = new BloomFilter(1000, 8);
        for (String key: map.navigableKeySet())
        {
            ssTable.append(key, map.get(key));
        }
        ssTable.close(bf);
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
        IFileWriter indexWriter = SequenceFile.bufferedWriter(indexFile, bufferSize);
        IFileReader dataReader = SequenceFile.bufferedReader(dataFile, bufferSize);
        DataOutputBuffer bufOut = new DataOutputBuffer();
        DataInputBuffer bufIn = new DataInputBuffer();
        /* BloomFilter of all data in the data file */
        BloomFilter bf = new BloomFilter((SSTable.indexInterval() + 1)*blockCount, 8);

        try
        {
            while ( !dataReader.isEOF() )
            {
                bufOut.reset();
                /* Record the position of the key. */
                long blockIndexOffset = dataReader.getCurrentPosition();
                dataReader.next(bufOut);
                bufIn.reset(bufOut.getData(), bufOut.getLength());
                /* Key just read */
                String key = bufIn.readUTF();
                if ( key.equals(SSTable.blockIndexKey_) )
                {
                    /* Ignore the size of the data associated with the block index */
                    bufIn.readInt();
                    /* Number of keys in the block. */
                    int blockSize = bufIn.readInt();
                    /* Largest key in the block */
                    String largestKey = null;

                    /*
                     * Read the keys in this block and find the largest key in
                     * this block. This is the key that gets written into the
                     * index file.
                    */
                    for ( int i = 0; i < blockSize; ++i )
                    {
                        String currentKey = bufIn.readUTF();
                        bf.add(currentKey);
                        if ( largestKey == null )
                        {
                            largestKey = currentKey;
                        }
                        else
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
        }
       
        File directory = new File(args[0]);
        File[] files = directory.listFiles();
        List<DataOutputBuffer> buffers = new ArrayList<DataOutputBuffer>();   
        BloomFilter bf = new BloomFilter(count_, 8);       
        int keyCount = 0;
       
        /* Process the list of files. */
        for ( File file : files )
        {
            System.out.println("Processing file " + file);
            BufferedReader bufReader = new BufferedReader( new InputStreamReader( new FileInputStream(file) ), ThreadListBuilder.bufSize_ );
            String line = null;
           
            while ( (line = bufReader.readLine()) != null )
            {
                /* After accumulating count_ keys reset the bloom filter. */
                if ( keyCount > 0 && keyCount % count_ == 0 )
                {                      
                    DataOutputBuffer bufOut = new DataOutputBuffer();
                    BloomFilter.serializer().serialize(bf, bufOut);
                    System.out.println("Finished serializing the bloom filter");
                    buffers.add(bufOut);
                    bf = new BloomFilter(count_, 8);
                }
                line = line.trim();               
                bf.add(line);
                ++keyCount;
            }
        }
       
        /* Add the bloom filter assuming the last one was left out */
 
View Full Code Here
628
629
630
631
632
633
634
635
            int size = file_.readInt();
            byte[] bytes = new byte[size];
            file_.readFully(bytes);
            DataInputBuffer bufIn = new DataInputBuffer();
            bufIn.reset(bytes, bytes.length);
            BloomFilter bf = BloomFilter.serializer().deserialize(bufIn);
            return bf;
        }
View Full Code Here
841
842
843
844
845
846
847
848
849
850
851
            }
            else
            {
                /* Read the bloom filter summarizing the columns */
                long preBfPos = file_.getFilePointer();
                BloomFilter bf = defreezeBloomFilter();
                long postBfPos = file_.getFilePointer();
                dataSize -= (postBfPos - preBfPos);

                List<IndexHelper.ColumnIndexInfo> columnIndexList = new ArrayList<IndexHelper.ColumnIndexInfo>();
                /* read the column name indexes if present */
 
View Full Code Here
306
307
308
309
310
311
312
313
314
315
316
317
     * key is not present then we skip processing this file.
    */
    public static boolean isKeyInFile(String clientKey, String filename)
    {
        boolean bVal = false;
        BloomFilter bf = bfs_.get(filename);
        if (bf != null)
        {
            bVal = bf.isPresent(clientKey);
        }
        return bVal;
    }
View Full Code Here
TOP

Related Classes of org.apache.cassandra.utils.BloomFilter

  • org.apache.cassandra.db.BinaryMemtable
  • org.apache.cassandra.db.ColumnFamilyStore
  • org.apache.cassandra.db.ColumnIndexer
  • org.apache.cassandra.db.columniterator.SSTableNamesIterator
  • org.apache.cassandra.db.filter.SSTableNamesIterator
  • org.apache.cassandra.db.Memtable
  • org.apache.cassandra.io.SequenceFile$AbstractReader
  • org.apache.cassandra.io.SSTable
  • org.apache.cassandra.io.SSTableTest
  • org.apache.cassandra.io.SSTableWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.