Package voldemort.utils

Examples of voldemort.utils.FnvHashFunction


        getRouter(0, 3).getNodes().containsAll(getTestNodes());
    }

    public void testTagAssignment() {
        List<Node> nodes = getTestNodes();
        ConsistentRoutingStrategy router = getRouter(new FnvHashFunction(), 3);
        for(Node n: nodes)
            for(Integer tag: n.getPartitionIds())
                assertEquals(router.getNodeByPartition(tag), n);
        for(int i = 0; i < nodes.size(); i++)
            assertEquals("Unexpected tag assignment for tag " + i + ": ",
View Full Code Here


        // use a seed so that this test is repeatable
        Random random = new Random(2158745224L);
        Collections.shuffle(nodes, random);

        ConsistentRoutingStrategy router = new ConsistentRoutingStrategy(new FnvHashFunction(),
                                                                         nodes,
                                                                         replicationFactor);
        for(Node n: nodes)
            assertEquals(tagsPerNode, router.getPartitionsByNode(n).size());
View Full Code Here

        if(argv.length != 1) {
            System.err.println("USAGE: java voldemort.partition.FnvHashFunctionTester filename");
            System.exit(1);
        }

        FnvHashFunction hash = new FnvHashFunction();
        BufferedReader reader = new BufferedReader(new FileReader(argv[0]));
        while(true) {
            String line = reader.readLine();
            if(line == null)
                break;
            System.out.println(hash.hash(line.trim().getBytes()));
        }

        reader.close();

    }
View Full Code Here

    private HashMap<Integer, Integer> zoneReplicationFactor;

    public ZoneRoutingStrategy(Cluster cluster,
                               HashMap<Integer, Integer> zoneReplicationFactor,
                               int numReplicas) {
        this(new FnvHashFunction(), cluster, zoneReplicationFactor, numReplicas);
    }
View Full Code Here

    private final HashFunction hash;

    private static final Logger logger = Logger.getLogger(ConsistentRoutingStrategy.class);

    public ConsistentRoutingStrategy(Cluster cluster, int numReplicas) {
        this(new FnvHashFunction(), cluster, numReplicas);
    }
View Full Code Here

    @Override
    public void transfer() throws Exception {
        cursor = srcDB.openCursor(null, null);
        DatabaseEntry keyEntry = new DatabaseEntry();
        DatabaseEntry valueEntry = new DatabaseEntry();
        HashFunction hash = new FnvHashFunction();
        int totalPartitions = cluster.getNumberOfPartitions();

        List<Versioned<byte[]>> vals;
        long startTime = System.currentTimeMillis();
        int scanCount = 0;
        int keyCount = 0;
        while(cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
            keyCount++;

            vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
            scanCount += vals.size();

            int partition = BdbConvertData.abs(hash.hash(keyEntry.getData()))
                            % (Math.max(1, totalPartitions));

            OperationStatus putStatus = dstDB.put(null,
                                                  new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(keyEntry.getData(),
                                                                                                      partition)),
View Full Code Here

        DatabaseEntry keyEntry = new DatabaseEntry();
        DatabaseEntry valueEntry = new DatabaseEntry();

        byte[] prevKey = null;
        List<Versioned<byte[]>> vals = new ArrayList<Versioned<byte[]>>();
        HashFunction hash = new FnvHashFunction();
        int totalPartitions = cluster.getNumberOfPartitions();

        long startTime = System.currentTimeMillis();
        int scanCount = 0;
        int keyCount = 0;
        while(cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
            scanCount++;
            if(scanCount % 1000000 == 0)
                logger.info("Converted " + scanCount + " entries in "
                            + (System.currentTimeMillis() - startTime) / 1000 + " secs");

            // read the value as a versioned Object
            VectorClock clock = new VectorClock(valueEntry.getData());
            byte[] bytes = ByteUtils.copy(valueEntry.getData(),
                                          clock.sizeInBytes(),
                                          valueEntry.getData().length);
            Versioned<byte[]> value = new Versioned<byte[]>(bytes, clock);
            byte[] key = keyEntry.getData();

            if(prevKey != null && (ByteUtils.compare(prevKey, key) != 0)) {
                // there is a new key; write out the buffered values and
                // previous key
                int partition = BdbConvertData.abs(hash.hash(prevKey))
                                % (Math.max(1, totalPartitions));

                OperationStatus putStatus = dstDB.put(null,
                                                      new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(prevKey,
                                                                                                          partition)),
                                                      new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
                if(OperationStatus.SUCCESS != putStatus) {
                    String errorStr = "Put failed with " + putStatus + " for key"
                                      + BdbConvertData.writeAsciiString(prevKey);
                    logger.error(errorStr);
                    throw new Exception(errorStr);
                }
                vals = new ArrayList<Versioned<byte[]>>();
                keyCount++;
            }

            vals.add(value);
            prevKey = key;
        }
        if(vals.size() > 0) {
            int partition = BdbConvertData.abs(hash.hash(prevKey)) % (Math.max(1, totalPartitions));
            OperationStatus putStatus = dstDB.put(null,
                                                  new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(prevKey,
                                                                                                      partition)),
                                                  new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
            if(OperationStatus.SUCCESS != putStatus) {
View Full Code Here

    public ScrambledZipfianGenerator(long min, long max, double zipfianConstant) {
        this.min = min;
        this.max = max;
        this.itemCount = this.max - this.min + 1;
        generator = new ZipfianGenerator(0, ITEM_COUNT, zipfianConstant, ZETAN);
        hash = new FnvHashFunction();
    }
View Full Code Here

TOP

Related Classes of voldemort.utils.FnvHashFunction

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.