Package com.netflix.bdp.s3mper.metastore.impl

Source Code of com.netflix.bdp.s3mper.metastore.impl.PathScannerTask

/*
*
*  Copyright 2013 Netflix, Inc.
*
*     Licensed under the Apache License, Version 2.0 (the "License");
*     you may not use this file except in compliance with the License.
*     You may obtain a copy of the License at
*
*         http://www.apache.org/licenses/LICENSE-2.0
*
*     Unless required by applicable law or agreed to in writing, software
*     distributed under the License is distributed on an "AS IS" BASIS,
*     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*     See the License for the specific language governing permissions and
*     limitations under the License.
*
*/


package com.netflix.bdp.s3mper.metastore.impl;

import com.amazonaws.services.dynamodb.AmazonDynamoDBClient;
import com.amazonaws.services.dynamodb.model.AttributeValue;
import com.amazonaws.services.dynamodb.model.ComparisonOperator;
import com.amazonaws.services.dynamodb.model.Condition;
import com.amazonaws.services.dynamodb.model.Key;
import com.amazonaws.services.dynamodb.model.ScanRequest;
import com.amazonaws.services.dynamodb.model.ScanResult;
import com.google.common.util.concurrent.RateLimiter;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import org.apache.log4j.Logger;

/**
* Scans entries by path in DynamoDb using the epoch field to determine their age and places
* old entries into a queue.
*
* @author dweeks
*/
public class PathScannerTask extends AbstractScannerTask {
    private static final Logger log = Logger.getLogger(PathScannerTask.class.getName());
   
    private final BlockingQueue<Key> deleteQueue;
    private long age;
    private int queueSize;
   
    public PathScannerTask(AmazonDynamoDBClient db, RateLimiter limiter, BlockingQueue<Key> deleteQueue, int queueSize, long age) {
        super(db, limiter);
        this.deleteQueue = deleteQueue;
        this.age = age;
        this.queueSize = queueSize;
    }

    @Override
    public Object call() throws Exception {
        running = true;
       
        long deleteEpoch = System.currentTimeMillis() - age;
       
        Map<String, Condition> filter = new HashMap<String, Condition>();
       
        AttributeValue value = new AttributeValue();
        value.setN(deleteEpoch + "");
       
        Condition c = new Condition();
        c.setComparisonOperator(ComparisonOperator.LT);
        c.setAttributeValueList(Collections.singletonList(value));
       
        filter.put("epoch", c);
       
        ScanRequest scan = new ScanRequest(MetastoreJanitor.tableName);
        scan.setScanFilter(filter);
        scan.setLimit( (int) limiter.getRate());
       
        ScanResult result;
       
        int scanTotal = 0;
        int matched = 0;
       
        do {
            //Can't set a hard limit on the queue since paths can be resubmitted by delete task
            synchronized (deleteQueue) {
                while (deleteQueue.size() >= queueSize) {
                    deleteQueue.wait();
                }
            }
           
            if(!running) {
                break;
            }
           
            result = db.scan(scan);
            scanTotal += result.getScannedCount();
            matched += result.getCount();
           
            log.info(String.format("Total scanned: %d, matched: %d, added: %d, queue size: %d, consumed capacity: %f, max rate: %f", scanTotal, matched, result.getCount(), deleteQueue.size(), result.getConsumedCapacityUnits(), limiter.getRate()));
           
            for (Map<String, AttributeValue> i : result.getItems()) {
                if (!i.containsKey("epoch")) {
                    continue;
                }
                deleteQueue.put(new Key(i.get(DynamoDBMetastore.HASH_KEY), i.get(DynamoDBMetastore.RANGE_KEY)));
            }
           
            limiter.acquire(result.getConsumedCapacityUnits().intValue());
           
            scan.setExclusiveStartKey(result.getLastEvaluatedKey());
        } while (running && result.getLastEvaluatedKey() != null);
       
        return Boolean.TRUE;
    }
   
}
TOP

Related Classes of com.netflix.bdp.s3mper.metastore.impl.PathScannerTask

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.