Package org.apache.pig.backend.hadoop.executionengine.tez.runtime

Examples of org.apache.pig.backend.hadoop.executionengine.tez.runtime.ObjectCache


    }

    @SuppressWarnings("unchecked")
    private void init() throws RuntimeException {

        ObjectCache cache = ObjectCache.getInstance();
        String isCachedKey = "sample-" + PigProcessor.sampleVertex + ".cached";
        String totalReducersCacheKey = "sample-" + PigProcessor.sampleVertex + ".totalReducers";
        String reducerMapCacheKey = "sample-" + PigProcessor.sampleVertex + ".reducerMap";
        if (cache.retrieve(isCachedKey) == Boolean.TRUE) {
            totalReducers = (Integer) cache.retrieve(totalReducersCacheKey);
            reducerMap = (Map<Object, Pair<Integer, Integer>>) cache.retrieve(reducerMapCacheKey);
            LOG.info("Found totalReducers and reducerMap in Tez cache. cachekey="
                    + totalReducersCacheKey + "," + reducerMapCacheKey);
            inited = true;
            return;
        }

        Map<String, Object> distMap = null;
        if (PigProcessor.sampleMap != null) {
            // We've already collected sampleMap in PigProcessor
            distMap = PigProcessor.sampleMap;
        } else {
            LOG.info("Key distribution map is empty");
            inited = true;
            return;
        }

        long start = System.currentTimeMillis();

        try {
            // The distMap is structured as (key, min, max) where min, max
            // being the index of the reducers
            DataBag partitionList = (DataBag) distMap.get(PartitionSkewedKeys.PARTITION_LIST);
            totalReducers = Integer.valueOf("" + distMap.get(PartitionSkewedKeys.TOTAL_REDUCERS));
            Iterator<Tuple> it = partitionList.iterator();
            while (it.hasNext()) {
                Tuple idxTuple = it.next();
                Integer maxIndex = (Integer) idxTuple.get(idxTuple.size() - 1);
                Integer minIndex = (Integer) idxTuple.get(idxTuple.size() - 2);
                // Used to replace the maxIndex with the number of reducers
                if (maxIndex < minIndex) {
                    maxIndex = totalReducers + maxIndex;
                }

                Object keyT;
                // if the join is on more than 1 key
                if (idxTuple.size() > 3) {
                // remove the last 2 fields of the tuple, i.e: minIndex
                // and maxIndex and store it in the reducer map
                Tuple keyTuple = tf.newTuple();
                for (int i=0; i < idxTuple.size() - 2; i++) {
                    keyTuple.append(idxTuple.get(i));
                }
                    keyT = keyTuple;
                } else {
                    keyT = idxTuple.get(0);
                }
                // number of reducers
                Integer cnt = maxIndex - minIndex;
                // 1 is added to account for the 0 index
                reducerMap.put(keyT, new Pair<Integer, Integer>(minIndex, cnt));
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }

        LOG.info("Initialized POPartitionRearrangeTez. Time taken: " + (System.currentTimeMillis() - start));
        cache.cache(isCachedKey, Boolean.TRUE);
        cache.cache(totalReducersCacheKey, totalReducers);
        cache.cache(reducerMapCacheKey, reducerMap);
        inited = true;
    }
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.tez.runtime.ObjectCache

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.