Examples of MemoryLimits

Examples of org.apache.pig.data.SelfSpillBag.MemoryLimits

215
216
217
218
219
220
221
222
223
224
225
    private void estimateMemThresholds() {
        if (!mapAggDisabled()) {
            LOG.info("Getting mem limits; considering " + ALL_POPARTS.size() + " POPArtialAgg objects.");

            float percent = getPercentUsageFromProp();
            memLimits = new MemoryLimits(ALL_POPARTS.size(), percent);
            int estTotalMem = 0;
            int estTuples = 0;
            for (Map.Entry<Object, List<Tuple>> entry : rawInputMap.entrySet()) {
                for (Tuple t : entry.getValue()) {
                    estTuples += 1;
View Full Code Here

Examples of org.apache.pig.data.SelfSpillBag.MemoryLimits

269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
    private void estimateMemThresholds() {
        if (!mapAggDisabled()) {
            LOG.info("Getting mem limits; considering " + ALL_POPARTS.size()
                    + " POPArtialAgg objects." + " with memory percentage "
                    + percentUsage);
            MemoryLimits memLimits = new MemoryLimits(ALL_POPARTS.size(), percentUsage);
            int estTotalMem = 0;
            int estTuples = 0;
            for (Map.Entry<Object, List<Tuple>> entry : rawInputMap.entrySet()) {
                for (Tuple t : entry.getValue()) {
                    estTuples += 1;
                    int mem = (int) t.getMemorySize();
                    estTotalMem += mem;
                    memLimits.addNewObjSize(mem);
                }
            }
            avgTupleSize = estTotalMem / estTuples;
            long totalTuples = memLimits.getCacheLimit();
            LOG.info("Estimated total tuples to buffer, based on " + estTuples + " tuples that took up " + estTotalMem + " bytes: " + totalTuples);
            firstTierThreshold = (int) (0.5 + totalTuples * (1f - (1f / sizeReduction)));
            secondTierThreshold = (int) (0.5 + totalTuples *  (1f / sizeReduction));
            LOG.info("Setting thresholds. Primary: " + firstTierThreshold + ". Secondary: " + secondTierThreshold);
            // The second tier should at least allow one tuple before it tries to aggregate.
View Full Code Here

Examples of org.apache.pig.data.SelfSpillBag.MemoryLimits

215
216
217
218
219
220
221
222
223
224
225
    private void estimateMemThresholds() {
        if (!mapAggDisabled()) {
            LOG.info("Getting mem limits; considering " + ALL_POPARTS.size() + " POPArtialAgg objects.");

            float percent = getPercentUsageFromProp();
            memLimits = new MemoryLimits(ALL_POPARTS.size(), percent);
            int estTotalMem = 0;
            int estTuples = 0;
            for (Map.Entry<Object, List<Tuple>> entry : rawInputMap.entrySet()) {
                for (Tuple t : entry.getValue()) {
                    estTuples += 1;
View Full Code Here

Examples of org.apache.pig.data.SelfSpillBag.MemoryLimits

341
342
343
344
345
346
347
348
349
        aggMap = new HashMap<Object, Tuple>();

        // TODO: keep track of actual number of objects that share the
        // memory limit. For now using a default of 3, which is what is
        // used by InternalCachedBag
        memLimits = new MemoryLimits(3, -1);
        maxHashMapSize = Integer.MAX_VALUE;

    }
View Full Code Here

Examples of org.apache.pig.data.SelfSpillBag.MemoryLimits

214
215
216
217
218
219
220
221
222
223
224
    private void estimateMemThresholds() {
        if (!mapAggDisabled()) {
            LOG.info("Getting mem limits; considering " + ALL_POPARTS.size() + " POPArtialAgg objects.");

            float percent = getPercentUsageFromProp();
            memLimits = new MemoryLimits(ALL_POPARTS.size(), percent);
            int estTotalMem = 0;
            int estTuples = 0;
            for (Map.Entry<Object, List<Tuple>> entry : rawInputMap.entrySet()) {
                for (Tuple t : entry.getValue()) {
                    estTuples += 1;
View Full Code Here

Examples of org.apache.pig.data.SelfSpillBag.MemoryLimits

215
216
217
218
219
220
221
222
223
224
225
    private void estimateMemThresholds() {
        if (!mapAggDisabled()) {
            LOG.info("Getting mem limits; considering " + ALL_POPARTS.size() + " POPArtialAgg objects.");

            float percent = getPercentUsageFromProp();
            memLimits = new MemoryLimits(ALL_POPARTS.size(), percent);
            int estTotalMem = 0;
            int estTuples = 0;
            for (Map.Entry<Object, List<Tuple>> entry : rawInputMap.entrySet()) {
                for (Tuple t : entry.getValue()) {
                    estTuples += 1;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.