Examples of MongoInputSplit

Examples of com.mongodb.hadoop.input.MongoInputSplit

52
53
54
55
56
57
58
59
60
61
62
        shardsMap = this.getShardsMap();

        for (Map.Entry<String, String> entry : shardsMap.entrySet()) {
            String shardHosts = entry.getValue();

            MongoInputSplit chunkSplit = createSplitFromBounds(null, null);
            chunkSplit.setInputURI(rewriteURI(inputURI, shardHosts));
            returnVal.add(chunkSplit);
        }
        return returnVal;
    }
View Full Code Here

Examples of com.mongodb.hadoop.input.MongoInputSplit

93
94
95
96
97
98
99
    public static class MongoHiveInputSplit extends FileSplit {
        private InputSplit delegate;
        private Path path;

        MongoHiveInputSplit() {
            this(new MongoInputSplit());
        }
View Full Code Here

Examples of com.mongodb.hadoop.input.MongoInputSplit

157
158
159
160
161
162
163
164
165
166
            returnVal.add(createSplitFromBounds(lastKey, currentKey));
            lastKey = currentKey;
        }

        // Last max split, with empty upper boundary
        MongoInputSplit lastSplit = createSplitFromBounds(lastKey, null);
        returnVal.add(lastSplit);

        return returnVal;
    }
View Full Code Here

Examples of com.mongodb.hadoop.input.MongoInputSplit

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
        init();

        MongoClientURI inputURI = MongoConfigUtil.getInputURI(getConfiguration());
        LOG.info("SingleMongoSplitter calculating splits for " + inputURI);
        final List<InputSplit> splits = new ArrayList<InputSplit>();
        MongoInputSplit mongoSplit = new MongoInputSplit();

        mongoSplit.setInputURI(MongoConfigUtil.getInputURI(getConfiguration()));
        mongoSplit.setAuthURI(MongoConfigUtil.getAuthURI(getConfiguration()));
        mongoSplit.setQuery(MongoConfigUtil.getQuery(getConfiguration()));
        mongoSplit.setNoTimeout(MongoConfigUtil.isNoTimeout(getConfiguration()));
        mongoSplit.setFields(MongoConfigUtil.getFields(getConfiguration()));
        mongoSplit.setSort(MongoConfigUtil.getSort(getConfiguration()));

        //Not using any index min/max bounds, so range query is
        //meaningless here - don't set it
        //mongoSplit.setUseRangeQuery(...)
View Full Code Here

Examples of com.mongodb.hadoop.input.MongoInputSplit

92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
        while (cur.hasNext()) {
            final BasicDBObject row = (BasicDBObject) cur.next();
            BasicDBObject chunkLowerBound = (BasicDBObject) row.get("min");
            BasicDBObject chunkUpperBound = (BasicDBObject) row.get("max");
            MongoInputSplit chunkSplit = createSplitFromBounds(chunkLowerBound, chunkUpperBound);
            chunkSplit.setInputURI(inputURI);
            String shard = (String) row.get("shard");
            if (targetShards) {
                //The job is configured to target shards, so replace the
                //mongos hostname with the host of the shard's servers
                String shardHosts = shardsMap.get(shard);
                if (shardHosts == null) {
                    throw new SplitFailedException("Couldn't find shard ID: " + shard + " in config.shards.");
                }

                MongoClientURI newURI = rewriteURI(inputURI, shardHosts);
                chunkSplit.setInputURI(newURI);
            } else if (mongosHostNames.size() > 0) {
                //Multiple mongos hosts are specified, so
                //choose a host name in round-robin fashion
                //and rewrite the URI using that hostname.
                //This evenly distributes the load to avoid
                //pegging a single mongos instance.
                String roundRobinHost = mongosHostNames.get(numChunks % mongosHostNames.size());
                MongoClientURI newURI = rewriteURI(inputURI, roundRobinHost);
                chunkSplit.setInputURI(newURI);
            }
            LinkedList<InputSplit> shardList = shardToSplits.get(shard);
            if (shardList == null) {
                shardList = new LinkedList<InputSplit>();
                shardToSplits.put(shard, shardList);
View Full Code Here

Examples of com.mongodb.hadoop.mapred.input.MongoInputSplit

39
40
41
42
43
44
45
46
47
                                                                    JobConf job,
                                                                    Reporter reporter) {
        if (!(split instanceof MongoInputSplit))
            throw new IllegalStateException("Creation of a new RecordReader requires a MongoInputSplit instance.");

        final MongoInputSplit mis = (MongoInputSplit) split;

        return new MongoRecordReader(mis);
    }
View Full Code Here

Examples of com.mongodb.hadoop.mapred.input.MongoInputSplit

51
52
53
54
55
56
57
58
59
        // TODO - Support allowing specification of numSplits to affect our ops?
        final List<org.apache.hadoop.mapreduce.InputSplit> splits = MongoSplitter.calculateSplits( conf );
        // TODO - Make me less egregiously inefficient.
        InputSplit[] classicSplits = new InputSplit[splits.size()];
        for ( int i = 0; i < splits.size(); i++ ) {
            classicSplits[i] = new MongoInputSplit( (com.mongodb.hadoop.input.MongoInputSplit) splits.get( i ) );
        }
        return classicSplits;
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.