Examples of com.mongodb.hadoop.splitter.BSONSplitter

    public FileSplit[] getSplits(final JobConf job, final int numSplits) throws IOException {


        FileStatus[] inputFiles = listStatus(job);
        List<FileSplit> results = new ArrayList<FileSplit>();
        for (FileStatus file : inputFiles) {
            BSONSplitter splitter = new BSONSplitter();
            splitter.setConf(job);
            splitter.setInputPath(file.getPath());
            Path splitFilePath;
            splitFilePath = new Path(file.getPath().getParent(), "." + file.getPath().getName() + ".splits");
            try {
                splitter.loadSplitsFromSplitFile(file, splitFilePath);
            } catch (BSONSplitter.NoSplitFileException nsfe) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(String.format("No split file for %s; building split file", file.getPath()));
                }
                splitter.readSplitsForFile(file);
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug(format("BSONSplitter found %d splits.", splitter.getAllSplits().size()));
            }


            for (org.apache.hadoop.mapreduce.lib.input.FileSplit split : splitter.getAllSplits()) {
                FileSplit fsplit =
                    new FileSplit(split.getPath(),
                                  split.getStart(),
                                  split.getLength(),
                                  split.getLocations());

                if (LOG.isDebugEnabled()) {
                    LOG.debug("processing file " + file.getPath());
                }
            }


            BSONSplitter splitter = new BSONSplitter();
            splitter.setConf(config);
            splitter.setInputPath(file.getPath());
            Path splitFilePath = new Path(file.getPath().getParent(), "." + file.getPath().getName() + ".splits");
            try {
                splitter.loadSplitsFromSplitFile(file, splitFilePath);
            } catch (BSONSplitter.NoSplitFileException nsfe) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(String.format("No split file for %s; building split file", file.getPath()));
                }
                splitter.readSplitsForFile(file);
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format("BSONSplitter found %d splits.", splitter.getAllSplits().size()));
            }
            splits.addAll(splitter.getAllSplits());
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug(String.format("Total of %d found.", splits.size()));
        }
        return splits;

Examples of com.mongodb.hadoop.splitter.BSONSplitter

Related Classes of com.mongodb.hadoop.splitter.BSONSplitter