Examples of com.flaptor.util.sort.RecordReader

        MergeSort.sort(beforeSort, afterSort, tcp, null);
        logger.info("Sorting categoryTokenCount.... done");


        logger.info("Folding categoryTokenCount.... ");
        // Now saves all this data to a FileCache        
        RecordReader tcrr= tcp.newRecordReader(afterSort);                
        TokenCounter catTc= new TokenCounter();
        TokenCounter nonCatTc= new TokenCounter();
        TokenCounterPersistence.TCRecord tcr= (TokenCounterPersistence.TCRecord) tcrr.readRecord();
        if (null==tcrr) return;
        catTc.update(tcr.getCatVal());
        nonCatTc.update(tcr.getNonCatVal());
        String prevToken= tcr.getToken();        
        while (null != (tcr= (TokenCounterPersistence.TCRecord) tcrr.readRecord())){
            if (tcr.getToken().equals(prevToken)){
                catTc.update(tcr.getCatVal());
                nonCatTc.update(tcr.getNonCatVal());
            } else {
                computeProbs(prevToken, catTc, nonCatTc, tokenProbabilities);

        WhoHasPersistence whp= new WhoHasPersistence();
        MergeSort.sort(beforeSort, afterSort, whp, null);
        logger.info("Sorting WhoHasPersistence... done");


        logger.info("Writing WhoHas Total to ... " + WHO_HAS_FILE_DONE);
        RecordReader whrr= whp.newRecordReader(afterSort); 
        WhoHasPersistence.WHRecord whr= (WhoHasPersistence.WHRecord) whrr.readRecord();        
        if (null==whr) return;
        
//        FileUtil.deleteFile(WHO_HAS_FILE_DONE);
        FileWriter whDone= new FileWriter(WHO_HAS_FILE_DONE);


        HashSet<String> urls= new HashSet<String>();
        for (String url: whr.getUrls()){
            urls.add(url);
        }
        String prevToken= whr.getToken();
        while (null != (whr= (WhoHasPersistence.WHRecord) whrr.readRecord())){
            if (whr.getToken().equals(prevToken)){
                for (String url: whr.getUrls()){
                    urls.add(url);
                }
            } else {

Examples of com.flaptor.util.sort.RecordReader

Related Classes of com.flaptor.util.sort.RecordReader