Package com.flaptor.util

Examples of com.flaptor.util.TextSignature

250
251
252
253
254
255
256
257
258
259
260
        bestPage.setLastAttempt(0);
        bestPage.setLastSuccess(0);
        bestPage.setLastChange(0);
        bestPage.setPriority(-Float.MAX_VALUE);
        bestPage.setEmitted(false);
        bestPage.setSignature(new TextSignature(""));
        bestPage.setAntiScore(0);
        boolean hasAntiScore = false;
        boolean unfetched = false;
        int inlinks = 0;
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49


    protected Pair<String,Integer> findGroup(Document doc) {
       
        TextSignature signature = new TextSignature(doc.get("text"));

        if (signatures.size() == 0 ) {
            signatures.add(signature);
            return new Pair<String,Integer>(criteria,0);
        }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
                page.setLastAttempt(Math.abs(rnd.nextLong()));
                long succ = (rnd.nextFloat() < 0.7f) ? Math.abs(rnd.nextLong()) : 0;
                page.setLastSuccess(succ);
                if (succ > 0) {
                    page.setLastChange(Math.abs(rnd.nextLong()));
                    page.setSignature(new TextSignature(TestUtils.randomText(100,500)));
                    page.setScore(rnd.nextFloat());
                    page.setEmitted(rnd.nextBoolean());
                    page.setLocal(rnd.nextBoolean());
                    page.setNumInlinks(rnd.nextInt(20));
                }
View Full Code Here
167
168
169
170
171
172
173
174
175
176
177
     * Determine if the page changed by analysing the text signature.
     */
    private void checkTextChanges() {
        textChanged = false;
        if (null != text) {
            TextSignature newSig = new TextSignature(text);
            textChanged = ! page.hasSimilarSignature(newSig);
            page.setSignature(newSig);
            if (textChanged) {
                page.setLastChange(System.currentTimeMillis());
            }
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
        numInlinks = 0;
        urlHash = new byte[16];
        anchors = new HashSet<String>();
        parents = new HashSet<String>();
        tags = new HashSet<String>();
        signature = new TextSignature("");
        try {
            digester = MessageDigest.getInstance("MD5");
        } catch (NoSuchAlgorithmException ex) {
            logger.error("Initializing message digester", ex);
        }
View Full Code Here
TOP

Related Classes of com.flaptor.util.TextSignature

  • com.flaptor.hounder.crawler.modules.FetchDocument
  • com.flaptor.hounder.crawler.pagedb.Page
  • com.flaptor.hounder.crawler.pagedb.PageTest
  • com.flaptor.hounder.crawler.PageDBTrimmer
  • com.flaptor.hounder.searcher.group.TextSignatureResultsGrouper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.