Examples of org.apache.nutch.crawl.TextProfileSignature$Token

org.openstack.docs.identity.api.v2.Token
An implementation of a page signature. It calculates an MD5 hash of a plain text "profile" of a page. In case there is no text, it calculates a hash using the {@link MD5Signature}.

The algorithm to calculate a page "profile" takes the plain text version of a page and performs the following steps:
- remove all characters except letters and digits, and bring all characters to lower case,
- split the text into tokens (all consecutive non-whitespace characters),
- discard tokens equal or shorter than MIN_TOKEN_LEN (default 2 characters),
- sort the list of tokens by decreasing frequency,
- round down the counts of tokens to the nearest multiple of QUANT (QUANT = QUANT_RATE * maxFreq, where QUANT_RATE is 0.01f by default, and maxFreq is the maximum token frequency). If maxFreq is higher than 1, then QUANT is always higher than 2 (which means that tokens with frequency 1 are always discarded).
- tokens, which frequency after quantization falls below QUANT, are discarded.
- create a list of tokens and their quantized frequency, separated by spaces, in the order of decreasing frequency.
This list is then submitted to an MD5 hash calculation. @author Andrzej Bialecki <ab@getopt.org>

  public void setTokenStopIndex(int index) {
  }


  public String toString() {
    if ( payload instanceof Token ) {
      Token t = (Token)payload;
      if ( t.getType() == Token.EOF ) {
        return "<EOF>";
      }
      return t.getText();
    }
    return payload.toString();
  }

View Full Code Here

   */
  public String toStringWithHiddenTokens() {
    StringBuffer buf = new StringBuffer();
    if ( hiddenTokens!=null ) {
      for (int i = 0; i < hiddenTokens.size(); i++) {
        Token hidden = (Token) hiddenTokens.get(i);
        buf.append(hidden.getText());
      }
    }
    String nodeText = this.toString();
    if ( !nodeText.equals("<EOF>") ) buf.append(nodeText);
    return buf.toString();

View Full Code Here

    }


    // Wildcard?
    if ( ttype == TreePatternLexer.DOT ) {
      ttype = tokenizer.nextToken();
      Token wildcardPayload = new CommonToken(0, ".");
      TreeWizard.TreePattern node =
        new TreeWizard.WildcardTreePattern(wildcardPayload);
      if ( label!=null ) {
        node.label = label;
      }

View Full Code Here

    Tree t = (Tree)create(fromToken);
    return t;
  }


  public Object create(int tokenType, String text) {
    Token fromToken = createToken(tokenType, text);
    Tree t = (Tree)create(fromToken);
    return t;
  }

View Full Code Here

  public void consume() {
    if ( initialStreamState ) {
      consumeInitialHiddenTokens();
    }
    int a = input.index();
    Token t = input.LT(1);
    input.consume();
    int b = input.index();
    dbg.consumeToken(t);
    if ( b>a+1 ) {
      // then we consumed more than one token; must be off channel tokens

View Full Code Here

    super(adaptor, elementDescription, elements);
  }


  /** Get next token from stream and make a node for it */
  public Object nextNode() {
    Token t = (Token)_next();
    return adaptor.create(t);
  }

View Full Code Here

  }
*/
  public void process() throws IOException {
    CharStream input = new ANTLRFileStream(dispelFile);
    DISPEL lexer = new DISPEL(input);
    Token token;
    while ((token = lexer.nextToken()) != Token.EOF_TOKEN) {
      tp.process(token);
    }
  }

View Full Code Here

    CharStream input;
    try {
      input = new ANTLRInputStream(is);
      
      DISPEL lexer = new DISPEL(input);
      Token token;
        
        while ((token = lexer.nextToken()).getType() !=
          org.antlr.runtime3_3_0.Token.EOF) {
    
          tp.process(token);

View Full Code Here

        client.listTenants(fakeToken);
    }


    private KeystoneAuthenticationToken buildFakeToken(String tokenCode) {
        Access auth = new Access();
        Token tokenObject = new Token();
        tokenObject.setId(tokenCode);
        auth.setToken(tokenObject);
        return new KeystoneAuthenticationToken(auth);
    }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.nutch.crawl.TextProfileSignature$Token

Analyzer.Lexical.LexicalAnalyzer

Analyzer.Syntax.AnalizadorRecursivoPaseo

Analyzer.Syntax.Interpreter

com.drighetto.essai.bouncycastle.Sample01

com.go.trove.io.SourceInfo

com.google.collide.client.code.autocomplete.codegraph.CodeGraphAutocompleterTest$MockProposalBuilder

com.google.collide.client.code.autocomplete.codegraph.ExplicitAutocompleterTest

com.google.collide.client.code.autocomplete.codegraph.js.JsIndexUpdater

com.google.collide.client.code.autocomplete.codegraph.js.JsParsingTask

com.google.collide.client.code.autocomplete.codegraph.JsIndexUpdaterTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.