Package org.python.pydev.core

Examples of org.python.pydev.core.IToken


                    addToResult(result, message);

                } else {
                    //the generator token has many associated messages - the messages may have different types,
                    //so, we need to get them by types
                    IToken generator = message.getGenerator();
                    CompositeMessage compositeMessage;
                    if (generator != null) {
                        compositeMessage = new CompositeMessage(message.getType(), generator, prefs);
                    } else {
                        compositeMessage = new CompositeMessage(message.getType(), message.getStartLine(document),
View Full Code Here


     * @param result
     * @param message
     */
    private void addToResult(List<IMessage> result, IMessage message) {
        if (isUnusedImportMessage(message.getType())) {
            IToken generator = message.getGenerator();
            if (generator instanceof SourceToken) {
                String asAbsoluteImport = generator.getAsAbsoluteImport();
                if (asAbsoluteImport.indexOf("__future__.") != -1 || asAbsoluteImport.indexOf("__metaclass__") != -1) {
                    //do not add from __future__ import xxx
                    return;
                }
            }
View Full Code Here

            requireTokensToBeImports = true;
        }

        ScopeItems m = scope.peek();
        for (Iterator<IToken> iter = list.iterator(); iter.hasNext();) {
            IToken o = iter.next();
            //System.out.println("adding: "+o.getRepresentation());
            Found found = addToken(generator, m, o, o.getRepresentation());
            if (withinExceptNode != null) {
                withinExceptNode.addFoundImportToTryExcept(found); //may mark previous as used...
            }

            //the token that we find here is either an import (in the case of some from xxx import yyy or import aa.bb)
            //or a Name, ClassDef, MethodDef, etc. (in the case of wild imports)
            if (requireTokensToBeImports) {
                if (!o.isImport()) {
                    throw new RuntimeException("Expecting import token");
                }
                importInfo = importChecker.visitImportToken(o, reportUndefinedImports, completionCache);
            }
            //can be either the one resolved in the wild import or in this token (if it is not a wild import)
View Full Code Here

        for (Iterator<Object> iter = iTokenList.iterator(); iter.hasNext();) {

            Object obj = iter.next();

            if (obj instanceof IToken) {
                IToken element = (IToken) obj;

                String name = element.getRepresentation();

                //GET the ARGS
                int l = name.length();

                String args = "";
                if (!importsTip) {
                    boolean getIt = true;
                    if (AbstractToken.isClassDef(element)) {
                        if (!request.isInCalltip) {
                            getIt = false;
                        }
                    }
                    if (getIt) {
                        args = getArgs(element, state);
                        if (args.length() > 0) {
                            l++; //cursor position is name + '('
                        }
                    }
                }
                //END

                if (name.equals(request.fullQualifier) && args.trim().length() == 0) {
                    //we don't want to get the tokens that are equal to the current 'full' qualifier
                    //...unless it adds the parameters to a call...
                    continue;
                }

                int type = element.getType();

                int priority = IPyCompletionProposal.PRIORITY_DEFAULT;
                if (type == IToken.TYPE_PARAM || type == IToken.TYPE_LOCAL
                        || type == IToken.TYPE_OBJECT_FOUND_INTERFACE) {
                    priority = IPyCompletionProposal.PRIORITY_LOCALS;
View Full Code Here

                                    if (module == null) {
                                        continue;
                                    }
                                    IToken[] comps = astManager.getCompletionsForModule(module, state, true, true);
                                    for (int i = 0; i < comps.length; i++) {
                                        IToken iToken = comps[i];
                                        String representation = iToken.getRepresentation();
                                        ImmutableTuple<IToken, String> curr = map.get(representation);
                                        if (curr != null && curr.o1 instanceof SourceToken) {
                                            continue; //source tokens are never reset!
                                        }

                                        int type = iToken.getType();
                                        if (iToken instanceof SourceToken
                                                && ((SourceToken) iToken).getAst() instanceof FunctionDef) {
                                            map.put(representation, new ImmutableTuple<IToken, String>(iToken,
                                                    baseClass));

                                        } else if (type == IToken.TYPE_FUNCTION || type == IToken.TYPE_UNKNOWN
                                                || type == IToken.TYPE_BUILTIN) {
                                            map.put(representation, new ImmutableTuple<IToken, String>(iToken,
                                                    baseClass));

                                        }
                                    }
                                } catch (Exception e) {
                                    Log.log(e);
                                }
                            }

                            for (ImmutableTuple<IToken, String> tokenAndBaseClass : map.values()) {
                                FunctionDef functionDef = null;

                                //No checkings needed for type (we already did that above).
                                if (tokenAndBaseClass.o1 instanceof SourceToken) {
                                    SourceToken sourceToken = (SourceToken) tokenAndBaseClass.o1;
                                    SimpleNode ast = sourceToken.getAst();
                                    if (ast instanceof FunctionDef) {
                                        functionDef = (FunctionDef) ast;
                                    } else {
                                        functionDef = sourceToken.getAliased().createCopy();
                                        NameTok t = (NameTok) functionDef.name;
                                        t.id = sourceToken.getRepresentation();
                                    }
                                } else {
                                    //unfortunately, for builtins we usually cannot trust the parameters.
                                    String representation = tokenAndBaseClass.o1.getRepresentation();
                                    PyAstFactory factory = new PyAstFactory(new AdapterPrefs(ps.getEndLineDelim(),
                                            request.nature));
                                    functionDef = factory.createFunctionDef(representation);
                                    functionDef.args = factory.createArguments(true);
                                    functionDef.args.vararg = new NameTok("args", NameTok.VarArg);
                                    functionDef.args.kwarg = new NameTok("kwargs", NameTok.KwArg);
                                    if (!representation.equals("__init__")) {
                                        functionDef.body = new stmtType[] { new Return(null) }; //signal that the return should be added
                                    }
                                }

                                if (functionDef != null) {
                                    ret.add(new OverrideMethodCompletionProposal(ps.getAbsoluteCursorOffset(), 0, 0,
                                            imageOverride, functionDef, tokenAndBaseClass.o2, //baseClass
                                            className));
                                }
                            }

                        }
                    }
                }
            }
            request.showTemplates = false;
            return ret;
        }

        try {
            IPythonNature pythonNature = request.nature;
            checkPythonNature(pythonNature);

            ICodeCompletionASTManager astManager = pythonNature.getAstManager();
            if (astManager == null) {
                //we're probably still loading it.
                return ret;
            }

            //list of Object[], IToken or ICompletionProposal
            List<Object> tokensList = new ArrayList<Object>();
            lazyStartShell(request);
            String trimmed = request.activationToken.replace('.', ' ').trim();

            ImportInfo importsTipper = getImportsTipperStr(request);

            int line = request.doc.getLineOfOffset(request.documentOffset);
            IRegion region = request.doc.getLineInformation(line);

            ICompletionState state = new CompletionState(line, request.documentOffset - region.getOffset(), null,
                    request.nature, request.qualifier);
            state.setIsInCalltip(request.isInCalltip);

            Map<String, IToken> alreadyChecked = new HashMap<String, IToken>();

            boolean importsTip = false;

            if (importsTipper.importsTipperStr.length() != 0) {
                //code completion in imports
                request.isInCalltip = false; //if found after (, but in an import, it is not a calltip!
                request.isInMethodKeywordParam = false; //if found after (, but in an import, it is not a calltip!
                importsTip = doImportCompletion(request, astManager, tokensList, importsTipper);

            } else if (trimmed.length() > 0 && request.activationToken.indexOf('.') != -1) {
                //code completion for a token
                doTokenCompletion(request, astManager, tokensList, trimmed, state);
                handleKeywordParam(request, line, alreadyChecked);

            } else {
                //go to globals
                doGlobalsCompletion(request, astManager, tokensList, state);

                //At this point, after doing the globals completion, we may also need to check if we need to show
                //keyword parameters to the user.
                handleKeywordParam(request, line, alreadyChecked);
            }

            String lowerCaseQual = request.qualifier.toLowerCase();
            if (lowerCaseQual.length() >= PyCodeCompletionPreferencesPage.getArgumentsDeepAnalysisNChars()) {
                //this can take some time on the analysis, so, let's let the user choose on how many chars does he
                //want to do the analysis...
                state.pushFindResolveImportMemoryCtx();
                try {
                    for (Iterator<Object> it = tokensList.listIterator(); it.hasNext();) {
                        Object o = it.next();
                        if (o instanceof IToken) {
                            it.remove(); // always remove the tokens from the list (they'll be re-added later once they are filtered)

                            IToken initialToken = (IToken) o;

                            IToken token = initialToken;
                            String strRep = token.getRepresentation();
                            IToken prev = alreadyChecked.get(strRep);

                            if (prev != null) {
                                if (prev.getArgs().length() != 0) {
                                    continue; // we already have a version with args... just keep going
                                }
                            }

                            if (!strRep.toLowerCase().startsWith(lowerCaseQual)) {
                                //just re-add it if we're going to actually use it (depending on the qualifier)
                                continue;
                            }

                            IModule current = request.getModule();

                            while (token.isImportFrom()) {
                                //we'll only add it here if it is an import from (so, set the flag to false for the outer add)

                                if (token.getArgs().length() > 0) {
                                    //if we already have the args, there's also no reason to do it (that's what we'll do here)
                                    break;
                                }
                                ICompletionState s = state.getCopyForResolveImportWithActTok(token.getRepresentation());
                                s.checkFindResolveImportMemory(token);

                                ImmutableTuple<IModule, IToken> modTok = astManager.resolveImport(s, token, current);
                                IToken token2 = modTok.o2;
                                current = modTok.o1;
                                if (token2 != null && initialToken != token2) {
                                    String args = token2.getArgs();
                                    if (args.length() > 0) {
                                        //put it into the map (may override previous if it didn't have args)
                                        initialToken.setArgs(args);
                                        initialToken.setDocStr(token2.getDocStr());
                                        if (initialToken instanceof SourceToken && token2 instanceof SourceToken) {
                                            SourceToken initialSourceToken = (SourceToken) initialToken;
                                            SourceToken token2SourceToken = (SourceToken) token2;
                                            initialSourceToken.setAst(token2SourceToken.getAst());
                                        }
                                        break;
                                    }
                                    if (token2 == null
                                            || (token2.equals(token) && token2.getArgs().equals(token.getArgs()) && token2
                                                    .getParentPackage().equals(token.getParentPackage()))) {
                                        break;
                                    }
                                    token = token2;
                                } else {
View Full Code Here

        //now, check for locals
        IToken[] localTokens = scopeVisitor.scope.getAllLocalTokens();
        int len = localTokens.length;
        for (int i = 0; i < len; i++) {
            IToken tok = localTokens[i];

            final String tokenRep = tok.getRepresentation();
            if (tokenRep.equals(actTok)) {
                return new Definition[] { new Definition(tok, scopeVisitor.scope, this, true) };
            } else if (actTok.startsWith(tokenRep + ".") && !actTok.startsWith("self.")) {
                final int tokenRepLen = tokenRep.length();
                //this means we have a declaration in the local scope and we're accessing a part of it
                //e.g.:
                //class B:           
                //    def met2(self):
                //        c = C()    
                //        c.met1
                state.checkFindLocalDefinedDefinitionMemory(this, tokenRep);
                ICompletionState copyWithActTok = state.getCopyWithActTok(tokenRep);

                Definition[] definitions = this.findDefinition(copyWithActTok, tok.getLineDefinition(),
                        tok.getColDefinition(), nature, innerFindPaths);
                ArrayList<Definition> ret = new ArrayList<Definition>();
                for (Definition definition : definitions) {
                    if (definition.module != null) {
                        if (definition.value.length() == 0) {
                            continue;
                        }
                        String checkFor = definition.value + actTok.substring(tokenRepLen);
                        if (this.equals(definition.module)) {
                            //no point in finding the starting point
                            if (actTok.equals(definition.value)) {
                                continue;
                            }
                            if (checkFor.equals(actTok)) {
                                continue;
                            }
                            if (checkFor.startsWith(actTok + '.')) {
                                //This means that to find some symbol we have a construct such as:
                                //a = a.strip().rjust()
                                //So, if we look for a.strip, and we resolve a as a.strip.rjust, we'll try to find:
                                //a.strip.rjust.strip, in which case we'd recurse.
                                continue;
                            }
                        }

                        //Note: I couldn't really reproduce this case, so, this fix is just a theoretical
                        //workaround. Hopefully sometime someone will provide some code to reproduce this.
                        //see: http://sourceforge.net/tracker/?func=detail&aid=2992629&group_id=85796&atid=577329
                        int dotsFound = StringUtils.count(checkFor, '.');
                        if (dotsFound > 15) {
                            throw new CompletionRecursionException("Trying to go to deep to find definition.\n"
                                    + "We probably started entering a recursion.\n" + "Module: "
                                    + definition.module.getName() + "\n" + "Token: " + checkFor);
                        }

                        Definition[] realDefinitions;
                        if (definition.module instanceof SourceModule) {
                            SourceModule sourceModule = (SourceModule) definition.module;
                            realDefinitions = (Definition[]) sourceModule.findDefinition(
                                    state.getCopyWithActTok(checkFor), definition.line, definition.col, nature,
                                    innerFindPaths);

                        } else {
                            realDefinitions = (Definition[]) definition.module.findDefinition(
                                    state.getCopyWithActTok(checkFor), definition.line, definition.col, nature);

                        }
                        for (Definition realDefinition : realDefinitions) {
                            ret.add(realDefinition);
                        }
                    }
                }
                return ret.toArray(new Definition[ret.size()]);
            }
        }

        //not found... check as local imports
        List<IToken> localImportedModules = scopeVisitor.scope.getLocalImportedModules(line, col, this.name);
        ICodeCompletionASTManager astManager = nature.getAstManager();
        int lenImportedModules = localImportedModules.size();
        for (int i = 0; i < lenImportedModules; i++) {
            IToken tok = localImportedModules.get(i);
            String importRep = tok.getRepresentation();
            if (importRep.equals(actTok) || actTok.startsWith(importRep + ".")) {
                Tuple3<IModule, String, IToken> o = astManager.findOnImportedMods(new IToken[] { tok },
                        state.getCopyWithActTok(actTok), this.getName(), this);
                if (o != null && o.o1 instanceof SourceModule) {
                    ICompletionState copy = state.getCopy();
                    copy.setActivationToken(o.o2);

                    findDefinitionsFromModAndTok(nature, toRet, null, (SourceModule) o.o1, copy);
                }
                if (toRet.size() > 0) {
                    return (Definition[]) toRet.toArray(new Definition[0]);
                }
            }
        }

        //ok, not assign nor import, let's check if it is some self (we do not check for only 'self' because that would map to a
        //local (which has already been covered).
        if (actTok.startsWith("self.")) {
            //ok, it is some self, now, that is only valid if we are in some class definition
            ClassDef classDef = (ClassDef) scopeVisitor.scope.getClassDef();
            if (classDef != null) {
                //ok, we are in a class, so, let's get the self completions
                String classRep = NodeUtils.getRepresentationString(classDef);
                IToken[] globalTokens = getGlobalTokens(new CompletionState(line - 1, col - 1, classRep, nature, "",
                        state), //use the old state as the cache
                        astManager);

                String withoutSelf = actTok.substring(5);
                for (IToken token : globalTokens) {
                    if (token.getRepresentation().equals(withoutSelf)) {
                        String parentPackage = token.getParentPackage();
                        IModule module = astManager.getModule(parentPackage, nature, true);

                        if (token instanceof SourceToken
                                && (module != null || this.name == null || this.name.equals(parentPackage))) {
                            if (module == null) {
                                module = this;
                            }

                            SimpleNode ast2 = ((SourceToken) token).getAst();
                            Tuple<Integer, Integer> def = getLineColForDefinition(ast2);
                            FastStack<SimpleNode> stack = new FastStack<SimpleNode>(5);
                            if (module instanceof SourceModule) {
                                stack.push(((SourceModule) module).getAst());
                            }
                            stack.push(classDef);
                            ILocalScope scope = new LocalScope(stack);
                            return new Definition[] { new Definition(def.o1, def.o2, token.getRepresentation(), ast2,
                                    scope, module) };

                        } else {
                            return new Definition[0];
                        }
                    }
                }
            }
        }

        //ok, it is not an assign, so, let's search the global tokens (and imports)
        String tok = actTok;
        SourceModule mod = this;

        Tuple3<IModule, String, IToken> o = astManager.findOnImportedMods(state.getCopyWithActTok(actTok), this);

        if (o != null) {
            if (o.o1 instanceof SourceModule) {
                mod = (SourceModule) o.o1;
                tok = o.o2;

            } else if (o.o1 instanceof CompiledModule) {
                //ok, we have to check the compiled module
                tok = o.o2;
                if (tok == null || tok.length() == 0) {
                    return new Definition[] { new Definition(1, 1, "", null, null, o.o1) };
                } else {
                    state.checkFindDefinitionMemory(o.o1, tok);
                    return (Definition[]) o.o1.findDefinition(state.getCopyWithActTok(tok), -1, -1, nature);
                }
View Full Code Here

    public boolean isBootstrapModule() {
        if (bootstrap == null) {
            IToken[] ret = getGlobalTokens();
            if (ret != null && (ret.length == 1 || ret.length == 2 || ret.length == 3) && this.file != null) { //also checking 2 or 3 tokens because of __file__ and __name__
                for (int i = 0; i < ret.length; i++) {
                    IToken tok = ret[i];
                    if ("__bootstrap__".equals(tok.getRepresentation())) {
                        //if we get here, we already know that it defined a __bootstrap__, so, let's see if it was also called
                        SimpleNode ast = this.getAst();
                        if (ast instanceof Module) {
                            Module module = (Module) ast;
                            if (module.body != null && module.body.length > 0) {
View Full Code Here

                    if (element.length > 0) {
                        o3 = element[2];
                    }

                    IToken t;
                    if (element.length > 0) {
                        t = new CompiledToken(o1, o2, o3, name, Integer.parseInt(element[3]));
                    } else {
                        t = new CompiledToken(o1, o2, o3, name, IToken.TYPE_BUILTIN);
                    }
View Full Code Here

                    ArrayList<IToken> array = new ArrayList<IToken>();

                    for (Iterator<String[]> iter = completions.iterator(); iter.hasNext();) {
                        String[] element = iter.next();
                        if (element.length >= 4) {//it might be a server error
                            IToken t = new CompiledToken(element[0], element[1], element[2], act,
                                    Integer.parseInt(element[3]));
                            array.add(t);
                        }

                    }
View Full Code Here

            return IModule.FOUND_TOKEN;
        }

        if (ifHasGetAttributeConsiderInTokens) {

            IToken token = cachedTokens.get("__getattribute__");
            if (token == null || isTokenFromBuiltins(token)) {
                token = cachedTokens.get("__getattr__");
            }
            if (token != null && !isTokenFromBuiltins(token)) {
                return IModule.FOUND_BECAUSE_OF_GETATTR;
            }

            //Try to determine if the user specified it has a @DynamicAttrs.
            String[] parentsHeadAndTail = FullRepIterable.headAndTail(generateTokensFor);
            cachedTokens = getCachedCompletions(tok, nature, searchSameLevelMods, completionCache,
                    parentsHeadAndTail[0]);
            IToken parentToken = cachedTokens.get(parentsHeadAndTail[1]);
            if (parentToken != null) {
                String docString = parentToken.getDocStr();
                if (docString != null) {
                    if (docString.indexOf("@DynamicAttrs") != -1) {
                        //class that has things dynamically defined.
                        return IModule.FOUND_BECAUSE_OF_GETATTR;
                    }
View Full Code Here

TOP

Related Classes of org.python.pydev.core.IToken

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.