cache specific namespace data to speedup reanalyzing of documents, some refactoring

Author: d-biehl

log.ini

@@ -14,7 +14,7 @@ keys: console
 format: %(name)s:%(levelname)s:  %(message)s
 
 [formatter_colored_simple]
-class=coloredlogs.ColoredFormatter
+class: coloredlogs.ColoredFormatter
 #format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
 format: %(name)s:%(levelname)s:  %(message)s
 

robotcode/language_server/robotframework/diagnostics/imports_manager.py

@@ -368,11 +368,11 @@ def __init__(
         self.parent_protocol.documents.did_save.add(self.resource_document_changed)
 
     @async_tasking_event
-    async def libraries_changed(sender, params: List[LibraryDoc]) -> None:
+    async def libraries_changed(sender, libraries: List[LibraryDoc]) -> None:
         ...
 
     @async_tasking_event
-    async def resources_changed(sender, params: List[LibraryDoc]) -> None:
+    async def resources_changed(sender, resources: List[LibraryDoc]) -> None:
         ...
 
     async def resource_document_changed(self, sender: Any, document: TextDocument) -> None:
@@ -636,7 +636,7 @@ async def find_file(self, name: str, base_dir: str, file_type: str = "Resource")
 
     @_logger.call
     async def _get_entry_for_resource_import(self, name: str, base_dir: str, sentinel: Any = None) -> _ResourcesEntry:
-        source = await self.find_file(name, base_dir, "Resource")
+        source = await self.find_file(name, base_dir)
 
         async def _get_document() -> TextDocument:
             self._logger.debug(lambda: f"Load resource {name} from source {source}")

robotcode/language_server/robotframework/diagnostics/namespace.py

@@ -37,7 +37,7 @@
 from ...common.text_document import TextDocument
 from ..utils.ast import (
     Token,
-    is_non_variable_token,
+    is_not_variable_token,
     range_from_node,
     range_from_token,
     range_from_token_or_node,
@@ -599,14 +599,14 @@ async def _analyse_run_keyword(
         if keyword_doc is None or not keyword_doc.is_any_run_keyword():
             return argument_tokens
 
-        if keyword_doc.is_run_keyword() and len(argument_tokens) > 0 and is_non_variable_token(argument_tokens[0]):
+        if keyword_doc.is_run_keyword() and len(argument_tokens) > 0 and is_not_variable_token(argument_tokens[0]):
             await self._analyze_keyword_call(argument_tokens[0].value, node, argument_tokens[0], argument_tokens[1:])
 
             return argument_tokens[1:]
         elif (
             keyword_doc.is_run_keyword_with_condition()
             and len(argument_tokens) > 1
-            and is_non_variable_token(argument_tokens[1])
+            and is_not_variable_token(argument_tokens[1])
         ):
             await self._analyze_keyword_call(argument_tokens[1].value, node, argument_tokens[1], argument_tokens[2:])
             return argument_tokens[2:]
@@ -627,7 +627,7 @@ async def _analyse_run_keyword(
                     )
                     continue
 
-                if not is_non_variable_token(t):
+                if not is_not_variable_token(t):
                     continue
 
                 and_token = next((e for e in argument_tokens if e.value == "AND"), None)
@@ -640,7 +640,7 @@ async def _analyse_run_keyword(
 
             return []
 
-        elif keyword_doc.is_run_keyword_if() and len(argument_tokens) > 1 and is_non_variable_token(argument_tokens[1]):
+        elif keyword_doc.is_run_keyword_if() and len(argument_tokens) > 1 and is_not_variable_token(argument_tokens[1]):
 
             def skip_args() -> None:
                 nonlocal argument_tokens
@@ -714,7 +714,7 @@ async def visit_Fixture(self, node: ast.AST) -> None:  # noqa: N802
 
         # TODO: calculate possible variables in NAME
 
-        if keyword_token is not None and is_non_variable_token(keyword_token):
+        if keyword_token is not None and is_not_variable_token(keyword_token):
             await self._analyze_keyword_call(
                 value.name, value, keyword_token, [cast(Token, e) for e in value.get_tokens(RobotToken.ARGUMENT)]
             )
@@ -730,7 +730,7 @@ async def visit_TestTemplate(self, node: ast.AST) -> None:  # noqa: N802
 
         # TODO: calculate possible variables in NAME
 
-        if keyword_token is not None and is_non_variable_token(keyword_token):
+        if keyword_token is not None and is_not_variable_token(keyword_token):
             await self._analyze_keyword_call(value.value, value, keyword_token, [])
 
         await self.generic_visit(node)
@@ -744,7 +744,7 @@ async def visit_Template(self, node: ast.AST) -> None:  # noqa: N802
 
         # TODO: calculate possible variables in NAME
 
-        if keyword_token is not None and is_non_variable_token(keyword_token):
+        if keyword_token is not None and is_not_variable_token(keyword_token):
             await self._analyze_keyword_call(value.value, value, keyword_token, [])
 
         await self.generic_visit(node)
@@ -908,6 +908,7 @@ def __init__(
         self._analyzed = False
         self._analyze_lock = asyncio.Lock()
         self._library_doc: Optional[LibraryDoc] = None
+        self._library_doc_lock = asyncio.Lock()
         self._imports: Optional[List[Import]] = None
         self._own_variables: Optional[List[VariableDefinition]] = None
         self._diagnostics: List[Diagnostic] = []
@@ -921,15 +922,14 @@ def __init__(
     def document(self) -> Optional[TextDocument]:
         return self._document() if self._document is not None else None
 
-    async def libraries_changed(self, sender: Any, params: List[LibraryDoc]) -> None:
-
-        for p in params:
+    async def libraries_changed(self, sender: Any, libraries: List[LibraryDoc]) -> None:
+        for p in libraries:
             if any(e for e in self._libraries.values() if e.library_doc == p):
                 self.invalidated_callback(self)
                 break
 
-    async def resources_changed(self, sender: Any, params: List[LibraryDoc]) -> None:
-        for p in params:
+    async def resources_changed(self, sender: Any, resources: List[LibraryDoc]) -> None:
+        for p in resources:
             if any(e for e in self._resources.values() if e.library_doc.source == p.source):
                 self.invalidated_callback(self)
                 break
@@ -972,34 +972,43 @@ async def get_library_doc(self) -> LibraryDoc:
         from ..parts.documents_cache import DocumentType
 
         if self._library_doc is None:
-
-            model_type = ""
-
-            if hasattr(self.model, "model_type"):
-                t = getattr(self.model, "model_type")
-
-                if t == DocumentType.RESOURCE:
-                    model_type = "RESOURCE"
-                elif t == DocumentType.GENERAL:
-                    model_type = "TESTCASE"
-                elif t == DocumentType.INIT:
-                    model_type = "INIT"
-
-            self._library_doc = await self.imports_manager.get_libdoc_from_model(
-                self.model, self.source, model_type=model_type
-            )
+            async with self._library_doc_lock:
+                if self._library_doc is None:
+                    model_type = ""
+
+                    if hasattr(self.model, "model_type"):
+                        t = getattr(self.model, "model_type")
+
+                        if t == DocumentType.RESOURCE:
+                            model_type = "RESOURCE"
+                        elif t == DocumentType.GENERAL:
+                            model_type = "TESTCASE"
+                        elif t == DocumentType.INIT:
+                            model_type = "INIT"
+
+                    self._library_doc = await self.imports_manager.get_libdoc_from_model(
+                        self.model, self.source, model_type=model_type
+                    )
 
         return self._library_doc
 
+    class DataEntry(NamedTuple):
+        libraries: OrderedDict[str, LibraryEntry] = OrderedDict()
+        resources: OrderedDict[str, ResourceEntry] = OrderedDict()
+        variables: OrderedDict[str, VariablesEntry] = OrderedDict()
+        diagnostics: List[Diagnostic] = []
+
     @_logger.call
     async def ensure_initialized(self) -> bool:
         if not self._initialized:
             async with self._initialize_lock:
                 if not self._initialized:
                     imports = await self.get_imports()
 
+                    data_entry: Optional[Namespace.DataEntry] = None
                     if self.document is not None:
-
+                        # check or save several data in documents data cache,
+                        # if imports are different, then the data is invalid
                         old_imports: List[Import] = self.document.get_data(Namespace)
                         if old_imports is None:
                             self.document.set_data(Namespace, imports)
@@ -1012,9 +1021,28 @@ async def ensure_initialized(self) -> bool:
                                 if e not in new_imports:
                                     new_imports.append(e)
                             self.document.set_data(Namespace, new_imports)
-
-                    await self._import_default_libraries()
-                    await self._import_imports(imports, str(Path(self.source).parent), top_level=True)
+                        else:
+                            data_entry = self.document.get_data(Namespace.DataEntry)
+
+                    if data_entry is not None:
+                        self._libraries = data_entry.libraries.copy()
+                        self._resources = data_entry.resources.copy()
+                        self._variables = data_entry.variables.copy()
+                        self._diagnostics = data_entry.diagnostics.copy()
+                    else:
+                        await self._import_default_libraries()
+                        await self._import_imports(imports, str(Path(self.source).parent), top_level=True)
+
+                        if self.document is not None:
+                            self.document.set_data(
+                                Namespace.DataEntry,
+                                Namespace.DataEntry(
+                                    self._libraries.copy(),
+                                    self._resources.copy(),
+                                    self._variables.copy(),
+                                    self._diagnostics.copy(),
+                                ),
+                            )
 
                     self._initialized = True
 

robotcode/language_server/robotframework/parts/model_helper.py

@@ -5,7 +5,7 @@
 from ...common.lsp_types import Position
 from ..diagnostics.library_doc import KeywordDoc, KeywordError
 from ..diagnostics.namespace import Namespace
-from ..utils.ast import Token, is_non_variable_token, range_from_token
+from ..utils.ast import Token, is_not_variable_token, range_from_token
 
 
 class ModelHelperMixin:
@@ -20,7 +20,7 @@ async def get_run_keyword_keyworddoc_and_token_from_position(
         if keyword_doc is None or not keyword_doc.is_any_run_keyword():
             return None, argument_tokens
 
-        if keyword_doc.is_run_keyword() and len(argument_tokens) > 0 and is_non_variable_token(argument_tokens[0]):
+        if keyword_doc.is_run_keyword() and len(argument_tokens) > 0 and is_not_variable_token(argument_tokens[0]):
             result = await self.get_keyworddoc_and_token_from_position(
                 argument_tokens[0].value, argument_tokens[0], argument_tokens[1:], namespace, position
             )
@@ -29,7 +29,7 @@ async def get_run_keyword_keyworddoc_and_token_from_position(
         elif (
             keyword_doc.is_run_keyword_with_condition()
             and len(argument_tokens) > 1
-            and is_non_variable_token(argument_tokens[1])
+            and is_not_variable_token(argument_tokens[1])
         ):
             result = await self.get_keyworddoc_and_token_from_position(
                 argument_tokens[1].value, argument_tokens[1], argument_tokens[2:], namespace, position
@@ -39,13 +39,12 @@ async def get_run_keyword_keyworddoc_and_token_from_position(
 
         elif keyword_doc.is_run_keywords():
             while argument_tokens:
-                # TODO: Parse "run keywords" with arguments using upper case AND
                 t = argument_tokens[0]
                 argument_tokens = argument_tokens[1:]
                 if t.value == "AND":
                     continue
 
-                if is_non_variable_token(t) and position.is_in_range(range_from_token(t)):
+                if is_not_variable_token(t) and position.is_in_range(range_from_token(t)):
                     result = await self.get_keyworddoc_and_token_from_position(t.value, t, [], namespace, position)
 
                     return result, argument_tokens
@@ -55,7 +54,7 @@ async def get_run_keyword_keyworddoc_and_token_from_position(
                     argument_tokens = argument_tokens[argument_tokens.index(and_token) + 1 :]
 
             return None, []
-        elif keyword_doc.is_run_keyword_if() and len(argument_tokens) > 1 and is_non_variable_token(argument_tokens[1]):
+        elif keyword_doc.is_run_keyword_if() and len(argument_tokens) > 1 and is_not_variable_token(argument_tokens[1]):
 
             def skip_args() -> None:
                 nonlocal argument_tokens

robotcode/language_server/robotframework/parts/references.py

@@ -35,6 +35,7 @@
     range_from_token_or_node,
     tokenize_variables,
 )
+from ..utils.async_ast import iter_nodes
 
 if TYPE_CHECKING:
     from ..protocol import RobotLanguageServerProtocol
@@ -280,7 +281,8 @@ def _yield_owner_and_kw_names(full_name: str) -> Iterator[Tuple[Optional[str], .
             for i in range(1, len(tokens)):
                 yield ".".join(tokens[:i]), ".".join(tokens[i:])
 
-    async def _find_keyword_references_from_file(
+    @_logger.call
+    async def _find_keyword_references_in_file(
         self, kw_doc: KeywordDoc, lib_doc: Optional[LibraryDoc], file: Path, cancel_token: CancelationToken
     ) -> List[Location]:
         from robot.parsing.lexer.tokens import Token as RobotToken
@@ -302,15 +304,15 @@ async def _find_keyword_references_from_file(
         ):
             return []
 
+        libraries_matchers = await namespace.get_libraries_matchers()
+        resources_matchers = await namespace.get_resources_matchers()
+
         async def _run() -> List[Location]:
             kw_matcher = KeywordMatcher(kw_doc.name)
 
             result: List[Location] = []
 
-            libraries_matchers = await namespace.get_libraries_matchers()
-            resources_matchers = await namespace.get_resources_matchers()
-
-            for node in ast.walk(namespace.model):
+            async for node in iter_nodes(namespace.model):
                 cancel_token.throw_if_canceled()
 
                 kw: Optional[KeywordDoc] = None
@@ -388,7 +390,7 @@ async def _find_keyword_references(self, document: TextDocument, kw_doc: Keyword
                 absolute=True,
             ):
                 futures.append(
-                    asyncio.create_task(self._find_keyword_references_from_file(kw_doc, lib_doc, f, cancel_token))
+                    asyncio.create_task(self._find_keyword_references_in_file(kw_doc, lib_doc, f, cancel_token))
                 )
 
             for e in await asyncio.gather(*futures, return_exceptions=True):

robotcode/language_server/robotframework/utils/ast.py

@@ -131,7 +131,7 @@ def range_from_token_or_node(node: ast.AST, token: Optional[Token]) -> Range:
     return Range.zero()
 
 
-def is_non_variable_token(token: Token) -> bool:
+def is_not_variable_token(token: Token) -> bool:
     from robot.errors import VariableError
 
     try: