autoloading of all scripts recursively below pyscript/scripts; fixes #97

Author: craigbarratt

custom_components/pyscript/__init__.py

@@ -32,10 +32,10 @@
 )
 from .eval import AstEval
 from .event import Event
-from .mqtt import Mqtt
 from .function import Function
 from .global_ctx import GlobalContext, GlobalContextMgr
 from .jupyter_kernel import Kernel
+from .mqtt import Mqtt
 from .requirements import install_requirements
 from .state import State, StateVal
 from .trigger import TrigTime
@@ -98,7 +98,7 @@ def start_global_contexts(global_ctx_only=None):
     start_list = []
     for global_ctx_name, global_ctx in GlobalContextMgr.items():
         idx = global_ctx_name.find(".")
-        if idx < 0 or global_ctx_name[0:idx] not in {"file", "apps"}:
+        if idx < 0 or global_ctx_name[0:idx] not in {"file", "apps", "scripts"}:
             continue
         if global_ctx_only is not None:
             if global_ctx_name != global_ctx_only and not global_ctx_name.startswith(global_ctx_only + "."):
@@ -252,7 +252,7 @@ async def unload_scripts(global_ctx_only=None, unload_all=False):
     for global_ctx_name, global_ctx in GlobalContextMgr.items():
         if not unload_all:
             idx = global_ctx_name.find(".")
-            if idx < 0 or global_ctx_name[0:idx] not in {"file", "apps", "modules"}:
+            if idx < 0 or global_ctx_name[0:idx] not in {"file", "apps", "modules", "scripts"}:
                 continue
         if global_ctx_only is not None:
             if global_ctx_name != global_ctx_only and not global_ctx_name.startswith(global_ctx_only + "."):
@@ -273,33 +273,41 @@ def glob_files(load_paths, data):
         source_files = []
         apps_config = data.get("apps", None)
         for path, match, check_config in load_paths:
-            for this_path in sorted(glob.glob(os.path.join(pyscript_dir, path, match))):
+            for this_path in sorted(glob.glob(os.path.join(pyscript_dir, path, match), recursive=True)):
                 rel_import_path = None
-                elts = this_path.split("/")
-                if match.find("/") < 0:
-                    # last entry without the .py
-                    mod_name = elts[-1][0:-3]
-                else:
-                    # 2nd last entry
-                    mod_name = elts[-2]
-                    rel_import_path = f"{path}/mod_name"
+                rel_path = this_path
+                if rel_path.startswith(pyscript_dir):
+                    rel_path = rel_path[len(pyscript_dir) :]
+                if rel_path.startswith("/"):
+                    rel_path = rel_path[1:]
+                if rel_path[0] == "#" or rel_path.find("/#") >= 0:
+                    continue
+                rel_path = rel_path[0:-3]
+                if rel_path.endswith("/__init__"):
+                    rel_path = rel_path[0 : -len("/__init__")]
+                    rel_import_path = rel_path
+                mod_name = rel_path.replace("/", ".")
                 if path == "":
                     global_ctx_name = f"file.{mod_name}"
                     fq_mod_name = mod_name
                 else:
-                    global_ctx_name = f"{path}.{mod_name}"
-                    fq_mod_name = global_ctx_name
+                    fq_mod_name = global_ctx_name = mod_name
+                    i = fq_mod_name.find(".")
+                    if i >= 0:
+                        fq_mod_name = fq_mod_name[i + 1 :]
                 if check_config:
-                    if not isinstance(apps_config, dict) or mod_name not in apps_config:
+                    if not isinstance(apps_config, dict) or fq_mod_name not in apps_config:
                         _LOGGER.debug("load_scripts: skipping %s because config not present", this_path)
                         continue
                 source_files.append([global_ctx_name, this_path, rel_import_path, fq_mod_name])
+
         return source_files
 
     load_paths = [
         ["apps", "*.py", True],
         ["apps", "*/__init__.py", True],
         ["", "*.py", False],
+        ["scripts", "**/*.py", False],
     ]
 
     source_files = await hass.async_add_executor_job(glob_files, load_paths, data)

docs/new_features.rst

@@ -34,13 +34,17 @@ The new features since 1.0.0 in master include:
   successful trigger. Setting this optional parameter makes state triggers edge triggered (ie,
   triggers only on transition from ``False`` to ``True``), instead of the default level trigger (ie,
   only has to evaluate to ``True``). Proposed by @tchef69 (#89).
-- ``del`` and new function ``state.delete()` can delete state variables and state variable attributes
+- Adding new decorator ``@mqtt_trigger`` by @dlashua (#98, #105).
+- All .py files below the pyscript/scripts directory are autoloaded, recursively into all subdirectories.
+  Any file name or directory starting with ``#`` is skipped, which is an in-place way of disabling
+  a specific file or directory tree (#97).
+- ``del`` and new function ``state.delete()` can delete state variables and state variable attributes.
 
 Bug fixes since 1.0.0 in master include:
 
-- state setting now copies the attributes, to avoid a strange ``MappingProxyType`` recursion error
+- State setting now copies the attributes, to avoid a strange ``MappingProxyType`` recursion error
   inside HASS, reported by @github392 (#87).
-- the deprecated function ``state.get_attr`` was missing an ``await``, which causes an exception; in 1.0.0 use
+- The deprecated function ``state.get_attr`` was missing an ``await``, which causes an exception; in 1.0.0 use
   ``state.getattr``, reported and fixed by @dlashua (#88).
-- the ``packaging`` module is installed if not found, since certain HASS configurations might not include it;
+- The ``packaging`` module is installed if not found, since certain HASS configurations might not include it;
   fixed by @raman325 (#90, #91).

docs/reference.rst

@@ -58,7 +58,12 @@ At startup, pyscript loads the following files. It also unloads and reloads thes
 the ``pyscript.reload`` service is called, which also reloads the ``yaml`` configuration.
 
 ``<config>/pyscript/*.py``
-  all files with a ``.py`` suffix are autoloaded
+  all files with a ``.py`` suffix are autoloaded.
+
+``<config>/pyscript/scripts/**/*.py``
+  all files with a ``.py`` suffix below the ``scripts`` directory, recursively to any depth,
+  are autoloaded. This is useful for organizing your scripts into subdirectories that are
+  related in some way.
 
 ``<config>/pyscript/apps/<app_name>.py``
   all files in the ``apps`` subdirectory with a ``.py`` suffix are autoloaded, provided ``app_name``
@@ -71,6 +76,12 @@ the ``pyscript.reload`` service is called, which also reloads the ``yaml`` confi
   This form is most convenient for sharing pyscript code, since all the files for one
   application are stored in its own directory.
 
+Any file name that starts with ``#`` is not loaded, and similarly scripts anywhere below a directory
+name that starts with ``#``, are not loaded. That's a convenient way to disable a specific script or
+entire directory below ``scripts`` - you can simply rename it with or without the leading ``#`` to
+disable or enable it at the next reload. Think of it as "commenting" the file name, rather than
+having to delete it or move it outside the pyscript directory.
+
 Like regular Python, functions within one source file can call each other, and can share global
 variables (if necessary), but just within that one file. Each file has its own separate global
 context. Each Jupyter session also has its own separate global context, so functions, triggers,
@@ -1027,7 +1038,8 @@ module or package that is explicitly imported). In normal use you don’t need t
 contexts. But for interactive debugging and development, you might want your Jupyter session to
 access variables and functions defined in a script file.
 
-Here is the naming convention for each file's global context:
+Here is the naming convention for each file's global context (upper case mean any value;
+lower case are actual fixed names):
 
   ======================================= ===========================
   pyscript file path                      global context name
@@ -1039,6 +1051,9 @@ Here is the naming convention for each file's global context:
   ``pyscript/apps/APP.py``                ``apps.APP``
   ``pyscript/apps/APP/__init__.py``       ``apps.APP.__init__``
   ``pyscript/apps/APP/FILE.py``           ``apps.APP.FILE``
+  ``pyscript/scripts/FILE.py``            ``scripts.FILE``
+  ``pyscript/scripts/DIR1/FILE.py``       ``scripts.DIR1.FILE``
+  ``pyscript/scripts/DIR1/DIR2/FILE.py``  ``scripts.DIR1.DIR2.FILE``
   ======================================= ===========================
 
 The logging path uses the global context name, so you can customize logging verbosity for each

tests/test_apps_modules.py

@@ -74,6 +74,8 @@ def func10():
         f"{conf_dir}/modules/xyz2/__init__.py": """
 from .other import f_minus, other_name
 
+log.info(f"modules/xyz2 global_ctx={pyscript.get_global_ctx()};")
+
 x = 99
 
 def f_add(a, b):
@@ -101,6 +103,41 @@ def other_name():
         f"{conf_dir}/modules/bad_module.py": """
 def func12()
     pass
+""",
+        #
+        # this script file should auto-load
+        #
+        f"{conf_dir}/scripts/func13.py": """
+@service
+def func13():
+    pass
+""",
+        #
+        # this script file should auto-load
+        #
+        f"{conf_dir}/scripts/a/b/c/d/func14.py": """
+@service
+def func14():
+    pass
+
+log.info(f"func14 global_ctx={pyscript.get_global_ctx()};")
+
+""",
+        #
+        # this script file should not auto-load
+        #
+        f"{conf_dir}/scripts/a/b/c/d/#func15.py": """
+@service
+def func15():
+    pass
+""",
+        #
+        # this script file should not auto-load
+        #
+        f"{conf_dir}/scripts/#a/b/c/d/func15.py": """
+@service
+def func15():
+    pass
 """,
     }
 
@@ -114,10 +151,10 @@ def isfile_side_effect(arg):
     def glob_side_effect(path, recursive=None):
         result = []
         path_re = path.replace("*", "[^/]*").replace(".", "\\.")
+        path_re = path_re.replace("[^/]*[^/]*/", ".*")
         for this_path in file_contents:
             if re.match(path_re, this_path):
                 result.append(this_path)
-        print(f"glob_side_effect: path={path}, path_re={path_re}, result={result}")
         return result
 
     conf = {"apps": {"world": {}}}
@@ -126,7 +163,7 @@ def glob_side_effect(path, recursive=None):
     ) as mock_glob, patch("custom_components.pyscript.global_ctx.open", mock_open), patch(
         "homeassistant.config.load_yaml_config_file", return_value={"pyscript": conf}
     ), patch(
-        "os.path.isfile"
+        "custom_components.pyscript.os.path.isfile"
     ) as mock_isfile:
         mock_isfile.side_effect = isfile_side_effect
         mock_glob.side_effect = glob_side_effect
@@ -145,7 +182,9 @@ async def state_changed(event):
 
     assert not hass.services.has_service("pyscript", "func10")
     assert not hass.services.has_service("pyscript", "func11")
-    assert not hass.services.has_service("pyscript", "func12")
+    assert hass.services.has_service("pyscript", "func13")
+    assert hass.services.has_service("pyscript", "func14")
+    assert not hass.services.has_service("pyscript", "func15")
 
     await hass.services.async_call("pyscript", "func1", {})
     ret = await wait_until_done(notify_q)
@@ -155,5 +194,7 @@ async def state_changed(event):
     ret = await wait_until_done(notify_q)
     assert literal_eval(ret) == [99, "xyz2", "xyz2.other", 1 + 5, 3 * 6, 10 + 30, 50 - 30]
 
+    assert "modules/xyz2 global_ctx=modules.xyz2.__init__;" in caplog.text
+    assert "func14 global_ctx=scripts.a.b.c.d.func14;" in caplog.text
     assert "ModuleNotFoundError: import of no_such_package not allowed" in caplog.text
     assert "SyntaxError: invalid syntax (bad_module.py, line 2)" in caplog.text

tests/test_decorator_errors.py

@@ -14,7 +14,7 @@
 async def setup_script(hass, notify_q, now, source):
     """Initialize and load the given pyscript."""
     scripts = [
-        "/some/config/dir/pyscripts/hello.py",
+        "/hello.py",
     ]
 
     with patch("custom_components.pyscript.os.path.isdir", return_value=True), patch(

tests/test_function.py

@@ -3,12 +3,14 @@
 from ast import literal_eval
 import asyncio
 from datetime import datetime as dt
+import re
 
-from custom_components.pyscript.const import CONF_ALLOW_ALL_IMPORTS, CONF_HASS_IS_GLOBAL, DOMAIN
+from custom_components.pyscript.const import CONF_ALLOW_ALL_IMPORTS, CONF_HASS_IS_GLOBAL, DOMAIN, FOLDER
 from custom_components.pyscript.function import Function
 import custom_components.pyscript.trigger as trigger
+from mock_open import MockOpen
 import pytest
-from pytest_homeassistant_custom_component.async_mock import MagicMock, Mock, mock_open, patch
+from pytest_homeassistant_custom_component.async_mock import MagicMock, Mock, patch
 
 from homeassistant.const import EVENT_HOMEASSISTANT_STARTED, EVENT_STATE_CHANGED
 from homeassistant.core import Context
@@ -100,21 +102,42 @@ async def test_service_completions(root, expected, hass, services):  # pylint: d
 
 async def setup_script(hass, notify_q, notify_q2, now, source, config=None):
     """Initialize and load the given pyscript."""
-    scripts = [
-        "/some/config/dir/pyscripts/hello.py",
-    ]
+
+    conf_dir = hass.config.path(FOLDER)
+
+    file_contents = {f"{conf_dir}/hello.py": source}
+
+    mock_open = MockOpen()
+    for key, value in file_contents.items():
+        mock_open[key].read_data = value
+
+    def isfile_side_effect(arg):
+        return arg in file_contents
+
+    def glob_side_effect(path, recursive=None):
+        result = []
+        path_re = path.replace("*", "[^/]*").replace(".", "\\.")
+        path_re = path_re.replace("[^/]*[^/]*/", ".*")
+        for this_path in file_contents:
+            if re.match(path_re, this_path):
+                result.append(this_path)
+        return result
 
     if not config:
         config = {DOMAIN: {CONF_ALLOW_ALL_IMPORTS: True}}
     with patch("custom_components.pyscript.os.path.isdir", return_value=True), patch(
-        "custom_components.pyscript.glob.iglob", return_value=scripts
-    ), patch("custom_components.pyscript.global_ctx.open", mock_open(read_data=source), create=True,), patch(
+        "custom_components.pyscript.glob.iglob"
+    ) as mock_glob, patch("custom_components.pyscript.global_ctx.open", mock_open), patch(
         "custom_components.pyscript.trigger.dt_now", return_value=now
     ), patch(
         "homeassistant.config.load_yaml_config_file", return_value=config
     ), patch(
         "custom_components.pyscript.install_requirements", return_value=None,
-    ):
+    ), patch(
+        "custom_components.pyscript.os.path.isfile"
+    ) as mock_isfile:
+        mock_isfile.side_effect = isfile_side_effect
+        mock_glob.side_effect = glob_side_effect
         assert await async_setup_component(hass, "pyscript", config)
 
     #

tests/test_init.py

@@ -23,7 +23,7 @@ async def setup_script(hass, notify_q, now, source):
     """Initialize and load the given pyscript."""
 
     scripts = [
-        "/some/config/dir/pyscript/hello.py",
+        "/hello.py",
     ]
 
     with patch("custom_components.pyscript.os.path.isdir", return_value=True), patch(
@@ -437,7 +437,7 @@ def func5(var_name=None, value=None):
         # now reload the other source file
         #
         scripts = [
-            "/some/config/dir/pyscript/hello.py",
+            "/hello.py",
         ]
 
         with patch("custom_components.pyscript.os.path.isdir", return_value=True), patch(

tests/test_jupyter.py

@@ -6,12 +6,14 @@
 import hashlib
 import hmac
 import json
+import re
 import uuid
 
-from custom_components.pyscript.const import DOMAIN
+from custom_components.pyscript.const import DOMAIN, FOLDER
 from custom_components.pyscript.jupyter_kernel import ZmqSocket
 import custom_components.pyscript.trigger as trigger
-from pytest_homeassistant_custom_component.async_mock import mock_open, patch
+from mock_open import MockOpen
+from pytest_homeassistant_custom_component.async_mock import patch
 
 from homeassistant.const import EVENT_HOMEASSISTANT_STARTED, EVENT_HOMEASSISTANT_STOP
 from homeassistant.setup import async_setup_component
@@ -107,19 +109,39 @@ def encode(msg):
 async def setup_script(hass, now, source, no_connect=False):
     """Initialize and load the given pyscript."""
 
-    scripts = [
-        "/some/config/dir/pyscripts/hello.py",
-    ]
+    conf_dir = hass.config.path(FOLDER)
+
+    file_contents = {f"{conf_dir}/hello.py": source}
+
+    mock_open = MockOpen()
+    for key, value in file_contents.items():
+        mock_open[key].read_data = value
+
+    def isfile_side_effect(arg):
+        return arg in file_contents
+
+    def glob_side_effect(path, recursive=None):
+        result = []
+        path_re = path.replace("*", "[^/]*").replace(".", "\\.")
+        path_re = path_re.replace("[^/]*[^/]*/", ".*")
+        for this_path in file_contents:
+            if re.match(path_re, this_path):
+                result.append(this_path)
+        return result
 
     with patch("custom_components.pyscript.os.path.isdir", return_value=True), patch(
-        "custom_components.pyscript.glob.iglob", return_value=scripts
-    ), patch("custom_components.pyscript.global_ctx.open", mock_open(read_data=source), create=True,), patch(
+        "custom_components.pyscript.glob.iglob"
+    ) as mock_glob, patch("custom_components.pyscript.global_ctx.open", mock_open), patch(
         "custom_components.pyscript.trigger.dt_now", return_value=now
     ), patch(
         "homeassistant.config.load_yaml_config_file", return_value={}
     ), patch(
         "custom_components.pyscript.install_requirements", return_value=None,
-    ):
+    ), patch(
+        "custom_components.pyscript.os.path.isfile"
+    ) as mock_isfile:
+        mock_isfile.side_effect = isfile_side_effect
+        mock_glob.side_effect = glob_side_effect
         assert await async_setup_component(hass, "pyscript", {DOMAIN: {}})
 
     #