Improve the process of installing Python deps for IDF

Author: valeros

builder/frameworks/espidf.py

@@ -24,6 +24,7 @@
 import json
 import subprocess
 import sys
+import shutil
 import os
 import pkg_resources
 
@@ -58,6 +59,7 @@
 
 # Required until Arduino switches to v5
 IDF5 = platform.get_package_version("framework-espidf").split(".")[1].startswith("5")
+IDF_ENV_VERSION = "1.0.0"
 FRAMEWORK_DIR = platform.get_package_dir("framework-espidf")
 TOOLCHAIN_DIR = platform.get_package_dir(
     "toolchain-%s" % ("riscv32-esp" if mcu == "esp32c3" else ("xtensa-%s" % mcu))
@@ -1147,40 +1149,71 @@ def _get_installed_pip_packages(python_exe_path):
                 )
             )
 
+def get_idf_venv_dir():
+    # The name of the IDF venv contains the IDF version to avoid possible conflicts and
+    # unnecessary reinstallation of Python dependencies in cases when Arduino
+    # as an IDF component requires a different version of the IDF package and
+    # hence a different set of Python deps or their versions
+    idf_version = get_original_version(platform.get_package_version("framework-espidf"))
+    return os.path.join(
+        env.subst("$PROJECT_CORE_DIR"), "penv", ".espidf-" + idf_version
+    )
+
+def ensure_python_venv_available():
+
+    def _is_venv_outdated(venv_data_file):
+        try:
+            with open(venv_data_file, "r", encoding="utf8") as fp:
+                venv_data = json.load(fp)
+                if venv_data.get("version", "") != IDF_ENV_VERSION:
+                    return True
+                return False
+        except:
+            return True
 
-def get_python_exe():
     def _create_venv(venv_dir):
         pip_path = os.path.join(
             venv_dir,
             "Scripts" if IS_WINDOWS else "bin",
             "pip" + (".exe" if IS_WINDOWS else ""),
         )
-        if not os.path.isfile(pip_path):
+
+        if os.path.isdir(venv_dir):
+            try:
+                print("Removing an oudated IDF virtual environment")
+                shutil.rmtree(venv_dir)
+            except OSError:
+                print(
+                    "Error: Cannot remove an outdated IDF virtual environment. " \
+                    "Please remove the `%s` folder manually!" % venv_dir
+                )
+                env.Exit(1)
+
             # Use the built-in PlatformIO Python to create a standalone IDF virtual env
             env.Execute(
                 env.VerboseAction(
                     '"$PYTHONEXE" -m venv --clear "%s"' % venv_dir,
-                    "Creating a virtual environment for IDF Python dependencies",
+                "Creating a new virtual environment for IDF Python dependencies",
                 )
             )
 
         assert os.path.isfile(
             pip_path
-        ), "Error: Failed to create a proper virtual environment. Missing the pip binary!"
+        ), "Error: Failed to create a proper virtual environment. Missing the `pip` binary!"
 
-    # The name of the IDF venv contains the IDF version to avoid possible conflicts and
-    # unnecessary reinstallation of Python dependencies in cases when Arduino
-    # as an IDF component requires a different version of the IDF package and
-    # hence a different set of Python deps or their versions
-    idf_version = get_original_version(platform.get_package_version("framework-espidf"))
-    venv_dir = os.path.join(
-        env.subst("$PROJECT_CORE_DIR"), "penv", ".espidf-" + idf_version)
 
-    if not os.path.isdir(venv_dir):
+    venv_dir = get_idf_venv_dir()
+    venv_data_file = os.path.join(venv_dir, "pio-idf-venv.json")
+    if not os.path.isfile(venv_data_file) or _is_venv_outdated(venv_data_file):
         _create_venv(venv_dir)
+        with open(venv_data_file, "w", encoding="utf8") as fp:
+            venv_info = {"version": IDF_ENV_VERSION}
+            json.dump(venv_info, fp, indent=2)
+
 
+def get_python_exe():
     python_exe_path = os.path.join(
-        venv_dir,
+        get_idf_venv_dir(),
         "Scripts" if IS_WINDOWS else "bin",
         "python" + (".exe" if IS_WINDOWS else ""),
     )
@@ -1196,6 +1229,7 @@ def _create_venv(venv_dir):
 # ESP-IDF requires Python packages with specific versions in a virtual environment
 #
 
+ensure_python_venv_available()
 install_python_deps()
 
 # ESP-IDF package doesn't contain .git folder, instead package version is specified