diff --git a/.github/scripts/docs_build_examples.py b/.github/scripts/docs_build_examples.py new file mode 100755 index 00000000000..15e256158c2 --- /dev/null +++ b/.github/scripts/docs_build_examples.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python3 +""" +Documentation build examples script for ESP32 Arduino. + +This script manages binary preparation and cleanup for documentation examples. +It can build examples with upload-binary configuration, clean up the binaries +directory, and optionally generate diagrams and LaunchPad configurations. + +The script requires Arduino CLI and user paths when building examples. +It processes all sketches that have 'upload-binary' configuration in their +ci.yml files and builds them for specified targets. +""" + +import click +from esp_docs.esp_extensions.docs_embed.tool.wokwi_tool import DiagramSync +import os +import shutil +import sys +from pathlib import Path +import subprocess +import yaml + +SCRIPT_DIR = Path(__file__).resolve().parent + +env_config_keys = [ + "DOCS_EMBED_PUBLIC_ROOT", + "DOCS_EMBED_GITHUB_BASE_URL", + "DOCS_EMBED_BINARIES_DIR", +] + +# load environment variables +env_config = {} +for key in env_config_keys: + value = os.environ.get(key) + if value is None: + raise EnvironmentError(f"{key} environment variable is not set") + env_config[key] = value + + +if env_config.get("DOCS_EMBED_BINARIES_DIR"): + DOCS_EMBED_BINARIES_PATH = Path(f"docs/{env_config['DOCS_EMBED_BINARIES_DIR']}") + +ARDUINO_ESP32_PATH = os.environ.get("ARDUINO_ESP32_PATH") +GITHUB_WORKSPACE = os.environ.get("GITHUB_WORKSPACE") + +if ( + ARDUINO_ESP32_PATH + and (Path(ARDUINO_ESP32_PATH) / "tools" / "esp32-arduino-libs").is_dir() +): + SDKCONFIG_DIR = Path(ARDUINO_ESP32_PATH) / "tools" / "esp32-arduino-libs" +elif ( + GITHUB_WORKSPACE + and (Path(GITHUB_WORKSPACE) / "tools" / "esp32-arduino-libs").is_dir() +): + SDKCONFIG_DIR = Path(GITHUB_WORKSPACE) / "tools" / "esp32-arduino-libs" +else: + raise EnvironmentError( + "Could not locate esp32-arduino-libs directory. " + "Set ARDUINO_ESP32_PATH or GITHUB_WORKSPACE environment variable." + ) + +KEEP_FILES = [ + "*.merged.bin", + "ci.yml", + "launchpad.toml", + "diagram*.json", +] + +SKETCH_UTILS = SCRIPT_DIR / "sketch_utils.sh" + + +def run_cmd(cmd, check=True, capture_output=False, text=True): + """Execute a shell command with error handling.""" + try: + return subprocess.run( + cmd, check=check, capture_output=capture_output, text=text + ) + except subprocess.CalledProcessError as e: + # CalledProcessError is raised only when check=True and the command exits non-zero + print(f"ERROR: Command failed: {' '.join(cmd)}") + print(f"Exit code: {e.returncode}") + if hasattr(e, "stdout") and e.stdout: + print("--- stdout ---") + print(e.stdout) + if hasattr(e, "stderr") and e.stderr: + print("--- stderr ---") + print(e.stderr) + # Exit the whole script with the same return code to mimic shell behavior + sys.exit(e.returncode) + except FileNotFoundError: + print(f"ERROR: Command not found: {cmd[0]}") + sys.exit(127) + + +def check_requirements(sketch_dir, sdkconfig_path): + """Check if sketch meets requirements for the given SDK config. + + Args: + sketch_dir (str): Path to the sketch directory + sdkconfig_path (str): Path to the SDK config file + + Returns: + bool: True if requirements are met, False otherwise + """ + cmd = [str(SKETCH_UTILS), "check_requirements", sketch_dir, str(sdkconfig_path)] + try: + res = run_cmd(cmd, check=False, capture_output=True) + return res.returncode == 0 + except Exception: + return False + + +def install_libs(*args): + """Install Arduino libraries using sketch_utils.sh""" + cmd = [str(SKETCH_UTILS), "install_libs"] + list(args) + return run_cmd(cmd, check=False) + + +def build_sketch(args_list): + """Build a sketch using sketch_utils.sh""" + cmd = [str(SKETCH_UTILS), "build"] + args_list + return run_cmd(cmd, check=False) + + + + +def validate_prerequisites(arduino_cli_path, user_path): + """Validate that required prerequisites are available and get paths from env vars if needed. + + Returns: + tuple: (arduino_cli_path, user_path) with values from env vars if not provided + """ + # Get paths from environment variables if not provided via arguments + if not arduino_cli_path or not user_path: + print("Getting Arduino paths from environment variables...") + arduino_ide_path = os.environ.get("ARDUINO_IDE_PATH") + arduino_usr_path = os.environ.get("ARDUINO_USR_PATH") + + if not arduino_cli_path: + if arduino_ide_path: + arduino_cli_path = arduino_ide_path + print(f" Arduino CLI path (ARDUINO_IDE_PATH): {arduino_ide_path}") + else: + print( + "ERROR: Arduino CLI path not found in ARDUINO_IDE_PATH env var and not provided (-ai option)" + ) + print("Run install-arduino-cli.sh first to set environment variables") + sys.exit(1) + + if not user_path: + if arduino_usr_path: + user_path = arduino_usr_path + print(f" Arduino user path (ARDUINO_USR_PATH): {arduino_usr_path}") + else: + print( + "ERROR: Arduino user path not found in ARDUINO_USR_PATH env var and not provided (-au option)" + ) + print("Run install-arduino-cli.sh first to set environment variables") + sys.exit(1) + + # Create Arduino user path if it doesn't exist + user_path_obj = Path(user_path) + if not user_path_obj.is_dir(): + try: + user_path_obj.mkdir(parents=True, exist_ok=True) + print(f"Created Arduino user directory: {user_path_obj}") + except Exception as e: + print(f"ERROR: Failed to create Arduino user path {user_path_obj}: {e}") + sys.exit(1) + + return arduino_cli_path, user_path + + +def cleanup_binaries(): + """Clean up the binaries directory, keeping only specified files. + + Removes all files except those matching patterns in KEEP_FILES. + Also removes empty directories after cleanup. + """ + print(f"Cleaning up binaries directory: {DOCS_EMBED_BINARIES_PATH}") + if not DOCS_EMBED_BINARIES_PATH.exists(): + print("Binaries directory does not exist, nothing to clean") + return + for root, dirs, files in os.walk(DOCS_EMBED_BINARIES_PATH): + for fname in files: + fpath = Path(root) / fname + parent = Path(root).name + # Always remove sketch/ci.yml + if parent == "sketch" and fname == "ci.yml": + fpath.unlink() + continue + keep = False + for pattern in KEEP_FILES: + if Path(fname).match(pattern): + keep = True + break + if not keep: + print(f"Removing: {fpath}") + fpath.unlink() + else: + print(f"Keeping: {fpath}") + # remove empty dirs + for root, dirs, files in os.walk(DOCS_EMBED_BINARIES_PATH, topdown=False): + if not os.listdir(root): + try: + os.rmdir(root) + except Exception: + pass + print("Cleanup completed") + + +def find_examples_with_upload_binary(): + """Find all Arduino sketches that have upload-binary configuration. + + Returns: + list: List of paths to .ino files that have upload-binary in ci.yml + """ + res = [] + for ino in Path(".").rglob("*.ino"): + sketch_dir = ino.parent + sketch_name = ino.stem + dir_name = sketch_dir.name + if dir_name != sketch_name: + continue + ci_yml = sketch_dir / "ci.yml" + if ci_yml.exists(): + try: + data = yaml.safe_load(ci_yml.read_text()) + if "upload-binary" in data and data["upload-binary"]: + res.append(str(ino)) + except Exception: + continue + return res + + +def get_upload_binary_targets(sketch_dir): + """Get the upload-binary targets from a sketch's ci.yml file. + + Args: + sketch_dir (str or Path): Path to the sketch directory + + Returns: + list: List of target names for upload-binary, empty if none found + """ + ci_yml = Path(sketch_dir) / "ci.yml" + try: + data = yaml.safe_load(ci_yml.read_text()) + targets = data.get("upload-binary", {}).get("targets", []) + return targets + except Exception: + return [] + + +def build_example_for_target(sketch_dir, target, relative_path, arduino_cli_path, user_path, generate_diagrams): + """Build a single example for a specific target. + + Args: + sketch_dir (Path): Path to the sketch directory + target (str): Target board/configuration name + relative_path (str): Relative path for output organization + arduino_cli_path (str): Path to Arduino CLI installation directory + user_path (str): Path to Arduino user directory + generate_diagrams (bool): Whether to generate diagrams + + Returns: + bool: True if build succeeded, False otherwise + """ + print(f"\n > Building example: {relative_path} for target: {target}") + output_dir = DOCS_EMBED_BINARIES_PATH / relative_path / target + output_dir.mkdir(parents=True, exist_ok=True) + + sdkconfig = SDKCONFIG_DIR / target / "sdkconfig" + if not check_requirements(str(sketch_dir), sdkconfig): + print( + f"Target {target} does not meet the requirements for {Path(sketch_dir).name}. Skipping." + ) + return True + + # Build the sketch using sketch_utils.sh build - pass args as in shell script + build_args = [ + "-ai", + arduino_cli_path, + "-au", + user_path, + "-s", + str(sketch_dir), + "-t", + target, + "-b", + str(output_dir), + "--first-only", + ] + res = build_sketch(build_args) + if res.returncode == 0: + print(f"Successfully built {relative_path} for {target}") + ci_yml = Path(sketch_dir) / "ci.yml" + if ci_yml.exists(): + shutil.copy(ci_yml, output_dir / "ci.yml") + if generate_diagrams: + print(f"Generating diagram for {relative_path} ({target})...") + try: + sync = DiagramSync(output_dir) + sync.generate_diagram_from_ci(target) + except Exception as e: + print( + f"WARNING: Failed to generate diagram for {relative_path} ({target}): {e}" + ) + else: + print(f"ERROR: Failed to build {relative_path} for {target}") + return False + return True + + +def build_all_examples(arduino_cli_path, user_path, generate_diagrams, generate_launchpad_config): + """Build all examples that have upload-binary configuration + + Prerequisites are validated in main() before calling this function + """ + total_built = 0 + total_failed = 0 + + if DOCS_EMBED_BINARIES_PATH.exists(): + shutil.rmtree(DOCS_EMBED_BINARIES_PATH) + print(f"Removed existing build directory: {DOCS_EMBED_BINARIES_PATH}") + + # add gitignore to binaries dir with * for new files + DOCS_EMBED_BINARIES_PATH.mkdir(parents=True, exist_ok=True) + gitignore_path = DOCS_EMBED_BINARIES_PATH / ".gitignore" + gitignore_path.write_text("*\n") + + examples = find_examples_with_upload_binary() + if not examples: + print("No examples found with upload-binary configuration") + return 0 + + print("\nExamples to be built:") + print("====================") + for i, example in enumerate(examples, start=1): + sketch_dir = Path(example).parent + relative_path = str(sketch_dir).lstrip("./") + targets = get_upload_binary_targets(sketch_dir) + if targets: + print(f"{i}. {relative_path} (targets: {' '.join(targets)})") + print() + + for example in examples: + sketch_dir = Path(example).parent + relative_path = str(sketch_dir).lstrip("./") + targets = get_upload_binary_targets(sketch_dir) + if not targets: + print(f"WARNING: No targets found for {relative_path}") + continue + print(f"Building {relative_path} for targets: {targets}") + for target in targets: + if build_example_for_target(sketch_dir, target, relative_path, arduino_cli_path, user_path, generate_diagrams): + total_built += 1 + else: + total_failed += 1 + + output_sketch_dir = DOCS_EMBED_BINARIES_PATH / relative_path + output_sketch_dir.mkdir(parents=True, exist_ok=True) + + # copy sketch ci.yml to output dir - parent of target dirs + ci_yml = sketch_dir / "ci.yml" + if ci_yml.exists(): + shutil.copy(ci_yml, output_sketch_dir / "ci.yml") + + if generate_launchpad_config: + print(f"Generating LaunchPad config for {relative_path}/{target}...") + try: + sync = DiagramSync(output_sketch_dir) + sync.generate_launchpad_config( + env_config["DOCS_EMBED_PUBLIC_ROOT"], + env_config["DOCS_EMBED_GITHUB_BASE_URL"], + True, + output_sketch_dir, + ) + except Exception as e: + print( + f"WARNING: Failed to generate LaunchPad config for {relative_path}/{target}: {e}" + ) + + print("\nBuild summary:") + print(f" Successfully built: {total_built}") + print(f" Failed builds: {total_failed}") + print(f" Output directory: {DOCS_EMBED_BINARIES_PATH}") + return total_failed + + +@click.command( + help="""Build examples that have ci.yml with upload-binary targets. + +\b +Examples: + docs_build_examples.py -c # Clean up binaries directory + docs_build_examples.py --build # Build all examples (use env vars) + docs_build_examples.py --build -ai /path/to/cli -au /path/to/user # Build with explicit paths + docs_build_examples.py --build --diagram --launchpad # Build with diagrams and LaunchPad (with env vars) + +\b +Path detection: + All paths can be set via environment variables or command line options. + ARDUINO_IDE_PATH and ARDUINO_USR_PATH environment variables are used by default. + Set by running install-arduino-cli.sh first, or use -ai/-au to override +""") +@click.option( + "-c", "--cleanup", + is_flag=True, + help="Clean up docs binaries directory and exit" +) +@click.option( + "-ai", "--arduino-cli-path", + default=None, + help="Path to Arduino CLI installation directory (overrides ARDUINO_IDE_PATH env var)" +) +@click.option( + "-au", "--arduino-user-path", + default=None, + help="Path to Arduino user directory (overrides ARDUINO_USR_PATH env var)" +) +@click.option( + "-b", "--build", + is_flag=True, + help="Build all examples" +) +@click.option( + "-d", "--diagram", + is_flag=True, + help="Generate diagrams for prepared examples using docs-embed" +) +@click.option( + "-l", "--launchpad", + is_flag=True, + help="Generate LaunchPad config for prepared examples" +) +def main(cleanup, arduino_cli_path, arduino_user_path, build, diagram, launchpad): + """Main entry point for the script""" + if cleanup: + cleanup_binaries() + return + + if build: + # Validate prerequisites and auto-detect paths if needed + arduino_cli_path, arduino_user_path = validate_prerequisites(arduino_cli_path, arduino_user_path) + + result = build_all_examples(arduino_cli_path, arduino_user_path, diagram, launchpad) + if result == 0: + print("\nAll examples built successfully!") + else: + print("\nSome builds failed. Check the output above for details.") + sys.exit(1) + return + + if diagram or launchpad: + print( + "ERROR: --diagram and --launchpad options are only available when building examples (--build)" + ) + sys.exit(1) + + # If no specific action is requested, show help + ctx = click.get_current_context() + click.echo(ctx.get_help()) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh index 668d2ab7bd8..f21b7cb2fe0 100755 --- a/.github/scripts/sketch_utils.sh +++ b/.github/scripts/sketch_utils.sh @@ -54,6 +54,8 @@ function check_requirements { # check_requirements } function build_sketch { # build_sketch [extra-options] + local first_only=false + while [ -n "$1" ]; do case "$1" in -ai ) @@ -92,6 +94,36 @@ function build_sketch { # build_sketch [ext shift debug_level="DebugLevel=$1" ;; + -b ) + shift + custom_build_dir=$1 + ;; + --first-only ) + first_only=true + ;; + -h ) + echo "Usage: build_sketch [options]" + echo "" + echo "Build an Arduino sketch for ESP32 targets." + echo "" + echo "Required options:" + echo " -ai Arduino IDE path" + echo " -au Arduino user path" + echo " -s Sketch directory containing .ino file and ci.yml" + echo " -t Target chip (esp32, esp32s2, esp32c3, esp32s3, esp32c6, esp32h2, esp32p4, esp32c5)" + echo " Required unless -fqbn is specified" + echo "" + echo "Optional options:" + echo " -fqbn Fully qualified board name (alternative to -t)" + echo " -o Board options (PSRAM, USBMode, etc.) [default: chip-specific]" + echo " -i Chunk index for parallel builds [default: none]" + echo " -l Log compilation output to JSON file [default: none]" + echo " -d Debug level (DebugLevel=...) [default: none]" + echo " -b Custom build directory [default: \$ARDUINO_BUILD_DIR or \$HOME/.arduino/tests/\$target/\$sketchname/build.tmp]" + echo " --first-only Build only the first FQBN from ci.yml configurations [default: false]" + echo " -h Show this help message" + exit 0 + ;; * ) break ;; @@ -128,7 +160,15 @@ function build_sketch { # build_sketch [ext len=$(yq eval ".fqbn.${target} | length" "$sketchdir"/ci.yml 2>/dev/null || echo 0) if [ "$len" -gt 0 ]; then - fqbn=$(yq eval ".fqbn.${target} | sort | @json" "$sketchdir"/ci.yml) + if [ "$first_only" = true ]; then + # Get only the first FQBN from the array (original order) + fqbn=$(yq eval ".fqbn.${target} | .[0]" "$sketchdir"/ci.yml 2>/dev/null) + fqbn="[\"$fqbn\"]" + len=1 + else + # Build all FQBNs + fqbn=$(yq eval ".fqbn.${target} | sort | @json" "$sketchdir"/ci.yml) + fi fi fi @@ -219,12 +259,14 @@ function build_sketch { # build_sketch [ext fi # The directory that will hold all the artifacts (the build directory) is - # provided through: - # 1. An env variable called ARDUINO_BUILD_DIR. - # 2. Created at the sketch level as "build" in the case of a single - # configuration test. - # 3. Created at the sketch level as "buildX" where X is the number - # of configuration built in case of a multiconfiguration test. + # determined by the following priority: + # 1. Custom build directory via -b flag: + # - If path contains "docs/_static/binaries", use as exact path (for docs builds) + # - Otherwise, append target name to custom directory + # 2. ARDUINO_BUILD_DIR environment variable (if set) + # 3. Default: $HOME/.arduino/tests/$target/$sketchname/build.tmp + # + # For multiple configurations, subsequent builds use .1, .2, etc. suffixes sketchname=$(basename "$sketchdir") local has_requirements @@ -253,11 +295,19 @@ function build_sketch { # build_sketch [ext fi ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp" - if [ -n "$ARDUINO_BUILD_DIR" ]; then - build_dir="$ARDUINO_BUILD_DIR" - elif [ "$len" -eq 1 ]; then - # build_dir="$sketchdir/build" - build_dir="$HOME/.arduino/tests/$target/$sketchname/build.tmp" + + # Determine base build directory + if [ -n "$custom_build_dir" ]; then + # If custom_build_dir contains docs/_static/binaries, use it as exact path + if [[ "$custom_build_dir" == *"docs/_static/binaries"* ]]; then + build_dir_base="$custom_build_dir" + else + build_dir_base="$custom_build_dir/$target" + fi + elif [ -n "$ARDUINO_BUILD_DIR" ]; then + build_dir_base="$ARDUINO_BUILD_DIR" + else + build_dir_base="$HOME/.arduino/tests/$target/$sketchname/build.tmp" fi output_file="$HOME/.arduino/cli_compile_output.txt" @@ -265,10 +315,14 @@ function build_sketch { # build_sketch [ext mkdir -p "$ARDUINO_CACHE_DIR" for i in $(seq 0 $((len - 1))); do - if [ "$len" -ne 1 ]; then - # build_dir="$sketchdir/build$i" - build_dir="$HOME/.arduino/tests/$target/$sketchname/build$i.tmp" + # Calculate build directory for this configuration + if [ "$i" -eq 0 ]; then + build_dir="$build_dir_base" + else + build_dir="${build_dir_base}.${i}" fi + + # Prepare build directory rm -rf "$build_dir" mkdir -p "$build_dir" diff --git a/.github/workflows/docs_build.yml b/.github/workflows/docs_build.yml index 5253c166f85..f0d58a258e0 100644 --- a/.github/workflows/docs_build.yml +++ b/.github/workflows/docs_build.yml @@ -5,6 +5,7 @@ on: branches: - master - release/v2.x + - docs-embed paths: - "docs/**" - ".github/workflows/docs_build.yml" @@ -34,18 +35,55 @@ jobs: cache: "pip" python-version: "3.10" - - name: Build + # Install Arduino CLI and set environment variables + - name: Install Arduino CLI + run: | + source .github/scripts/install-arduino-cli.sh + echo "ARDUINO_IDE_PATH=$ARDUINO_IDE_PATH" >> $GITHUB_ENV + echo "ARDUINO_USR_PATH=$ARDUINO_USR_PATH" >> $GITHUB_ENV + + # Install ESP32 Arduino Core + - name: Install ESP32 Arduino Core + run: | + source .github/scripts/install-arduino-core-esp32.sh + + # Install Python dependencies needed for docs_build_examples.py + - name: Install Python Dependencies run: | sudo apt update sudo apt install python3-pip python3-setuptools - # GitHub CI installs pip3 and setuptools outside the path. - # Update the path to include them and run. cd ./docs PATH=/home/runner/.local/bin:$PATH pip3 install -r requirements.txt --prefer-binary + + # Set documentation embedding environment variables + - name: Set Documentation Environment Variables + run: | + # configure env vars for docs_build_examples.py and docs build + echo "DOCS_EMBED_GITHUB_BASE_URL=https://github.com/${{ github.repository }}" >> $GITHUB_ENV + echo "DOCS_EMBED_GITHUB_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV + echo "DOCS_EMBED_PUBLIC_ROOT=https://${{ secrets.DOCS_SERVER }}/${{ secrets.DOCS_PATH }}/en/${{ github.ref_name == 'master' && 'latest' || github.ref_name }}/" >> $GITHUB_ENV + echo "DOCS_EMBED_BINARIES_DIR=${{ vars.DOCS_EMBED_BINARIES_DIR }}" >> $GITHUB_ENV + echo "DOCS_EMBED_LAUNCHPAD_URL=${{ vars.DOCS_EMBED_LAUNCHPAD_URL }}" >> $GITHUB_ENV + echo "DOCS_EMBED_WOKWI_VIEWER_URL=${{ vars.DOCS_EMBED_WOKWI_VIEWER_URL }}" >> $GITHUB_ENV + echo "DOCS_EMBED_ABOUT_WOKWI_URL=${{ vars.DOCS_EMBED_ABOUT_WOKWI_URL }}" >> $GITHUB_ENV + + # Build examples using environment variables from install script + - name: Build Examples + run: | + PATH=/home/runner/.local/bin:$PATH python3 .github/scripts/docs_build_examples.py --build --diagram --launchpad + + - name: Cleanup Binaries + run: | + PATH=/home/runner/.local/bin:$PATH python3 .github/scripts/docs_build_examples.py -c + + + - name: Build + run: | + cd ./docs PATH=/home/runner/.local/bin:$PATH SPHINXOPTS="-W" build-docs -l en - name: Archive Docs uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: docs - path: docs + path: docs \ No newline at end of file diff --git a/.github/workflows/docs_deploy.yml b/.github/workflows/docs_deploy.yml index 01eb2b773dc..98a58b76486 100644 --- a/.github/workflows/docs_deploy.yml +++ b/.github/workflows/docs_deploy.yml @@ -1,6 +1,7 @@ name: Documentation Build and Production Deploy CI on: + workflow_dispatch: workflow_run: workflows: ["ESP32 Arduino Release"] types: @@ -9,6 +10,7 @@ on: branches: - release/v2.x - master + - docs-embed paths: - "docs/**" - ".github/workflows/docs_deploy.yml" @@ -19,7 +21,7 @@ permissions: jobs: deploy-prod-docs: name: Deploy Documentation on Production - if: github.repository == 'espressif/arduino-esp32' + # if: github.repository == 'espressif/arduino-esp32' runs-on: ubuntu-22.04 defaults: run: @@ -41,6 +43,47 @@ jobs: cache: "pip" python-version: "3.10" + # Install Arduino CLI and set environment variables + - name: Install Arduino CLI + run: | + source .github/scripts/install-arduino-cli.sh + echo "ARDUINO_IDE_PATH=$ARDUINO_IDE_PATH" >> $GITHUB_ENV + echo "ARDUINO_USR_PATH=$ARDUINO_USR_PATH" >> $GITHUB_ENV + + # Install ESP32 Arduino Core + - name: Install ESP32 Arduino Core + run: | + source .github/scripts/install-arduino-core-esp32.sh + + # Install Python dependencies needed for docs_build_examples.py + - name: Install Python Dependencies + run: | + sudo apt update + sudo apt install python3-pip python3-setuptools + cd ./docs + PATH=/home/runner/.local/bin:$PATH pip3 install -r requirements.txt --prefer-binary + + # Set documentation embedding environment variables + - name: Set Documentation Environment Variables + run: | + # configure env vars for docs_build_examples.py and docs deployment + echo "DOCS_EMBED_GITHUB_BASE_URL=https://github.com/${{ github.repository }}" >> $GITHUB_ENV + echo "DOCS_EMBED_GITHUB_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV + echo "DOCS_EMBED_PUBLIC_ROOT=https://${{ secrets.DOCS_SERVER }}/${{ secrets.DOCS_PATH }}/en/${{ github.ref_name == 'master' && 'latest' || github.ref_name }}/" >> $GITHUB_ENV + echo "DOCS_EMBED_BINARIES_DIR=${{ vars.DOCS_EMBED_BINARIES_DIR }}" >> $GITHUB_ENV + echo "DOCS_EMBED_LAUNCHPAD_URL=${{ vars.DOCS_EMBED_LAUNCHPAD_URL }}" >> $GITHUB_ENV + echo "DOCS_EMBED_WOKWI_VIEWER_URL=${{ vars.DOCS_EMBED_WOKWI_VIEWER_URL }}" >> $GITHUB_ENV + echo "DOCS_EMBED_ABOUT_WOKWI_URL=${{ vars.DOCS_EMBED_ABOUT_WOKWI_URL }}" >> $GITHUB_ENV + + # Build examples using environment variables from install script + - name: Build Examples + run: | + PATH=/home/runner/.local/bin:$PATH python3 .github/scripts/docs_build_examples.py --build --diagram --launchpad + + - name: Cleanup Binaries + run: | + PATH=/home/runner/.local/bin:$PATH python3 .github/scripts/docs_build_examples.py -c + - name: Deploy Documentation env: # Deploy to production server diff --git a/.gitignore b/.gitignore index 66b8f1b8031..84312d174c8 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,7 @@ libraries/Insights/examples/*/*.ino.zip /affected_sketches.txt /affected_examples.txt /ctags_* + +# Disallow Wokwi diagram files except for diagram in tests folder +**/diagram.*.json +!tests/**/diagram.*.json diff --git a/docs/conf_common.py b/docs/conf_common.py index 0960d7c4315..c252a4d13f1 100644 --- a/docs/conf_common.py +++ b/docs/conf_common.py @@ -38,6 +38,7 @@ "sphinx_tabs.tabs", "sphinx_substitution_extensions", # For allowing substitutions inside code blocks "esp_docs.esp_extensions.dummy_build_system", + "esp_docs.esp_extensions.docs_embed", ] # ESP32_DOCS = [ diff --git a/docs/en/api/adc.rst b/docs/en/api/adc.rst index a0c2677854f..1530a76cc2f 100644 --- a/docs/en/api/adc.rst +++ b/docs/en/api/adc.rst @@ -274,8 +274,7 @@ Example Applications Here is an example of how to use the ADC in OneShot mode or you can run Arduino example 01.Basics -> AnalogReadSerial. -.. literalinclude:: ../../../libraries/ESP32/examples/AnalogRead/AnalogRead.ino - :language: arduino +.. wokwi-example:: libraries/ESP32/examples/AnalogRead Here is an example of how to use the ADC in Continuous mode. diff --git a/docs/en/api/gpio.rst b/docs/en/api/gpio.rst index ebf31088ffd..a7a9779aa4c 100644 --- a/docs/en/api/gpio.rst +++ b/docs/en/api/gpio.rst @@ -141,30 +141,12 @@ Example Code GPIO Input and Output Modes *************************** -.. code-block:: arduino - - #define LED 12 - #define BUTTON 2 - - uint8_t stateLED = 0; - - void setup() { - pinMode(LED, OUTPUT); - pinMode(BUTTON,INPUT_PULLUP); - } - - void loop() { - - if(!digitalRead(BUTTON)){ - stateLED = stateLED^1; - digitalWrite(LED,stateLED); - } - } +.. wokwi-example:: libraries/ESP32/examples/GPIO/Blink GPIO Interrupt ************** -.. literalinclude:: ../../../libraries/ESP32/examples/GPIO/GPIOInterrupt/GPIOInterrupt.ino - :language: arduino +.. wokwi-example:: libraries/ESP32/examples/GPIO/GPIOInterrupt + .. _datasheet: https://www.espressif.com/sites/default/files/documentation/esp32_datasheet_en.pdf diff --git a/docs/en/api/i2c.rst b/docs/en/api/i2c.rst index 06d4d1953a6..6f356a60c08 100644 --- a/docs/en/api/i2c.rst +++ b/docs/en/api/i2c.rst @@ -509,3 +509,17 @@ Here is an example of how to use the I2C in Slave Mode. :language: arduino .. _Arduino Wire Library: https://www.arduino.cc/en/reference/wire + + +External libraries - examples +***************************** + + +LiquidCrystal_I2C +^^^^^^^^^^^^^^^^^ + +This example demonstrates how to use the `LiquidCrystal_I2C`_ library to control an LCD display via I2C. + +.. wokwi-example:: libraries/Wire/examples/LiquidCrystal_I2C + +.. _LiquidCrystal_I2C: https://github.com/johnrickman/LiquidCrystal_I2C diff --git a/docs/en/api/timer.rst b/docs/en/api/timer.rst index 2637f9eec1d..09e66471d4d 100644 --- a/docs/en/api/timer.rst +++ b/docs/en/api/timer.rst @@ -215,10 +215,8 @@ There are 2 examples uses of Timer: Repeat timer example: -.. literalinclude:: ../../../libraries/ESP32/examples/Timer/RepeatTimer/RepeatTimer.ino - :language: arduino +.. wokwi-example:: libraries/ESP32/examples/Timer/RepeatTimer Watchdog timer example: -.. literalinclude:: ../../../libraries/ESP32/examples/Timer/WatchdogTimer/WatchdogTimer.ino - :language: arduino +.. wokwi-example:: libraries/ESP32/examples/Timer/WatchdogTimer diff --git a/docs/en/contributing.rst b/docs/en/contributing.rst index c9f83530d36..43a065d958f 100644 --- a/docs/en/contributing.rst +++ b/docs/en/contributing.rst @@ -109,6 +109,93 @@ Also: const char * WIFI_FTM_SSID = "WiFi_FTM_Responder"; // SSID of AP that has FTM Enabled const char * WIFI_FTM_PASS = "ftm_responder"; // STA Password + +Examples +******** + +All libraries in the Arduino ESP32 core have their own examples. You can find them in the ``libraries//examples/`` folder. + +The purpose of these examples is to demonstrate how to use the library features and provide a starting point for users. + +If you want to show the example in the documentation, you have two options: +1. link just the source code file in the documentation using the ``literalinclude`` directive: + +.. code-block:: rst + + .. literalinclude:: ../../../libraries//examples//.ino + :language: arduino + +2. Source code with Wokwi simulation embedded in the documentation using the ``wokwi-example`` directive: + +.. code-block:: rst + + .. wokwi-example:: libraries//examples/ + +To enable compiling the example in the CI system, you need to add a ``ci.yml`` file in the same folder as the sketch. +The ``ci.yml`` file is used to specify some configurations for the CI system, like required configurations, supported targets, and more. +You can enable compilation of the example by adding targets under the ``upload-binary`` directive in the ``ci.yml`` file. + +Here is an example of a ``ci.yml`` file that enables compilation for ESP32 and ESP32-S3 targets. +This configuration adds default diagrams for Wokwi simulations with just dev boards. + +.. code-block:: yaml + + upload-binary: + targets: + - esp32 + - esp32s3 + +If you want to add custom diagrams for Wokwi simulations, you can add the ``diagram..json`` file in the same folder as the sketch. +The ```` is the target name (e.g., ``esp32``, ``esp32s3``, etc.). You can create the diagram using ``docs-embed`` tool installed together with documentation building tools. + +To create the diagram, run the ``docs-embed init-diagram --platforms esp32`` command in the sketch folder. +Before you run the documentation build command, you have to convert the diagram config to the ``ci.yml`` file format by running: + +.. code-block:: bash + + docs-embed ci-from-diagram + # OR + docs-embed ci-from-diagram --override + +There is also an opposite command to generate diagram from ``ci.yml`` file: + +.. code-block:: bash + + docs-embed diagram-from-ci + + +The documentation building tools is working only with `ci.yml` files, so `diagram..json` files are just for configuration using a GUI tool. + +Please keep in mind that the ``ci.yml`` does not store the chip and its position on the diagram, just the components and their connections (to save space). +The chip is added automatically and positioned vertically in the center of the diagram (same as the default behavior of the `docs-embed init-diagram` command). + +To run the documentation build command locally and in the CI system, you need to add those environment variables: + +* ``DOCS_EMBED_ABOUT_WOKWI_URL``: URL to the info about Wokwi (default: ``https://docs.espressif.com/projects/arduino-esp32/en/latest/third_party/wokwi.html``) +* ``DOCS_EMBED_BINARIES_DIR``: Path to the folder where the pre-compiled binaries are stored (default: ``_static/binaries``) +* ``DOCS_EMBED_LAUNCHPAD_URL``: URL to the launchpad page (default: ``https://espressif.github.io/esp-launchpad/``) +* ``DOCS_EMBED_WOKWI_VIEWER_URL``: URL to the Wokwi iframe viewer (default: ``https://wokwi.com/experimental/viewer``) + + +CI/CD +***** + +This repository uses GitHub Actions for Continuous Integration and Continuous Deployment (CI/CD). +If you forked the repository, you can enable them under the `Actions` tab in your forked repository. + +To enable building the documentation, you need to set up additional secrets and environment variables in your forked repository. + +Secrets +^^^^^^^^^^^^ +* ``DOCS_SERVER``: The server where the documentation will be deployed +* ``DOCS_PATH``: The path where the documentation will be deployed on the server + + +Variables +^^^^^^^^^ + +They are described above in the `Examples` section. + Testing ******* @@ -424,6 +511,9 @@ The ``ci.yml`` file is used to specify how the test suite and sketches will hand Libraries are installed using the ``arduino-cli lib install`` command, so you can specify libraries by name + version (e.g., ``AudioZero@1.0.0``) or by URL (e.g., ``https://github.com/arduino-libraries/WiFi101.git``). More information can be found in the `Arduino CLI documentation `_. +* ``upload-binary``: This directive is used ESP-DOCS - Docs Embed to integrate Wokwi simulations into the documentation. + Together with custom Sphinx plugin and its directives, it allows embedding Wokwi simulator into the documentation pages with pre-compiled binaries. + For more information, please refer to the `ESP-DOCS - Docs Embed contributing guide `_. The ``wifi`` test suite is a good example of how to use the ``ci.yml`` file: diff --git a/docs/requirements.txt b/docs/requirements.txt index ef2ab88cb65..3d03ff23596 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,9 @@ -sphinx==4.5.0 -esp-docs>=1.4.0 +sphinx~=7.1.2 + +esp-docs @ git+https://github.com/JakubAndrysek/esp-docs.git@extensions/wokwi_embed + sphinx-copybutton==0.5.0 -sphinx-tabs==3.2.0 +sphinx-tabs==3.4.7 numpydoc==1.5.0 standard-imghdr==3.13.0 Sphinx-Substitution-Extensions==2022.2.16 diff --git a/libraries/ESP32/examples/AnalogRead/ci.yml b/libraries/ESP32/examples/AnalogRead/ci.yml new file mode 100644 index 00000000000..bb2d96c7ddf --- /dev/null +++ b/libraries/ESP32/examples/AnalogRead/ci.yml @@ -0,0 +1,53 @@ +upload-binary: + targets: + - esp32s3 + - esp32 + diagram: + esp32s3: + parts: + - type: wokwi-potentiometer + id: pot1 + top: 65.9 + left: -115.4 + attrs: {} + connections: + - - pot1:VCC + - esp:3V3.2 + - red + - - v19.2 + - h47.2 + - v-115.2 + - - pot1:SIG + - esp:2 + - green + - - v144 + - h191.6 + - v-115.2 + - - esp:GND.1 + - pot1:GND + - black + - - h0 + esp32: + parts: + - type: wokwi-potentiometer + id: pot1 + top: 37.1 + left: -125 + attrs: {} + connections: + - - pot1:VCC + - esp:3V3 + - red + - - v19.2 + - h47.2 + - v-96 + - - pot1:SIG + - esp:2 + - green + - - v124.8 + - h210.8 + - v-76.8 + - - esp:GND.1 + - pot1:GND + - black + - - h0 diff --git a/libraries/ESP32/examples/GPIO/Blink/Blink.ino b/libraries/ESP32/examples/GPIO/Blink/Blink.ino new file mode 100644 index 00000000000..00064ab9d03 --- /dev/null +++ b/libraries/ESP32/examples/GPIO/Blink/Blink.ino @@ -0,0 +1,25 @@ +// Define pin numbers +#define LED_PIN 4 +#define BUTTON_PIN 0 + +void setup() { + // Initialize the LED pin as an output + pinMode(LED_PIN, OUTPUT); + + // Initialize the button pin as an input with internal pull-up resistor + pinMode(BUTTON_PIN, INPUT_PULLUP); +} + +void loop() { + // Check if button is pressed (LOW because of INPUT_PULLUP) + if (digitalRead(BUTTON_PIN) == LOW) { + // Blink LED while button is held down + digitalWrite(LED_PIN, HIGH); // Turn LED on + delay(500); // Wait 500ms + digitalWrite(LED_PIN, LOW); // Turn LED off + delay(500); // Wait 500ms + } else { + // Keep LED off when button is not pressed + digitalWrite(LED_PIN, LOW); + } +} diff --git a/libraries/ESP32/examples/GPIO/Blink/ci.yml b/libraries/ESP32/examples/GPIO/Blink/ci.yml new file mode 100644 index 00000000000..eb31ec19671 --- /dev/null +++ b/libraries/ESP32/examples/GPIO/Blink/ci.yml @@ -0,0 +1,78 @@ +upload-binary: + targets: + - esp32 + - esp32s3 + diagram: + esp32: + parts: + - type: wokwi-pushbutton + id: btn1 + top: 140.6 + left: 144 + attrs: + color: green + - type: wokwi-led + id: led1 + top: 66 + left: 131.4 + rotate: 90 + attrs: + color: red + connections: + - - btn1:1.l + - esp:0 + - blue + - - h-19.2 + - v-19.2 + - - btn1:2.r + - esp:GND.1 + - black + - - h19.4 + - v48.2 + - h-240 + - v-67.2 + - - esp:GND.2 + - led1:C + - black + - - h33.64 + - v114.8 + - - esp:4 + - led1:A + - green + - - h0 + esp32s3: + parts: + - type: wokwi-pushbutton + id: btn1 + top: 140.6 + left: 115.2 + attrs: + color: green + - type: wokwi-led + id: led1 + top: 66.4 + left: -56.2 + rotate: 270 + attrs: + color: red + flip: "" + connections: + - - btn1:1.l + - esp:0 + - blue + - - h-19.2 + - v0 + - h-4.57 + - - btn1:2.r + - esp:GND.3 + - green + - - h19.4 + - v48.38 + - - esp:4 + - led1:A + - green + - - h0 + - - esp:GND.1 + - led1:C + - black + - - h0 diff --git a/libraries/ESP32/examples/GPIO/GPIOInterrupt/GPIOInterrupt.ino b/libraries/ESP32/examples/GPIO/GPIOInterrupt/GPIOInterrupt.ino index 2e94176224e..b9435d559f2 100644 --- a/libraries/ESP32/examples/GPIO/GPIOInterrupt/GPIOInterrupt.ino +++ b/libraries/ESP32/examples/GPIO/GPIOInterrupt/GPIOInterrupt.ino @@ -6,8 +6,8 @@ struct Button { bool pressed; }; -Button button1 = {23, 0, false}; -Button button2 = {18, 0, false}; +Button button1 = {0, 0, false}; +Button button2 = {4, 0, false}; void ARDUINO_ISR_ATTR isr(void *arg) { Button *s = static_cast