From ba8a3964b514bfd667f332841f0de43a68c9b210 Mon Sep 17 00:00:00 2001 From: Andrew Schilling Date: Tue, 7 Oct 2025 17:22:12 +0000 Subject: [PATCH 1/4] Fixing documentation layout Signed-off-by: Andrew Schilling --- .vscode/settings.json | 3 +- docs/{source => }/conf.py | 23 ++++-- docs/{source => }/faqs.md | 0 docs/{source => }/guides/cli.md | 21 +++--- docs/{source => }/guides/configuration.md | 5 +- docs/{source => }/guides/execution.md | 20 ++++-- docs/{source => }/guides/index.md | 12 ++-- docs/{source => }/guides/management.md | 6 +- docs/{source => }/guides/ray.md | 0 docs/{source => }/guides/why-use-nemo-run.md | 0 docs/{source/index.rst => index.md} | 74 +++++++++++++------- docs/{source => }/project.json | 0 docs/{source => }/versions1.json | 2 +- pyproject.toml | 1 + uv.lock | 31 +++++--- 15 files changed, 130 insertions(+), 68 deletions(-) rename docs/{source => }/conf.py (79%) rename docs/{source => }/faqs.md (100%) rename docs/{source => }/guides/cli.md (98%) rename docs/{source => }/guides/configuration.md (98%) rename docs/{source => }/guides/execution.md (92%) rename docs/{source => }/guides/index.md (88%) rename docs/{source => }/guides/management.md (95%) rename docs/{source => }/guides/ray.md (100%) rename docs/{source => }/guides/why-use-nemo-run.md (100%) rename docs/{source/index.rst => index.md} (55%) rename docs/{source => }/project.json (100%) rename docs/{source => }/versions1.json (56%) diff --git a/.vscode/settings.json b/.vscode/settings.json index 49ae8905..6008d4a3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -20,5 +20,6 @@ "test" ], "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "python.testing.pytestEnabled": true, + "liveServer.settings.port": 5501 } diff --git a/docs/source/conf.py b/docs/conf.py similarity index 79% rename from docs/source/conf.py rename to docs/conf.py index 9609288f..624c4b39 100644 --- a/docs/source/conf.py +++ b/docs/conf.py @@ -33,10 +33,25 @@ "sphinx.ext.githubpages", "sphinx.ext.napoleon", "sphinxcontrib.mermaid", + "sphinx_copybutton", ] templates_path = ["_templates"] -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "documentation.md"] + +# -- Options for MyST Parser (Markdown) -------------------------------------- +# MyST Parser settings +myst_enable_extensions = [ + "dollarmath", # Enables dollar math for inline math + "amsmath", # Enables LaTeX math for display mode + "colon_fence", # Enables code blocks using ::: delimiters instead of ``` + "deflist", # Supports definition lists with term: definition format + "fieldlist", # Enables field lists for metadata like :author: Name + "tasklist", # Adds support for GitHub-style task lists with [ ] and [x] +] +myst_heading_anchors = 5 # Generates anchor links for headings up to level 5 +myst_fence_as_directive = ["mermaid"] + python_maximum_signature_line_length = 88 # Autoapi settings @@ -44,7 +59,7 @@ autoapi_keep_files = False autoapi_add_toctree_entry = False autoapi_type = "python" -autoapi_dirs = ["../../nemo_run"] +autoapi_dirs = ["../nemo_run"] autoapi_file_pattern = "*.py" autoapi_root = "api" autoapi_options = [ @@ -58,10 +73,6 @@ # Autodoc settings autodoc_typehints = "signature" -# MyST settings -myst_heading_anchors = 3 -myst_fence_as_directive = ["mermaid"] - # Napoleon settings napoleon_google_docstring = True napoleon_numpy_docstring = True diff --git a/docs/source/faqs.md b/docs/faqs.md similarity index 100% rename from docs/source/faqs.md rename to docs/faqs.md diff --git a/docs/source/guides/cli.md b/docs/guides/cli.md similarity index 98% rename from docs/source/guides/cli.md rename to docs/guides/cli.md index 052a61ca..226b7d5e 100644 --- a/docs/source/guides/cli.md +++ b/docs/guides/cli.md @@ -2,7 +2,7 @@ NeMo Run CLI is a Python-based command-line tool designed to efficiently configure and execute machine learning experiments. It provides a type-safe, Python-centric alternative to argparse and Hydra, streamlining workflows from prototyping to scaling across diverse environments. -## 1. Introduction +## Introduction NeMo Run CLI simplifies experiment management by leveraging Python's capabilities: @@ -65,7 +65,7 @@ def train(): - **Typer**: General-purpose CLIs with good documentation that don't require nested configuration - **argparse**: Simple scripts with minimal configuration needs and standard library requirements -## 2. Core Concepts +## Core Concepts - **Entrypoints**: Python functions decorated with `@run.cli.entrypoint` serving as primary CLI commands. - **Factories**: Functions decorated with `@run.cli.factory` that configure complex objects (e.g., models, optimizers). @@ -73,7 +73,7 @@ def train(): - **Experiments**: Groups of tasks executed sequentially or concurrently. - **RunContext**: Manages execution settings, including executor configurations. -## 3. Getting Started +## Getting Started ### Example 1: Basic Entrypoint @@ -116,7 +116,7 @@ Output: Unknown argument 'epocks'. Did you mean 'epochs'? ``` -## 4. Advanced Configuration +## Advanced Configuration ### Nested Configurations with Dataclasses @@ -213,25 +213,25 @@ File contents: │ 2 target = "main.train_model" │ 3 batch_size = 32 │ 4 epochs = 10 -│ 5 +│ 5 │ 6 [model] │ 7 target = "main.Model" │ 8 activation = "relu" │ 9 hidden_size = 256 │ 10 num_layers = 5 -│ 11 +│ 11 │ 12 [optimizer] │ 13 target = "main.Optimizer" │ 14 betas = [ 0.9, 0.999,] │ 15 learning_rate = 0.001 │ 16 weight_decay = 1e-5 -│ 17 +│ 17 ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ Export complete. Skipping execution. ``` -## 5. Executors +## Executors Executors determine where your code runs, such as local environments, Docker containers, or Slurm clusters. @@ -283,7 +283,7 @@ def slurm_cluster() -> run.Executor: job_dir=BASE_DIR, container_image="nvcr.io/nvidia/nemo:dev", container_mounts=[ - f"/home/{USER}:/home/{USER}", + f"/home/{USER}:/home/{USER}", "/lustre:/lustre", ], time="4:00:00", @@ -298,7 +298,7 @@ Execute lazily: python script.py --lazy model=alexnet epochs=5 run.executor=slurm_cluster run.executor.nodes=2 ``` -## 6. Advanced CLI Features +## Advanced CLI Features ### Dry Runs and Help Messages @@ -393,4 +393,3 @@ The help output clearly shows: 5. Registered factory functions for each complex argument type This makes it easy for users to discover what factory functions they can use to configure complex arguments like `model` and `optimizer`, along with information about where these factories are defined (module name and line number). - diff --git a/docs/source/guides/configuration.md b/docs/guides/configuration.md similarity index 98% rename from docs/source/guides/configuration.md rename to docs/guides/configuration.md index 7d7b91b0..134f5708 100644 --- a/docs/source/guides/configuration.md +++ b/docs/guides/configuration.md @@ -113,7 +113,10 @@ In our context, this is equivalent to: _target_: nemo.collections.llm.gpt.model.llama.Llama3Config8B seq_length: 16384 ``` -> Note: we've used the [Hydra instantiation](https://hydra.cc/docs/advanced/instantiate_objects/overview/) syntax here. + +```{note} +We've used the [Hydra instantiation](https://hydra.cc/docs/advanced/instantiate_objects/overview/) syntax here. +``` Python operations are performed on the config rather than directly on the class. For example: diff --git a/docs/source/guides/execution.md b/docs/guides/execution.md similarity index 92% rename from docs/source/guides/execution.md rename to docs/guides/execution.md index cd7c49d6..7f34893d 100644 --- a/docs/source/guides/execution.md +++ b/docs/guides/execution.md @@ -14,9 +14,13 @@ A tuple of task and executor form an execution unit. A key goal of NeMo-Run is t Once an execution unit is created, the next step is to run it. The `run.run` function executes a single task, whereas `run.Experiment` offers more fine-grained control to define complex experiments. `run.run` wraps `run.Experiment` with a single task. `run.Experiment` is an API to launch and manage multiple tasks all using pure Python. The `run.Experiment` takes care of storing the run metadata, launching it on the specified cluster, and syncing the logs, etc. Additionally, `run.Experiment` also provides management tools to easily inspect and reproduce past experiments. The `run.Experiment` is inspired from [xmanager](https://github.com/google-deepmind/xmanager/tree/main) and uses [TorchX](https://pytorch.org/torchx/latest/) under the hood to handle execution. -> **_NOTE:_** NeMo-Run assumes familiarity with Docker and uses a docker image as the environment for remote execution. This means you must provide a Docker image that includes all necessary dependencies and configurations when using a remote executor. +```{note} +NeMo-Run assumes familiarity with Docker and uses a docker image as the environment for remote execution. This means you must provide a Docker image that includes all necessary dependencies and configurations when using a remote executor. +``` -> **_NOTE:_** All the experiment metadata is stored under `NEMORUN_HOME` env var on the machine where you launch the experiments. By default, the value for `NEMORUN_HOME` value is `~/.run`. Be sure to change this according to your needs. +```{note} +All the experiment metadata is stored under `NEMORUN_HOME` env var on the machine where you launch the experiments. By default, the value for `NEMORUN_HOME` value is `~/.run`. Be sure to change this according to your needs. +``` ## Executors Executors are dataclasses that configure your remote executor and set up the packaging of your code. All supported executors inherit from the base class `run.Executor`, but have configuration parameters specific to their execution environment. There is an initial cost to understanding the specifics of your executor and setting it up, but this effort is easily amortized over time. @@ -29,7 +33,9 @@ We support the following `launchers`: - `torchrun` or `run.Torchrun`: This will launch the task using `torchrun`. See the `Torchrun` class for configuration options. You can use it using `executor.launcher = "torchrun"` or `executor.launcher = Torchrun(...)`. - `ft` or `run.core.execution.FaultTolerance`: This will launch the task using NVIDIA's fault tolerant launcher. See the `FaultTolerance` class for configuration options. You can use it using `executor.launcher = "ft"` or `executor.launcher = FaultTolerance(...)`. -> **_NOTE:_** Launcher may not work very well with `run.Script`. Please report any issues at https://github.com/NVIDIA-NeMo/Run/issues. +```{attention} +Launcher may not work very well with `run.Script`. Please report any issues at [https://github.com/NVIDIA-NeMo/Run/issues](https://github.com/NVIDIA-NeMo/Run/issues). +``` ### Packagers @@ -65,7 +71,9 @@ Your working directory at the time of execution will look like: ``` If you're executing a Python function, this working directory will automatically be included in your Python path. -> **_NOTE:_** git archive doesn't package uncommitted changes. In the future, we may add support for including uncommitted changes while honoring `.gitignore`. +```{note} +Git archive doesn't package uncommitted changes. In the future, we may add support for including uncommitted changes while honoring `.gitignore`. +``` `run.PatternPackager` is a packager that uses a pattern to package your code. It is useful for packaging code that is not under version control. For example, if you have a directory structure like this: ``` @@ -228,7 +236,9 @@ As demonstrated in the examples, defining executors in Python offers great flexi The `DGXCloudExecutor` integrates with a DGX Cloud cluster's Run:ai API to launch distributed jobs. It uses REST API calls to authenticate, identify the target project and cluster, and submit the job specification. -> **_WARNING:_** Currently, the `DGXCloudExecutor` is only supported when launching experiments *from* a pod running on the DGX Cloud cluster itself. Furthermore, this launching pod must have access to a Persistent Volume Claim (PVC) where the experiment/job directories will be created, and this same PVC must also be configured to be mounted by the job being launched. +```{warning} +Currently, the `DGXCloudExecutor` is only supported when launching experiments *from* a pod running on the DGX Cloud cluster itself. Furthermore, this launching pod must have access to a Persistent Volume Claim (PVC) where the experiment/job directories will be created, and this same PVC must also be configured to be mounted by the job being launched. +``` Here's an example configuration: diff --git a/docs/source/guides/index.md b/docs/guides/index.md similarity index 88% rename from docs/source/guides/index.md rename to docs/guides/index.md index 912d7ce8..f8d5fcf1 100644 --- a/docs/source/guides/index.md +++ b/docs/guides/index.md @@ -1,7 +1,7 @@ -Guides -================= +# Guides -```{toctree} + +:::{toctree} :maxdepth: 2 :hidden: @@ -11,7 +11,7 @@ execution management ray cli -``` +::: Welcome to the NeMo-Run guides! This section provides comprehensive documentation on how to use NeMo-Run effectively for your machine learning experiments. @@ -36,7 +36,7 @@ For more advanced usage: NeMo-Run is built around three core responsibilities: 1. **Configuration** - Define your ML experiments using a flexible, Pythonic configuration system. -2. **Execution** - Run your experiments seamlessly across local machines, Slurm clusters, cloud providers, and more. -3. **Management** - Track, reproduce, and organize your experiments with built-in experiment management. +1. **Execution** - Run your experiments seamlessly across local machines, Slurm clusters, cloud providers, and more. +1. **Management** - Track, reproduce, and organize your experiments with built-in experiment management. Each guide dives deep into these concepts with practical examples and best practices. Choose a guide above to get started! diff --git a/docs/source/guides/management.md b/docs/guides/management.md similarity index 95% rename from docs/source/guides/management.md rename to docs/guides/management.md index 30e7af6b..66a4f25e 100644 --- a/docs/source/guides/management.md +++ b/docs/guides/management.md @@ -12,7 +12,9 @@ exp = Experiment("My Experiment") When executed, it will automatically generate a unique experiment ID for you, which represents one unique run of the experiment. -> [!NOTE] > `Experiment` is a context manager and `Experiment.add` and `Experiment.run` methods can currently only be used after entering the context manager. +```{note} +`Experiment` is a context manager and `Experiment.add` and `Experiment.run` methods can currently only be used after entering the context manager. +``` ## Add Tasks @@ -73,7 +75,7 @@ You can check the status of an experiment using the `status` method: exp.status() ``` -This method will display information about the status of each task in the experiment. The following is a sample output from the status of experiment in [hello_scripts.py](../../../examples/hello-world/hello_scripts.py): +This method will display information about the status of each task in the experiment. The following is a sample output from the status of experiment in [hello_scripts.py](../../examples/hello-world/hello_scripts.py): ```bash Experiment Status for experiment_with_scripts_1730761155 diff --git a/docs/source/guides/ray.md b/docs/guides/ray.md similarity index 100% rename from docs/source/guides/ray.md rename to docs/guides/ray.md diff --git a/docs/source/guides/why-use-nemo-run.md b/docs/guides/why-use-nemo-run.md similarity index 100% rename from docs/source/guides/why-use-nemo-run.md rename to docs/guides/why-use-nemo-run.md diff --git a/docs/source/index.rst b/docs/index.md similarity index 55% rename from docs/source/index.rst rename to docs/index.md index 4329df4f..db5ca539 100644 --- a/docs/source/index.rst +++ b/docs/index.md @@ -1,56 +1,52 @@ -.. NeMo-Run documentation master file, created by - sphinx-quickstart on Thu Jul 25 17:57:46 2024. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +# NeMo-Run Documentation -NeMo-Run Documentation -====================== NeMo-Run is a powerful tool designed to streamline the configuration, execution and management of Machine Learning experiments across various computing environments. NeMo Run has three core responsibilities: -1. :doc:`Configuration ` -2. :doc:`Execution ` -3. :doc:`Management ` +1. [Configuration](guides/configuration.md) +1. [Execution](guides/execution.md) +1. [Management](guides/management.md) Please click into each link to learn more. This is also the typical order Nemo Run users will follow to setup and launch experiments. -.. toctree:: - :maxdepth: 1 +## Install the Project - guides/index - API Reference - faqs - -Install the Project -------------------- To install the project, use the following command: -``pip install git+https://github.com/NVIDIA-NeMo/Run.git`` +```bash +ip install git+https://github.com/NVIDIA-NeMo/Run.git +``` To install Skypilot with optional features, use one of the following commands: - To install Skypilot with Kubernetes support: - ``pip install git+https://github.com/NVIDIA-NeMo/Run.git[skypilot]`` + ```bash + pip install git+https://github.com/NVIDIA-NeMo/Run.git[skypilot] + ``` - To install Skypilot with support for all cloud platforms: - ``pip install git+https://github.com/NVIDIA-NeMo/Run.git[skypilot-all]`` + ```bash + pip install git+https://github.com/NVIDIA-NeMo/Run.git[skypilot-all] + ``` You can also manually install Skypilot from https://skypilot.readthedocs.io/en/latest/getting-started/installation.html If using DGX Cloud Lepton, use the following command to install the Lepton CLI: -``pip install leptonai`` +```bash +pip install leptonai +``` To authenticate with the DGX Cloud Lepton cluster, navigate to the **Settings > Tokens** page in the DGX Cloud Lepton UI and copy the ``lep login`` command shown on the page and run it in the terminal. -Make sure you have `pip` installed and configured properly. +Make sure you have ``pip`` installed and configured properly. + +## Tutorials -Tutorials ---------- The ``hello_world`` tutorial series provides a comprehensive introduction to NeMo-Run, demonstrating its capabilities through a simple example. The tutorial covers: @@ -61,6 +57,30 @@ The ``hello_world`` tutorial series provides a comprehensive introduction to NeM You can find the tutorial series below: -1. `Part 1: Hello World `_ -2. `Part 2: Hello Experiments `_ -3. `Part 3: Hello Scripts `_ +1. [Part 1: Hello World](https://github.com/NVIDIA-NeMo/Run/blob/main/examples/hello-world/hello_world.ipynb) +1. [Part 2: Hello Experiments](https://github.com/NVIDIA-NeMo/Run/blob/main/examples/hello-world/hello_experiments.ipynb) +1. [Part 3: Hello Scripts](https://github.com/NVIDIA-NeMo/Run/blob/main/examples/hello-world/hello_scripts.py) + + +:::{toctree} +:hidden: +Home +::: + +:::{toctree} +:caption: Get Started +:maxdepth: 2 +:hidden: + +guides/index + +::: + +:::{toctree} +:hidden: +:caption: Reference +:maxdepth: 2 + +faqs +API Reference +::: diff --git a/docs/source/project.json b/docs/project.json similarity index 100% rename from docs/source/project.json rename to docs/project.json diff --git a/docs/source/versions1.json b/docs/versions1.json similarity index 56% rename from docs/source/versions1.json rename to docs/versions1.json index 604af762..c987bebc 100644 --- a/docs/source/versions1.json +++ b/docs/versions1.json @@ -2,6 +2,6 @@ { "preferred": true, "version": "0.1.0", - "url": "../0.1.0" + "url": "http://docs.nvidia.com/nemo/run/0.1.0" } ] diff --git a/pyproject.toml b/pyproject.toml index 05887f0f..0e5ee09b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,7 @@ docs = [ "sphinx-autoapi>=3.0.0", "nvidia-sphinx-theme", "sphinxcontrib-mermaid", + "sphinx-copybutton>=0.5.2", ] [build-system] diff --git a/uv.lock b/uv.lock index 363caffc..195e1c2f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.13' and sys_platform == 'darwin' and extra != 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all' and extra != 'group-8-nemo-run-docs'", @@ -250,10 +250,10 @@ name = "anyio" version = "4.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, { name = "idna" }, { name = "sniffio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/73/199a98fc2dae33535d6b8e8e6ec01f8c1d76c9adb096c6b7d64823038cde/anyio-4.8.0.tar.gz", hash = "sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a", size = 181126, upload-time = "2025-01-05T13:13:11.095Z" } wheels = [ @@ -2338,7 +2338,7 @@ name = "cryptography" version = "42.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, + { name = "cffi", marker = "platform_python_implementation != 'PyPy' or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/93/a7/1498799a2ea06148463a9a2c10ab2f6a921a74fb19e231b27dc412a748e2/cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2", size = 671250, upload-time = "2024-06-04T19:55:08.609Z" } wheels = [ @@ -2474,7 +2474,7 @@ name = "docker" version = "7.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "sys_platform == 'win32' or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, { name = "requests" }, { name = "urllib3" }, ] @@ -4449,6 +4449,7 @@ docs = [ { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs')" }, { name = "sphinx-autoapi" }, + { name = "sphinx-copybutton" }, { name = "sphinxcontrib-mermaid" }, ] lint = [ @@ -4497,6 +4498,7 @@ docs = [ { name = "nvidia-sphinx-theme" }, { name = "sphinx", specifier = ">=7" }, { name = "sphinx-autoapi", specifier = ">=3.0.0" }, + { name = "sphinx-copybutton", specifier = ">=0.5.2" }, { name = "sphinxcontrib-mermaid" }, ] lint = [{ name = "ruff", specifier = ">=0.4.4" }] @@ -6100,7 +6102,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "rpds-py" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" } wheels = [ @@ -6168,7 +6170,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py" }, { name = "pygments" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ab/3a/0316b28d0761c6734d6bc14e770d85506c986c85ffb239e688eeaab2c2bc/rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098", size = 223149, upload-time = "2024-11-01T16:43:57.873Z" } wheels = [ @@ -6764,6 +6766,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/17/0eda9dc80fcaf257222b506844207e71b5d59567c41bbdcca2a72da119b9/sphinx_autoapi-3.6.0-py3-none-any.whl", hash = "sha256:f3b66714493cab140b0e896d33ce7137654a16ac1edb6563edcbd47bf975f711", size = 35281, upload-time = "2025-02-18T01:50:52.789Z" }, ] +[[package]] +name = "sphinx-copybutton" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/2b/a964715e7f5295f77509e59309959f4125122d648f86b4fe7d70ca1d882c/sphinx-copybutton-0.5.2.tar.gz", hash = "sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd", size = 23039, upload-time = "2023-04-14T08:10:22.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/48/1ea60e74949eecb12cdd6ac43987f9fd331156388dcc2319b45e2ebb81bf/sphinx_copybutton-0.5.2-py3-none-any.whl", hash = "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e", size = 13343, upload-time = "2023-04-14T08:10:20.844Z" }, +] + [[package]] name = "sphinxcontrib-applehelp" version = "2.0.0" @@ -7311,7 +7326,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4b/4d/938bd85e5bf2edeec766267a5015ad969730bb91e31b44021dfe8b22df6c/uvicorn-0.34.0.tar.gz", hash = "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9", size = 76568, upload-time = "2024-12-15T13:33:30.42Z" } wheels = [ From 2734b00e44807bbe0bef9ac1e94af8ff81749072 Mon Sep 17 00:00:00 2001 From: Andrew Schilling Date: Tue, 7 Oct 2025 17:29:44 +0000 Subject: [PATCH 2/4] documentation.md Signed-off-by: Andrew Schilling --- docs/conf.py | 1 + docs/documentation.md | 47 +++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + uv.lock | 29 ++++++++++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 docs/documentation.md diff --git a/docs/conf.py b/docs/conf.py index 624c4b39..f222af11 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,6 +34,7 @@ "sphinx.ext.napoleon", "sphinxcontrib.mermaid", "sphinx_copybutton", + "sphinx_new_tab_link", ] templates_path = ["_templates"] diff --git a/docs/documentation.md b/docs/documentation.md new file mode 100644 index 00000000..99399ea9 --- /dev/null +++ b/docs/documentation.md @@ -0,0 +1,47 @@ +# Documentation Development + +- [Documentation Development](#documentation-development) + - [Build the Documentation](#build-the-documentation) + - [Live Building](#live-building) + + +## Build the Documentation + +The following sections describe how to set up and build the NeMo RL documentation. + +Switch to the documentation source folder and generate HTML output. + +```sh +cd docs/ +uv run --group docs sphinx-build . _build/html +``` + +* The resulting HTML files are generated in a `_build/html` folder that is created under the project `docs/` folder. +* The generated python API docs are placed in `apidocs` under the `docs/` folder. + +## Checking for Broken Links + +To check for broken http links in the docs, run this command: + +```sh +cd docs/ +uv run --group docs sphinx-build --builder linkcheck . _build/linkcheck +``` + +It will output a JSON file at `_build/linkcheck/output.json` with links it found while building the +docs. Records will have a status of `broken` if the link is not reachable. The `docs/conf.py` file is +configured to ignore github links because the CI test will often experience rate limit errors. +Comment out the `linkcheck_ignore` variable there to check all the links. + +## Live Building + +When writing documentation, it can be helpful to serve the documentation and have it update live while you edit. + +To do so, run: + +```sh +cd docs/ +uv run --group docs sphinx-autobuild . _build/html --port 12345 --host 0.0.0.0 +``` + +Open a web browser and go to `http://${HOST_WHERE_SPHINX_COMMAND_RUN}:12345` to view the output. diff --git a/pyproject.toml b/pyproject.toml index 0e5ee09b..34e636b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ docs = [ "nvidia-sphinx-theme", "sphinxcontrib-mermaid", "sphinx-copybutton>=0.5.2", + "sphinx-new-tab-link>=0.8.0", ] [build-system] diff --git a/uv.lock b/uv.lock index 195e1c2f..06869599 100644 --- a/uv.lock +++ b/uv.lock @@ -4450,6 +4450,7 @@ docs = [ { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs')" }, { name = "sphinx-autoapi" }, { name = "sphinx-copybutton" }, + { name = "sphinx-new-tab-link" }, { name = "sphinxcontrib-mermaid" }, ] lint = [ @@ -4499,6 +4500,7 @@ docs = [ { name = "sphinx", specifier = ">=7" }, { name = "sphinx-autoapi", specifier = ">=3.0.0" }, { name = "sphinx-copybutton", specifier = ">=0.5.2" }, + { name = "sphinx-new-tab-link", specifier = ">=0.8.0" }, { name = "sphinxcontrib-mermaid" }, ] lint = [{ name = "ruff", specifier = ">=0.4.4" }] @@ -6779,6 +6781,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/48/1ea60e74949eecb12cdd6ac43987f9fd331156388dcc2319b45e2ebb81bf/sphinx_copybutton-0.5.2-py3-none-any.whl", hash = "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e", size = 13343, upload-time = "2023-04-14T08:10:20.844Z" }, ] +[[package]] +name = "sphinx-new-tab-link" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs')" }, + { name = "sphinxcontrib-extdevhelper-kasane" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/de/f62360114d605d1c7c5fba060c76a3521655d388e3fa03747d31b9452a69/sphinx_new_tab_link-0.8.0.tar.gz", hash = "sha256:6c757d99f559224a04142c3971c8baa6ac90aca905f15b129d57eeca0ece9582", size = 6637, upload-time = "2025-04-01T14:01:18.88Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/c7/b35261707bc72ce2dff1c4b66a391542be4ba80faded83b6e8e00fb14af9/sphinx_new_tab_link-0.8.0-py3-none-any.whl", hash = "sha256:c74b873d6c8a1ec089015dc414a75f6908e87f66ce4ab8d9f2c7268f13afc593", size = 5622, upload-time = "2025-04-01T14:01:17.091Z" }, +] + [[package]] name = "sphinxcontrib-applehelp" version = "2.0.0" @@ -6797,6 +6813,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, ] +[[package]] +name = "sphinxcontrib-extdevhelper-kasane" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs')" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot-all' and extra == 'group-8-nemo-run-docs') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'extra-8-nemo-run-skypilot-all') or (extra == 'extra-8-nemo-run-skypilot' and extra == 'group-8-nemo-run-docs')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/74/d5b2650ca859664400603d0db08b6bfce507a9606284137ab0d7bcec4e02/sphinxcontrib-extdevhelper-kasane-0.2.0.tar.gz", hash = "sha256:4dc7b00327f33c7b421c27122b40278eeaca43f24601b572cee5616d31b206a9", size = 4496, upload-time = "2024-03-16T07:23:03.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/91/41a07c91e2adee3463b443cf924778f5a2d92a1166f3f7208959d8b1fabf/sphinxcontrib_extdevhelper_kasane-0.2.0-py3-none-any.whl", hash = "sha256:20f94e3b209cddec24596234458ea3887e7a7ad45b54a4d0a5bf169ff45a38f1", size = 3918, upload-time = "2024-03-16T07:23:01.026Z" }, +] + [[package]] name = "sphinxcontrib-htmlhelp" version = "2.1.0" From b833920d21a5faf7548e6a7bf003cdbc2f330bf0 Mon Sep 17 00:00:00 2001 From: Andrew Schilling Date: Tue, 7 Oct 2025 17:31:53 +0000 Subject: [PATCH 3/4] Removing live-server Signed-off-by: Andrew Schilling --- .vscode/settings.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 6008d4a3..3806c61e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,5 +21,4 @@ ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, - "liveServer.settings.port": 5501 } From 9d5f394a0bfd110c6eaba96ef6ab6d72ec5f07a5 Mon Sep 17 00:00:00 2001 From: Andrew Schilling Date: Tue, 7 Oct 2025 18:46:20 +0000 Subject: [PATCH 4/4] Correctin .vscode Signed-off-by: Andrew Schilling --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 3806c61e..49ae8905 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -20,5 +20,5 @@ "test" ], "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, + "python.testing.pytestEnabled": true }