From d78009c960ebb652d86dcfc6cd3b55b1ae28659c Mon Sep 17 00:00:00 2001 From: Michael Ruoss Date: Wed, 5 Nov 2025 20:38:43 +0100 Subject: [PATCH 1/3] make uv_init return a struct with required paths --- lib/pythonx.ex | 25 +++++++++- lib/pythonx/uv.ex | 118 +++++++++++++++++++++++++--------------------- 2 files changed, 88 insertions(+), 55 deletions(-) diff --git a/lib/pythonx.ex b/lib/pythonx.ex index 1e5453a..587c819 100644 --- a/lib/pythonx.ex +++ b/lib/pythonx.ex @@ -8,8 +8,21 @@ defmodule Pythonx do @moduledoc readme_docs + defstruct [ + :python_dl_path, + :python_home_path, + :python_executable_path, + :sys_paths + ] + alias Pythonx.Object + @type python_config :: %__MODULE__{ + python_dl_path: String.t(), + python_home_path: String.t(), + python_executable_path: String.t(), + sys_paths: [String.t()] + } @type encoder :: (term(), encoder() -> Object.t()) @doc ~s''' @@ -90,7 +103,7 @@ defmodule Pythonx do # (`sys.path`). Defaults to `[]`. # @doc false - @spec init(String.t(), String.t(), keyword()) :: :ok + @spec init(String.t(), String.t(), String.t(), keyword()) :: :ok def init(python_dl_path, python_home_path, python_executable_path, opts \\ []) when is_binary(python_dl_path) and is_binary(python_home_path) when is_binary(python_executable_path) and is_list(opts) do @@ -111,6 +124,16 @@ defmodule Pythonx do Pythonx.NIF.init(python_dl_path, python_home_path, python_executable_path, opts[:sys_paths]) end + @spec init(python_config()) :: :ok + def init(%__MODULE__{ + python_dl_path: python_dl_path, + python_home_path: python_home_path, + python_executable_path: python_executable_path, + sys_paths: sys_paths + }) do + init(python_dl_path, python_home_path, python_executable_path, sys_paths: sys_paths) + end + @doc ~S''' Evaluates the Python `code`. diff --git a/lib/pythonx/uv.ex b/lib/pythonx/uv.ex index 68a648a..fcebd56 100644 --- a/lib/pythonx/uv.ex +++ b/lib/pythonx/uv.ex @@ -66,7 +66,7 @@ defmodule Pythonx.Uv do Initializes the interpreter using Python and dependencies previously fetched by `fetch/3`. """ - @spec init(String.t(), boolean()) :: :ok + @spec init(String.t(), boolean()) :: Pythonx.python_config() def init(pyproject_toml, priv?, opts \\ []) do opts = Keyword.validate!(opts, uv_version: default_uv_version()) project_dir = project_dir(pyproject_toml, priv?, opts[:uv_version]) @@ -95,59 +95,69 @@ defmodule Pythonx.Uv do root_dir = Path.join(python_install_dir(priv?, opts[:uv_version]), versioned_dir_name) - case :os.type() do - {:win32, _osname} -> - # Note that we want the version-specific DLL, rather than the - # "forwarder DLL" python3.dll, otherwise symbols cannot be - # found directly. - python_dl_path = - root_dir - |> Path.join("python3?*.dll") - |> wildcard_one!() - |> make_windows_slashes() - - python_home_path = make_windows_slashes(root_dir) - - python_executable_path = - project_dir - |> Path.join(".venv/Scripts/python.exe") - |> make_windows_slashes() - - venv_packages_path = - project_dir - |> Path.join(".venv/Lib/site-packages") - |> make_windows_slashes() - - Pythonx.init(python_dl_path, python_home_path, python_executable_path, - sys_paths: [venv_packages_path] - ) - - {:unix, osname} -> - dl_extension = - case osname do - :darwin -> ".dylib" - :linux -> ".so" - end - - python_dl_path = - root_dir - |> Path.join("lib/libpython3.*" <> dl_extension) - |> wildcard_one!() - |> Path.expand() - - python_home_path = root_dir - - python_executable_path = Path.join(project_dir, ".venv/bin/python") - - venv_packages_path = - project_dir - |> Path.join(".venv/lib/python3*/site-packages") - |> wildcard_one!() - - Pythonx.init(python_dl_path, python_home_path, python_executable_path, - sys_paths: [venv_packages_path] - ) - end + python_config = + case :os.type() do + {:win32, _osname} -> + # Note that we want the version-specific DLL, rather than the + # "forwarder DLL" python3.dll, otherwise symbols cannot be + # found directly. + python_dl_path = + root_dir + |> Path.join("python3?*.dll") + |> wildcard_one!() + |> make_windows_slashes() + + python_home_path = make_windows_slashes(root_dir) + + python_executable_path = + project_dir + |> Path.join(".venv/Scripts/python.exe") + |> make_windows_slashes() + + venv_packages_path = + project_dir + |> Path.join(".venv/Lib/site-packages") + |> make_windows_slashes() + + %Pythonx{ + python_dl_path: python_dl_path, + python_home_path: python_home_path, + python_executable_path: python_executable_path, + sys_paths: [venv_packages_path] + } + + {:unix, osname} -> + dl_extension = + case osname do + :darwin -> ".dylib" + :linux -> ".so" + end + + python_dl_path = + root_dir + |> Path.join("lib/libpython3.*" <> dl_extension) + |> wildcard_one!() + |> Path.expand() + + python_home_path = root_dir + + python_executable_path = Path.join(project_dir, ".venv/bin/python") + + venv_packages_path = + project_dir + |> Path.join(".venv/lib/python3*/site-packages") + |> wildcard_one!() + + %Pythonx{ + python_dl_path: python_dl_path, + python_home_path: python_home_path, + python_executable_path: python_executable_path, + sys_paths: [venv_packages_path] + } + end + + Pythonx.init(python_config) + python_config end defp wildcard_one!(path) do From e6d21763263fd6d7a868eb2531fee61abbbe2828 Mon Sep 17 00:00:00 2001 From: Michael Ruoss Date: Sat, 29 Nov 2025 16:33:16 +0100 Subject: [PATCH 2/3] store init_state as persistent_term and add flame_env() to retrieve it --- lib/pythonx.ex | 48 +++++++++++++++++++++++++++++++++++--- lib/pythonx/application.ex | 13 ++++++++++- lib/pythonx/uv.ex | 8 +++---- 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/lib/pythonx.ex b/lib/pythonx.ex index 587c819..83b7b8d 100644 --- a/lib/pythonx.ex +++ b/lib/pythonx.ex @@ -17,7 +17,9 @@ defmodule Pythonx do alias Pythonx.Object - @type python_config :: %__MODULE__{ + @install_env_name "PYTHONX_FLAME_INIT_STATE" + + @type init_state :: %__MODULE__{ python_dl_path: String.t(), python_home_path: String.t(), python_executable_path: String.t(), @@ -71,7 +73,47 @@ defmodule Pythonx do opts = Keyword.validate!(opts, force: false, uv_version: Pythonx.Uv.default_uv_version()) Pythonx.Uv.fetch(pyproject_toml, false, opts) - Pythonx.Uv.init(pyproject_toml, false, Keyword.take(opts, [:uv_version])) + init_state = Pythonx.Uv.init(pyproject_toml, false, Keyword.take(opts, [:uv_version])) + :persistent_term.put(:pythonx_init_state, init_state) + end + + @spec init_state() :: init_state() + defp init_state() do + :persistent_term.get(:pythonx_init_state) + end + + @doc ~s''' + Fetches the pythonx init state from the system environment variable. + ''' + @spec init_state_from_env() :: String.t() | nil + def init_state_from_env(), do: System.get_env(@install_env_name) + + @doc ~s''' + Returns a map containing the environment variables required to initialize Pythonx. + ''' + @spec install_env() :: map() + def install_env() do + init_state = + init_state() + |> :erlang.term_to_binary() + |> Base.encode64() + + %{name: @install_env_name, value: init_state} + end + + @doc ~s''' + Returns a list of paths to copy to the flame runner. + ''' + @spec install_paths() :: list(String.t()) + def install_paths() do + init_state = init_state() + + [ + init_state.python_dl_path, + init_state.python_executable_path + ] ++ + init_state.sys_paths ++ + Path.wildcard(Path.join(init_state.python_home_path, "**"), match_dot: true) end # Initializes the Python interpreter. @@ -124,7 +166,7 @@ defmodule Pythonx do Pythonx.NIF.init(python_dl_path, python_home_path, python_executable_path, opts[:sys_paths]) end - @spec init(python_config()) :: :ok + @spec init(init_state()) :: :ok def init(%__MODULE__{ python_dl_path: python_dl_path, python_home_path: python_home_path, diff --git a/lib/pythonx/application.ex b/lib/pythonx/application.ex index e6bea9f..e280c9d 100644 --- a/lib/pythonx/application.ex +++ b/lib/pythonx/application.ex @@ -31,7 +31,18 @@ defmodule Pythonx.Application do Pythonx.Uv.fetch(pyproject_toml, true, opts) defp maybe_uv_init(), do: Pythonx.Uv.init(unquote(pyproject_toml), true, unquote(opts)) else - defp maybe_uv_init(), do: :noop + defp maybe_uv_init() do + case Pythonx.init_state_from_env() do + nil -> + :noop + + init_state_env_value -> + init_state_env_value + |> Base.decode64!() + |> :erlang.binary_to_term() + |> Pythonx.init() + end + end end defp enable_sigchld() do diff --git a/lib/pythonx/uv.ex b/lib/pythonx/uv.ex index fcebd56..93d6626 100644 --- a/lib/pythonx/uv.ex +++ b/lib/pythonx/uv.ex @@ -66,7 +66,7 @@ defmodule Pythonx.Uv do Initializes the interpreter using Python and dependencies previously fetched by `fetch/3`. """ - @spec init(String.t(), boolean()) :: Pythonx.python_config() + @spec init(String.t(), boolean()) :: Pythonx.init_state() def init(pyproject_toml, priv?, opts \\ []) do opts = Keyword.validate!(opts, uv_version: default_uv_version()) project_dir = project_dir(pyproject_toml, priv?, opts[:uv_version]) @@ -95,7 +95,7 @@ defmodule Pythonx.Uv do root_dir = Path.join(python_install_dir(priv?, opts[:uv_version]), versioned_dir_name) - python_config = + init_state = case :os.type() do {:win32, _osname} -> # Note that we want the version-specific DLL, rather than the @@ -156,8 +156,8 @@ defmodule Pythonx.Uv do } end - Pythonx.init(python_config) - python_config + Pythonx.init(init_state) + init_state end defp wildcard_one!(path) do From 90696dcceb3f1bf5b8d8c5dddebd028c288c3900 Mon Sep 17 00:00:00 2001 From: Michael Ruoss Date: Wed, 3 Dec 2025 18:49:34 +0100 Subject: [PATCH 3/3] refactoring --- lib/pythonx.ex | 94 ++++++++++++++++------------- lib/pythonx/application.ex | 13 +--- lib/pythonx/uv.ex | 118 +++++++++++++++++-------------------- 3 files changed, 110 insertions(+), 115 deletions(-) diff --git a/lib/pythonx.ex b/lib/pythonx.ex index 83b7b8d..156cc1c 100644 --- a/lib/pythonx.ex +++ b/lib/pythonx.ex @@ -8,23 +8,10 @@ defmodule Pythonx do @moduledoc readme_docs - defstruct [ - :python_dl_path, - :python_home_path, - :python_executable_path, - :sys_paths - ] - alias Pythonx.Object - @install_env_name "PYTHONX_FLAME_INIT_STATE" + @install_env_name "PYTHONX_INIT_STATE" - @type init_state :: %__MODULE__{ - python_dl_path: String.t(), - python_home_path: String.t(), - python_executable_path: String.t(), - sys_paths: [String.t()] - } @type encoder :: (term(), encoder() -> Object.t()) @doc ~s''' @@ -73,47 +60,84 @@ defmodule Pythonx do opts = Keyword.validate!(opts, force: false, uv_version: Pythonx.Uv.default_uv_version()) Pythonx.Uv.fetch(pyproject_toml, false, opts) - init_state = Pythonx.Uv.init(pyproject_toml, false, Keyword.take(opts, [:uv_version])) + install_paths = Pythonx.Uv.init(pyproject_toml, false, Keyword.take(opts, [:uv_version])) + + init_state = %{ + type: :uv_init, + pyproject_toml: pyproject_toml, + opts: Keyword.drop(opts, [:force]), + install_paths: install_paths + } + :persistent_term.put(:pythonx_init_state, init_state) end - @spec init_state() :: init_state() + @spec init_state() :: map() defp init_state() do :persistent_term.get(:pythonx_init_state) end - @doc ~s''' - Fetches the pythonx init state from the system environment variable. - ''' @spec init_state_from_env() :: String.t() | nil - def init_state_from_env(), do: System.get_env(@install_env_name) + defp init_state_from_env(), do: System.get_env(@install_env_name) @doc ~s''' - Returns a map containing the environment variables required to initialize Pythonx. + Returns a map with opaque environment variables to initialize Pythonx in + the same way as the current initialization. + + When those environment variables are set, Pythonx is initialized on boot. + + In particular, this can be used to make Pythonx initialize on `FLAME` nodes. ''' @spec install_env() :: map() def install_env() do + init_state = init_state() + + if init_state == nil do + raise "before calling Pythonx.install_env/0, you must initialize Pythonx" + end + init_state = - init_state() + init_state |> :erlang.term_to_binary() |> Base.encode64() - %{name: @install_env_name, value: init_state} + %{@install_env_name => init_state} end @doc ~s''' - Returns a list of paths to copy to the flame runner. + Returns a list of paths that `install_env/0` initialization depends on. + + In particular, this can be used to make Pythonx initialize on `FLAME` nodes. ''' @spec install_paths() :: list(String.t()) def install_paths() do init_state = init_state() - [ - init_state.python_dl_path, - init_state.python_executable_path - ] ++ - init_state.sys_paths ++ - Path.wildcard(Path.join(init_state.python_home_path, "**"), match_dot: true) + if init_state == nil do + raise "before calling Pythonx.install_paths/0, you must initialize Pythonx" + end + + init_state.install_paths + end + + @doc false + def maybe_init_from_env() do + case init_state_from_env() do + nil -> + :noop + + init_state_env_value -> + %{ + type: :uv_init, + pyproject_toml: pyproject_toml, + opts: opts + } = + init_state_env_value + |> Base.decode64!() + |> :erlang.binary_to_term() + + uv_init(pyproject_toml, opts) + end end # Initializes the Python interpreter. @@ -166,16 +190,6 @@ defmodule Pythonx do Pythonx.NIF.init(python_dl_path, python_home_path, python_executable_path, opts[:sys_paths]) end - @spec init(init_state()) :: :ok - def init(%__MODULE__{ - python_dl_path: python_dl_path, - python_home_path: python_home_path, - python_executable_path: python_executable_path, - sys_paths: sys_paths - }) do - init(python_dl_path, python_home_path, python_executable_path, sys_paths: sys_paths) - end - @doc ~S''' Evaluates the Python `code`. diff --git a/lib/pythonx/application.ex b/lib/pythonx/application.ex index e280c9d..f950ba5 100644 --- a/lib/pythonx/application.ex +++ b/lib/pythonx/application.ex @@ -31,18 +31,7 @@ defmodule Pythonx.Application do Pythonx.Uv.fetch(pyproject_toml, true, opts) defp maybe_uv_init(), do: Pythonx.Uv.init(unquote(pyproject_toml), true, unquote(opts)) else - defp maybe_uv_init() do - case Pythonx.init_state_from_env() do - nil -> - :noop - - init_state_env_value -> - init_state_env_value - |> Base.decode64!() - |> :erlang.binary_to_term() - |> Pythonx.init() - end - end + defp maybe_uv_init(), do: Pythonx.maybe_init_from_env() end defp enable_sigchld() do diff --git a/lib/pythonx/uv.ex b/lib/pythonx/uv.ex index 93d6626..78312b9 100644 --- a/lib/pythonx/uv.ex +++ b/lib/pythonx/uv.ex @@ -66,7 +66,7 @@ defmodule Pythonx.Uv do Initializes the interpreter using Python and dependencies previously fetched by `fetch/3`. """ - @spec init(String.t(), boolean()) :: Pythonx.init_state() + @spec init(String.t(), boolean()) :: list(String.t()) def init(pyproject_toml, priv?, opts \\ []) do opts = Keyword.validate!(opts, uv_version: default_uv_version()) project_dir = project_dir(pyproject_toml, priv?, opts[:uv_version]) @@ -95,69 +95,61 @@ defmodule Pythonx.Uv do root_dir = Path.join(python_install_dir(priv?, opts[:uv_version]), versioned_dir_name) - init_state = - case :os.type() do - {:win32, _osname} -> - # Note that we want the version-specific DLL, rather than the - # "forwarder DLL" python3.dll, otherwise symbols cannot be - # found directly. - python_dl_path = - root_dir - |> Path.join("python3?*.dll") - |> wildcard_one!() - |> make_windows_slashes() - - python_home_path = make_windows_slashes(root_dir) - - python_executable_path = - project_dir - |> Path.join(".venv/Scripts/python.exe") - |> make_windows_slashes() - - venv_packages_path = - project_dir - |> Path.join(".venv/Lib/site-packages") - |> make_windows_slashes() - - %Pythonx{ - python_dl_path: python_dl_path, - python_home_path: python_home_path, - python_executable_path: python_executable_path, - sys_paths: [venv_packages_path] - } - - {:unix, osname} -> - dl_extension = - case osname do - :darwin -> ".dylib" - :linux -> ".so" - end - - python_dl_path = - root_dir - |> Path.join("lib/libpython3.*" <> dl_extension) - |> wildcard_one!() - |> Path.expand() - - python_home_path = root_dir - - python_executable_path = Path.join(project_dir, ".venv/bin/python") - - venv_packages_path = - project_dir - |> Path.join(".venv/lib/python3*/site-packages") - |> wildcard_one!() - - %Pythonx{ - python_dl_path: python_dl_path, - python_home_path: python_home_path, - python_executable_path: python_executable_path, - sys_paths: [venv_packages_path] - } - end + case :os.type() do + {:win32, _osname} -> + # Note that we want the version-specific DLL, rather than the + # "forwarder DLL" python3.dll, otherwise symbols cannot be + # found directly. + python_dl_path = + root_dir + |> Path.join("python3?*.dll") + |> wildcard_one!() + |> make_windows_slashes() + + python_home_path = make_windows_slashes(root_dir) + + python_executable_path = + project_dir + |> Path.join(".venv/Scripts/python.exe") + |> make_windows_slashes() + + venv_packages_path = + project_dir + |> Path.join(".venv/Lib/site-packages") + |> make_windows_slashes() + + Pythonx.init(python_dl_path, python_home_path, python_executable_path, + sys_paths: [venv_packages_path] + ) + + {:unix, osname} -> + dl_extension = + case osname do + :darwin -> ".dylib" + :linux -> ".so" + end + + python_dl_path = + root_dir + |> Path.join("lib/libpython3.*" <> dl_extension) + |> wildcard_one!() + |> Path.expand() + + python_home_path = root_dir + + python_executable_path = Path.join(project_dir, ".venv/bin/python") + + venv_packages_path = + project_dir + |> Path.join(".venv/lib/python3*/site-packages") + |> wildcard_one!() + + Pythonx.init(python_dl_path, python_home_path, python_executable_path, + sys_paths: [venv_packages_path] + ) + end - Pythonx.init(init_state) - init_state + [root_dir, project_dir] end defp wildcard_one!(path) do