From e7041f70408e2959e91962a260ca528c57048c2d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 2 Dec 2025 23:16:07 +0100 Subject: [PATCH 1/3] Remove support for Python < 3.9 and Tmod3 --- .github/workflows/main.yml | 27 +- .github/workflows/test-flux.yaml | 3 +- ci-scripts/configs/cscs-ci.py | 153 --------- ci-scripts/configs/envmod.py | 23 ++ ci-scripts/configs/lmod.py | 53 +-- ci-scripts/configs/spack.py | 53 +-- ci-scripts/configs/tmod32.py | 70 ---- ci-scripts/configs/tmod4.py | 70 ---- ci-scripts/dockerfiles/Lmod.dockerfile | 6 +- .../dockerfiles/eb-spack-howto.dockerfile | 12 +- ...Tmod4.dockerfile => envmodules.dockerfile} | 12 +- ...ckerfile => reframe-envmodules.dockerfile} | 5 +- .../dockerfiles/reframe-lmod.dockerfile | 3 +- .../dockerfiles/reframe-lmod77.dockerfile | 25 -- .../dockerfiles/reframe-python.dockerfile | 3 +- .../dockerfiles/reframe-tmod32.dockerfile | 23 -- docs/config_reference.rst | 17 +- docs/howto.rst | 32 +- docs/requirements.txt | 15 +- docs/started.rst | 9 +- .../tutorial/dockerfiles/eb-spack.dockerfile | 2 +- examples/tutorial/easybuild/eb_test.py | 12 +- examples/tutorial/spack/spack_test.py | 12 +- reframe/__init__.py | 2 +- reframe/core/logging.py | 17 +- reframe/core/modules.py | 319 ++++-------------- reframe/core/pipeline.py | 2 +- reframe/frontend/autodetect.py | 2 +- reframe/schemas/config.json | 2 +- reframe/utility/osext.py | 63 +--- reframe/utility/profile.py | 8 +- requirements.txt | 38 +-- setup.cfg | 13 +- unittests/test_cli.py | 2 +- unittests/test_loader.py | 3 +- unittests/test_modules.py | 9 +- unittests/test_utility.py | 16 +- 37 files changed, 224 insertions(+), 912 deletions(-) delete mode 100644 ci-scripts/configs/cscs-ci.py create mode 100644 ci-scripts/configs/envmod.py delete mode 100644 ci-scripts/configs/tmod32.py delete mode 100644 ci-scripts/configs/tmod4.py rename ci-scripts/dockerfiles/{Tmod4.dockerfile => envmodules.dockerfile} (62%) rename ci-scripts/dockerfiles/{reframe-tmod4.dockerfile => reframe-envmodules.dockerfile} (78%) delete mode 100644 ci-scripts/dockerfiles/reframe-lmod77.dockerfile delete mode 100644 ci-scripts/dockerfiles/reframe-tmod32.dockerfile diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3831e4257a..f2cd1a20ab 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -24,28 +24,11 @@ jobs: - name: Upload coverage reports uses: codecov/codecov-action@v4.2.0 - unittest-py-eol: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.6', '3.7'] - steps: - - uses: actions/checkout@v4 - - name: Build Image for Python ${{ matrix.python-version }} - run: | - docker build --build-arg PYTHON_VERSION=${{ matrix.python-version }} -f ci-scripts/dockerfiles/reframe-python.dockerfile -t reframe-python${{ matrix.python-version }}:latest . - - name: Run Unittests - run: | - docker run --name reframe-python${{ matrix.python-version }} reframe-python${{ matrix.python-version }}:latest - docker cp reframe-python${{ matrix.python-version }}:/home/rfmuser/reframe/coverage.xml . - - name: Upload coverage reports - uses: codecov/codecov-action@v4.2.0 - unittest-macos: runs-on: macos-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -66,7 +49,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - modules-version: [lmod, lmod77, tmod32, tmod4] + modules-version: [lmod, envmodules] steps: - uses: actions/checkout@v4 - name: Login to GitHub Container Registry @@ -121,7 +104,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup up Python ${{ matrix.python-version }} @@ -144,7 +127,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/test-flux.yaml b/.github/workflows/test-flux.yaml index cd10473543..c1eecd329c 100644 --- a/.github/workflows/test-flux.yaml +++ b/.github/workflows/test-flux.yaml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - container: ['fluxrm/flux-sched:focal'] + container: ['fluxrm/flux-sched:noble'] container: image: ${{ matrix.container }} @@ -30,7 +30,6 @@ jobs: run: | apt-get update && apt-get install -y python3-pip ./bootstrap.sh - pip install pytest-cov export PATH=$PWD/bin:$PATH which reframe diff --git a/ci-scripts/configs/cscs-ci.py b/ci-scripts/configs/cscs-ci.py deleted file mode 100644 index 221b58a1c9..0000000000 --- a/ci-scripts/configs/cscs-ci.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# -# CSCS CI settings -# - -import reframe.utility.osext as osext - - -site_configuration = { - 'systems': [ - { - 'name': 'daint', - 'descr': 'Piz Daint CI nodes', - 'hostnames': [ - 'daint' - ], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': [ - { - 'name': 'gpu', - 'scheduler': 'slurm', - 'time_limit': '10m', - 'access': [ - '--constraint=gpu', - '--partition=cscsci', - f'--account={osext.osgroup()}' - ], - 'environs': [ - 'builtin' - ], - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100, - 'resources': [ - { - 'name': 'switches', - 'options': [ - '--switches={num_switches}' - ] - } - ], - 'launcher': 'srun' - } - ] - }, - { - 'name': 'dom', - 'descr': 'Dom TDS', - 'hostnames': [ - 'dom' - ], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': [ - { - 'name': 'slurm', - 'scheduler': 'slurm', - 'time_limit': '10m', - 'access': [ - '--constraint=gpu', - f'--account={osext.osgroup()}' - ], - 'environs': [ - 'builtin' - ], - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100, - 'resources': [ - { - 'name': 'switches', - 'options': [ - '--switches={num_switches}' - ] - } - ], - 'launcher': 'srun' - }, - { - 'name': 'pbs', - 'scheduler': 'pbs', - 'time_limit': '10m', - 'access': [ - 'proc=gpu', - f'-A {osext.osgroup()}' - ], - 'environs': [ - 'builtin' - ], - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100, - 'launcher': 'mpiexec' - }, - { - 'name': 'torque', - 'scheduler': 'torque', - 'time_limit': '10m', - 'access': [ - '-l proc=gpu', - f'-A {osext.osgroup()}' - ], - 'environs': [ - 'builtin' - ], - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100, - 'launcher': 'mpiexec' - } - ] - }, - { - 'name': 'tsa', - 'descr': 'Tsa MCH', - 'hostnames': [ - r'tsa-\w+\d+' - ], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': [ - { - 'name': 'cn', - 'scheduler': 'slurm', - 'access': [ - '--partition=cn-regression' - ], - 'environs': [ - 'builtin' - ], - 'descr': 'Tsa compute nodes', - 'max_jobs': 20, - 'resources': [ - { - 'name': '_rfm_gpu', - 'options': [ - '--gres=gpu:{num_gpus_per_node}' - ] - } - ], - 'launcher': 'srun' - } - ] - }, - ], - 'general': [ - { - 'check_search_path': ['checks/'], - 'check_search_recursive': True - } - ] -} diff --git a/ci-scripts/configs/envmod.py b/ci-scripts/configs/envmod.py new file mode 100644 index 0000000000..6d31563ca0 --- /dev/null +++ b/ci-scripts/configs/envmod.py @@ -0,0 +1,23 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'envmodsys', + 'descr': 'Generic system using Environment Modules', + 'hostnames': ['.*'], + 'modules_system': 'envmod', + 'partitions': [ + { + 'name': 'default', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['builtin'] + } + ] + } + ] +} diff --git a/ci-scripts/configs/lmod.py b/ci-scripts/configs/lmod.py index e5fe1983ee..0ac569b197 100644 --- a/ci-scripts/configs/lmod.py +++ b/ci-scripts/configs/lmod.py @@ -3,15 +3,11 @@ # # SPDX-License-Identifier: BSD-3-Clause -# -# Generic fallback configuration -# - site_configuration = { 'systems': [ { - 'name': 'generic', - 'descr': 'Generic example system', + 'name': 'lmodsys', + 'descr': 'Generic system using Lmod', 'hostnames': ['.*'], 'modules_system': 'lmod', 'partitions': [ @@ -22,49 +18,6 @@ 'environs': ['builtin'] } ] - }, - ], - 'environments': [ - { - 'name': 'builtin', - 'cc': 'cc', - 'cxx': '', - 'ftn': '' - }, - ], - 'logging': [ - { - 'handlers': [ - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'info', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'level': 'debug', - 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 - 'append': False - } - ], - 'handlers_perflog': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'info', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'append': True - } - ] } - ], + ] } diff --git a/ci-scripts/configs/spack.py b/ci-scripts/configs/spack.py index 7f2862bbac..d95d8c885a 100644 --- a/ci-scripts/configs/spack.py +++ b/ci-scripts/configs/spack.py @@ -3,15 +3,11 @@ # # SPDX-License-Identifier: BSD-3-Clause -# -# Generic fallback configuration -# - site_configuration = { 'systems': [ { - 'name': 'generic', - 'descr': 'Generic example system', + 'name': 'spacksys', + 'descr': 'Generic system using Spack', 'hostnames': ['.*'], 'modules_system': 'spack', 'partitions': [ @@ -22,49 +18,6 @@ 'environs': ['builtin'] } ] - }, - ], - 'environments': [ - { - 'name': 'builtin', - 'cc': 'cc', - 'cxx': '', - 'ftn': '' - }, - ], - 'logging': [ - { - 'handlers': [ - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'info', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'level': 'debug', - 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 - 'append': False - } - ], - 'handlers_perflog': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'info', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'append': True - } - ] } - ], + ] } diff --git a/ci-scripts/configs/tmod32.py b/ci-scripts/configs/tmod32.py deleted file mode 100644 index c82594db06..0000000000 --- a/ci-scripts/configs/tmod32.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# -# Generic fallback configuration -# - -site_configuration = { - 'systems': [ - { - 'name': 'generic', - 'descr': 'Generic example system', - 'hostnames': ['.*'], - 'modules_system': 'tmod32', - 'partitions': [ - { - 'name': 'default', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['builtin'] - } - ] - }, - ], - 'environments': [ - { - 'name': 'builtin', - 'cc': 'cc', - 'cxx': '', - 'ftn': '' - }, - ], - 'logging': [ - { - 'handlers': [ - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'info', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'level': 'debug', - 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 - 'append': False - } - ], - 'handlers_perflog': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'info', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'append': True - } - ] - } - ], -} diff --git a/ci-scripts/configs/tmod4.py b/ci-scripts/configs/tmod4.py deleted file mode 100644 index 1b9d4c6531..0000000000 --- a/ci-scripts/configs/tmod4.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# -# Generic fallback configuration -# - -site_configuration = { - 'systems': [ - { - 'name': 'generic', - 'descr': 'Generic example system', - 'hostnames': ['.*'], - 'modules_system': 'tmod4', - 'partitions': [ - { - 'name': 'default', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['builtin'] - } - ] - }, - ], - 'environments': [ - { - 'name': 'builtin', - 'cc': 'cc', - 'cxx': '', - 'ftn': '' - }, - ], - 'logging': [ - { - 'handlers': [ - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'info', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'level': 'debug', - 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 - 'append': False - } - ], - 'handlers_perflog': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'info', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'append': True - } - ] - } - ], -} diff --git a/ci-scripts/dockerfiles/Lmod.dockerfile b/ci-scripts/dockerfiles/Lmod.dockerfile index b908d76022..4240d9eb5a 100644 --- a/ci-scripts/dockerfiles/Lmod.dockerfile +++ b/ci-scripts/dockerfiles/Lmod.dockerfile @@ -1,8 +1,8 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 ENV TZ=Europe/Zurich ENV DEBIAN_FRONTEND=noninteractive -ENV _LMOD_VER=8.4.12 +ENV _LMOD_VER=9.0.4 # Setup apt RUN \ @@ -11,7 +11,7 @@ RUN \ update-ca-certificates # Required utilities -RUN apt-get -y install wget +RUN apt-get -y install bc wget # Install Lmod RUN \ diff --git a/ci-scripts/dockerfiles/eb-spack-howto.dockerfile b/ci-scripts/dockerfiles/eb-spack-howto.dockerfile index f308189495..25777111bc 100644 --- a/ci-scripts/dockerfiles/eb-spack-howto.dockerfile +++ b/ci-scripts/dockerfiles/eb-spack-howto.dockerfile @@ -3,10 +3,10 @@ # -FROM ghcr.io/reframe-hpc/lmod:8.4.12 +FROM ghcr.io/reframe-hpc/lmod:9.0.4 -ENV _SPACK_VER=0.22.2 -ENV _EB_VER=4.9.4 +ENV _SPACK_VER=1.1.0 +ENV _EB_VER=5.1.2 # Install ReFrame unit test requirements @@ -22,7 +22,7 @@ USER rfmuser RUN git clone --branch v${_SPACK_VER} https://github.com/spack/spack ~/spack && \ cd ~/spack -RUN pip3 install easybuild==${_EB_VER} +RUN pip3 install --break-system-packages easybuild==${_EB_VER} ENV PATH="/home/rfmuser/.local/bin:${PATH}" @@ -35,6 +35,6 @@ RUN ./bootstrap.sh RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/rfmuser/spack/share/spack/setup-env.sh' > /home/rfmuser/setup.sh -ENV BASH_ENV /home/rfmuser/setup.sh +ENV BASH_ENV=/home/rfmuser/setup.sh -CMD ["/bin/bash", "-c", "./bin/reframe --system=tutorialsys -r -C examples/tutorial/config/baseline_modules.py -R -c examples/tutorial/easybuild/eb_test.py -c examples/tutorial/spack/spack_test.py"] +CMD ["/bin/bash", "-c", "./bin/reframe --system=tutorialsys --exec-policy=serial -r -C examples/tutorial/config/baseline_modules.py -R -c examples/tutorial/easybuild/eb_test.py -c examples/tutorial/spack/spack_test.py"] diff --git a/ci-scripts/dockerfiles/Tmod4.dockerfile b/ci-scripts/dockerfiles/envmodules.dockerfile similarity index 62% rename from ci-scripts/dockerfiles/Tmod4.dockerfile rename to ci-scripts/dockerfiles/envmodules.dockerfile index d8f7fb66aa..e9360e38c0 100644 --- a/ci-scripts/dockerfiles/Tmod4.dockerfile +++ b/ci-scripts/dockerfiles/envmodules.dockerfile @@ -1,8 +1,8 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 ENV TZ=Europe/Zurich ENV DEBIAN_FRONTEND=noninteractive -ENV _TMOD_VER=4.6.0 +ENV _ENVMOD_VER=5.6.1 # Setup apt RUN \ @@ -13,14 +13,14 @@ RUN \ # Required utilities RUN apt-get -y install wget less -# Install Tmod4 +# Install Environment Modules RUN \ apt-get -y install autoconf tcl-dev && \ - wget -q https://github.com/cea-hpc/modules/archive/v${_TMOD_VER}.tar.gz -O tmod.tar.gz && \ + wget -q https://github.com/cea-hpc/modules/archive/v${_ENVMOD_VER}.tar.gz -O tmod.tar.gz && \ tar xzf tmod.tar.gz && \ - cd modules-${_TMOD_VER} && \ + cd modules-${_ENVMOD_VER} && \ ./configure && make install && \ - cd .. && rm -rf tmod.tar.gz modules-${_TMOD_VER} && \ + cd .. && rm -rf tmod.tar.gz modules-${_ENVMOD_VER} && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/ci-scripts/dockerfiles/reframe-tmod4.dockerfile b/ci-scripts/dockerfiles/reframe-envmodules.dockerfile similarity index 78% rename from ci-scripts/dockerfiles/reframe-tmod4.dockerfile rename to ci-scripts/dockerfiles/reframe-envmodules.dockerfile index 360a448991..bedeb42389 100644 --- a/ci-scripts/dockerfiles/reframe-tmod4.dockerfile +++ b/ci-scripts/dockerfiles/reframe-envmodules.dockerfile @@ -2,7 +2,7 @@ # Execute this from the top-level ReFrame source directory # -FROM ghcr.io/reframe-hpc/tmod:4.6.0 +FROM ghcr.io/reframe-hpc/envmodules:5.6.1 # ReFrame requirements @@ -21,6 +21,5 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh -RUN pip install pytest-cov -CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/tmod4.py"] +CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/envmod.py"] diff --git a/ci-scripts/dockerfiles/reframe-lmod.dockerfile b/ci-scripts/dockerfiles/reframe-lmod.dockerfile index f95dae4b67..007e60a3e6 100644 --- a/ci-scripts/dockerfiles/reframe-lmod.dockerfile +++ b/ci-scripts/dockerfiles/reframe-lmod.dockerfile @@ -3,7 +3,7 @@ # -FROM ghcr.io/reframe-hpc/lmod:8.4.12 +FROM ghcr.io/reframe-hpc/lmod:9.0.4 # Install ReFrame unit test requirements RUN apt-get -y update && \ @@ -20,6 +20,5 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh -RUN pip install pytest-cov CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/lmod.py"] diff --git a/ci-scripts/dockerfiles/reframe-lmod77.dockerfile b/ci-scripts/dockerfiles/reframe-lmod77.dockerfile deleted file mode 100644 index 5ee2500394..0000000000 --- a/ci-scripts/dockerfiles/reframe-lmod77.dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# -# Execute this from the top-level ReFrame source directory -# - - -FROM ghcr.io/reframe-hpc/lmod:7.7 - -# Install ReFrame unit test requirements -RUN apt-get -y update && \ - apt-get -y install gcc git make python3 python3-pip - -# ReFrame user -RUN useradd -ms /bin/bash rfmuser - -USER rfmuser - -# Install ReFrame from the current directory -COPY --chown=rfmuser . /home/rfmuser/reframe/ - -WORKDIR /home/rfmuser/reframe - -RUN ./bootstrap.sh -RUN pip install pytest-cov - -CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/lmod.py"] diff --git a/ci-scripts/dockerfiles/reframe-python.dockerfile b/ci-scripts/dockerfiles/reframe-python.dockerfile index d80d09f542..d70cfa221e 100644 --- a/ci-scripts/dockerfiles/reframe-python.dockerfile +++ b/ci-scripts/dockerfiles/reframe-python.dockerfile @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -ARG PYTHON_VERSION=3.6 +ARG PYTHON_VERSION=3.9 FROM docker.io/python:${PYTHON_VERSION} @@ -18,6 +18,5 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh +docs -RUN pip install pytest-cov CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml"] diff --git a/ci-scripts/dockerfiles/reframe-tmod32.dockerfile b/ci-scripts/dockerfiles/reframe-tmod32.dockerfile deleted file mode 100644 index df9418589f..0000000000 --- a/ci-scripts/dockerfiles/reframe-tmod32.dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# -# Execute this from the top-level ReFrame source directory -# - -FROM ghcr.io/reframe-hpc/tmod:3.2.10 - -# ReFrame requirements -RUN yum -y install gcc make git python3 python3-pip - -# ReFrame user -RUN useradd -ms /bin/bash rfmuser -RUN pip3 install pytest-cov - -USER rfmuser - -# Install ReFrame from the current directory -COPY --chown=rfmuser . /home/rfmuser/reframe/ - -WORKDIR /home/rfmuser/reframe - -RUN ./bootstrap.sh - -CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/tmod32.py"] diff --git a/docs/config_reference.rst b/docs/config_reference.rst index f8e8723df1..2aaa042242 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -161,25 +161,28 @@ System Configuration The modules system that should be used for loading environment modules on this system. Available values are the following: - - ``tmod``: The classic Tcl implementation of the `environment modules `__ (version 3.2). - - ``tmod31``: The classic Tcl implementation of the `environment modules `__ (version 3.1). - A separate backend is required for Tmod 3.1, because Python bindings are different from Tmod 3.2. - - ``tmod32``: A synonym of ``tmod``. - - ``tmod4``: The `new environment modules `__ implementation (versions older than 4.1 are not supported). + - ``envmod``: The `new environment modules `__ implementation (versions older than 4.1 are not supported). - ``lmod``: The `Lua implementation `__ of the environment modules. - ``spack``: `Spack `__'s built-in mechanism for managing modules. + - ``tmod4``: (deprecated) Synonym of ``envmod``. - ``nomod``: This is to denote that no modules system is used by this system. Normally, upon loading the configuration of the system ReFrame checks that a sane installation exists for the modules system requested and will issue an error if it fails to find one. The modules system sanity check is skipped when the :attr:`~config.general.resolve_module_conflicts` is set to :obj:`False`. This is useful in cases where the current system does not have a modules system but the remote partitions have one and you would like ReFrame to generate the module commands. - .. versionadded:: 3.4 + .. versionadded:: 3.4 The ``spack`` backend is added. - .. versionchanged:: 4.5.0 + .. versionchanged:: 4.5.0 The modules system sanity check is skipped when the :attr:`config.general.resolve_module_conflicts` is not set. + .. versionchanged:: 4.10 + The ``tmod``, ``tmod31``, ``tmod32`` backends are no more supported. + + .. deprecated:: 4.10 + The ``tmod4`` backend is deprecated; please use ``envmod`` instead. + .. py:attribute:: systems.modules diff --git a/docs/howto.rst b/docs/howto.rst index 97c72bf55b..9e3acece64 100644 --- a/docs/howto.rst +++ b/docs/howto.rst @@ -97,7 +97,7 @@ Integrating with EasyBuild ReFrame integrates with the `EasyBuild `__ build automation framework, which allows you to use EasyBuild for building the source code of your test. -Let's consider a simple ReFrame test that installs ``bzip2-1.0.6`` given the easyconfig `bzip2-1.0.6.eb `__ and checks that the installed version is correct. +Let's consider a simple ReFrame test that installs ``zlib-1.3.1`` given the easyconfig `zlib-1.3.1.eb `__ and checks that the installed version is correct. The following code block shows the check, highlighting the lines specific to this tutorial: .. literalinclude:: ../examples/tutorial/easybuild/eb_test.py @@ -138,7 +138,7 @@ ReFrame generates the following commands to build and install the easyconfig: .. code-block:: bash :caption: Run in the EasyBuild+Spack container. - cat output/tutorialsys/default/builtin/BZip2EBCheck/rfm_build.sh + cat output/tutorialsys/default/builtin/ZlibEBCheck/rfm_build.sh .. code-block:: bash @@ -147,7 +147,7 @@ ReFrame generates the following commands to build and install the easyconfig: export EASYBUILD_INSTALLPATH=${stagedir}/easybuild export EASYBUILD_PREFIX=${stagedir}/easybuild export EASYBUILD_SOURCEPATH=${stagedir}/easybuild - eb bzip2-1.0.6.eb -f + eb zlib-1.3.1.eb -f All the files generated by EasyBuild (sources, temporary files, installed software and the corresponding modules) are kept under the test's stage directory, thus the relevant EasyBuild environment variables are set. @@ -169,13 +169,13 @@ This generated final run script is the following: .. code-block:: bash :caption: Run in the EasyBuild+Spack container. - cat output/tutorialsys/default/builtin/BZip2EBCheck/rfm_job.sh + cat output/tutorialsys/default/builtin/ZlibEBCheck/rfm_job.sh .. code-block:: bash module use ${stagedir}/easybuild/modules/all - module load bzip/1.0.6 - bzip2 --help + module load zlib/1.3.1 + ls $LD_LIBRARY_PATH/libz.so.1.3.1 Packaging the installation @@ -204,7 +204,7 @@ Integrating with Spack ReFrame can also use `Spack `__ to build a software package and test it. -The example shown here is the equivalent to the `EasyBuild <#integrating-with-easybuild>`__ one that built ``bzip2``. +The example shown here is the equivalent to the `EasyBuild <#integrating-with-easybuild>`__ one that built ``zlib``. Here is the test code: .. literalinclude:: ../examples/tutorial/spack/spack_test.py @@ -244,7 +244,7 @@ Here is what ReFrame generates as a build script for this example: spack env create -d rfm_spack_env spack -e rfm_spack_env config add "config:install_tree:root:opt/spack" - spack -e rfm_spack_env add bzip2@1.0.6 + spack -e rfm_spack_env add zlib@1.3.1 spack -e rfm_spack_env install As you might have noticed ReFrame expects that Spack is already installed on the system. @@ -262,12 +262,12 @@ Here is the stage directory structure: │   │      └── darwin-catalina-skylake │   ├── spack.lock │   └── spack.yaml - ├── rfm_BZip2SpackCheck_build.err - ├── rfm_BZip2SpackCheck_build.out - ├── rfm_BZip2SpackCheck_build.sh - ├── rfm_BZip2SpackCheck_job.err - ├── rfm_BZip2SpackCheck_job.out - └── rfm_BZip2SpackCheck_job.sh + ├── rfm_ZlibSpackCheck_build.err + ├── rfm_ZlibSpackCheck_build.out + ├── rfm_ZlibSpackCheck_build.sh + ├── rfm_ZlibSpackCheck_job.err + ├── rfm_ZlibSpackCheck_job.out + └── rfm_ZlibSpackCheck_job.sh Finally, here is the generated run script that ReFrame uses to run the test, once its build has succeeded: @@ -276,8 +276,8 @@ Finally, here is the generated run script that ReFrame uses to run the test, onc #!/bin/bash spack env create -d rfm_spack_env - eval `spack -e rfm_spack_env load --sh bzip2@1.0.6` - bzip2 --help + eval `spack -e rfm_spack_env load --sh zlib@1.3.1` + pkg-config --libs zlib From this point on, sanity and performance checking are exactly identical to any other ReFrame test. diff --git a/docs/requirements.txt b/docs/requirements.txt index 7447db2b40..a24877df97 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,20 +1,13 @@ archspec==0.2.5 ClusterShell==1.9.3 -docutils==0.18.1; python_version < '3.9' docutils==0.21.2; python_version >= '3.9' fasteners==0.19; python_version < '3.10' fasteners==0.20; python_version >= '3.10' -jinja2==3.0.3; python_version == '3.6' -jinja2==3.1.6; python_version >= '3.7' +jinja2==3.1.6 jsonschema==3.2.0 -PyYAML==6.0.1; python_version < '3.8' -PyYAML==6.0.3; python_version >= '3.8' -semver==2.13.0; python_version == '3.6' -semver==3.0.4; python_version >= '3.7' -Sphinx==5.3.0; python_version < '3.8' -Sphinx==7.1.2; python_version == '3.8' +PyYAML==6.0.3 +semver==3.0.4 Sphinx==7.4.7; python_version == '3.9' Sphinx==8.1.3; python_version == '3.10' Sphinx==8.2.3; python_version >= '3.11' -sphinx-rtd-theme==2.0.0; python_version < '3.9' -sphinx-rtd-theme==3.0.2; python_version >= '3.9' +sphinx-rtd-theme==3.0.2 diff --git a/docs/started.rst b/docs/started.rst index 3ab07c0295..fa6bd84aa5 100644 --- a/docs/started.rst +++ b/docs/started.rst @@ -5,7 +5,7 @@ Getting Started Requirements ------------ -* Python 3.6 or higher. +* Python 3.9 or higher. Python 2 is not supported. * The required Python packages are the following: @@ -16,12 +16,11 @@ Requirements .. note:: .. versionchanged:: 3.0 - Support for Python 3.5 has been dropped. + Support for Python 3.5 is dropped. + .. versionchanged:: 4.10 -.. warning:: - Although ReFrame supports Python 3.6 and 3.7, you should note that these Python versions have reached end-of-life and you are strongly advised to use a newer version. - ReFrame installations on these Python versions may use out-of-date dependencies due to incompatibilities. + Support for Python < 3.9 is dropped. Getting the Framework diff --git a/examples/tutorial/dockerfiles/eb-spack.dockerfile b/examples/tutorial/dockerfiles/eb-spack.dockerfile index 645507a887..11acf45041 100644 --- a/examples/tutorial/dockerfiles/eb-spack.dockerfile +++ b/examples/tutorial/dockerfiles/eb-spack.dockerfile @@ -36,4 +36,4 @@ RUN mkdir .local && cd .local && \ git clone --branch releases/v${_SPACK_VER} --depth 1 https://github.com/spack/spack RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/user/.local/spack/share/spack/setup-env.sh' > /home/user/.profile -ENV BASH_ENV /home/user/.profile +ENV BASH_ENV=/home/user/.profile diff --git a/examples/tutorial/easybuild/eb_test.py b/examples/tutorial/easybuild/eb_test.py index 8e50d915f8..d859ab394a 100644 --- a/examples/tutorial/easybuild/eb_test.py +++ b/examples/tutorial/easybuild/eb_test.py @@ -8,17 +8,17 @@ @rfm.simple_test -class BZip2EBCheck(rfm.RegressionTest): +class ZlibEBCheck(rfm.RegressionTest): descr = 'Demo test using EasyBuild to build the test code' valid_systems = ['*'] valid_prog_environs = ['builtin'] - executable = 'bzip2' - executable_opts = ['--help'] + executable = 'ls' + executable_opts = ['$LD_LIBRARY_PATH/libz.so.1.3.1'] build_system = 'EasyBuild' @run_before('compile') def setup_build_system(self): - self.build_system.easyconfigs = ['bzip2-1.0.6.eb'] + self.build_system.easyconfigs = ['zlib-1.3.1.eb'] self.build_system.options = ['-f'] @run_before('run') @@ -26,5 +26,5 @@ def prepare_run(self): self.modules = self.build_system.generated_modules @sanity_function - def assert_version(self): - return sn.assert_found(r'Version 1.0.6', self.stderr) + def assert_exists(self): + return sn.assert_eq(self.job.exitcode, 0) diff --git a/examples/tutorial/spack/spack_test.py b/examples/tutorial/spack/spack_test.py index a095b11d99..e7ae86b1e4 100644 --- a/examples/tutorial/spack/spack_test.py +++ b/examples/tutorial/spack/spack_test.py @@ -8,18 +8,20 @@ @rfm.simple_test -class BZip2SpackCheck(rfm.RegressionTest): +class ZlibSpackCheck(rfm.RegressionTest): descr = 'Demo test using Spack to build the test code' valid_systems = ['*'] valid_prog_environs = ['builtin'] - executable = 'bzip2' - executable_opts = ['--help'] + executable = 'pkg-config' + executable_opts = ['--libs', 'zlib'] build_system = 'Spack' @run_before('compile') def setup_build_system(self): - self.build_system.specs = ['bzip2@1.0.6'] + self.build_system.specs = ['zlib@1.3.1'] @sanity_function def assert_version(self): - return sn.assert_found(r'Version 1.0.6', self.stderr) + return sn.assert_found( + r'-L.*/spack/linux-.*/zlib-1.3.1-.*/lib -lz', self.stdout + ) diff --git a/reframe/__init__.py b/reframe/__init__.py index a661177c12..aeff4e1f58 100644 --- a/reframe/__init__.py +++ b/reframe/__init__.py @@ -10,7 +10,7 @@ INSTALL_PREFIX = os.path.normpath( os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) ) -MIN_PYTHON_VERSION = (3, 6, 0) +MIN_PYTHON_VERSION = (3, 9, 0) # Check python version if sys.version_info[:3] < MIN_PYTHON_VERSION: diff --git a/reframe/core/logging.py b/reframe/core/logging.py index ec810e061a..bf1e1d6c48 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -337,12 +337,8 @@ class CheckFieldFormatter(logging.Formatter): # NOTE: This formatter will work only for the '%' style def __init__(self, fmt=None, datefmt=None, perffmt=None, ignore_keys=None, style='%'): - if sys.version_info[:2] <= (3, 7): - super().__init__(fmt, datefmt, style) - else: - super().__init__(fmt, datefmt, style, - validate=(fmt != '%(check_#ALL)s')) - + super().__init__(fmt, datefmt, style, + validate=(fmt != '%(check_#ALL)s')) self.__fmt = fmt self.__fmtperf = perffmt[:-1] if perffmt else '' self.__specs = re.findall(r'\%\((\S+?)\)s', fmt) @@ -809,11 +805,10 @@ def __init__(self, name, level=logging.NOTSET): def setLevel(self, level): self.level = _check_level(level) - if sys.version_info[:2] >= (3, 7): - # Clear the internal cache of the base logger, otherwise the - # logger will remain disabled if its level is raised and then - # lowered again - self._cache.clear() + # Clear the internal cache of the base logger, otherwise the + # logger will remain disabled if its level is raised and then + # lowered again + self._cache.clear() def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None): diff --git a/reframe/core/modules.py b/reframe/core/modules.py index e8c9c504b1..292acd8843 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -112,13 +112,15 @@ def create(cls, modules_kind=None, validate=True): modules_impl = { None: NoModImpl, 'nomod': NoModImpl, - 'tmod31': TMod31Impl, - 'tmod': TModImpl, - 'tmod32': TModImpl, - 'tmod4': TMod4Impl, + 'tmod4': EnvModulesImpl, + 'envmod': EnvModulesImpl, 'lmod': LModImpl, 'spack': SpackImpl } + if modules_kind == 'tmod4': + getlogger().warning("'tmod4' backend is deprecated; " + "please use 'envmod' instead") + try: impl_cls = modules_impl[modules_kind] except KeyError: @@ -583,244 +585,8 @@ def __str__(self): return self.name() + ' ' + self.version() -class TModImpl(ModulesSystemImpl): - '''Base class for TMod Module system (Tcl).''' - - MIN_VERSION = (3, 2) - - def __init__(self): - self._version = None - self._validated = False - if self.validate: - self._do_validate() - - def _do_validate(self): - # Try to figure out if we are indeed using the TCL version - try: - completed = osext.run_command('modulecmd -V') - except OSError as e: - raise ConfigError( - 'could not find a sane TMod installation') from e - - version_match = re.search(r'^VERSION=(\S+)', completed.stdout, - re.MULTILINE) - tcl_version_match = re.search(r'^TCL_VERSION=(\S+)', completed.stdout, - re.MULTILINE) - - if version_match is None or tcl_version_match is None: - raise ConfigError('could not find a sane TMod installation') - - version = version_match.group(1) - try: - ver_major, ver_minor = [int(v) for v in version.split('.')[:2]] - except ValueError: - raise ConfigError( - 'could not parse TMod version string: ' + version) from None - - if (ver_major, ver_minor) < self.MIN_VERSION: - raise ConfigError( - f'unsupported TMod version: ' - f'{version} (required >= {self.MIN_VERSION})' - ) - - self._version = version - try: - # Try the Python bindings now - completed = osext.run_command(self.modulecmd()) - except OSError as e: - raise ConfigError( - f'could not get the Python bindings for TMod: {e}' - ) from e - - if re.search(r'Unknown shell type', completed.stderr): - raise ConfigError( - 'Python is not supported by this TMod installation' - ) - - self._validated = True - - def name(self): - return 'tmod' - - def version(self): - return self._version - - def modulecmd(self, *args): - return ' '.join(['modulecmd', 'python', *args]) - - def _execute(self, cmd, *args): - if not self._validated: - self._do_validate() - - modulecmd = self.modulecmd(cmd, *args) - completed = osext.run_command(modulecmd) - if re.search(r'\bERROR\b', completed.stderr) is not None: - raise SpawnedProcessError(modulecmd, - completed.stdout, - completed.stderr, - completed.returncode) - - exec(self.process(completed.stdout)) - return completed.stderr - - def available_modules(self, substr): - output = self.execute('avail', '-t', substr) - ret = [] - for line in output.split('\n'): - if not line or line[-1] == ':': - # Ignore empty lines and path entries - continue - - module = re.sub(r'\(default\)', '', line) - ret.append(Module(module)) - - return ret - - def loaded_modules(self): - try: - # LOADEDMODULES may be defined but empty - return [Module(m) - for m in os.environ['LOADEDMODULES'].split(':') if m] - except KeyError: - return [] - - def conflicted_modules(self, module): - output = self.execute_with_path('show', str(module), path=module.path) - return [Module(m.group(1)) - for m in re.finditer(r'^conflict\s+(\S+)', - output, re.MULTILINE)] - - def is_module_loaded(self, module): - return module in self.loaded_modules() - - def load_module(self, module): - self.execute_with_path('load', str(module), path=module.path) - - def unload_module(self, module): - self.execute('unload', str(module)) - - def unload_all(self): - self.execute('purge') - - def searchpath(self): - path = os.getenv('MODULEPATH', '') - return path.split(':') - - def searchpath_add(self, *dirs): - if dirs: - self.execute('use', *dirs) - - def searchpath_remove(self, *dirs): - if dirs: - self.execute('unuse', *dirs) - - def emit_load_instr(self, module): - commands = [] - if module.path: - commands.append(f'module use {module.path}') - - commands.append(f'module load {module.fullname}') - if module.path: - commands.append(f'module unuse {module.path}') - - return commands - - def emit_unload_instr(self, module): - return [f'module unload {module}'] - - -class TMod31Impl(TModImpl): - '''Module system for TMod (Tcl).''' - - MIN_VERSION = (3, 1) - - def __init__(self): - self._version = None - self._command = None - self._validated = False - if self.validate: - self._do_validate() - - def _do_validate(self): - # Try to figure out if we are indeed using the TCL version - try: - modulecmd = os.getenv('MODULESHOME') - modulecmd = os.path.join(modulecmd, 'modulecmd.tcl') - completed = osext.run_command(modulecmd) - except OSError as e: - raise ConfigError( - f'could not find a sane TMod31 installation: {e}' - ) from e - - version_match = re.search(r'Release Tcl (\S+)', completed.stderr, - re.MULTILINE) - tcl_version_match = version_match - - if version_match is None or tcl_version_match is None: - raise ConfigError('could not find a sane TMod31 installation') - - version = version_match.group(1) - try: - ver_major, ver_minor = [int(v) for v in version.split('.')[:2]] - except ValueError: - raise ConfigError( - 'could not parse TMod31 version string: ' + version) from None - - if (ver_major, ver_minor) < self.MIN_VERSION: - raise ConfigError( - f'unsupported TMod version: {version} ' - f'(required >= {self.MIN_VERSION})' - ) - - self._version = version - self._command = f'{modulecmd} python' - try: - # Try the Python bindings now - completed = osext.run_command(self._command) - except OSError as e: - raise ConfigError( - f'could not get the Python bindings for TMod31: {e}' - ) - - if re.search(r'Unknown shell type', completed.stderr): - raise ConfigError( - 'Python is not supported by this TMod installation' - ) - - self._validated = True - - def name(self): - return 'tmod31' - - def modulecmd(self, *args): - return ' '.join([self._command, *args]) - - def _execute(self, cmd, *args): - if not self._validated: - self._do_validate() - - modulecmd = self.modulecmd(cmd, *args) - completed = osext.run_command(modulecmd) - if re.search(r'\bERROR\b', completed.stderr) is not None: - raise SpawnedProcessError(modulecmd, - completed.stdout, - completed.stderr, - completed.returncode) - - exec_match = re.search(r"^exec\s'(\S+)'", completed.stdout, - re.MULTILINE) - if exec_match is None: - raise ConfigError('could not use the python bindings') - - with open(exec_match.group(1), 'r') as content_file: - cmd = content_file.read() - - exec(self.process(cmd)) - return completed.stderr - - -class TMod4Impl(TModImpl): - '''Module system for TMod 4.''' +class EnvModulesImpl(ModulesSystemImpl): + '''Module system for Environment Modules.''' MIN_VERSION = (4, 1) @@ -867,7 +633,10 @@ def _do_validate(self): self._validated = True def name(self): - return 'tmod4' + return 'envmod' + + def version(self): + return self._version def modulecmd(self, *args): return ' '.join(['modulecmd', 'python', *args]) @@ -899,20 +668,34 @@ def load_module(self, module): # 'restore' discards previous module path manipulations for op, mp in self._extra_module_paths: if op == '+': - super().searchpath_add(mp) + self.execute('use', mp) else: - super().searchpath_remove(mp) + self.execute('unuse', mp) return [] else: - return super().load_module(module) + self.execute_with_path('load', str(module), path=module.path) def unload_module(self, module): if module.collection: - # Module collection are not unloaded + # Module collections are not unloaded return - super().unload_module(module) + self.execute('unload', str(module)) + + def loaded_modules(self): + try: + # LOADEDMODULES may be defined but empty + return [Module(m) + for m in os.environ['LOADEDMODULES'].split(':') if m] + except KeyError: + return [] + + def is_module_loaded(self, module): + return module in self.loaded_modules() + + def unload_all(self): + self.execute('purge') def conflicted_modules(self, module): if module.collection: @@ -921,7 +704,10 @@ def conflicted_modules(self, module): # collection return [] - return super().conflicted_modules(module) + output = self.execute_with_path('show', str(module), path=module.path) + return [Module(m.group(1)) + for m in re.finditer(r'^conflict\s+(\S+)', + output, re.MULTILINE)] def _emit_restore_instr(self, module): cmds = [f'module restore {module}'] @@ -938,28 +724,51 @@ def emit_load_instr(self, module): if module.collection: return self._emit_restore_instr(module) - return super().emit_load_instr(module) + commands = [] + if module.path: + commands.append(f'module use {module.path}') + + commands.append(f'module load {module.fullname}') + if module.path: + commands.append(f'module unuse {module.path}') + + return commands def emit_unload_instr(self, module): if module.collection: return [] - return super().emit_unload_instr(module) + return [f'module unload {module}'] + + def searchpath(self): + path = os.getenv('MODULEPATH', '') + return path.split(':') def searchpath_add(self, *dirs): if dirs: self._extra_module_paths += [('+', mp) for mp in dirs] - - super().searchpath_add(*dirs) + self.execute('use', *dirs) def searchpath_remove(self, *dirs): if dirs: self._extra_module_paths += [('-', mp) for mp in dirs] + self.execute('unuse', *dirs) + + def available_modules(self, substr): + output = self.execute('avail', '-t', substr) + ret = [] + for line in output.split('\n'): + if not line or line[-1] == ':': + # Ignore empty lines and path entries + continue - super().searchpath_remove(*dirs) + module = re.sub(r'\(default\)', '', line) + ret.append(Module(module)) + + return ret -class LModImpl(TMod4Impl): +class LModImpl(EnvModulesImpl): '''Module system for Lmod (Tcl/Lua).''' def __init__(self): diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 09f03f0470..76790c4306 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -2616,7 +2616,7 @@ def _copy_to_outputdir(self): dst = os.path.join( self.outputdir, os.path.relpath(f, self.stagedir) ) - osext.copytree(f, dst, dirs_exist_ok=True) + shutil.copytree(f, dst, dirs_exist_ok=True) else: shutil.copy2(f, self.outputdir) diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py index 3ef7a0b9d1..a84dbd72fd 100644 --- a/reframe/frontend/autodetect.py +++ b/reframe/frontend/autodetect.py @@ -59,7 +59,7 @@ def __enter__(self): src = os.path.join(rfm.INSTALL_PREFIX, p) if os.path.isdir(src): dst = os.path.join(self._workdir, p) - osext.copytree(src, dst, dirs_exist_ok=True) + shutil.copytree(src, dst, dirs_exist_ok=True) else: shutil.copy2(src, self._workdir) except FileNotFoundError: diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index ac03ea0fda..255acf948a 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -270,7 +270,7 @@ "max_local_jobs": {"type": "number"}, "modules_system": { "type": "string", - "enum": ["tmod", "tmod31", "tmod32", "tmod4", + "enum": ["tmod4", "envmod", "lmod", "nomod", "spack"] }, "modules": {"$ref": "#/defs/modules_list"}, diff --git a/reframe/utility/osext.py b/reframe/utility/osext.py index 544c47900b..1b58972f4c 100644 --- a/reframe/utility/osext.py +++ b/reframe/utility/osext.py @@ -27,6 +27,7 @@ import reframe.utility as util from reframe.core.exceptions import (ReframeError, SpawnedProcessError, SpawnedProcessTimeout) +from reframe.core.warnings import user_deprecation_warning from . import OrderedSet @@ -409,52 +410,20 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=shutil.copy2, This function will automatically delegate to :py:func:`shutil.copytree` for Python versions >= 3.8. + + .. deprecated:: 4.10 + + Please use :py:func:`shutil.copytree` directly. ''' + user_deprecation_warning('`osext.copytree()` is deprecated; ' + 'please use `shutil.copytree()` directly') + if src == os.path.commonpath([src, dst]): raise ValueError("cannot copy recursively the parent directory " "`%s' into one of its descendants `%s'" % (src, dst)) - if sys.version_info[1] >= 8: - return shutil.copytree(src, dst, symlinks, ignore, copy_function, - ignore_dangling_symlinks, dirs_exist_ok) - - if not dirs_exist_ok: - return shutil.copytree(src, dst, symlinks, ignore, copy_function, - ignore_dangling_symlinks) - - # dirs_exist_ok=True and Python < 3.8 - if not os.path.exists(dst): - return shutil.copytree(src, dst, symlinks, ignore, copy_function, - ignore_dangling_symlinks) - - # dst exists; manually descend into the subdirectories, but do some sanity - # checking first - - # We raise the following errors to comply with the copytree()'s behaviour - - if not os.path.isdir(dst): - raise FileExistsError(errno.EEXIST, 'File exists', dst) - - if not os.path.exists(src): - raise FileNotFoundError(errno.ENOENT, 'No such file or directory', src) - - if not os.path.isdir(src): - raise NotADirectoryError(errno.ENOTDIR, 'Not a directory', src) - - _, subdirs, files = list(os.walk(src))[0] - ignore_paths = ignore(src, os.listdir(src)) if ignore else {} - for f in files: - if f not in ignore_paths: - copy_function(os.path.join(src, f), os.path.join(dst, f), - follow_symlinks=not symlinks) - - for d in subdirs: - if d not in ignore_paths: - copytree(os.path.join(src, d), os.path.join(dst, d), - symlinks, ignore, copy_function, - ignore_dangling_symlinks, dirs_exist_ok) - - return dst + return shutil.copytree(src, dst, symlinks, ignore, copy_function, + ignore_dangling_symlinks, dirs_exist_ok) def copytree_virtual(src, dst, file_links=None, @@ -464,10 +433,10 @@ def copytree_virtual(src, dst, file_links=None, ``file_links``. If ``file_links`` is empty or :class:`None`, this is equivalent to - :func:`copytree()`. The rest of the arguments are passed as-is to - :func:`copytree()`. Paths in ``file_links`` must be relative to ``src``. - If you try to pass ``'.'`` in ``file_links``, an :py:class:`OSError` will - be raised. + :py:func:`shutil.copytree()`. The rest of the arguments are passed as-is + to :py:func:`shutil.copytree()`. Paths in ``file_links`` must be relative + to ``src``. If you try to pass ``'.'`` in ``file_links``, an + :py:class:`OSError` will be raised. ''' @@ -510,8 +479,8 @@ def ignore(dir, contents): if os.path.join(dir, c) in link_targets} # Copy to dst ignoring the file_links - copytree(src, dst, symlinks, ignore, - copy_function, ignore_dangling_symlinks, dirs_exist_ok) + shutil.copytree(src, dst, symlinks, ignore, + copy_function, ignore_dangling_symlinks, dirs_exist_ok) # Now create the symlinks for f in link_targets: diff --git a/reframe/utility/profile.py b/reframe/utility/profile.py index 144970df0d..c6b94d52eb 100644 --- a/reframe/utility/profile.py +++ b/reframe/utility/profile.py @@ -6,9 +6,6 @@ # A lightweight time profiler import time -import sys - -from collections import OrderedDict class ProfilerError(Exception): @@ -33,10 +30,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class TimeProfiler: def __init__(self): self._region_stack = ['root'] - if sys.version_info[:2] < (3, 8): - self._region_times = OrderedDict() - else: - self._region_times = {} + self._region_times = {} @property def current_region(self): diff --git a/requirements.txt b/requirements.txt index 4b5a2c45a9..93cac76459 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,22 @@ archspec==0.2.5 -argcomplete==3.1.2; python_version < '3.8' argcomplete==3.6.3; python_version >= '3.8' ClusterShell==1.9.3 fasteners==0.19; python_version < '3.10' fasteners==0.20; python_version >= '3.10' -importlib_metadata==4.0.1; python_version < '3.8' -jinja2==3.0.3; python_version == '3.6' -jinja2==3.1.6; python_version >= '3.7' +jinja2==3.1.6 jsonschema==3.2.0 -lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64' -lxml==5.4.0; python_version < '3.8' and platform_machine != 'aarch64' -lxml==6.0.2; python_version >= '3.8' -pytest==7.0.1; python_version < '3.8' -pytest==8.3.5; python_version == '3.8' +lxml==6.0.2 pytest==8.4.2; python_version == '3.9' pytest==9.0.1; python_version >= '3.10' -pytest-forked==1.4.0; python_version == '3.6' -pytest-forked==1.6.0; python_version >= '3.7' +pytest-cov==7.0.0 +pytest-forked==1.6.0 pytest-parallel==0.1.1 -pytest-rerunfailures==10.3; python_version == '3.6' -pytest-rerunfailures==13.0; python_version == '3.7' -pytest-rerunfailures==14.0; python_version == '3.8' pytest-rerunfailures==16.0.1; python_version == '3.9' pytest-rerunfailures==16.1; python_version >= '3.10' -PyYAML==6.0.1; python_version < '3.8' -PyYAML==6.0.3; python_version >= '3.8' -requests==2.27.1; python_version == '3.6' -requests==2.31.0; python_version == '3.7' -requests==2.32.4; python_version >= '3.8' -semver==2.13.0; python_version == '3.6' -semver==3.0.4; python_version >= '3.7' -setuptools==59.6.0; python_version == '3.6' -setuptools==68.0.0; python_version == '3.7' -setuptools==75.3.0; python_version == '3.8' -setuptools==80.9.0; python_version >= '3.9' -tabulate==0.8.10; python_version == '3.6' -tabulate==0.9.0; python_version >= '3.7' +PyYAML==6.0.3 +requests==2.32.4 +semver==3.0.4 +setuptools==80.9.0 +tabulate==0.9.0 wcwidth==0.2.14 -#+pygelf%pygelf==0.4.0 +#+pygelf%pygelf==0.4.3 diff --git a/setup.cfg b/setup.cfg index a3db8c3f05..d63c748f29 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,9 +9,6 @@ long_description = file: README_minimal.md long_description_content_type = text/markdown classifiers = Development Status :: 5 - Production/Stable - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 @@ -24,29 +21,21 @@ classifiers = [options] packages = find_namespace: -python_requires = >=3.6 +python_requires = >=3.9 scripts = bin/reframe install_requires = archspec >= 0.2.4 argcomplete - argcomplete <= 3.1.2; python_version < '3.8' ClusterShell fasteners==0.19; python_version < '3.10' fasteners - jinja2==3.0.3; python_version == '3.6' jinja2 jsonschema - lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64' - lxml==5.4.0; python_version < '3.8' and platform_machine != 'aarch64' lxml - PyYAML==6.0.1; python_version < '3.8' PyYAML requests - requests <= 2.27.1; python_version == '3.6' semver - semver <= 2.13.0; python_version == '3.6' tabulate - tabulate <= 0.8.10; python_version == '3.6' [options.packages.find] include = reframe,reframe.*,hpctestlib.* diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 8742dc173b..640be0b94d 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -634,7 +634,7 @@ def test_timestamp_option_default(run_reframe): assert returncode == 0 matches = re.findall( - r'(stage|output) directory: .*\/(\d{8}T\d{6}\+\d{4})', stdout + r'(stage|output) directory: .*\/(\d{8}T\d{6}(\+|-)\d{4})', stdout ) assert len(matches) == 2 diff --git a/unittests/test_loader.py b/unittests/test_loader.py index 729708bde7..ba32e5bc31 100644 --- a/unittests/test_loader.py +++ b/unittests/test_loader.py @@ -8,7 +8,6 @@ import shutil import reframe as rfm -import reframe.utility.osext as osext from reframe.core.exceptions import ReframeSyntaxError from reframe.frontend.loader import RegressionCheckLoader @@ -148,7 +147,7 @@ def test_relative_import_outside_rfm_prefix(loader, tmp_path): # imported as a hierarchical module. If not, we want to make sure that # reframe will still load its parent modules - osext.copytree( + shutil.copytree( os.path.abspath('unittests/resources/checks_unlisted/testlib'), tmp_path / 'testlib', dirs_exist_ok=True ) diff --git a/unittests/test_modules.py b/unittests/test_modules.py index feffdc5eb3..5f09e7dd7d 100644 --- a/unittests/test_modules.py +++ b/unittests/test_modules.py @@ -9,10 +9,11 @@ import reframe.core.environments as env import reframe.core.modules as modules import unittests.utility as test_util +from reframe.utility.versioning import parse as parse_version from reframe.core.exceptions import ConfigError, EnvironError -@pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'spack', 'nomod']) +@pytest.fixture(params=['envmod', 'lmod', 'spack', 'nomod']) def modules_system_nopath(request, monkeypatch): # Always pretend to be on a clean modules environment monkeypatch.setenv('MODULEPATH', '') @@ -77,7 +78,11 @@ def module_collection(modules_system, tmp_path, monkeypatch): # Remove the temporary collection if modules_system.name == 'lmod': - prefix = os.path.join(os.environ['HOME'], '.lmod.d') + lmod_version = parse_version(modules_system.version) + if lmod_version < (9,): + prefix = os.path.join(os.environ['HOME'], '.lmod.d') + else: + prefix = os.path.join(os.environ['HOME'], '.config', 'lmod') else: prefix = os.path.join(os.environ['HOME'], '.module') diff --git a/unittests/test_utility.py b/unittests/test_utility.py index ca0939bd97..fb0a204ebb 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -22,6 +22,7 @@ from reframe.core.exceptions import (ConfigError, SpawnedProcessError, SpawnedProcessTimeout) +from reframe.core.warnings import ReframeDeprecationWarning def test_command_success(): @@ -281,7 +282,8 @@ def test_copytree(tmp_path): dir_src.mkdir() dir_dst = tmp_path / 'dst' dir_dst.mkdir() - osext.copytree(str(dir_src), str(dir_dst), dirs_exist_ok=True) + with pytest.warns(ReframeDeprecationWarning): + osext.copytree(str(dir_src), str(dir_dst), dirs_exist_ok=True) def test_copytree_src_parent_of_dst(tmp_path): @@ -289,7 +291,8 @@ def test_copytree_src_parent_of_dst(tmp_path): src_path = (dst_path / '..').resolve() with pytest.raises(ValueError): - osext.copytree(str(src_path), str(dst_path)) + with pytest.warns(ReframeDeprecationWarning): + osext.copytree(str(src_path), str(dst_path)) @pytest.fixture(params=['dirs_exist_ok=True', 'dirs_exist_ok=False']) @@ -303,7 +306,8 @@ def test_copytree_dst_notdir(tmp_path, dirs_exist_ok): dst = tmp_path / 'dst' dst.touch() with pytest.raises(FileExistsError, match=fr'{dst}'): - osext.copytree(str(dir_src), str(dst), dirs_exist_ok=dirs_exist_ok) + with pytest.warns(ReframeDeprecationWarning): + osext.copytree(str(dir_src), str(dst), dirs_exist_ok=dirs_exist_ok) def test_copytree_src_notdir(tmp_path, dirs_exist_ok): @@ -312,7 +316,8 @@ def test_copytree_src_notdir(tmp_path, dirs_exist_ok): dst = tmp_path / 'dst' dst.mkdir() with pytest.raises(NotADirectoryError, match=fr'{src}'): - osext.copytree(str(src), str(dst), dirs_exist_ok=dirs_exist_ok) + with pytest.warns(ReframeDeprecationWarning): + osext.copytree(str(src), str(dst), dirs_exist_ok=dirs_exist_ok) def test_copytree_src_does_not_exist(tmp_path, dirs_exist_ok): @@ -320,7 +325,8 @@ def test_copytree_src_does_not_exist(tmp_path, dirs_exist_ok): dst = tmp_path / 'dst' dst.mkdir() with pytest.raises(FileNotFoundError, match=fr'{src}'): - osext.copytree(str(src), str(dst), dirs_exist_ok=dirs_exist_ok) + with pytest.warns(ReframeDeprecationWarning): + osext.copytree(str(src), str(dst), dirs_exist_ok=dirs_exist_ok) @pytest.fixture From b8c7eac2d720f4dff826aa37af55b6aadb436c6b Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 5 Dec 2025 00:52:04 +0100 Subject: [PATCH 2/3] Use coverage directly instead of pytest-cov --- .github/pseudo-cluster/reframe/docker-entrypoint.sh | 6 ++++-- .github/workflows/main.yml | 10 ++++++---- .github/workflows/test-flux.yaml | 3 ++- ci-scripts/dockerfiles/reframe-envmodules.dockerfile | 4 +++- ci-scripts/dockerfiles/reframe-lmod.dockerfile | 4 +++- ci-scripts/dockerfiles/reframe-python.dockerfile | 6 ++++-- requirements.txt | 1 - 7 files changed, 22 insertions(+), 12 deletions(-) diff --git a/.github/pseudo-cluster/reframe/docker-entrypoint.sh b/.github/pseudo-cluster/reframe/docker-entrypoint.sh index 71eb08fc03..a01dba35f2 100755 --- a/.github/pseudo-cluster/reframe/docker-entrypoint.sh +++ b/.github/pseudo-cluster/reframe/docker-entrypoint.sh @@ -8,11 +8,13 @@ sudo service munge start cp -r /usr/local/share/reframe . cd reframe ./bootstrap.sh -pip install pytest-cov +pip install coverage +source $HOME/.profile echo "Running unittests with backend scheduler: ${BACKEND}" tempdir=$(mktemp -d -p /scratch) -TMPDIR=$tempdir ./test_reframe.py --cov=reframe --cov-report=xml \ +TMPDIR=$tempdir coverage run --source=reframe ./test_reframe.py \ --rfm-user-config=ci-scripts/configs/ci-cluster.py \ --rfm-user-system=pseudo-cluster:compute-${BACKEND:-squeue} +coverage xml -o coverage.xml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f2cd1a20ab..330c6c4744 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,8 +19,9 @@ jobs: ./bootstrap.sh - name: Generic Unittests run: | - pip install pytest-cov - ./test_reframe.py --cov=reframe --cov-report=xml + pip install coverage + coverage run --source=reframe ./test_reframe.py + coverage xml -o coverage.xml - name: Upload coverage reports uses: codecov/codecov-action@v4.2.0 @@ -40,8 +41,9 @@ jobs: ./bootstrap.sh - name: Generic Unittests run: | - pip install pytest-cov - ./test_reframe.py --cov=reframe --cov-report=xml + pip install coverage + coverage run --source=reframe ./test_reframe.py + coverage xml -o coverage.xml - name: Upload coverage reports uses: codecov/codecov-action@v4.2.0 diff --git a/.github/workflows/test-flux.yaml b/.github/workflows/test-flux.yaml index c1eecd329c..fbe6b0dd4d 100644 --- a/.github/workflows/test-flux.yaml +++ b/.github/workflows/test-flux.yaml @@ -40,6 +40,7 @@ jobs: which reframe flux start reframe -c examples/howto/flux -C examples/howto/flux/settings.py -l flux start reframe -c examples/howto/flux -C examples/howto/flux/settings.py -r - flux start python3 ./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=examples/howto/flux/settings.py + flux start coverage run --source=reframe ./test_reframe.py --rfm-user-config=examples/howto/flux/settings.py + coverage xml -o coverage.xml - name: Upload coverage reports uses: codecov/codecov-action@v4.2.0 diff --git a/ci-scripts/dockerfiles/reframe-envmodules.dockerfile b/ci-scripts/dockerfiles/reframe-envmodules.dockerfile index bedeb42389..6ab319c745 100644 --- a/ci-scripts/dockerfiles/reframe-envmodules.dockerfile +++ b/ci-scripts/dockerfiles/reframe-envmodules.dockerfile @@ -21,5 +21,7 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh +RUN pip install --break-system-packages coverage +ENV BASH_ENV=/home/rfmuser/.profile -CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/envmod.py"] +CMD ["/bin/bash", "-c", "coverage run --source=reframe ./test_reframe.py --rfm-user-config=ci-scripts/configs/envmod.py; coverage xml -o coverage.xml"] diff --git a/ci-scripts/dockerfiles/reframe-lmod.dockerfile b/ci-scripts/dockerfiles/reframe-lmod.dockerfile index 007e60a3e6..71625e68d8 100644 --- a/ci-scripts/dockerfiles/reframe-lmod.dockerfile +++ b/ci-scripts/dockerfiles/reframe-lmod.dockerfile @@ -20,5 +20,7 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh +RUN pip install --break-system-packages coverage +ENV BASH_ENV=/home/rfmuser/.profile -CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/lmod.py"] +CMD ["/bin/bash", "-c", "coverage run --source=reframe ./test_reframe.py --rfm-user-config=ci-scripts/configs/lmod.py; coverage xml -o coverage.xml"] diff --git a/ci-scripts/dockerfiles/reframe-python.dockerfile b/ci-scripts/dockerfiles/reframe-python.dockerfile index d70cfa221e..13156b8ed8 100644 --- a/ci-scripts/dockerfiles/reframe-python.dockerfile +++ b/ci-scripts/dockerfiles/reframe-python.dockerfile @@ -17,6 +17,8 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/ WORKDIR /home/rfmuser/reframe -RUN ./bootstrap.sh +docs +RUN ./bootstrap.sh +RUN pip install --break-system-packages coverage +ENV BASH_ENV=/home/rfmuser/.profile -CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml"] +CMD ["/bin/bash", "-c", "coverage run --source=reframe ./test_reframe.py; coverage xml -o coverage.xml"] diff --git a/requirements.txt b/requirements.txt index 93cac76459..0edca43782 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ jsonschema==3.2.0 lxml==6.0.2 pytest==8.4.2; python_version == '3.9' pytest==9.0.1; python_version >= '3.10' -pytest-cov==7.0.0 pytest-forked==1.6.0 pytest-parallel==0.1.1 pytest-rerunfailures==16.0.1; python_version == '3.9' From f99e54491316accdfadab9215d6f94bf494e9d23 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 6 Dec 2025 18:11:53 +0100 Subject: [PATCH 3/3] Test Spack module backend in CI --- .github/workflows/main.yml | 2 +- .../dockerfiles/eb-spack-howto.dockerfile | 4 +-- .../dockerfiles/reframe-lmod.dockerfile | 3 +- .../dockerfiles/reframe-spack.dockerfile | 33 +++++++++++++++++++ .../tutorial/dockerfiles/eb-spack.dockerfile | 12 +++---- 5 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 ci-scripts/dockerfiles/reframe-spack.dockerfile diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 330c6c4744..5bda327307 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - modules-version: [lmod, envmodules] + modules-version: [envmodules, lmod, spack] steps: - uses: actions/checkout@v4 - name: Login to GitHub Container Registry diff --git a/ci-scripts/dockerfiles/eb-spack-howto.dockerfile b/ci-scripts/dockerfiles/eb-spack-howto.dockerfile index 25777111bc..c0a92e2962 100644 --- a/ci-scripts/dockerfiles/eb-spack-howto.dockerfile +++ b/ci-scripts/dockerfiles/eb-spack-howto.dockerfile @@ -19,9 +19,7 @@ RUN useradd -ms /bin/bash rfmuser USER rfmuser # Install Spack -RUN git clone --branch v${_SPACK_VER} https://github.com/spack/spack ~/spack && \ - cd ~/spack - +RUN git clone --branch v${_SPACK_VER} --depth 1 https://github.com/spack/spack ~/spack RUN pip3 install --break-system-packages easybuild==${_EB_VER} ENV PATH="/home/rfmuser/.local/bin:${PATH}" diff --git a/ci-scripts/dockerfiles/reframe-lmod.dockerfile b/ci-scripts/dockerfiles/reframe-lmod.dockerfile index 71625e68d8..2bd098d4a9 100644 --- a/ci-scripts/dockerfiles/reframe-lmod.dockerfile +++ b/ci-scripts/dockerfiles/reframe-lmod.dockerfile @@ -21,6 +21,7 @@ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh RUN pip install --break-system-packages coverage +RUN echo '. /usr/local/lmod/lmod/init/profile' >> /home/rfmuser/.profile ENV BASH_ENV=/home/rfmuser/.profile -CMD ["/bin/bash", "-c", "coverage run --source=reframe ./test_reframe.py --rfm-user-config=ci-scripts/configs/lmod.py; coverage xml -o coverage.xml"] +CMD ["/bin/bash", "-c", "coverage run --source=reframe ./test_reframe.py -v --rfm-user-config=ci-scripts/configs/lmod.py; coverage xml -o coverage.xml"] diff --git a/ci-scripts/dockerfiles/reframe-spack.dockerfile b/ci-scripts/dockerfiles/reframe-spack.dockerfile new file mode 100644 index 0000000000..c8c2bebd97 --- /dev/null +++ b/ci-scripts/dockerfiles/reframe-spack.dockerfile @@ -0,0 +1,33 @@ +# +# Execute this from the top-level ReFrame source directory +# + + +FROM ubuntu:24.04 + +ENV _SPACK_VER=1.1.0 + +# Install ReFrame unit test requirements +RUN apt-get -y update && \ + apt-get -y install gcc git make python3 python3-pip + +# ReFrame user +RUN useradd -ms /bin/bash rfmuser + +USER rfmuser + +# Install Spack +RUN git clone --branch v${_SPACK_VER} https://github.com/spack/spack ~/spack + +# Install ReFrame from the current directory +COPY --chown=rfmuser . /home/rfmuser/reframe/ + +WORKDIR /home/rfmuser/reframe + +RUN ./bootstrap.sh +RUN pip install --break-system-packages coverage + +RUN echo '. /home/rfmuser/spack/share/spack/setup-env.sh' >> /home/rfmuser/.profile +ENV BASH_ENV=/home/rfmuser/.profile + +CMD ["/bin/bash", "-c", "coverage run --source=reframe ./test_reframe.py -v --rfm-user-config=ci-scripts/configs/spack.py; coverage xml -o coverage.xml"] diff --git a/examples/tutorial/dockerfiles/eb-spack.dockerfile b/examples/tutorial/dockerfiles/eb-spack.dockerfile index 11acf45041..34a6fe1217 100644 --- a/examples/tutorial/dockerfiles/eb-spack.dockerfile +++ b/examples/tutorial/dockerfiles/eb-spack.dockerfile @@ -3,10 +3,10 @@ # -FROM ghcr.io/reframe-hpc/lmod:8.4.12 +FROM ghcr.io/reframe-hpc/lmod:9.0.4 -ENV _SPACK_VER=0.16 -ENV _EB_VER=4.4.1 +ENV _SPACK_VER=1.1.0 +ENV _EB_VER=5.1.2 RUN apt-get -y update && \ apt-get -y install curl && \ @@ -22,7 +22,7 @@ RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/reframe-hpc/ref ENV PATH=/usr/local/share/reframe/bin:$PATH # Install EasyBuild -RUN pip3 install easybuild==${_EB_VER} +RUN pip3 install --break-system-packages easybuild==${_EB_VER} # Add tutorial user RUN useradd -ms /bin/bash -G sudo user && \ @@ -33,7 +33,7 @@ WORKDIR /home/user # Install Spack RUN mkdir .local && cd .local && \ - git clone --branch releases/v${_SPACK_VER} --depth 1 https://github.com/spack/spack + git clone --branch v${_SPACK_VER} --depth 1 https://github.com/spack/spack -RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/user/.local/spack/share/spack/setup-env.sh' > /home/user/.profile +RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/user/.local/spack/share/spack/setup-env.sh' >> /home/user/.profile ENV BASH_ENV=/home/user/.profile