From af7021c736c184093b106583728f9d2e70fd4a48 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 13:08:30 +0100 Subject: [PATCH 01/46] Changed pylustrequota to a generic python lustreclient extension --- .../{pylustrequota => lustreclient}/README | 6 +- .../lustreclient}/__init__.py | 27 +- .../lustreclient}/lfs.c | 6 +- .../{pylustrequota => lustreclient}/setup.cfg | 4 +- .../{pylustrequota => lustreclient}/setup.py | 17 +- mig/src/pylustrequota/bin/miglustrequota.py | 567 ------------------ 6 files changed, 29 insertions(+), 598 deletions(-) rename mig/src/{pylustrequota => lustreclient}/README (83%) rename mig/src/{pylustrequota/pylustrequota => lustreclient/lustreclient}/__init__.py (77%) rename mig/src/{pylustrequota/pylustrequota => lustreclient/lustreclient}/lfs.c (98%) rename mig/src/{pylustrequota => lustreclient}/setup.cfg (84%) rename mig/src/{pylustrequota => lustreclient}/setup.py (86%) delete mode 100755 mig/src/pylustrequota/bin/miglustrequota.py diff --git a/mig/src/pylustrequota/README b/mig/src/lustreclient/README similarity index 83% rename from mig/src/pylustrequota/README rename to mig/src/lustreclient/README index 22be7628b..46dc4d186 100644 --- a/mig/src/pylustrequota/README +++ b/mig/src/lustreclient/README @@ -1,7 +1,7 @@ -This folder contains a module for MiG lustre quota +This folder contains a module for MiG lustre client python extension To suppert containerized MiG (where lustre is mounted outside the container) -lustre quota functionality is compiled statically into this module. +lustre client functionality is compiled statically into this module. Install lustre dependencies (Rocky 9): ====================================== @@ -18,7 +18,7 @@ dnf --enablerepo=crb install \ libnl3-devel.x86_64 \ libyaml-devel \ krb5-devel.x86_64 -git clone git://git.whamcloud.com/fs/lustre-release.git +git clone https://github.com/lustre/lustre-release cd lustre-release && git checkout ${VERSION} ; cd - cd lustre-release && sh ./autogen.sh ; cd - cd lustre-release && ./configure --disable-server --enable-quota --enable-utils --enable-gss ; cd - diff --git a/mig/src/pylustrequota/pylustrequota/__init__.py b/mig/src/lustreclient/lustreclient/__init__.py similarity index 77% rename from mig/src/pylustrequota/pylustrequota/__init__.py rename to mig/src/lustreclient/lustreclient/__init__.py index 10abf6cd7..46385bde5 100644 --- a/mig/src/pylustrequota/pylustrequota/__init__.py +++ b/mig/src/lustreclient/lustreclient/__init__.py @@ -3,7 +3,7 @@ # # --- BEGIN_HEADER --- # -# __init__ - luste quota python extensions +# __init__ - luste client python extension # Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. @@ -24,7 +24,7 @@ # # -- END_HEADER --- # -"""This package provide luste quota functionality""" +"""This package provide luste client functionality""" __dummy = True @@ -37,8 +37,8 @@ # Collect all package information here for easy use from scripts and helpers -package_name = 'Lustre Quota Python extension' -short_name = 'pylustrequota' +package_name = 'Lustre Client Python extension' +short_name = 'lustreclient' # IMPORTANT: Please keep version in sync with doc-src/README.t2t @@ -46,18 +46,18 @@ version_suffix = '' version_string = '.'.join([str(i) for i in version_tuple]) + version_suffix package_version = '%s %s' % (package_name, version_string) -project_team = 'The MiG Project lead by Brian Vinter' -project_email = 'info@erda.dk' -maintainer_team = 'The pylustrequota maintainers' -maintainer_email = 'info@erda.dk' -project_url = 'https://github.com/ucphhpc/pylustrequota' -download_url = 'https://github.com/ucphhpc/pylustrequota/releases' +project_team = 'The MiG Project by the Science HPC Center at UCPH' +project_email = 'info@migrid.org' +maintainer_team = 'The migrid.org maintainers' +maintainer_email = 'info@migrid.org' +project_url = 'https://github.com/ucphhpc/migrid-sync' +download_url = 'https://github.com/ucphhpc/migrid-sync/releases' license_name = 'GNU GPL v2' short_desc = \ - 'Python quota extension for lustre' + 'Lustre client python extension' long_desc = \ - """Python quota extension for for lustre: -Documentation: https://github.com/ucphhpc/pylustrequota + """Lustre client python extension: +Documentation: https://github.com/ucphhpc/migrid-sync """ project_class = [ 'Development Status :: 1 - Beta', @@ -72,7 +72,6 @@ 'Python', 'Python C extensions', 'lustre', - 'rsync', ] # Requirements diff --git a/mig/src/pylustrequota/pylustrequota/lfs.c b/mig/src/lustreclient/lustreclient/lfs.c similarity index 98% rename from mig/src/pylustrequota/pylustrequota/lfs.c rename to mig/src/lustreclient/lustreclient/lfs.c index 1082d9deb..5e0b0437d 100644 --- a/mig/src/pylustrequota/pylustrequota/lfs.c +++ b/mig/src/lustreclient/lustreclient/lfs.c @@ -1,7 +1,7 @@ /* --- BEGIN_HEADER --- -lfs - Shared lustre library functions for Python lustre quota -Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +lfs - Shared lustre library functions for Python lustre client +Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH This file is part of MiG. MiG is free software: you can redistribute it and/or modify @@ -385,4 +385,4 @@ PyMODINIT_FUNC PyInit_lfs(void) { } return module; -} \ No newline at end of file +} diff --git a/mig/src/pylustrequota/setup.cfg b/mig/src/lustreclient/setup.cfg similarity index 84% rename from mig/src/pylustrequota/setup.cfg rename to mig/src/lustreclient/setup.cfg index 2118cc8e3..75012b93e 100644 --- a/mig/src/pylustrequota/setup.cfg +++ b/mig/src/lustreclient/setup.cfg @@ -1,7 +1,7 @@ # --- BEGIN_HEADER --- # -# setup.cfg - setup configuration file for python lustre quota -# Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +# setup.cfg - setup configuration file for lustre client python extension +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # diff --git a/mig/src/pylustrequota/setup.py b/mig/src/lustreclient/setup.py similarity index 86% rename from mig/src/pylustrequota/setup.py rename to mig/src/lustreclient/setup.py index 830eeca69..7693e4fd5 100644 --- a/mig/src/pylustrequota/setup.py +++ b/mig/src/lustreclient/setup.py @@ -3,8 +3,8 @@ # # --- BEGIN_HEADER --- # -# setup.py - Setup for python luste quota -# Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +# setup.py - Setup for lustre client python extension +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # @@ -27,7 +27,7 @@ from setuptools import setup, Extension -from pylustrequota import version_string, short_name, project_team, \ +from lustreclient import version_string, short_name, project_team, \ project_email, short_desc, long_desc, project_url, download_url, \ license_name, project_class, project_keywords, versioned_requires, \ project_requires, project_extras, project_platforms, maintainer_team, \ @@ -51,14 +51,13 @@ install_requires=versioned_requires, requires=project_requires, extras_require=project_extras, - scripts=['bin/miglustrequota.py', - ], - packages=['pylustrequota'], - package_dir={'pylustrequota': 'pylustrequota', + scripts=[], + packages=['lustreclient'], + package_dir={'lustreclient': 'lustreclient', }, package_data={}, ext_modules=[ - Extension('pylustrequota.lfs', + Extension('lustreclient.lfs', include_dirs=['/usr/include', '/usr/include/python3', 'lustre-release/libcfs/include', @@ -69,7 +68,7 @@ ], library_dirs=[], libraries=[], - sources=['pylustrequota/lfs.c', + sources=['lustreclient/lfs.c', 'lustre-release/lustre/utils/lfs_project.c'], extra_objects=[ 'lustre-release/lustre/utils/.libs/liblustreapi.a'], diff --git a/mig/src/pylustrequota/bin/miglustrequota.py b/mig/src/pylustrequota/bin/miglustrequota.py deleted file mode 100755 index b471bce4a..000000000 --- a/mig/src/pylustrequota/bin/miglustrequota.py +++ /dev/null @@ -1,567 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# --- BEGIN_HEADER --- -# -# mig_lustre_quota - MiG lustre quota manager -# Copyright (C) 2003-2025 The MiG Project lead by the Science HPC Center at UCPH -# -# This file is part of MiG. -# -# MiG is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# MiG is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. -# -# --- END_HEADER --- -# - -"""Assign lustre project id's to new users and vgrids, -set default quota on new entries and update existing quotas if changed. -Fetch the number of files and bytes used by each project id. -""" - -import os -import sys -import time -import stat -import getopt -import shlex -import subprocess -import psutil - -from mig.shared.base import force_unicode -from mig.shared.conf import get_configuration_object -from mig.shared.fileio import unpickle, pickle, save_json, makedirs_rec, \ - make_symlink -from mig.shared.logger import daemon_logger - -from pylustrequota.lfs import lfs_set_project_id, lfs_get_project_quota, \ - lfs_set_project_quota - - -supported_quota_backends = ['lustre', 'lustre-gocryptfs'] - - -def usage(name=sys.argv[0]): - """Usage help""" - msg = """Usage: %(name)s [OPTIONS] -Where OPTIONS may be one or more of: - -h|--help Show this help - -v|--verbose Verbose output - -q|--quiet No stdout/stderr output - -c PATH|--config=PATH Path to config file - -l PATH|--lustre-basepath Path to lustre base - -g PATH|--gocryptfs-sock Path to gocryptfs socket -""" % {'name': name} - print(msg, file=sys.stderr) - - -def INFO(configuration, msg, verbose=False): - """log info and print to stdout on verbose""" - configuration.logger.info(msg) - if verbose: - print(msg) - - -def ERROR(configuration, msg, quiet=False): - """log error and print to stderr on verbose""" - configuration.logger.error(msg) - if not quiet: - print("ERROR: %s" % msg, file=sys.stderr) - - -def DEBUG(configuration, msg, verbose=False): - """log debug and print to stderr on verbose""" - configuration.logger.debug(msg) - if verbose and configuration.loglevel == 'debug': - print("DEBUG: %s" % msg, file=sys.stderr) - - -def __shellexec(configuration, - command, - args=[], - stdin_str=None, - stdout_filepath=None, - stderr_filepath=None): - """Execute shell command - Returns (exit_code, stdout, stderr) of subprocess""" - result = 0 - logger = configuration.logger - stdin_handle = subprocess.PIPE - stdout_handle = subprocess.PIPE - stderr_handle = subprocess.PIPE - if stdout_filepath is not None: - stdout_handle = open(stdout_filepath, "w+") - if stderr_filepath is not None: - stderr_handle = open(stderr_filepath, "w+") - __args = shlex.split(command) - __args.extend(args) - logger.debug("__args: %s" % __args) - process = subprocess.Popen( - __args, - stdin=stdin_handle, - stdout=stdout_handle, - stderr=stderr_handle) - if stdin_str: - process.stdin.write(stdin_str.encode()) - stdout, stderr = process.communicate() - rc = process.wait() - - if stdout_filepath: - stdout = stdout_filepath - stdout_handle.close() - if stderr_filepath: - stderr = stderr_filepath - stderr_handle.close() - - # Close stdin, stdout and stderr FDs if they exists - if process.stdin: - process.stdin.close() - if process.stdout: - process.stdout.close() - if process.stderr: - process.stderr.close() - - if stdout: - stdout = force_unicode(stdout) - if stderr: - stderr = force_unicode(stderr) - if result == 0: - logger.debug("%s %s: rc: %s, stdout: %s, error: %s" - % (command, - " ".join(args), - rc, - stdout, - stderr)) - else: - logger.error("shellexec: %s %s: rc: %s, stdout: %s, error: %s" - % (command, - " ".join(__args), - rc, - stdout, - stderr)) - - return (rc, stdout, stderr) - - -def __update_quota(configuration, - lustre_basepath, - lustre_setting, - quota_name, - quota_type, - gocryptfs_sock, - timestamp, - verbose, - quiet): - """Update quota for *quota_name*, if new entry then - assign lustre project id and set default quota. - If existing entry then update quota settings if changed - and fetch file and bytes usage and store it as pickle and json - """ - logger = configuration.logger - quota_limits_changed = False - next_lustre_pid = lustre_setting.get('next_pid', -1) - if next_lustre_pid == -1: - msg = "Invalid lustre quota next_pid: %d for: %r" \ - % (next_lustre_pid, quota_name) - ERROR(configuration, msg, quiet) - return False - if quota_type == 'vgrid': - default_quota_limit = configuration.quota_vgrid_limit - data_basepath = configuration.vgrid_files_writable - # NOTE: Old vgrids stored data directly in 'vgrid_files_home' - if not os.path.isdir(os.path.join(data_basepath, quota_name)): - data_basepath = configuration.vgrid_files_home - else: - default_quota_limit = configuration.quota_user_limit - data_basepath = configuration.user_home - - # Load quota if it exists otherwise new quota - - quota_filepath = os.path.join(configuration.quota_home, - configuration.quota_backend, - quota_type, - "%s.pck" % quota_name) - - if os.path.exists(quota_filepath): - quota = unpickle(quota_filepath, logger) - if not quota: - msg = "Failed to load quota settings for: %r from %r" \ - % (quota_name, quota_filepath) - ERROR(configuration, msg, quiet) - return False - else: - quota = {'lustre_pid': next_lustre_pid, - 'files': -1, - 'bytes': -1, - 'softlimit_bytes': -1, - 'hardlimit_bytes': -1, - } - - quota_lustre_pid = quota.get('lustre_pid', -1) - if quota_lustre_pid == -1: - msg = "Invalid quota lustre pid: %d for %r" \ - % (quota_lustre_pid, quota_name) - ERROR(configuration, msg, quiet) - return False - - # Resolve quota data path - # if gocryptfs then resolve encrypted path - # otherwise use plain path - - if configuration.quota_backend == "lustre": - quota_datapath = os.path.join(data_basepath, - quota_name) - elif configuration.quota_backend == "lustre-gocryptfs": - rel_data_basepath = data_basepath. \ - replace(configuration.state_path + os.sep, "") - stdin_str = os.path.join(rel_data_basepath, quota_name) - cmd = "gocryptfs-xray -encrypt-paths %s" % gocryptfs_sock - (rc, stdout, stderr) = __shellexec(configuration, - cmd, - stdin_str=stdin_str) - if rc == 0 and stdout: - encoded_path = stdout.strip() - quota_datapath = os.path.join(lustre_basepath, - encoded_path) - else: - msg = "Failed to resolve encrypted path for: %r" \ - % quota_name \ - + ", rc: %d, error: %s" \ - % (rc, stderr) - ERROR(configuration, msg, quiet) - return False - else: - ERROR(configuration, - "Invalid quota backend: %r" % configuration.quota_backend, - quiet) - return False - - # Skip non-dir entries - - if not os.path.isdir(quota_datapath): - msg = "Skipping non-dir entry: %r: %r" \ - % (quota_name, quota_datapath) - DEBUG(configuration, msg, verbose) - return True - - # If new entry then set lustre project id - - if quota_lustre_pid == next_lustre_pid: - # TODO: Mask out path's from log if gocryptfs ? - msg = "Setting lustre project id: %d for %r: %r" \ - % (quota_lustre_pid, quota_name, quota_datapath) - INFO(configuration, msg) - rc = lfs_set_project_id(quota_datapath, quota_lustre_pid, 1) - if rc == 0: - lustre_setting['next_pid'] = quota_lustre_pid + 1 - else: - msg = "Failed to set lustre project id: %d for %r: %r" \ - % (quota_lustre_pid, quota_name, quota_datapath) \ - + ", rc: %d" \ - % rc - ERROR(configuration, msg, quiet) - return False - - # Get current quota values for lustre_pid - - (rc, currfiles, currbytes, softlimit_bytes, hardlimit_bytes) \ - = lfs_get_project_quota(quota_datapath, quota_lustre_pid) - if rc != 0: - msg = "Failed to fetch quota for lustre project id: %d, %r, %r" \ - % (quota_lustre_pid, quota_name, quota_datapath) \ - + ", rc: %d" \ - % rc - ERROR(configuration, msg, quiet) - return False - - # Update quota info - - quota['mtime'] = timestamp - quota['files'] = currfiles - quota['bytes'] = currbytes - - # If new entry use default quota - # and update quota if changed - - if quota_lustre_pid == next_lustre_pid: - quota_limits_changed = True - quota['softlimit_bytes'] = default_quota_limit - quota['hardlimit_bytes'] = default_quota_limit - elif hardlimit_bytes != quota.get('hardlimit_bytes', -1) \ - or softlimit_bytes != quota.get('softlimit_bytes', -1): - quota_limits_changed = True - quota['softlimit_bytes'] = softlimit_bytes - quota['hardlimit_bytes'] = hardlimit_bytes - - if quota_limits_changed: - rc = lfs_set_project_quota(quota_datapath, - quota_lustre_pid, - quota['softlimit_bytes'], - quota['hardlimit_bytes'], - ) - if rc != 0: - msg = "Failed to set quota limit: %d/%d" \ - % (softlimit_bytes, - hardlimit_bytes) \ - + " for lustre project id: %d, %r, %r, rc: %d" \ - % (quota_lustre_pid, - quota_name, - quota_datapath, - rc) - ERROR(configuration, msg, quiet) - return False - - # Save current quota - - new_quota_basepath = os.path.join(configuration.quota_home, - configuration.quota_backend, - quota_type, - str(timestamp)) - if not os.path.exists(new_quota_basepath) \ - and not makedirs_rec(new_quota_basepath, configuration): - msg = "Failed to create new quota base path: %r" \ - % new_quota_basepath - ERROR(configuration, msg, quiet) - return False - - new_quota_filepath_pck = os.path.join(new_quota_basepath, - "%s.pck" % quota_name) - status = pickle(quota, new_quota_filepath_pck, logger) - if not status: - msg = "Failed to save quota for: %r to %r" \ - % (quota_name, new_quota_filepath_pck) - ERROR(configuration, msg, quiet) - return False - new_quota_filepath_json = os.path.join(new_quota_basepath, - "%s.json" % quota_name) - status = save_json(quota, - new_quota_filepath_json, - logger) - if not status: - msg = "Failed to save quota for: %r to %r" \ - % (quota_name, new_quota_filepath_json) - ERROR(configuration, msg, quiet) - return False - - # Create symlink to new quota - - status = make_symlink(new_quota_filepath_pck, - quota_filepath, - logger, - force=True) - if not status: - msg = "Failed to make quota symlink for: %r: %r -> %r" \ - % (quota_name, new_quota_filepath_pck, quota_filepath) - ERROR(configuration, msg, quiet) - return False - - return True - - -def update_quota(configuration, - lustre_basepath, - gocryptfs_sock, - verbose, - quiet): - """Update lustre quotas for users and vgrids""" - logger = configuration.logger - retval = True - timestamp = int(time.time()) - - # Load lustre quota settings - - lustre_setting_filepath = os.path.join(configuration.quota_home, - '%s.pck' - % configuration.quota_backend) - if os.path.exists(lustre_setting_filepath): - lustre_setting = unpickle(lustre_setting_filepath, - logger) - if not lustre_setting: - msg = "Failed to load lustre quota: %r" % lustre_setting_filepath - ERROR(configuration, msg, quiet) - return False - else: - lustre_setting = {'next_pid': 1, - 'mtime': 0} - - # Update quotas - - for quota_type in ('vgrid', 'user'): - if quota_type == 'vgrid': - scandir = configuration.vgrid_home - else: - scandir = configuration.user_home - - # Scan for new and modified entries - - with os.scandir(scandir) as it: - for entry in it: - if not os.path.isdir(entry.path): - # Only take dirs into account - msg = "Skiping non-dir path: %r" % entry.path - DEBUG(configuration, msg, verbose) - continue - status = __update_quota(configuration, - lustre_basepath, - lustre_setting, - entry.name, - quota_type, - gocryptfs_sock, - timestamp, - verbose, - quiet, - ) - if not status: - retval = False - - # Save updated lustre quota settings - - lustre_setting['mtime'] = timestamp - status = pickle(lustre_setting, - lustre_setting_filepath, - logger) - if not status: - msg = "Failed to save lustra quota settings: %r" \ - % lustre_setting_filepath - ERROR(configuration, msg, quiet) - - return retval - - -def main(): - retval = True - verbose = False - quiet = False - config_file = None - lustre_basepath = None - gocryptfs_sock = None - try: - opts, args = getopt.getopt(sys.argv[1:], "hvqc:l:g:", - ["help", "verbose", "quiet", "config=", - "--lustre-basepath", "--gocryptfs-sock="]) - for opt, arg in opts: - if opt in ("-h", "--help"): - usage() - sys.exit() - elif opt in ("-v", "--verbose"): - verbose = True - elif opt in ("-q", "--quiet"): - quiet = True - elif opt in ("-c", "--config"): - config_file = arg - elif opt in ("-l", "--lustre-basepath"): - lustre_basepath = arg - elif opt in ("-g", "--gocryptfs-sock"): - gocryptfs_sock = arg - except Exception as err: - print(err, file=sys.stderr) - usage() - return 1 - - if quiet: - verbose = False - - # Initialize configuration - - try: - configuration = get_configuration_object(config_file=config_file) - except Exception as err: - print(err, file=sys.stderr) - usage() - return 1 - - # Use separate logger - - logger = daemon_logger("quota", - configuration.user_quota_log, - configuration.loglevel) - configuration.logger = logger - if configuration.quota_backend not in supported_quota_backends: - msg = "Quota backend: %s not in supported backends: %s" \ - % (configuration.quota_backend, - ", ".join(supported_quota_backends)) - ERROR(configuration, msg, quiet) - return False - - # If lustre_basepath is provided then check it, - # otherwise try to resolve it - - valid_lustre_basepath = None - for mount in psutil.disk_partitions(all=True): - if mount.fstype == "lustre": - if lustre_basepath \ - and lustre_basepath.startswith(mount.mountpoint) \ - and os.path.isdir(lustre_basepath): - valid_lustre_basepath = lustre_basepath - break - elif mount.mountpoint.endswith(configuration.server_fqdn): - valid_lustre_basepath = mount.mountpoint - else: - check_lustre_basepath = os.path.join(mount.mountpoint, - configuration.server_fqdn) - if os.path.isdir(check_lustre_basepath): - valid_lustre_basepath = check_lustre_basepath - break - - if valid_lustre_basepath is None: - if lustre_basepath: - msg = "Lustre base: %r is NOT mounted" % lustre_basepath - else: - msg = "Found no valid lustre mounts for: %s" \ - % configuration.server_fqdn - ERROR(configuration, msg, quiet) - return False - - INFO(configuration, - "Using lustre basepath: %r" % valid_lustre_basepath, - verbose) - - # Check gocryptfs socket - - if configuration.quota_backend == "lustre-gocryptfs": - check_gocryptfs_sock = gocryptfs_sock - if check_gocryptfs_sock is None: - check_gocryptfs_sock = "/var/run/gocryptfs.%s.sock" \ - % configuration.server_fqdn - if os.path.exists(check_gocryptfs_sock): - gocryptfs_sock_stat = os.lstat(check_gocryptfs_sock) - if stat.S_ISSOCK(gocryptfs_sock_stat.st_mode): - gocryptfs_sock = check_gocryptfs_sock - if gocryptfs_sock: - INFO(configuration, - "Using gocryptfs socket: %r" % gocryptfs_sock, - verbose) - else: - ERROR(configuration, - "Missing gocryptfs socket: %r" % check_gocryptfs_sock, - quiet) - return False - - # Perform update - - retval = update_quota(configuration, - valid_lustre_basepath, - gocryptfs_sock, - verbose, - quiet) - return retval - - -if __name__ == "__main__": - status = main() - if status: - sys.exit(0) - else: - sys.exit(1) From 437a6a78cc92f693d67c0adbae29edc38830b743 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 14:57:53 +0100 Subject: [PATCH 02/46] Changed lustre quota from cron job to grid daemon --- .../miglustrequota-template.sh.cronjob | 62 --- mig/install/migrid-init.d-deb-template | 56 +- mig/install/migrid-init.d-rh-template | 53 +- mig/lib/lustrequota.py | 480 ++++++++++++++++++ mig/lib/quota.py | 46 ++ mig/server/grid_quota.py | 1 + mig/shared/configuration.py | 4 + mig/shared/install.py | 12 +- sbin/grid_quota.py | 144 ++++++ 9 files changed, 786 insertions(+), 72 deletions(-) delete mode 100755 mig/install/miglustrequota-template.sh.cronjob create mode 100644 mig/lib/lustrequota.py create mode 100644 mig/lib/quota.py create mode 120000 mig/server/grid_quota.py create mode 100755 sbin/grid_quota.py diff --git a/mig/install/miglustrequota-template.sh.cronjob b/mig/install/miglustrequota-template.sh.cronjob deleted file mode 100755 index 48f18050a..000000000 --- a/mig/install/miglustrequota-template.sh.cronjob +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# -# Run lustre quota for MiG servers -# -# The script depends on a miglustrequota setup -# (please refer to mig/src/pylustrequota/README). -# -# IMPORTANT: if placed in /etc/cron.X the script filename must be -# something consisting entirely of upper and lower case letters, digits, -# underscores, and hyphens. I.e. if the script name contains e.g. a period, -# '.', it will be silently ignored! -# This is a limitation on the run-parts wrapper used by cron -# (see man run-parts for the rationale behind this). - -# By default bash silently ignores and continues on most errors but we can set -# options to e.g. catch uninitialized variables and errors as explained in: -# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ -# NOTE: 'set -eE' exits on non-zero exit codes to add safety and as recommended -# best-practice (CWE-252, CWE-248, ...), yet, in some cases it hurts more to -# exit midway, so it can be a trade-off. -set -eEuo pipefail - -# Send output to another email address -#MAILTO="root" - -MIG_CONF=__MIG_CODE__/server/MiGserver.conf - -# Specify if migrid runs natively or inside containers with lustre at host. -# Value is the container manager (docker, podman, or empty string for none) -container_manager="" -container="migrid-lustre-quota" - -# Look in miglustrequota install dir first -export PATH="/usr/local/bin:${PATH}" - -if [[ $(id -u) -ne 0 ]]; then - echo "Please run $0 as root" - exit 1 -fi - -if [ -z "${container_manager}" ]; then - miglustrequota=$(which "miglustrequota.py" 2>/dev/null) - if [ ! -x "${miglustrequota}" ]; then - echo "ERROR: Missing miglustrequota.py" - exit 1 - fi - quota_cmd="${miglustrequota} -c ${MIG_CONF}" -else - check_cmd="${container_manager} container ls -a | grep -q '${container}'" - eval "$check_cmd" - ret=$? - if [ "$ret" -ne 0 ]; then - echo "ERROR: Missing ${container} container" - exit 1 - fi - quota_cmd="${container_manager} start -a ${container}" -fi - -eval "$quota_cmd" -ret=$? - -exit $ret diff --git a/mig/install/migrid-init.d-deb-template b/mig/install/migrid-init.d-deb-template index d6ac02101..87351aa07 100755 --- a/mig/install/migrid-init.d-deb-template +++ b/mig/install/migrid-init.d-deb-template @@ -43,6 +43,9 @@ if [ -z "$PYTHONPATH" ]; then else PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -62,13 +65,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -264,6 +268,18 @@ start_vmproxy() { log_end_msg 1 || true fi } +start_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Starting MiG quota daemon" ${SHORT_NAME} || true + if start-stop-daemon --start --quiet --oknodo --pidfile ${PID_FILE} --make-pidfile --user root --chuid root --background --name ${SHORT_NAME} --startas ${DAEMON_PATH} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} start_sftpsubsys() { check_enabled "sftp_subsys" || return 0 DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -292,6 +308,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -524,6 +541,19 @@ stop_vmproxy() { log_end_msg 1 || true fi } +stop_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Stopping MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --quiet --oknodo --pidfile ${PID_FILE} ; then + rm -f ${PID_FILE} + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -563,6 +593,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -735,6 +766,18 @@ reload_vmproxy() { log_end_msg 1 || true fi } +reload_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Reloading MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile ${PID_FILE} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -787,6 +830,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -891,6 +935,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} } +status_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -929,6 +980,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -940,7 +992,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') diff --git a/mig/install/migrid-init.d-rh-template b/mig/install/migrid-init.d-rh-template index c4bfad648..911c5bb3c 100755 --- a/mig/install/migrid-init.d-rh-template +++ b/mig/install/migrid-init.d-rh-template @@ -35,6 +35,7 @@ # processname: grid_notify.py # processname: grid_imnotify.py # processname: grid_vmproxy.py +# processname: grid_quota.py # processname: sshd # config: /etc/sysconfig/migrid # @@ -74,6 +75,9 @@ if [ -z "$PYTHONPATH" ]; then else export PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -93,13 +97,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -353,6 +358,21 @@ start_vmproxy() { [ $RET2 -ne 0 ] && echo "Warning: vmproxy not started." echo } +start_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Starting MiG quota daemon: $SHORT_NAME" + daemon --user root --pidfile ${PID_FILE} \ + "${DAEMON_PATH} >> ${MIG_LOG}/quota.out 2>&1 &" + fallback_save_pid "$DAEMON_PATH" "$PID_FILE" "$!" + RET2=$? + [ $RET2 -eq 0 ] && success + echo + [ $RET2 -ne 0 ] && echo "Warning: quota not started." + echo +} start_sftpsubsys() { check_enabled "sftp_subsys" || return DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -385,6 +405,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -545,6 +566,15 @@ stop_vmproxy() { killproc ${DAEMON_PATH} echo } +stop_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Shutting down MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} + echo +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -588,6 +618,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -717,6 +748,15 @@ reload_vmproxy() { killproc ${DAEMON_PATH} -HUP echo } +reload_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Reloading MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} -HUP + echo +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -773,6 +813,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -877,6 +918,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status ${DAEMON_PATH} } +status_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status ${DAEMON_PATH} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -916,6 +964,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -927,7 +976,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') diff --git a/mig/lib/lustrequota.py b/mig/lib/lustrequota.py new file mode 100644 index 000000000..cd652814e --- /dev/null +++ b/mig/lib/lustrequota.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# lustrequota - helpers to support lustre quota +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""helpers to support lustre quota""" + +import os +import stat +import time +import shlex +import subprocess +import psutil + +from mig.shared.base import force_unicode +from mig.shared.fileio import unpickle, pickle, save_json, makedirs_rec, \ + make_symlink +from lustreclient.lfs import lfs_set_project_id, lfs_get_project_quota, \ + lfs_set_project_quota + + +def __get_lustre_basepath(configuration, lustre_basepath=None): + """If *lustre_basepath* is provided then check it, + otherwise try to resolve it""" + valid_lustre_basepath = None + for mount in psutil.disk_partitions(all=True): + if mount.fstype == "lustre": + if lustre_basepath \ + and lustre_basepath.startswith(mount.mountpoint) \ + and os.path.isdir(lustre_basepath): + valid_lustre_basepath = lustre_basepath + break + elif mount.mountpoint.endswith(configuration.server_fqdn): + valid_lustre_basepath = mount.mountpoint + else: + check_lustre_basepath = os.path.join(mount.mountpoint, + configuration.server_fqdn) + if os.path.isdir(check_lustre_basepath): + valid_lustre_basepath = check_lustre_basepath + break + + return valid_lustre_basepath + + +def __get_gocryptfs_socket(configuration, gocryptfs_sock=None): + """If *gocryptfs_sock* is provided then check it, + otherwise return default if it exists""" + valid_gocryptfs_sock = None + if gocryptfs_sock is None: + gocryptfs_sock = "/var/run/gocryptfs.%s.sock" \ + % configuration.server_fqdn + if os.path.exists(gocryptfs_sock): + gocryptfs_sock_stat = os.lstat(gocryptfs_sock) + if stat.S_ISSOCK(gocryptfs_sock_stat.st_mode): + valid_gocryptfs_sock = gocryptfs_sock + + return valid_gocryptfs_sock + + +def __shellexec(configuration, + command, + args=[], + stdin_str=None, + stdout_filepath=None, + stderr_filepath=None): + """Execute shell command + Returns (exit_code, stdout, stderr) of subprocess""" + result = 0 + logger = configuration.logger + stdin_handle = subprocess.PIPE + stdout_handle = subprocess.PIPE + stderr_handle = subprocess.PIPE + if stdout_filepath is not None: + stdout_handle = open(stdout_filepath, "w+") + if stderr_filepath is not None: + stderr_handle = open(stderr_filepath, "w+") + __args = shlex.split(command) + __args.extend(args) + logger.debug("__args: %s" % __args) + process = subprocess.Popen( + __args, + stdin=stdin_handle, + stdout=stdout_handle, + stderr=stderr_handle) + if stdin_str: + process.stdin.write(stdin_str.encode()) + stdout, stderr = process.communicate() + rc = process.wait() + + if stdout_filepath: + stdout = stdout_filepath + stdout_handle.close() + if stderr_filepath: + stderr = stderr_filepath + stderr_handle.close() + + # Close stdin, stdout and stderr FDs if they exists + if process.stdin: + process.stdin.close() + if process.stdout: + process.stdout.close() + if process.stderr: + process.stderr.close() + + if stdout: + stdout = force_unicode(stdout) + if stderr: + stderr = force_unicode(stderr) + if result == 0: + logger.debug("%s %s: rc: %s, stdout: %s, error: %s" + % (command, + " ".join(args), + rc, + stdout, + stderr)) + else: + logger.error("shellexec: %s %s: rc: %s, stdout: %s, error: %s" + % (command, + " ".join(__args), + rc, + stdout, + stderr)) + + return (rc, stdout, stderr) + + +def __set_project_id(configuration, + lustre_basepath, + quota_datapath, + quota_name, + quota_lustre_pid): + """Set lustre project *quota_lustre_pid* + Find the next *free* project id (PID) if *quota_lustre_pid* is occupied + NOTE: lustre uses a global counter for project id's (PID) + That means that different datasets and sub-mounts + share the same project id counter + # TODO: Add 'lustre_pid' offset support to configuration ? + """ + + # Find next unused lustre project id + + max_lustre_pid = 4294967294 + logger = configuration.logger + next_lustre_pid = quota_lustre_pid + while next_lustre_pid < max_lustre_pid: + (rc, currfiles, _, _, _) \ + = lfs_get_project_quota(lustre_basepath, next_lustre_pid) + if rc != 0: + logger.error("Failed to fetch quota for lustre project id: %d, %r" + % (next_lustre_pid, lustre_basepath) + + ", rc: %d" % rc) + return -1 + if currfiles == 0: + break + logger.info("Skipping project id: %d" \ + % next_lustre_pid \ + + " already registered with %d files" \ + % currfiles) + next_lustre_pid += 1 + + if next_lustre_pid == max_lustre_pid: + logger.error("Reached max lustre project id: %d" % max_lustre_pid) + return -1 + + # Set new project id + + logger.info("Setting lustre project id: %d for %r: %r" + % (next_lustre_pid, quota_name, quota_datapath)) + rc = lfs_set_project_id(quota_datapath, next_lustre_pid, 1) + if rc != 0: + logger.error("Failed to set lustre project id: %d for %r: %r" + % (next_lustre_pid, quota_name, quota_datapath) + + ", rc: %d" % rc) + return -1 + + return next_lustre_pid + + +def __update_quota(configuration, + lustre_basepath, + lustre_setting, + quota_name, + quota_type, + gocryptfs_sock, + timestamp): + """Update quota for *quota_name*, if new entry then + assign lustre project id and set default quota. + If existing entry then update quota settings if changed + and fetch file and bytes usage and store it as pickle and json + """ + logger = configuration.logger + quota_limits_changed = False + next_lustre_pid = lustre_setting.get('next_pid', -1) + if next_lustre_pid == -1: + logger.error("Invalid lustre quota next_pid: %d for: %r" + % (next_lustre_pid, quota_name)) + return False + if quota_type == 'vgrid': + default_quota_limit = configuration.quota_vgrid_limit + data_basepath = configuration.vgrid_files_writable + # NOTE: Old vgrids stored data directly in 'vgrid_files_home' + if not os.path.isdir(os.path.join(data_basepath, quota_name)): + data_basepath = configuration.vgrid_files_home + else: + default_quota_limit = configuration.quota_user_limit + data_basepath = configuration.user_home + + # Load quota if it exists otherwise new quota + + quota_filepath = os.path.join(configuration.quota_home, + configuration.quota_backend, + quota_type, + "%s.pck" % quota_name) + + if os.path.exists(quota_filepath): + quota = unpickle(quota_filepath, logger) + if not quota: + logger.error("Failed to load quota settings for: %r from %r" + % (quota_name, quota_filepath)) + return False + else: + quota = {'lustre_pid': next_lustre_pid, + 'files': -1, + 'bytes': -1, + 'softlimit_bytes': -1, + 'hardlimit_bytes': -1, + } + + quota_lustre_pid = quota.get('lustre_pid', -1) + if quota_lustre_pid == -1: + logger.error("Invalid quota lustre pid: %d for %r" + % (quota_lustre_pid, quota_name)) + return False + + # Resolve quota data path + # if gocryptfs then resolve encrypted path + # otherwise use plain path + + if configuration.quota_backend == "lustre": + quota_datapath = os.path.join(data_basepath, + quota_name) + elif configuration.quota_backend == "lustre-gocryptfs": + rel_data_basepath = data_basepath. \ + replace(configuration.state_path + os.sep, "") + stdin_str = os.path.join(rel_data_basepath, quota_name) + cmd = "gocryptfs-xray -encrypt-paths %s" % gocryptfs_sock + (rc, stdout, stderr) = __shellexec(configuration, + cmd, + stdin_str=stdin_str) + if rc == 0 and stdout: + encoded_path = stdout.strip() + quota_datapath = os.path.join(lustre_basepath, + encoded_path) + else: + logger.error("Failed to resolve encrypted path for: %r" + % quota_name + + ", rc: %d, error: %s" + % (rc, stderr)) + return False + else: + logger.error("Invalid quota backend: %r" + % configuration.quota_backend) + return False + + # Skip non-dir entries + + if not os.path.isdir(quota_datapath): + logger.debug("Skipping non-dir entry: %r: %r" + % (quota_name, quota_datapath)) + return True + + # If new entry then set lustre project id + new_lustre_pid = -1 + if quota_lustre_pid == next_lustre_pid: + new_lustre_pid = __set_project_id(configuration, + lustre_basepath, + quota_datapath, + quota_name, + quota_lustre_pid) + if new_lustre_pid == -1: + logger.error("Failed to set project id: %d, %r, %r" + % (new_lustre_pid, quota_name, quota_datapath)) + return False + lustre_setting['next_pid'] = new_lustre_pid + 1 + quota_lustre_pid = new_lustre_pid + + # Get current quota values for lustre_pid + + (rc, currfiles, currbytes, softlimit_bytes, hardlimit_bytes) \ + = lfs_get_project_quota(quota_datapath, quota_lustre_pid) + if rc != 0: + logger.error("Failed to fetch quota for lustre project id: %d, %r, %r" + % (quota_lustre_pid, quota_name, quota_datapath) + + ", rc: %d" % rc) + return False + + # Update quota info + + quota['mtime'] = timestamp + quota['files'] = currfiles + quota['bytes'] = currbytes + + # If new entry use default quota + # and update quota if changed + + if new_lustre_pid > -1: + quota_limits_changed = True + quota['softlimit_bytes'] = default_quota_limit + quota['hardlimit_bytes'] = default_quota_limit + elif hardlimit_bytes != quota.get('hardlimit_bytes', -1) \ + or softlimit_bytes != quota.get('softlimit_bytes', -1): + quota_limits_changed = True + quota['softlimit_bytes'] = softlimit_bytes + quota['hardlimit_bytes'] = hardlimit_bytes + + if quota_limits_changed: + rc = lfs_set_project_quota(quota_datapath, + quota_lustre_pid, + quota['softlimit_bytes'], + quota['hardlimit_bytes'], + ) + if rc != 0: + logger.error("Failed to set quota limit: %d/%d" + % (softlimit_bytes, + hardlimit_bytes) + + " for lustre project id: %d, %r, %r, rc: %d" + % (quota_lustre_pid, + quota_name, + quota_datapath, + rc)) + return False + + # Save current quota + + new_quota_basepath = os.path.join(configuration.quota_home, + configuration.quota_backend, + quota_type, + str(timestamp)) + if not os.path.exists(new_quota_basepath) \ + and not makedirs_rec(new_quota_basepath, configuration): + logger.error("Failed to create new quota base path: %r" + % new_quota_basepath) + return False + + new_quota_filepath_pck = os.path.join(new_quota_basepath, + "%s.pck" % quota_name) + status = pickle(quota, new_quota_filepath_pck, logger) + if not status: + logger.error("Failed to save quota for: %r to %r" + % (quota_name, new_quota_filepath_pck)) + return False + + new_quota_filepath_json = os.path.join(new_quota_basepath, + "%s.json" % quota_name) + status = save_json(quota, + new_quota_filepath_json, + logger) + if not status: + logger.error("Failed to save quota for: %r to %r" + % (quota_name, new_quota_filepath_json)) + return False + + # Create symlink to new quota + + status = make_symlink(new_quota_filepath_pck, + quota_filepath, + logger, + force=True) + if not status: + logger.error("Failed to make quota symlink for: %r: %r -> %r" + % (quota_name, new_quota_filepath_pck, quota_filepath)) + return False + + return True + + +def update_lustre_quota(configuration): + """Update lustre quota for users and vgrids""" + logger = configuration.logger + retval = True + timestamp = int(time.time()) + + # Get lustre_basepath + + lustre_basepath = __get_lustre_basepath(configuration) + if lustre_basepath: + logger.debug("Using lustre basepath: %r" + % lustre_basepath) + else: + logger.error("Found no valid lustre mounts for: %s" + % configuration.server_fqdn) + return False + + # Get gocryptfs socket if enabled + + if configuration.quota_backend == "lustre-gocryptfs": + gocryptfs_sock = __get_gocryptfs_socket(configuration) + if gocryptfs_sock: + logger.debug("Using gocryptfs socket: %r" + % gocryptfs_sock) + else: + logger.error("Missing gocryptfs socket") + return False + + # Load lustre quota settings + + lustre_setting_filepath = os.path.join(configuration.quota_home, + '%s.pck' + % configuration.quota_backend) + if os.path.exists(lustre_setting_filepath): + lustre_setting = unpickle(lustre_setting_filepath, + logger) + if not lustre_setting: + logger.error("Failed to load lustre quota: %r" + % lustre_setting_filepath) + return False + else: + lustre_setting = {'next_pid': 1, + 'mtime': 0} + + # Update quota + + for quota_type in ('vgrid', 'user'): + if quota_type == 'vgrid': + scandir = configuration.vgrid_home + else: + scandir = configuration.user_home + + # Scan for new and modified entries + + with os.scandir(scandir) as it: + for entry in it: + if not os.path.isdir(entry.path): + # Only take dirs into account + logger.debug("Skiping non-dir path: %r" % entry.path) + continue + status = __update_quota(configuration, + lustre_basepath, + lustre_setting, + entry.name, + quota_type, + gocryptfs_sock, + timestamp) + if not status: + retval = False + + # Save updated lustre quota settings + + lustre_setting['mtime'] = timestamp + status = pickle(lustre_setting, + lustre_setting_filepath, + logger) + if not status: + logger.error("Failed to save lustra quota settings: %r" + % lustre_setting_filepath) + + return retval diff --git a/mig/lib/quota.py b/mig/lib/quota.py new file mode 100644 index 000000000..a1dec8e45 --- /dev/null +++ b/mig/lib/quota.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# quota - helpers to support storage quota +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""helpers to support storage quota""" + +from mig.lib.lustrequota import update_lustre_quota + + +supported_quota_backends = ['lustre', 'lustre-gocryptfs'] + + +def update_quota(configuration): + """Update quota for users and vgrids""" + logger = configuration.logger + retval = False + + if configuration.quota_backend == 'lustre' \ + or configuration.quota_backend == 'lustre-gocryptfs': + retval = update_lustre_quota(configuration) + + return retval diff --git a/mig/server/grid_quota.py b/mig/server/grid_quota.py new file mode 120000 index 000000000..5d3c8e420 --- /dev/null +++ b/mig/server/grid_quota.py @@ -0,0 +1 @@ +../../sbin/grid_quota.py \ No newline at end of file diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py index 2f872694a..84d7d110d 100644 --- a/mig/shared/configuration.py +++ b/mig/shared/configuration.py @@ -397,6 +397,7 @@ def fix_missing(config_file, verbose=True): 'vgrid_recipes_home': '.workflow_recipes_home/', 'vgrid_history_home': '.workflow_history_home/'} quota_section = {'backend': 'lustre', + 'update_interval': 3600, 'user_limit': 1024**4, 'vgrid_limit': 1024**4} defaults = { @@ -2002,6 +2003,9 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, if config.has_option('QUOTA', 'backend'): self.quota_backend = config.get( 'QUOTA', 'backend') + if config.has_option('QUOTA', 'update_interval'): + self.quota_update_interval = config.getint( + 'QUOTA', 'update_interval') if config.has_option('QUOTA', 'user_limit'): self.quota_user_limit = config.getint( 'QUOTA', 'user_limit') diff --git a/mig/shared/install.py b/mig/shared/install.py index 92a1ab997..c9579fc60 100644 --- a/mig/shared/install.py +++ b/mig/shared/install.py @@ -533,6 +533,7 @@ def generate_confs( gdp_id_scramble='safe_hash', gdp_path_scramble='safe_encrypt', quota_backend='lustre', + quota_update_interval=3600, quota_user_limit=(1024**4), quota_vgrid_limit=(1024**4), ca_fqdn='', @@ -859,6 +860,7 @@ def _generate_confs_prepare( gdp_id_scramble, gdp_path_scramble, quota_backend, + quota_update_interval, quota_user_limit, quota_vgrid_limit, ca_fqdn, @@ -1117,6 +1119,7 @@ def _generate_confs_prepare( user_dict['__PUBLIC_ALIAS_HTTPS_LISTEN__'] = listen_clause user_dict['__STATUS_ALIAS_HTTPS_LISTEN__'] = listen_clause user_dict['__QUOTA_BACKEND__'] = quota_backend + user_dict['__QUOTA_UPDATE_INTERVAL__'] = "%s" % quota_update_interval user_dict['__QUOTA_USER_LIMIT__'] = "%s" % quota_user_limit user_dict['__QUOTA_VGRID_LIMIT__'] = "%s" % quota_vgrid_limit user_dict['__CA_FQDN__'] = ca_fqdn @@ -2343,7 +2346,6 @@ def _generate_confs_writefiles(options, user_dict, insert_list=[], cleanup_list= ("migacctexpire-template.sh.cronjob", "migacctexpire"), ("migverifyarchives-template.sh.cronjob", "migverifyarchives"), ("migstats-template.sh.cronjob", "migstats"), - ("miglustrequota-template.sh.cronjob", "miglustrequota"), ] overrides_out_name = { 'apache.initd': _override_apache_initd @@ -2522,11 +2524,11 @@ def _generate_confs_instructions(options, user_dict): /etc/cron.daily/ until the janitor service is ready to take care of those tasks. -The migcheckssl, migverifyarchives, migstats, migacctexpire and miglustrequota +The migcheckssl, migverifyarchives, migstats and migacctexpire files are cron scripts to automatically check for LetsEncrypt certificate renewal, run pending archive verification before sending a copy to tape, save -various usage stats, generate account expire stats and create/update lustre -quota. +various usage stats and generate account expire stats. + You can install them with: chmod 700 %(destination)s/migcheckssl sudo cp %(destination)s/migcheckssl /etc/cron.daily/ @@ -2536,8 +2538,6 @@ def _generate_confs_instructions(options, user_dict): sudo cp %(destination)s/migstats /etc/cron.weekly/ chmod 700 %(destination)s/migacctexpire sudo cp %(destination)s/migacctexpire /etc/cron.monthly/ -chmod 700 %(destination)s/miglustrequota -sudo cp %(destination)s/miglustrequota /etc/cron.hourly/ ''' % instructions_dict instructions_path = os.path.join( diff --git a/sbin/grid_quota.py b/sbin/grid_quota.py new file mode 100755 index 000000000..1e228566c --- /dev/null +++ b/sbin/grid_quota.py @@ -0,0 +1,144 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# grid_quota - daemon to create storage quota +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +"""Daemon that create storage quota""" + +from __future__ import absolute_import, print_function + +import os +import sys +import time +import traceback +import datetime + +from mig.lib.daemon import check_run, check_stop, interruptible_sleep, \ + register_run_handler, register_stop_handler, reset_run, stop_running +from mig.lib.quota import update_quota, supported_quota_backends +from mig.shared.conf import get_configuration_object +from mig.shared.logger import daemon_logger, register_hangup_handler + + +if __name__ == "__main__": + print( + """This is the MiG lustre quota daemon which collect storage quota + information for users and vgrids. + +Set the MIG_CONF environment to the server configuration path +unless it is available in mig/server/MiGserver.conf +""" + ) + # Force no log init since we use separate logger + configuration = get_configuration_object(skip_log=True) + + log_level = configuration.loglevel + if sys.argv[1:] and sys.argv[1] in ["debug", "info", "warning", "error"]: + log_level = sys.argv[1] + + # Use separate logger + + logger = daemon_logger("quota", + configuration.user_quota_log, + log_level) + configuration.logger = logger + + # Check if quota is enabled + + if not configuration.site_enable_quota: + msg = "Quota support is disabled in configuration!" + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + sys.exit(1) + + # Check quota backend + + if configuration.quota_backend not in supported_quota_backends: + msg = "Quota backend: %s not in supported backends: %s" \ + % (configuration.quota_backend, + ", ".join(supported_quota_backends)) + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + sys.exit(1) + + # Allow e.g. logrotate to force log re-open after rotates + register_hangup_handler(configuration) + + # Allow trigger next run on SIGCONT to main process + register_run_handler(configuration) + + # Allow clean shutdown on SIGINT only to main process + register_stop_handler(configuration) + + throttle_secs = float(configuration.quota_update_interval) + main_pid = os.getpid() + msg = "(%s) Starting quota daemon with throttle: %d secs" \ + % (main_pid, throttle_secs) + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + + throttle = False + while not check_stop(): + try: + if throttle: + interruptible_sleep(configuration, throttle_secs, + (check_run, check_stop)) + reset_run() + if check_stop(): + break + t1 = time.time() + status = update_quota(configuration) + t2 = time.time() + msg = "(%s) Updated quota in %d secs with status: %s" \ + % (os.getpid(), int(t2-t1), status) + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + throttle = True + except KeyboardInterrupt: + stop_running() + # NOTE: we can't be sure if SIGINT was sent to only main process + # so we make sure to propagate to monitor child + msg = "(%s) Interrupt requested - shutdown" \ + % os.getpid() + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + except Exception as exc: + throttle = True + msg = "(%s) Caught unexpected exception:\n%s" \ + % (os.getpid(), traceback.format_exc()) + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + + msg = "(%s) Quota daemon shutting down" % main_pid + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + + sys.exit(0) From fabce46c1beaea15ce2cbb9834a64cd155aa938e Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 15:06:44 +0100 Subject: [PATCH 03/46] Updated unit tests to match new grid quota daemon setup --- tests/fixture/confs-stdlocal/miglustrequota | 62 ------------------- .../fixture/confs-stdlocal/migrid-init.d-deb | 56 ++++++++++++++++- tests/fixture/confs-stdlocal/migrid-init.d-rh | 53 +++++++++++++++- 3 files changed, 105 insertions(+), 66 deletions(-) delete mode 100644 tests/fixture/confs-stdlocal/miglustrequota diff --git a/tests/fixture/confs-stdlocal/miglustrequota b/tests/fixture/confs-stdlocal/miglustrequota deleted file mode 100644 index 062f246bc..000000000 --- a/tests/fixture/confs-stdlocal/miglustrequota +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# -# Run lustre quota for MiG servers -# -# The script depends on a miglustrequota setup -# (please refer to mig/src/pylustrequota/README). -# -# IMPORTANT: if placed in /etc/cron.X the script filename must be -# something consisting entirely of upper and lower case letters, digits, -# underscores, and hyphens. I.e. if the script name contains e.g. a period, -# '.', it will be silently ignored! -# This is a limitation on the run-parts wrapper used by cron -# (see man run-parts for the rationale behind this). - -# By default bash silently ignores and continues on most errors but we can set -# options to e.g. catch uninitialized variables and errors as explained in: -# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ -# NOTE: 'set -eE' exits on non-zero exit codes to add safety and as recommended -# best-practice (CWE-252, CWE-248, ...), yet, in some cases it hurts more to -# exit midway, so it can be a trade-off. -set -eEuo pipefail - -# Send output to another email address -#MAILTO="root" - -MIG_CONF=/home/mig/mig/server/MiGserver.conf - -# Specify if migrid runs natively or inside containers with lustre at host. -# Value is the container manager (docker, podman, or empty string for none) -container_manager="" -container="migrid-lustre-quota" - -# Look in miglustrequota install dir first -export PATH="/usr/local/bin:${PATH}" - -if [[ $(id -u) -ne 0 ]]; then - echo "Please run $0 as root" - exit 1 -fi - -if [ -z "${container_manager}" ]; then - miglustrequota=$(which "miglustrequota.py" 2>/dev/null) - if [ ! -x "${miglustrequota}" ]; then - echo "ERROR: Missing miglustrequota.py" - exit 1 - fi - quota_cmd="${miglustrequota} -c ${MIG_CONF}" -else - check_cmd="${container_manager} container ls -a | grep -q '${container}'" - eval "$check_cmd" - ret=$? - if [ "$ret" -ne 0 ]; then - echo "ERROR: Missing ${container} container" - exit 1 - fi - quota_cmd="${container_manager} start -a ${container}" -fi - -eval "$quota_cmd" -ret=$? - -exit $ret diff --git a/tests/fixture/confs-stdlocal/migrid-init.d-deb b/tests/fixture/confs-stdlocal/migrid-init.d-deb index d6ac02101..87351aa07 100755 --- a/tests/fixture/confs-stdlocal/migrid-init.d-deb +++ b/tests/fixture/confs-stdlocal/migrid-init.d-deb @@ -43,6 +43,9 @@ if [ -z "$PYTHONPATH" ]; then else PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -62,13 +65,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -264,6 +268,18 @@ start_vmproxy() { log_end_msg 1 || true fi } +start_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Starting MiG quota daemon" ${SHORT_NAME} || true + if start-stop-daemon --start --quiet --oknodo --pidfile ${PID_FILE} --make-pidfile --user root --chuid root --background --name ${SHORT_NAME} --startas ${DAEMON_PATH} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} start_sftpsubsys() { check_enabled "sftp_subsys" || return 0 DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -292,6 +308,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -524,6 +541,19 @@ stop_vmproxy() { log_end_msg 1 || true fi } +stop_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Stopping MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --quiet --oknodo --pidfile ${PID_FILE} ; then + rm -f ${PID_FILE} + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -563,6 +593,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -735,6 +766,18 @@ reload_vmproxy() { log_end_msg 1 || true fi } +reload_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Reloading MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile ${PID_FILE} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -787,6 +830,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -891,6 +935,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} } +status_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -929,6 +980,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -940,7 +992,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') diff --git a/tests/fixture/confs-stdlocal/migrid-init.d-rh b/tests/fixture/confs-stdlocal/migrid-init.d-rh index c4bfad648..911c5bb3c 100755 --- a/tests/fixture/confs-stdlocal/migrid-init.d-rh +++ b/tests/fixture/confs-stdlocal/migrid-init.d-rh @@ -35,6 +35,7 @@ # processname: grid_notify.py # processname: grid_imnotify.py # processname: grid_vmproxy.py +# processname: grid_quota.py # processname: sshd # config: /etc/sysconfig/migrid # @@ -74,6 +75,9 @@ if [ -z "$PYTHONPATH" ]; then else export PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -93,13 +97,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -353,6 +358,21 @@ start_vmproxy() { [ $RET2 -ne 0 ] && echo "Warning: vmproxy not started." echo } +start_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Starting MiG quota daemon: $SHORT_NAME" + daemon --user root --pidfile ${PID_FILE} \ + "${DAEMON_PATH} >> ${MIG_LOG}/quota.out 2>&1 &" + fallback_save_pid "$DAEMON_PATH" "$PID_FILE" "$!" + RET2=$? + [ $RET2 -eq 0 ] && success + echo + [ $RET2 -ne 0 ] && echo "Warning: quota not started." + echo +} start_sftpsubsys() { check_enabled "sftp_subsys" || return DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -385,6 +405,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -545,6 +566,15 @@ stop_vmproxy() { killproc ${DAEMON_PATH} echo } +stop_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Shutting down MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} + echo +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -588,6 +618,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -717,6 +748,15 @@ reload_vmproxy() { killproc ${DAEMON_PATH} -HUP echo } +reload_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Reloading MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} -HUP + echo +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -773,6 +813,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -877,6 +918,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status ${DAEMON_PATH} } +status_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status ${DAEMON_PATH} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -916,6 +964,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -927,7 +976,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') From b6c56f15fb862dba7bb4a4349cc22ac24ab8bd01 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 15:44:17 +0100 Subject: [PATCH 04/46] Added 'quota_update_interval' option to generateconfs --- mig/install/generateconfs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mig/install/generateconfs.py b/mig/install/generateconfs.py index 411af14b3..2299233cb 100755 --- a/mig/install/generateconfs.py +++ b/mig/install/generateconfs.py @@ -252,6 +252,7 @@ def main(argv, _generate_confs=generate_confs, _print=print): 'seafile_seafhttp_port', 'seafile_client_port', 'seafile_quota', + 'quota_update_interval', 'quota_user_limit', 'quota_vgrid_limit', 'wwwserve_max_bytes', From 5261194e607b70619c90703553d5b9047137d0d8 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 17:56:45 +0100 Subject: [PATCH 05/46] Addwed missing 'user_quota_log' entry to configuration --- mig/shared/configuration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py index 84d7d110d..8ff46f01d 100644 --- a/mig/shared/configuration.py +++ b/mig/shared/configuration.py @@ -1738,6 +1738,9 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, if config.has_option('GLOBAL', 'user_shared_dhparams'): self.user_shared_dhparams = config.get('GLOBAL', 'user_shared_dhparams') + if config.has_option('GLOBAL', 'user_quota_log'): + self.user_quota_log = config.get('GLOBAL', + 'user_quota_log') if config.has_option('GLOBAL', 'public_key_file'): self.public_key_file = config.get('GLOBAL', 'public_key_file') if config.has_option('GLOBAL', 'smtp_sender'): From 375f634031f83357e06a4f38f1ea0bc295eed2cb Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 19:01:18 +0100 Subject: [PATCH 06/46] Added missing 'update_interval' to MiGserver-template.conf --- mig/install/MiGserver-template.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/mig/install/MiGserver-template.conf b/mig/install/MiGserver-template.conf index 3f02388a5..568cc737f 100644 --- a/mig/install/MiGserver-template.conf +++ b/mig/install/MiGserver-template.conf @@ -550,6 +550,7 @@ default_mount_re = SSHFS-2.X-1 [QUOTA] backend = __QUOTA_BACKEND__ +update_interval = __QUOTA_UPDATE_INTERVAL__ user_limit = __QUOTA_USER_LIMIT__ vgrid_limit = __QUOTA_VGRID_LIMIT__ From 9c9e976a9b8b9269007f85d7833e47c65b28416f Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 19:19:10 +0100 Subject: [PATCH 07/46] Added 'update_interval' to MiGserver.conf fixture --- tests/fixture/confs-stdlocal/MiGserver.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fixture/confs-stdlocal/MiGserver.conf b/tests/fixture/confs-stdlocal/MiGserver.conf index 9520491ab..a9983263f 100644 --- a/tests/fixture/confs-stdlocal/MiGserver.conf +++ b/tests/fixture/confs-stdlocal/MiGserver.conf @@ -550,6 +550,7 @@ default_mount_re = SSHFS-2.X-1 [QUOTA] backend = lustre +update_interval = 3600 user_limit = 1099511627776 vgrid_limit = 1099511627776 From aa83180d32989826db62e5cc27faf5830b6c9fa4 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Thu, 27 Nov 2025 14:11:22 +0100 Subject: [PATCH 08/46] Minor comment corrections thanks to @jonasbardino --- mig/src/lustreclient/lustreclient/__init__.py | 4 ++-- sbin/grid_quota.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mig/src/lustreclient/lustreclient/__init__.py b/mig/src/lustreclient/lustreclient/__init__.py index 46385bde5..8723f555f 100644 --- a/mig/src/lustreclient/lustreclient/__init__.py +++ b/mig/src/lustreclient/lustreclient/__init__.py @@ -3,7 +3,7 @@ # # --- BEGIN_HEADER --- # -# __init__ - luste client python extension +# __init__ - lustre client python extension # Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. @@ -24,7 +24,7 @@ # # -- END_HEADER --- # -"""This package provide luste client functionality""" +"""This package provide lustre client functionality""" __dummy = True diff --git a/sbin/grid_quota.py b/sbin/grid_quota.py index 1e228566c..48d2d401e 100755 --- a/sbin/grid_quota.py +++ b/sbin/grid_quota.py @@ -44,7 +44,7 @@ if __name__ == "__main__": print( - """This is the MiG lustre quota daemon which collect storage quota + """This is the MiG quota daemon which collects storage quota information for users and vgrids. Set the MIG_CONF environment to the server configuration path From 1132a2d2bb3b8e81567f820213d81cf3acaa5973 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Mon, 1 Dec 2025 13:34:49 +0100 Subject: [PATCH 09/46] Added 'lustreclient' to 'c-ext-sanity-check' ignore paths as it depends on the 'lustre' source code which we do not wan't to include in our code path. --- .github/workflows/python-c-ext-sanity-check.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-c-ext-sanity-check.yml b/.github/workflows/python-c-ext-sanity-check.yml index becc1bf2c..1f7be9442 100644 --- a/.github/workflows/python-c-ext-sanity-check.yml +++ b/.github/workflows/python-c-ext-sanity-check.yml @@ -24,6 +24,7 @@ on: - 'mig/apache/**' - 'mig/bin/**' - 'mig/java-bin/**' + - 'mig/src/lustreclient/**' - '**/*.py' - '**/*.js' branches: @@ -52,6 +53,7 @@ on: - 'mig/apache/**' - 'mig/bin/**' - 'mig/java-bin/**' + - 'mig/src/lustreclient/**' - '**/*.py' - '**/*.js' branches: From be7f1bc40510db6b6246d3bc1df7a7eb15e24775 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 10:03:36 +0100 Subject: [PATCH 10/46] Added check for 'lustreclient' module import --- mig/lib/lustrequota.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/mig/lib/lustrequota.py b/mig/lib/lustrequota.py index cd652814e..026769074 100644 --- a/mig/lib/lustrequota.py +++ b/mig/lib/lustrequota.py @@ -38,8 +38,13 @@ from mig.shared.base import force_unicode from mig.shared.fileio import unpickle, pickle, save_json, makedirs_rec, \ make_symlink -from lustreclient.lfs import lfs_set_project_id, lfs_get_project_quota, \ - lfs_set_project_quota +try: + from lustreclient.lfs import lfs_set_project_id, lfs_get_project_quota, \ + lfs_set_project_quota +except: + lfs_set_project_id = None + lfs_get_project_quota = None + lfs_set_project_quota = None def __get_lustre_basepath(configuration, lustre_basepath=None): @@ -175,10 +180,10 @@ def __set_project_id(configuration, return -1 if currfiles == 0: break - logger.info("Skipping project id: %d" \ - % next_lustre_pid \ - + " already registered with %d files" \ - % currfiles) + logger.info("Skipping project id: %d" + % next_lustre_pid + + " already registered with %d files" + % currfiles) next_lustre_pid += 1 if next_lustre_pid == max_lustre_pid: @@ -400,6 +405,15 @@ def __update_quota(configuration, def update_lustre_quota(configuration): """Update lustre quota for users and vgrids""" logger = configuration.logger + + # Check if lustreclient module was imported correctly + + if lfs_set_project_id is None \ + or lfs_get_project_quota is None \ + or lfs_set_project_quota is None: + logger.error("Failed to import lustreclient module") + return False + retval = True timestamp = int(time.time()) From d5a9d3cca599ec57093c4c5487949881f9276e4d Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 10:04:00 +0100 Subject: [PATCH 11/46] Added log error message if requested 'quota' backend is unsupported --- mig/lib/quota.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mig/lib/quota.py b/mig/lib/quota.py index a1dec8e45..e93830d28 100644 --- a/mig/lib/quota.py +++ b/mig/lib/quota.py @@ -36,11 +36,14 @@ def update_quota(configuration): """Update quota for users and vgrids""" - logger = configuration.logger retval = False - + logger = configuration.logger if configuration.quota_backend == 'lustre' \ or configuration.quota_backend == 'lustre-gocryptfs': retval = update_lustre_quota(configuration) + else: + logger.error("quota_backend: %r not in supported_quota_backends: %r" + % (configuration.quota_backend, + supported_quota_backends)) return retval From 3933cd897aca91cd36acd2ede3b6886aa63c88aa Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 13:16:47 +0100 Subject: [PATCH 12/46] Removed trailing whitespace --- mig/src/lustreclient/lustreclient/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mig/src/lustreclient/lustreclient/__init__.py b/mig/src/lustreclient/lustreclient/__init__.py index 8723f555f..78e0d2121 100644 --- a/mig/src/lustreclient/lustreclient/__init__.py +++ b/mig/src/lustreclient/lustreclient/__init__.py @@ -34,7 +34,7 @@ # All sub modules to load in case of 'from X import *' __all__ = [] - + # Collect all package information here for easy use from scripts and helpers package_name = 'Lustre Client Python extension' From 8312ebdafd85ac85a806eb900c435b558043a8fa Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 13:50:02 +0100 Subject: [PATCH 13/46] Restrict 'lustreclient.lfs' import exception to 'ImportError' to satisfy pylint --- mig/lib/lustrequota.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mig/lib/lustrequota.py b/mig/lib/lustrequota.py index 026769074..3b0e8f319 100644 --- a/mig/lib/lustrequota.py +++ b/mig/lib/lustrequota.py @@ -29,6 +29,7 @@ """helpers to support lustre quota""" import os +import sys import stat import time import shlex @@ -41,7 +42,7 @@ try: from lustreclient.lfs import lfs_set_project_id, lfs_get_project_quota, \ lfs_set_project_quota -except: +except ImportError: lfs_set_project_id = None lfs_get_project_quota = None lfs_set_project_quota = None From 9dba87064d196f503aa843ca4cea2007aaf4b8f7 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 14:51:29 +0100 Subject: [PATCH 14/46] Removed absolete 'import sys' --- mig/lib/lustrequota.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mig/lib/lustrequota.py b/mig/lib/lustrequota.py index 3b0e8f319..c9e9a6a9e 100644 --- a/mig/lib/lustrequota.py +++ b/mig/lib/lustrequota.py @@ -29,7 +29,6 @@ """helpers to support lustre quota""" import os -import sys import stat import time import shlex From 559f60cde21e7192d99ae0ecc8e3e2f8b9ce06fe Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 15:35:16 +0100 Subject: [PATCH 15/46] lustrequota: Added check for 'psutil' module --- mig/lib/lustrequota.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mig/lib/lustrequota.py b/mig/lib/lustrequota.py index c9e9a6a9e..d335554d3 100644 --- a/mig/lib/lustrequota.py +++ b/mig/lib/lustrequota.py @@ -33,7 +33,12 @@ import time import shlex import subprocess -import psutil + +# NOTE: we rely on psutil to resolve lustre mount point +try: + import psutil +except ImportError: + psutil = None from mig.shared.base import force_unicode from mig.shared.fileio import unpickle, pickle, save_json, makedirs_rec, \ @@ -50,6 +55,9 @@ def __get_lustre_basepath(configuration, lustre_basepath=None): """If *lustre_basepath* is provided then check it, otherwise try to resolve it""" + if psutil is None: + return None + valid_lustre_basepath = None for mount in psutil.disk_partitions(all=True): if mount.fstype == "lustre": From 17f4194dce6f662ead2ac0e4bbdf7d1124e350db Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 15:36:01 +0100 Subject: [PATCH 16/46] Added quota unittest --- tests/test_mig_lib_quota.py | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/test_mig_lib_quota.py diff --git a/tests/test_mig_lib_quota.py b/tests/test_mig_lib_quota.py new file mode 100644 index 000000000..25dd1328a --- /dev/null +++ b/tests/test_mig_lib_quota.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# test_mig_lib_quota - unit test of the corresponding mig lib module +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""Unit tests for the migrid module pointed to in the filename""" + +from mig.lib.quota import update_quota +from tests.support import MigTestCase + + +class MigLibQouta(MigTestCase): + """Unit tests for quota related helper functions""" + + def _provide_configuration(self): + """Prepare isolated test config""" + return 'testconfig' + + def before_each(self): + """Set up test configuration and reset state before each test""" + pass + + def test_invalid_quota_backend(self): + """Test invalid quota_backend in configuration""" + self.configuration.quota_backend = "NEVERNEVER" + with self.assertLogs(level='DEBUG') as log_capture: + update_quota(self.configuration) + self.assertTrue(any("quota_backend: 'NEVERNEVER' not in supported_quota_backends" in msg + for msg in log_capture.output)) From f888463af1d49c030f74fab86d7ef8b203cd6a52 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Thu, 11 Dec 2025 15:02:53 +0100 Subject: [PATCH 17/46] python-c-ext-sanity-check: Added support for skipping paths specified in env.skip --- .github/workflows/python-c-ext-sanity-check.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-c-ext-sanity-check.yml b/.github/workflows/python-c-ext-sanity-check.yml index 1f7be9442..6482609a0 100644 --- a/.github/workflows/python-c-ext-sanity-check.yml +++ b/.github/workflows/python-c-ext-sanity-check.yml @@ -6,6 +6,12 @@ name: Python C-Extension Sanity Checks on: + env: + # env.skip: Skip the following paths + # Format: PATH1|PATH2|PATH3 + env: + - skip: "mig/src/lustreclient/.*$" + # Triggers the workflow on push or pull request events but only for this git branch push: paths-ignore: @@ -92,9 +98,9 @@ jobs: run: | # NOTE: we only run splint error check for changed C files to limit noise # NOTE: point splint to Ubuntu's custom /usr/include/python3.x for Python.h - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$')" # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list $(python3-config --includes) &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list $(python3-config --includes) &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log @@ -118,7 +124,7 @@ jobs: run: | # NOTE: we only run splint error check for changed C files to limit noise # NOTE: point splint to Ubuntu's custom /usr/include/python3.x for Python.h - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$')" # NOTE: splint complains about NATIVE_TSS_KEY_T in system header here # NOTE: show splint warnings but don't fail unless it found critical errors git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true @@ -158,10 +164,10 @@ jobs: # NOTE: perms are not right inside container so repeat what checkout module does. git config --global --add safe.directory "$PWD" # NOTE: we only run splint error check for changed C files to limit noise - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$')" echo "with splint from $(which splint)" ls -l /bin/splint # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log From 72655fd212a5f9e190e70b9cd3147b2921c7ca22 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Thu, 11 Dec 2025 15:18:54 +0100 Subject: [PATCH 18/46] python-c-ext-sanity-check: Removed double 'env' --- .github/workflows/python-c-ext-sanity-check.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-c-ext-sanity-check.yml b/.github/workflows/python-c-ext-sanity-check.yml index 6482609a0..d0d035ec7 100644 --- a/.github/workflows/python-c-ext-sanity-check.yml +++ b/.github/workflows/python-c-ext-sanity-check.yml @@ -6,11 +6,10 @@ name: Python C-Extension Sanity Checks on: + # env.skip: Skip the following paths + # Format: PATH1|PATH2|PATH3 env: - # env.skip: Skip the following paths - # Format: PATH1|PATH2|PATH3 - env: - - skip: "mig/src/lustreclient/.*$" + - skip: "mig/src/lustreclient/.*$" # Triggers the workflow on push or pull request events but only for this git branch push: From 7dd5ed42f13fa7d985166637b869b6d79f567f1f Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Thu, 11 Dec 2025 15:51:20 +0100 Subject: [PATCH 19/46] python-c-ext-sanity-check: Added missing 'filter' --- .github/workflows/python-c-ext-sanity-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-c-ext-sanity-check.yml b/.github/workflows/python-c-ext-sanity-check.yml index d0d035ec7..99a3d66c6 100644 --- a/.github/workflows/python-c-ext-sanity-check.yml +++ b/.github/workflows/python-c-ext-sanity-check.yml @@ -126,7 +126,7 @@ jobs: echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$')" # NOTE: splint complains about NATIVE_TSS_KEY_T in system header here # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log From af450ff9e5df02dc25452f807082daab8b13a1b1 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Thu, 11 Dec 2025 16:00:03 +0100 Subject: [PATCH 20/46] python-c-ext-sanity-check: Changed 'env.skip' to 'env.paths-ignore-splint' --- .github/workflows/python-c-ext-sanity-check.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-c-ext-sanity-check.yml b/.github/workflows/python-c-ext-sanity-check.yml index 99a3d66c6..de33c52f2 100644 --- a/.github/workflows/python-c-ext-sanity-check.yml +++ b/.github/workflows/python-c-ext-sanity-check.yml @@ -6,10 +6,10 @@ name: Python C-Extension Sanity Checks on: - # env.skip: Skip the following paths + # env.paths-ignore-splint: Don't run 'splint' on these paths # Format: PATH1|PATH2|PATH3 env: - - skip: "mig/src/lustreclient/.*$" + - paths-ignore-splint: "mig/src/lustreclient/.*$" # Triggers the workflow on push or pull request events but only for this git branch push: @@ -97,9 +97,9 @@ jobs: run: | # NOTE: we only run splint error check for changed C files to limit noise # NOTE: point splint to Ubuntu's custom /usr/include/python3.x for Python.h - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$')" # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list $(python3-config --includes) &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list $(python3-config --includes) &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log @@ -123,10 +123,10 @@ jobs: run: | # NOTE: we only run splint error check for changed C files to limit noise # NOTE: point splint to Ubuntu's custom /usr/include/python3.x for Python.h - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$')" # NOTE: splint complains about NATIVE_TSS_KEY_T in system header here # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log @@ -163,10 +163,10 @@ jobs: # NOTE: perms are not right inside container so repeat what checkout module does. git config --global --add safe.directory "$PWD" # NOTE: we only run splint error check for changed C files to limit noise - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$')" echo "with splint from $(which splint)" ls -l /bin/splint # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.skip }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log From 1a90bbd47b81f70fc7132e0136633fd7c26a13e8 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Fri, 12 Dec 2025 09:09:23 +0100 Subject: [PATCH 21/46] python-c-ext-sanity-check: Test --- mig/src/libpam-mig/libpam_mig.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mig/src/libpam-mig/libpam_mig.c b/mig/src/libpam-mig/libpam_mig.c index b030fb340..4ca6f89a1 100644 --- a/mig/src/libpam-mig/libpam_mig.c +++ b/mig/src/libpam-mig/libpam_mig.c @@ -32,6 +32,7 @@ * */ + #include #include #include From f6e9e7b243b82da39361a6d6e6d23fe2030428fb Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Wed, 17 Dec 2025 09:13:23 +0100 Subject: [PATCH 22/46] migrid init: Append /usr/local/(s)bin to existing PATH variable as suggested by @jonasbardino --- mig/install/migrid-init.d-deb-template | 4 ++-- mig/install/migrid-init.d-rh-template | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mig/install/migrid-init.d-deb-template b/mig/install/migrid-init.d-deb-template index 87351aa07..2d5e262fb 100755 --- a/mig/install/migrid-init.d-deb-template +++ b/mig/install/migrid-init.d-deb-template @@ -43,8 +43,8 @@ if [ -z "$PYTHONPATH" ]; then else PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi -# Make sure '/usr/local/(s)bin' is in path and force lookup order -export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# Make sure '/usr/local/(s)bin' is in PATH +export PATH="$PATH:/usr/local/sbin:/usr/local/bin" # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} diff --git a/mig/install/migrid-init.d-rh-template b/mig/install/migrid-init.d-rh-template index 911c5bb3c..cc4b3e15d 100755 --- a/mig/install/migrid-init.d-rh-template +++ b/mig/install/migrid-init.d-rh-template @@ -75,8 +75,8 @@ if [ -z "$PYTHONPATH" ]; then else export PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi -# Make sure '/usr/local/(s)bin' is in path and force lookup order -export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# Make sure '/usr/local/(s)bin' is in PATH +export PATH="$PATH:/usr/local/sbin:/usr/local/bin" # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} From 247ea4f38686d5e5e410b9b196cedef9636ebf5b Mon Sep 17 00:00:00 2001 From: Martin-Rehr Date: Wed, 17 Dec 2025 11:02:01 +0100 Subject: [PATCH 23/46] python-c-ext-sanity-check: Added 'PATHS_IGNORE_SPLINT' (#397) * python-c-ext-sanity-check: Added 'PATHS_IGNORE_SPLINT' * python-c-ext-sanity-check: github apt require 'sudo' --- .../workflows/python-c-ext-sanity-check.yml | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python-c-ext-sanity-check.yml b/.github/workflows/python-c-ext-sanity-check.yml index becc1bf2c..dec1e4843 100644 --- a/.github/workflows/python-c-ext-sanity-check.yml +++ b/.github/workflows/python-c-ext-sanity-check.yml @@ -5,6 +5,11 @@ name: Python C-Extension Sanity Checks +# Ignore the following paths +# Format: PATH1|PATH2|PATH3 +env: + PATHS_IGNORE_SPLINT: mig/src/lustreclient/.*$ + on: # Triggers the workflow on push or pull request events but only for this git branch push: @@ -71,16 +76,19 @@ jobs: if: ${{ false }} # Disabled until we figure out how to fix system headers runs-on: ubuntu-latest steps: + - name: Update packages + run: | + DEBIAN_FRONTEND='noninteractive' sudo apt update -y - name: Set up latest stable python 3.x uses: actions/setup-python@v5 with: python-version: "3.x" - name: Set up git, findutils and make with apt run: | - sudo apt install -y git findutils make + DEBIAN_FRONTEND='noninteractive' sudo apt install -y git findutils make - name: Install dependencies run: | - sudo apt install -y libnss3-dev libpam-dev splint + DEBIAN_FRONTEND='noninteractive' sudo apt install -y libnss3-dev libpam-dev splint # We may need git installed to get a full repo clone rather than unpacked archive - name: Check out source repository uses: actions/checkout@v4 @@ -90,9 +98,9 @@ jobs: run: | # NOTE: we only run splint error check for changed C files to limit noise # NOTE: point splint to Ubuntu's custom /usr/include/python3.x for Python.h - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$')" # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list $(python3-config --includes) &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list $(python3-config --includes) &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log @@ -101,12 +109,15 @@ jobs: name: Sanity check c-extension module code in latest Ubuntu LTS python3 runs-on: ubuntu-24.04 steps: + - name: Update packages + run: | + DEBIAN_FRONTEND='noninteractive' sudo apt update -y - name: Set up git, findutils and make with apt run: | - sudo apt install -y git findutils make + DEBIAN_FRONTEND='noninteractive' sudo apt install -y git findutils make - name: Install dependencies run: | - sudo apt install -y python3-dev libnss3-dev libpam-dev splint + DEBIAN_FRONTEND='noninteractive' sudo apt install -y python3-dev libnss3-dev libpam-dev splint # We may need git installed to get a full repo clone rather than unpacked archive - name: Check out source repository uses: actions/checkout@v4 @@ -116,10 +127,10 @@ jobs: run: | # NOTE: we only run splint error check for changed C files to limit noise # NOTE: point splint to Ubuntu's custom /usr/include/python3.x for Python.h - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$')" # NOTE: splint complains about NATIVE_TSS_KEY_T in system header here # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log @@ -135,6 +146,9 @@ jobs: container: image: rockylinux/rockylinux:9 steps: + - name: Update packages + run: | + dnf update -y - name: Set up git, findutils, make and python3 with dnf and make the latter default run: | dnf install -y git findutils make python3 python3-pip python-unversioned-command @@ -156,10 +170,10 @@ jobs: # NOTE: perms are not right inside container so repeat what checkout module does. git config --global --add safe.directory "$PWD" # NOTE: we only run splint error check for changed C files to limit noise - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$')" echo "with splint from $(which splint)" ls -l /bin/splint # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log From 3245588b2a1f5fce89251a3036a6b0765d8905d8 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 13:08:30 +0100 Subject: [PATCH 24/46] Changed pylustrequota to a generic python lustreclient extension --- .../{pylustrequota => lustreclient}/README | 6 +- .../lustreclient}/__init__.py | 27 +- .../lustreclient}/lfs.c | 6 +- .../{pylustrequota => lustreclient}/setup.cfg | 4 +- .../{pylustrequota => lustreclient}/setup.py | 17 +- mig/src/pylustrequota/bin/miglustrequota.py | 567 ------------------ 6 files changed, 29 insertions(+), 598 deletions(-) rename mig/src/{pylustrequota => lustreclient}/README (83%) rename mig/src/{pylustrequota/pylustrequota => lustreclient/lustreclient}/__init__.py (77%) rename mig/src/{pylustrequota/pylustrequota => lustreclient/lustreclient}/lfs.c (98%) rename mig/src/{pylustrequota => lustreclient}/setup.cfg (84%) rename mig/src/{pylustrequota => lustreclient}/setup.py (86%) delete mode 100755 mig/src/pylustrequota/bin/miglustrequota.py diff --git a/mig/src/pylustrequota/README b/mig/src/lustreclient/README similarity index 83% rename from mig/src/pylustrequota/README rename to mig/src/lustreclient/README index 22be7628b..46dc4d186 100644 --- a/mig/src/pylustrequota/README +++ b/mig/src/lustreclient/README @@ -1,7 +1,7 @@ -This folder contains a module for MiG lustre quota +This folder contains a module for MiG lustre client python extension To suppert containerized MiG (where lustre is mounted outside the container) -lustre quota functionality is compiled statically into this module. +lustre client functionality is compiled statically into this module. Install lustre dependencies (Rocky 9): ====================================== @@ -18,7 +18,7 @@ dnf --enablerepo=crb install \ libnl3-devel.x86_64 \ libyaml-devel \ krb5-devel.x86_64 -git clone git://git.whamcloud.com/fs/lustre-release.git +git clone https://github.com/lustre/lustre-release cd lustre-release && git checkout ${VERSION} ; cd - cd lustre-release && sh ./autogen.sh ; cd - cd lustre-release && ./configure --disable-server --enable-quota --enable-utils --enable-gss ; cd - diff --git a/mig/src/pylustrequota/pylustrequota/__init__.py b/mig/src/lustreclient/lustreclient/__init__.py similarity index 77% rename from mig/src/pylustrequota/pylustrequota/__init__.py rename to mig/src/lustreclient/lustreclient/__init__.py index 10abf6cd7..46385bde5 100644 --- a/mig/src/pylustrequota/pylustrequota/__init__.py +++ b/mig/src/lustreclient/lustreclient/__init__.py @@ -3,7 +3,7 @@ # # --- BEGIN_HEADER --- # -# __init__ - luste quota python extensions +# __init__ - luste client python extension # Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. @@ -24,7 +24,7 @@ # # -- END_HEADER --- # -"""This package provide luste quota functionality""" +"""This package provide luste client functionality""" __dummy = True @@ -37,8 +37,8 @@ # Collect all package information here for easy use from scripts and helpers -package_name = 'Lustre Quota Python extension' -short_name = 'pylustrequota' +package_name = 'Lustre Client Python extension' +short_name = 'lustreclient' # IMPORTANT: Please keep version in sync with doc-src/README.t2t @@ -46,18 +46,18 @@ version_suffix = '' version_string = '.'.join([str(i) for i in version_tuple]) + version_suffix package_version = '%s %s' % (package_name, version_string) -project_team = 'The MiG Project lead by Brian Vinter' -project_email = 'info@erda.dk' -maintainer_team = 'The pylustrequota maintainers' -maintainer_email = 'info@erda.dk' -project_url = 'https://github.com/ucphhpc/pylustrequota' -download_url = 'https://github.com/ucphhpc/pylustrequota/releases' +project_team = 'The MiG Project by the Science HPC Center at UCPH' +project_email = 'info@migrid.org' +maintainer_team = 'The migrid.org maintainers' +maintainer_email = 'info@migrid.org' +project_url = 'https://github.com/ucphhpc/migrid-sync' +download_url = 'https://github.com/ucphhpc/migrid-sync/releases' license_name = 'GNU GPL v2' short_desc = \ - 'Python quota extension for lustre' + 'Lustre client python extension' long_desc = \ - """Python quota extension for for lustre: -Documentation: https://github.com/ucphhpc/pylustrequota + """Lustre client python extension: +Documentation: https://github.com/ucphhpc/migrid-sync """ project_class = [ 'Development Status :: 1 - Beta', @@ -72,7 +72,6 @@ 'Python', 'Python C extensions', 'lustre', - 'rsync', ] # Requirements diff --git a/mig/src/pylustrequota/pylustrequota/lfs.c b/mig/src/lustreclient/lustreclient/lfs.c similarity index 98% rename from mig/src/pylustrequota/pylustrequota/lfs.c rename to mig/src/lustreclient/lustreclient/lfs.c index 1082d9deb..5e0b0437d 100644 --- a/mig/src/pylustrequota/pylustrequota/lfs.c +++ b/mig/src/lustreclient/lustreclient/lfs.c @@ -1,7 +1,7 @@ /* --- BEGIN_HEADER --- -lfs - Shared lustre library functions for Python lustre quota -Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +lfs - Shared lustre library functions for Python lustre client +Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH This file is part of MiG. MiG is free software: you can redistribute it and/or modify @@ -385,4 +385,4 @@ PyMODINIT_FUNC PyInit_lfs(void) { } return module; -} \ No newline at end of file +} diff --git a/mig/src/pylustrequota/setup.cfg b/mig/src/lustreclient/setup.cfg similarity index 84% rename from mig/src/pylustrequota/setup.cfg rename to mig/src/lustreclient/setup.cfg index 2118cc8e3..75012b93e 100644 --- a/mig/src/pylustrequota/setup.cfg +++ b/mig/src/lustreclient/setup.cfg @@ -1,7 +1,7 @@ # --- BEGIN_HEADER --- # -# setup.cfg - setup configuration file for python lustre quota -# Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +# setup.cfg - setup configuration file for lustre client python extension +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # diff --git a/mig/src/pylustrequota/setup.py b/mig/src/lustreclient/setup.py similarity index 86% rename from mig/src/pylustrequota/setup.py rename to mig/src/lustreclient/setup.py index 830eeca69..7693e4fd5 100644 --- a/mig/src/pylustrequota/setup.py +++ b/mig/src/lustreclient/setup.py @@ -3,8 +3,8 @@ # # --- BEGIN_HEADER --- # -# setup.py - Setup for python luste quota -# Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +# setup.py - Setup for lustre client python extension +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # @@ -27,7 +27,7 @@ from setuptools import setup, Extension -from pylustrequota import version_string, short_name, project_team, \ +from lustreclient import version_string, short_name, project_team, \ project_email, short_desc, long_desc, project_url, download_url, \ license_name, project_class, project_keywords, versioned_requires, \ project_requires, project_extras, project_platforms, maintainer_team, \ @@ -51,14 +51,13 @@ install_requires=versioned_requires, requires=project_requires, extras_require=project_extras, - scripts=['bin/miglustrequota.py', - ], - packages=['pylustrequota'], - package_dir={'pylustrequota': 'pylustrequota', + scripts=[], + packages=['lustreclient'], + package_dir={'lustreclient': 'lustreclient', }, package_data={}, ext_modules=[ - Extension('pylustrequota.lfs', + Extension('lustreclient.lfs', include_dirs=['/usr/include', '/usr/include/python3', 'lustre-release/libcfs/include', @@ -69,7 +68,7 @@ ], library_dirs=[], libraries=[], - sources=['pylustrequota/lfs.c', + sources=['lustreclient/lfs.c', 'lustre-release/lustre/utils/lfs_project.c'], extra_objects=[ 'lustre-release/lustre/utils/.libs/liblustreapi.a'], diff --git a/mig/src/pylustrequota/bin/miglustrequota.py b/mig/src/pylustrequota/bin/miglustrequota.py deleted file mode 100755 index b471bce4a..000000000 --- a/mig/src/pylustrequota/bin/miglustrequota.py +++ /dev/null @@ -1,567 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# --- BEGIN_HEADER --- -# -# mig_lustre_quota - MiG lustre quota manager -# Copyright (C) 2003-2025 The MiG Project lead by the Science HPC Center at UCPH -# -# This file is part of MiG. -# -# MiG is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# MiG is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. -# -# --- END_HEADER --- -# - -"""Assign lustre project id's to new users and vgrids, -set default quota on new entries and update existing quotas if changed. -Fetch the number of files and bytes used by each project id. -""" - -import os -import sys -import time -import stat -import getopt -import shlex -import subprocess -import psutil - -from mig.shared.base import force_unicode -from mig.shared.conf import get_configuration_object -from mig.shared.fileio import unpickle, pickle, save_json, makedirs_rec, \ - make_symlink -from mig.shared.logger import daemon_logger - -from pylustrequota.lfs import lfs_set_project_id, lfs_get_project_quota, \ - lfs_set_project_quota - - -supported_quota_backends = ['lustre', 'lustre-gocryptfs'] - - -def usage(name=sys.argv[0]): - """Usage help""" - msg = """Usage: %(name)s [OPTIONS] -Where OPTIONS may be one or more of: - -h|--help Show this help - -v|--verbose Verbose output - -q|--quiet No stdout/stderr output - -c PATH|--config=PATH Path to config file - -l PATH|--lustre-basepath Path to lustre base - -g PATH|--gocryptfs-sock Path to gocryptfs socket -""" % {'name': name} - print(msg, file=sys.stderr) - - -def INFO(configuration, msg, verbose=False): - """log info and print to stdout on verbose""" - configuration.logger.info(msg) - if verbose: - print(msg) - - -def ERROR(configuration, msg, quiet=False): - """log error and print to stderr on verbose""" - configuration.logger.error(msg) - if not quiet: - print("ERROR: %s" % msg, file=sys.stderr) - - -def DEBUG(configuration, msg, verbose=False): - """log debug and print to stderr on verbose""" - configuration.logger.debug(msg) - if verbose and configuration.loglevel == 'debug': - print("DEBUG: %s" % msg, file=sys.stderr) - - -def __shellexec(configuration, - command, - args=[], - stdin_str=None, - stdout_filepath=None, - stderr_filepath=None): - """Execute shell command - Returns (exit_code, stdout, stderr) of subprocess""" - result = 0 - logger = configuration.logger - stdin_handle = subprocess.PIPE - stdout_handle = subprocess.PIPE - stderr_handle = subprocess.PIPE - if stdout_filepath is not None: - stdout_handle = open(stdout_filepath, "w+") - if stderr_filepath is not None: - stderr_handle = open(stderr_filepath, "w+") - __args = shlex.split(command) - __args.extend(args) - logger.debug("__args: %s" % __args) - process = subprocess.Popen( - __args, - stdin=stdin_handle, - stdout=stdout_handle, - stderr=stderr_handle) - if stdin_str: - process.stdin.write(stdin_str.encode()) - stdout, stderr = process.communicate() - rc = process.wait() - - if stdout_filepath: - stdout = stdout_filepath - stdout_handle.close() - if stderr_filepath: - stderr = stderr_filepath - stderr_handle.close() - - # Close stdin, stdout and stderr FDs if they exists - if process.stdin: - process.stdin.close() - if process.stdout: - process.stdout.close() - if process.stderr: - process.stderr.close() - - if stdout: - stdout = force_unicode(stdout) - if stderr: - stderr = force_unicode(stderr) - if result == 0: - logger.debug("%s %s: rc: %s, stdout: %s, error: %s" - % (command, - " ".join(args), - rc, - stdout, - stderr)) - else: - logger.error("shellexec: %s %s: rc: %s, stdout: %s, error: %s" - % (command, - " ".join(__args), - rc, - stdout, - stderr)) - - return (rc, stdout, stderr) - - -def __update_quota(configuration, - lustre_basepath, - lustre_setting, - quota_name, - quota_type, - gocryptfs_sock, - timestamp, - verbose, - quiet): - """Update quota for *quota_name*, if new entry then - assign lustre project id and set default quota. - If existing entry then update quota settings if changed - and fetch file and bytes usage and store it as pickle and json - """ - logger = configuration.logger - quota_limits_changed = False - next_lustre_pid = lustre_setting.get('next_pid', -1) - if next_lustre_pid == -1: - msg = "Invalid lustre quota next_pid: %d for: %r" \ - % (next_lustre_pid, quota_name) - ERROR(configuration, msg, quiet) - return False - if quota_type == 'vgrid': - default_quota_limit = configuration.quota_vgrid_limit - data_basepath = configuration.vgrid_files_writable - # NOTE: Old vgrids stored data directly in 'vgrid_files_home' - if not os.path.isdir(os.path.join(data_basepath, quota_name)): - data_basepath = configuration.vgrid_files_home - else: - default_quota_limit = configuration.quota_user_limit - data_basepath = configuration.user_home - - # Load quota if it exists otherwise new quota - - quota_filepath = os.path.join(configuration.quota_home, - configuration.quota_backend, - quota_type, - "%s.pck" % quota_name) - - if os.path.exists(quota_filepath): - quota = unpickle(quota_filepath, logger) - if not quota: - msg = "Failed to load quota settings for: %r from %r" \ - % (quota_name, quota_filepath) - ERROR(configuration, msg, quiet) - return False - else: - quota = {'lustre_pid': next_lustre_pid, - 'files': -1, - 'bytes': -1, - 'softlimit_bytes': -1, - 'hardlimit_bytes': -1, - } - - quota_lustre_pid = quota.get('lustre_pid', -1) - if quota_lustre_pid == -1: - msg = "Invalid quota lustre pid: %d for %r" \ - % (quota_lustre_pid, quota_name) - ERROR(configuration, msg, quiet) - return False - - # Resolve quota data path - # if gocryptfs then resolve encrypted path - # otherwise use plain path - - if configuration.quota_backend == "lustre": - quota_datapath = os.path.join(data_basepath, - quota_name) - elif configuration.quota_backend == "lustre-gocryptfs": - rel_data_basepath = data_basepath. \ - replace(configuration.state_path + os.sep, "") - stdin_str = os.path.join(rel_data_basepath, quota_name) - cmd = "gocryptfs-xray -encrypt-paths %s" % gocryptfs_sock - (rc, stdout, stderr) = __shellexec(configuration, - cmd, - stdin_str=stdin_str) - if rc == 0 and stdout: - encoded_path = stdout.strip() - quota_datapath = os.path.join(lustre_basepath, - encoded_path) - else: - msg = "Failed to resolve encrypted path for: %r" \ - % quota_name \ - + ", rc: %d, error: %s" \ - % (rc, stderr) - ERROR(configuration, msg, quiet) - return False - else: - ERROR(configuration, - "Invalid quota backend: %r" % configuration.quota_backend, - quiet) - return False - - # Skip non-dir entries - - if not os.path.isdir(quota_datapath): - msg = "Skipping non-dir entry: %r: %r" \ - % (quota_name, quota_datapath) - DEBUG(configuration, msg, verbose) - return True - - # If new entry then set lustre project id - - if quota_lustre_pid == next_lustre_pid: - # TODO: Mask out path's from log if gocryptfs ? - msg = "Setting lustre project id: %d for %r: %r" \ - % (quota_lustre_pid, quota_name, quota_datapath) - INFO(configuration, msg) - rc = lfs_set_project_id(quota_datapath, quota_lustre_pid, 1) - if rc == 0: - lustre_setting['next_pid'] = quota_lustre_pid + 1 - else: - msg = "Failed to set lustre project id: %d for %r: %r" \ - % (quota_lustre_pid, quota_name, quota_datapath) \ - + ", rc: %d" \ - % rc - ERROR(configuration, msg, quiet) - return False - - # Get current quota values for lustre_pid - - (rc, currfiles, currbytes, softlimit_bytes, hardlimit_bytes) \ - = lfs_get_project_quota(quota_datapath, quota_lustre_pid) - if rc != 0: - msg = "Failed to fetch quota for lustre project id: %d, %r, %r" \ - % (quota_lustre_pid, quota_name, quota_datapath) \ - + ", rc: %d" \ - % rc - ERROR(configuration, msg, quiet) - return False - - # Update quota info - - quota['mtime'] = timestamp - quota['files'] = currfiles - quota['bytes'] = currbytes - - # If new entry use default quota - # and update quota if changed - - if quota_lustre_pid == next_lustre_pid: - quota_limits_changed = True - quota['softlimit_bytes'] = default_quota_limit - quota['hardlimit_bytes'] = default_quota_limit - elif hardlimit_bytes != quota.get('hardlimit_bytes', -1) \ - or softlimit_bytes != quota.get('softlimit_bytes', -1): - quota_limits_changed = True - quota['softlimit_bytes'] = softlimit_bytes - quota['hardlimit_bytes'] = hardlimit_bytes - - if quota_limits_changed: - rc = lfs_set_project_quota(quota_datapath, - quota_lustre_pid, - quota['softlimit_bytes'], - quota['hardlimit_bytes'], - ) - if rc != 0: - msg = "Failed to set quota limit: %d/%d" \ - % (softlimit_bytes, - hardlimit_bytes) \ - + " for lustre project id: %d, %r, %r, rc: %d" \ - % (quota_lustre_pid, - quota_name, - quota_datapath, - rc) - ERROR(configuration, msg, quiet) - return False - - # Save current quota - - new_quota_basepath = os.path.join(configuration.quota_home, - configuration.quota_backend, - quota_type, - str(timestamp)) - if not os.path.exists(new_quota_basepath) \ - and not makedirs_rec(new_quota_basepath, configuration): - msg = "Failed to create new quota base path: %r" \ - % new_quota_basepath - ERROR(configuration, msg, quiet) - return False - - new_quota_filepath_pck = os.path.join(new_quota_basepath, - "%s.pck" % quota_name) - status = pickle(quota, new_quota_filepath_pck, logger) - if not status: - msg = "Failed to save quota for: %r to %r" \ - % (quota_name, new_quota_filepath_pck) - ERROR(configuration, msg, quiet) - return False - new_quota_filepath_json = os.path.join(new_quota_basepath, - "%s.json" % quota_name) - status = save_json(quota, - new_quota_filepath_json, - logger) - if not status: - msg = "Failed to save quota for: %r to %r" \ - % (quota_name, new_quota_filepath_json) - ERROR(configuration, msg, quiet) - return False - - # Create symlink to new quota - - status = make_symlink(new_quota_filepath_pck, - quota_filepath, - logger, - force=True) - if not status: - msg = "Failed to make quota symlink for: %r: %r -> %r" \ - % (quota_name, new_quota_filepath_pck, quota_filepath) - ERROR(configuration, msg, quiet) - return False - - return True - - -def update_quota(configuration, - lustre_basepath, - gocryptfs_sock, - verbose, - quiet): - """Update lustre quotas for users and vgrids""" - logger = configuration.logger - retval = True - timestamp = int(time.time()) - - # Load lustre quota settings - - lustre_setting_filepath = os.path.join(configuration.quota_home, - '%s.pck' - % configuration.quota_backend) - if os.path.exists(lustre_setting_filepath): - lustre_setting = unpickle(lustre_setting_filepath, - logger) - if not lustre_setting: - msg = "Failed to load lustre quota: %r" % lustre_setting_filepath - ERROR(configuration, msg, quiet) - return False - else: - lustre_setting = {'next_pid': 1, - 'mtime': 0} - - # Update quotas - - for quota_type in ('vgrid', 'user'): - if quota_type == 'vgrid': - scandir = configuration.vgrid_home - else: - scandir = configuration.user_home - - # Scan for new and modified entries - - with os.scandir(scandir) as it: - for entry in it: - if not os.path.isdir(entry.path): - # Only take dirs into account - msg = "Skiping non-dir path: %r" % entry.path - DEBUG(configuration, msg, verbose) - continue - status = __update_quota(configuration, - lustre_basepath, - lustre_setting, - entry.name, - quota_type, - gocryptfs_sock, - timestamp, - verbose, - quiet, - ) - if not status: - retval = False - - # Save updated lustre quota settings - - lustre_setting['mtime'] = timestamp - status = pickle(lustre_setting, - lustre_setting_filepath, - logger) - if not status: - msg = "Failed to save lustra quota settings: %r" \ - % lustre_setting_filepath - ERROR(configuration, msg, quiet) - - return retval - - -def main(): - retval = True - verbose = False - quiet = False - config_file = None - lustre_basepath = None - gocryptfs_sock = None - try: - opts, args = getopt.getopt(sys.argv[1:], "hvqc:l:g:", - ["help", "verbose", "quiet", "config=", - "--lustre-basepath", "--gocryptfs-sock="]) - for opt, arg in opts: - if opt in ("-h", "--help"): - usage() - sys.exit() - elif opt in ("-v", "--verbose"): - verbose = True - elif opt in ("-q", "--quiet"): - quiet = True - elif opt in ("-c", "--config"): - config_file = arg - elif opt in ("-l", "--lustre-basepath"): - lustre_basepath = arg - elif opt in ("-g", "--gocryptfs-sock"): - gocryptfs_sock = arg - except Exception as err: - print(err, file=sys.stderr) - usage() - return 1 - - if quiet: - verbose = False - - # Initialize configuration - - try: - configuration = get_configuration_object(config_file=config_file) - except Exception as err: - print(err, file=sys.stderr) - usage() - return 1 - - # Use separate logger - - logger = daemon_logger("quota", - configuration.user_quota_log, - configuration.loglevel) - configuration.logger = logger - if configuration.quota_backend not in supported_quota_backends: - msg = "Quota backend: %s not in supported backends: %s" \ - % (configuration.quota_backend, - ", ".join(supported_quota_backends)) - ERROR(configuration, msg, quiet) - return False - - # If lustre_basepath is provided then check it, - # otherwise try to resolve it - - valid_lustre_basepath = None - for mount in psutil.disk_partitions(all=True): - if mount.fstype == "lustre": - if lustre_basepath \ - and lustre_basepath.startswith(mount.mountpoint) \ - and os.path.isdir(lustre_basepath): - valid_lustre_basepath = lustre_basepath - break - elif mount.mountpoint.endswith(configuration.server_fqdn): - valid_lustre_basepath = mount.mountpoint - else: - check_lustre_basepath = os.path.join(mount.mountpoint, - configuration.server_fqdn) - if os.path.isdir(check_lustre_basepath): - valid_lustre_basepath = check_lustre_basepath - break - - if valid_lustre_basepath is None: - if lustre_basepath: - msg = "Lustre base: %r is NOT mounted" % lustre_basepath - else: - msg = "Found no valid lustre mounts for: %s" \ - % configuration.server_fqdn - ERROR(configuration, msg, quiet) - return False - - INFO(configuration, - "Using lustre basepath: %r" % valid_lustre_basepath, - verbose) - - # Check gocryptfs socket - - if configuration.quota_backend == "lustre-gocryptfs": - check_gocryptfs_sock = gocryptfs_sock - if check_gocryptfs_sock is None: - check_gocryptfs_sock = "/var/run/gocryptfs.%s.sock" \ - % configuration.server_fqdn - if os.path.exists(check_gocryptfs_sock): - gocryptfs_sock_stat = os.lstat(check_gocryptfs_sock) - if stat.S_ISSOCK(gocryptfs_sock_stat.st_mode): - gocryptfs_sock = check_gocryptfs_sock - if gocryptfs_sock: - INFO(configuration, - "Using gocryptfs socket: %r" % gocryptfs_sock, - verbose) - else: - ERROR(configuration, - "Missing gocryptfs socket: %r" % check_gocryptfs_sock, - quiet) - return False - - # Perform update - - retval = update_quota(configuration, - valid_lustre_basepath, - gocryptfs_sock, - verbose, - quiet) - return retval - - -if __name__ == "__main__": - status = main() - if status: - sys.exit(0) - else: - sys.exit(1) From 34b2473a694a2da3c32ba37bc8e31252fe6eee10 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 14:57:53 +0100 Subject: [PATCH 25/46] Changed lustre quota from cron job to grid daemon --- .../miglustrequota-template.sh.cronjob | 62 --- mig/install/migrid-init.d-deb-template | 56 +- mig/install/migrid-init.d-rh-template | 53 +- mig/lib/lustrequota.py | 480 ++++++++++++++++++ mig/lib/quota.py | 46 ++ mig/server/grid_quota.py | 1 + mig/shared/configuration.py | 4 + mig/shared/install.py | 12 +- sbin/grid_quota.py | 144 ++++++ 9 files changed, 786 insertions(+), 72 deletions(-) delete mode 100755 mig/install/miglustrequota-template.sh.cronjob create mode 100644 mig/lib/lustrequota.py create mode 100644 mig/lib/quota.py create mode 120000 mig/server/grid_quota.py create mode 100755 sbin/grid_quota.py diff --git a/mig/install/miglustrequota-template.sh.cronjob b/mig/install/miglustrequota-template.sh.cronjob deleted file mode 100755 index 48f18050a..000000000 --- a/mig/install/miglustrequota-template.sh.cronjob +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# -# Run lustre quota for MiG servers -# -# The script depends on a miglustrequota setup -# (please refer to mig/src/pylustrequota/README). -# -# IMPORTANT: if placed in /etc/cron.X the script filename must be -# something consisting entirely of upper and lower case letters, digits, -# underscores, and hyphens. I.e. if the script name contains e.g. a period, -# '.', it will be silently ignored! -# This is a limitation on the run-parts wrapper used by cron -# (see man run-parts for the rationale behind this). - -# By default bash silently ignores and continues on most errors but we can set -# options to e.g. catch uninitialized variables and errors as explained in: -# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ -# NOTE: 'set -eE' exits on non-zero exit codes to add safety and as recommended -# best-practice (CWE-252, CWE-248, ...), yet, in some cases it hurts more to -# exit midway, so it can be a trade-off. -set -eEuo pipefail - -# Send output to another email address -#MAILTO="root" - -MIG_CONF=__MIG_CODE__/server/MiGserver.conf - -# Specify if migrid runs natively or inside containers with lustre at host. -# Value is the container manager (docker, podman, or empty string for none) -container_manager="" -container="migrid-lustre-quota" - -# Look in miglustrequota install dir first -export PATH="/usr/local/bin:${PATH}" - -if [[ $(id -u) -ne 0 ]]; then - echo "Please run $0 as root" - exit 1 -fi - -if [ -z "${container_manager}" ]; then - miglustrequota=$(which "miglustrequota.py" 2>/dev/null) - if [ ! -x "${miglustrequota}" ]; then - echo "ERROR: Missing miglustrequota.py" - exit 1 - fi - quota_cmd="${miglustrequota} -c ${MIG_CONF}" -else - check_cmd="${container_manager} container ls -a | grep -q '${container}'" - eval "$check_cmd" - ret=$? - if [ "$ret" -ne 0 ]; then - echo "ERROR: Missing ${container} container" - exit 1 - fi - quota_cmd="${container_manager} start -a ${container}" -fi - -eval "$quota_cmd" -ret=$? - -exit $ret diff --git a/mig/install/migrid-init.d-deb-template b/mig/install/migrid-init.d-deb-template index d6ac02101..87351aa07 100755 --- a/mig/install/migrid-init.d-deb-template +++ b/mig/install/migrid-init.d-deb-template @@ -43,6 +43,9 @@ if [ -z "$PYTHONPATH" ]; then else PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -62,13 +65,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -264,6 +268,18 @@ start_vmproxy() { log_end_msg 1 || true fi } +start_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Starting MiG quota daemon" ${SHORT_NAME} || true + if start-stop-daemon --start --quiet --oknodo --pidfile ${PID_FILE} --make-pidfile --user root --chuid root --background --name ${SHORT_NAME} --startas ${DAEMON_PATH} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} start_sftpsubsys() { check_enabled "sftp_subsys" || return 0 DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -292,6 +308,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -524,6 +541,19 @@ stop_vmproxy() { log_end_msg 1 || true fi } +stop_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Stopping MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --quiet --oknodo --pidfile ${PID_FILE} ; then + rm -f ${PID_FILE} + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -563,6 +593,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -735,6 +766,18 @@ reload_vmproxy() { log_end_msg 1 || true fi } +reload_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Reloading MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile ${PID_FILE} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -787,6 +830,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -891,6 +935,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} } +status_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -929,6 +980,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -940,7 +992,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') diff --git a/mig/install/migrid-init.d-rh-template b/mig/install/migrid-init.d-rh-template index c4bfad648..911c5bb3c 100755 --- a/mig/install/migrid-init.d-rh-template +++ b/mig/install/migrid-init.d-rh-template @@ -35,6 +35,7 @@ # processname: grid_notify.py # processname: grid_imnotify.py # processname: grid_vmproxy.py +# processname: grid_quota.py # processname: sshd # config: /etc/sysconfig/migrid # @@ -74,6 +75,9 @@ if [ -z "$PYTHONPATH" ]; then else export PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -93,13 +97,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -353,6 +358,21 @@ start_vmproxy() { [ $RET2 -ne 0 ] && echo "Warning: vmproxy not started." echo } +start_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Starting MiG quota daemon: $SHORT_NAME" + daemon --user root --pidfile ${PID_FILE} \ + "${DAEMON_PATH} >> ${MIG_LOG}/quota.out 2>&1 &" + fallback_save_pid "$DAEMON_PATH" "$PID_FILE" "$!" + RET2=$? + [ $RET2 -eq 0 ] && success + echo + [ $RET2 -ne 0 ] && echo "Warning: quota not started." + echo +} start_sftpsubsys() { check_enabled "sftp_subsys" || return DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -385,6 +405,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -545,6 +566,15 @@ stop_vmproxy() { killproc ${DAEMON_PATH} echo } +stop_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Shutting down MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} + echo +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -588,6 +618,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -717,6 +748,15 @@ reload_vmproxy() { killproc ${DAEMON_PATH} -HUP echo } +reload_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Reloading MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} -HUP + echo +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -773,6 +813,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -877,6 +918,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status ${DAEMON_PATH} } +status_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status ${DAEMON_PATH} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -916,6 +964,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -927,7 +976,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') diff --git a/mig/lib/lustrequota.py b/mig/lib/lustrequota.py new file mode 100644 index 000000000..cd652814e --- /dev/null +++ b/mig/lib/lustrequota.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# lustrequota - helpers to support lustre quota +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""helpers to support lustre quota""" + +import os +import stat +import time +import shlex +import subprocess +import psutil + +from mig.shared.base import force_unicode +from mig.shared.fileio import unpickle, pickle, save_json, makedirs_rec, \ + make_symlink +from lustreclient.lfs import lfs_set_project_id, lfs_get_project_quota, \ + lfs_set_project_quota + + +def __get_lustre_basepath(configuration, lustre_basepath=None): + """If *lustre_basepath* is provided then check it, + otherwise try to resolve it""" + valid_lustre_basepath = None + for mount in psutil.disk_partitions(all=True): + if mount.fstype == "lustre": + if lustre_basepath \ + and lustre_basepath.startswith(mount.mountpoint) \ + and os.path.isdir(lustre_basepath): + valid_lustre_basepath = lustre_basepath + break + elif mount.mountpoint.endswith(configuration.server_fqdn): + valid_lustre_basepath = mount.mountpoint + else: + check_lustre_basepath = os.path.join(mount.mountpoint, + configuration.server_fqdn) + if os.path.isdir(check_lustre_basepath): + valid_lustre_basepath = check_lustre_basepath + break + + return valid_lustre_basepath + + +def __get_gocryptfs_socket(configuration, gocryptfs_sock=None): + """If *gocryptfs_sock* is provided then check it, + otherwise return default if it exists""" + valid_gocryptfs_sock = None + if gocryptfs_sock is None: + gocryptfs_sock = "/var/run/gocryptfs.%s.sock" \ + % configuration.server_fqdn + if os.path.exists(gocryptfs_sock): + gocryptfs_sock_stat = os.lstat(gocryptfs_sock) + if stat.S_ISSOCK(gocryptfs_sock_stat.st_mode): + valid_gocryptfs_sock = gocryptfs_sock + + return valid_gocryptfs_sock + + +def __shellexec(configuration, + command, + args=[], + stdin_str=None, + stdout_filepath=None, + stderr_filepath=None): + """Execute shell command + Returns (exit_code, stdout, stderr) of subprocess""" + result = 0 + logger = configuration.logger + stdin_handle = subprocess.PIPE + stdout_handle = subprocess.PIPE + stderr_handle = subprocess.PIPE + if stdout_filepath is not None: + stdout_handle = open(stdout_filepath, "w+") + if stderr_filepath is not None: + stderr_handle = open(stderr_filepath, "w+") + __args = shlex.split(command) + __args.extend(args) + logger.debug("__args: %s" % __args) + process = subprocess.Popen( + __args, + stdin=stdin_handle, + stdout=stdout_handle, + stderr=stderr_handle) + if stdin_str: + process.stdin.write(stdin_str.encode()) + stdout, stderr = process.communicate() + rc = process.wait() + + if stdout_filepath: + stdout = stdout_filepath + stdout_handle.close() + if stderr_filepath: + stderr = stderr_filepath + stderr_handle.close() + + # Close stdin, stdout and stderr FDs if they exists + if process.stdin: + process.stdin.close() + if process.stdout: + process.stdout.close() + if process.stderr: + process.stderr.close() + + if stdout: + stdout = force_unicode(stdout) + if stderr: + stderr = force_unicode(stderr) + if result == 0: + logger.debug("%s %s: rc: %s, stdout: %s, error: %s" + % (command, + " ".join(args), + rc, + stdout, + stderr)) + else: + logger.error("shellexec: %s %s: rc: %s, stdout: %s, error: %s" + % (command, + " ".join(__args), + rc, + stdout, + stderr)) + + return (rc, stdout, stderr) + + +def __set_project_id(configuration, + lustre_basepath, + quota_datapath, + quota_name, + quota_lustre_pid): + """Set lustre project *quota_lustre_pid* + Find the next *free* project id (PID) if *quota_lustre_pid* is occupied + NOTE: lustre uses a global counter for project id's (PID) + That means that different datasets and sub-mounts + share the same project id counter + # TODO: Add 'lustre_pid' offset support to configuration ? + """ + + # Find next unused lustre project id + + max_lustre_pid = 4294967294 + logger = configuration.logger + next_lustre_pid = quota_lustre_pid + while next_lustre_pid < max_lustre_pid: + (rc, currfiles, _, _, _) \ + = lfs_get_project_quota(lustre_basepath, next_lustre_pid) + if rc != 0: + logger.error("Failed to fetch quota for lustre project id: %d, %r" + % (next_lustre_pid, lustre_basepath) + + ", rc: %d" % rc) + return -1 + if currfiles == 0: + break + logger.info("Skipping project id: %d" \ + % next_lustre_pid \ + + " already registered with %d files" \ + % currfiles) + next_lustre_pid += 1 + + if next_lustre_pid == max_lustre_pid: + logger.error("Reached max lustre project id: %d" % max_lustre_pid) + return -1 + + # Set new project id + + logger.info("Setting lustre project id: %d for %r: %r" + % (next_lustre_pid, quota_name, quota_datapath)) + rc = lfs_set_project_id(quota_datapath, next_lustre_pid, 1) + if rc != 0: + logger.error("Failed to set lustre project id: %d for %r: %r" + % (next_lustre_pid, quota_name, quota_datapath) + + ", rc: %d" % rc) + return -1 + + return next_lustre_pid + + +def __update_quota(configuration, + lustre_basepath, + lustre_setting, + quota_name, + quota_type, + gocryptfs_sock, + timestamp): + """Update quota for *quota_name*, if new entry then + assign lustre project id and set default quota. + If existing entry then update quota settings if changed + and fetch file and bytes usage and store it as pickle and json + """ + logger = configuration.logger + quota_limits_changed = False + next_lustre_pid = lustre_setting.get('next_pid', -1) + if next_lustre_pid == -1: + logger.error("Invalid lustre quota next_pid: %d for: %r" + % (next_lustre_pid, quota_name)) + return False + if quota_type == 'vgrid': + default_quota_limit = configuration.quota_vgrid_limit + data_basepath = configuration.vgrid_files_writable + # NOTE: Old vgrids stored data directly in 'vgrid_files_home' + if not os.path.isdir(os.path.join(data_basepath, quota_name)): + data_basepath = configuration.vgrid_files_home + else: + default_quota_limit = configuration.quota_user_limit + data_basepath = configuration.user_home + + # Load quota if it exists otherwise new quota + + quota_filepath = os.path.join(configuration.quota_home, + configuration.quota_backend, + quota_type, + "%s.pck" % quota_name) + + if os.path.exists(quota_filepath): + quota = unpickle(quota_filepath, logger) + if not quota: + logger.error("Failed to load quota settings for: %r from %r" + % (quota_name, quota_filepath)) + return False + else: + quota = {'lustre_pid': next_lustre_pid, + 'files': -1, + 'bytes': -1, + 'softlimit_bytes': -1, + 'hardlimit_bytes': -1, + } + + quota_lustre_pid = quota.get('lustre_pid', -1) + if quota_lustre_pid == -1: + logger.error("Invalid quota lustre pid: %d for %r" + % (quota_lustre_pid, quota_name)) + return False + + # Resolve quota data path + # if gocryptfs then resolve encrypted path + # otherwise use plain path + + if configuration.quota_backend == "lustre": + quota_datapath = os.path.join(data_basepath, + quota_name) + elif configuration.quota_backend == "lustre-gocryptfs": + rel_data_basepath = data_basepath. \ + replace(configuration.state_path + os.sep, "") + stdin_str = os.path.join(rel_data_basepath, quota_name) + cmd = "gocryptfs-xray -encrypt-paths %s" % gocryptfs_sock + (rc, stdout, stderr) = __shellexec(configuration, + cmd, + stdin_str=stdin_str) + if rc == 0 and stdout: + encoded_path = stdout.strip() + quota_datapath = os.path.join(lustre_basepath, + encoded_path) + else: + logger.error("Failed to resolve encrypted path for: %r" + % quota_name + + ", rc: %d, error: %s" + % (rc, stderr)) + return False + else: + logger.error("Invalid quota backend: %r" + % configuration.quota_backend) + return False + + # Skip non-dir entries + + if not os.path.isdir(quota_datapath): + logger.debug("Skipping non-dir entry: %r: %r" + % (quota_name, quota_datapath)) + return True + + # If new entry then set lustre project id + new_lustre_pid = -1 + if quota_lustre_pid == next_lustre_pid: + new_lustre_pid = __set_project_id(configuration, + lustre_basepath, + quota_datapath, + quota_name, + quota_lustre_pid) + if new_lustre_pid == -1: + logger.error("Failed to set project id: %d, %r, %r" + % (new_lustre_pid, quota_name, quota_datapath)) + return False + lustre_setting['next_pid'] = new_lustre_pid + 1 + quota_lustre_pid = new_lustre_pid + + # Get current quota values for lustre_pid + + (rc, currfiles, currbytes, softlimit_bytes, hardlimit_bytes) \ + = lfs_get_project_quota(quota_datapath, quota_lustre_pid) + if rc != 0: + logger.error("Failed to fetch quota for lustre project id: %d, %r, %r" + % (quota_lustre_pid, quota_name, quota_datapath) + + ", rc: %d" % rc) + return False + + # Update quota info + + quota['mtime'] = timestamp + quota['files'] = currfiles + quota['bytes'] = currbytes + + # If new entry use default quota + # and update quota if changed + + if new_lustre_pid > -1: + quota_limits_changed = True + quota['softlimit_bytes'] = default_quota_limit + quota['hardlimit_bytes'] = default_quota_limit + elif hardlimit_bytes != quota.get('hardlimit_bytes', -1) \ + or softlimit_bytes != quota.get('softlimit_bytes', -1): + quota_limits_changed = True + quota['softlimit_bytes'] = softlimit_bytes + quota['hardlimit_bytes'] = hardlimit_bytes + + if quota_limits_changed: + rc = lfs_set_project_quota(quota_datapath, + quota_lustre_pid, + quota['softlimit_bytes'], + quota['hardlimit_bytes'], + ) + if rc != 0: + logger.error("Failed to set quota limit: %d/%d" + % (softlimit_bytes, + hardlimit_bytes) + + " for lustre project id: %d, %r, %r, rc: %d" + % (quota_lustre_pid, + quota_name, + quota_datapath, + rc)) + return False + + # Save current quota + + new_quota_basepath = os.path.join(configuration.quota_home, + configuration.quota_backend, + quota_type, + str(timestamp)) + if not os.path.exists(new_quota_basepath) \ + and not makedirs_rec(new_quota_basepath, configuration): + logger.error("Failed to create new quota base path: %r" + % new_quota_basepath) + return False + + new_quota_filepath_pck = os.path.join(new_quota_basepath, + "%s.pck" % quota_name) + status = pickle(quota, new_quota_filepath_pck, logger) + if not status: + logger.error("Failed to save quota for: %r to %r" + % (quota_name, new_quota_filepath_pck)) + return False + + new_quota_filepath_json = os.path.join(new_quota_basepath, + "%s.json" % quota_name) + status = save_json(quota, + new_quota_filepath_json, + logger) + if not status: + logger.error("Failed to save quota for: %r to %r" + % (quota_name, new_quota_filepath_json)) + return False + + # Create symlink to new quota + + status = make_symlink(new_quota_filepath_pck, + quota_filepath, + logger, + force=True) + if not status: + logger.error("Failed to make quota symlink for: %r: %r -> %r" + % (quota_name, new_quota_filepath_pck, quota_filepath)) + return False + + return True + + +def update_lustre_quota(configuration): + """Update lustre quota for users and vgrids""" + logger = configuration.logger + retval = True + timestamp = int(time.time()) + + # Get lustre_basepath + + lustre_basepath = __get_lustre_basepath(configuration) + if lustre_basepath: + logger.debug("Using lustre basepath: %r" + % lustre_basepath) + else: + logger.error("Found no valid lustre mounts for: %s" + % configuration.server_fqdn) + return False + + # Get gocryptfs socket if enabled + + if configuration.quota_backend == "lustre-gocryptfs": + gocryptfs_sock = __get_gocryptfs_socket(configuration) + if gocryptfs_sock: + logger.debug("Using gocryptfs socket: %r" + % gocryptfs_sock) + else: + logger.error("Missing gocryptfs socket") + return False + + # Load lustre quota settings + + lustre_setting_filepath = os.path.join(configuration.quota_home, + '%s.pck' + % configuration.quota_backend) + if os.path.exists(lustre_setting_filepath): + lustre_setting = unpickle(lustre_setting_filepath, + logger) + if not lustre_setting: + logger.error("Failed to load lustre quota: %r" + % lustre_setting_filepath) + return False + else: + lustre_setting = {'next_pid': 1, + 'mtime': 0} + + # Update quota + + for quota_type in ('vgrid', 'user'): + if quota_type == 'vgrid': + scandir = configuration.vgrid_home + else: + scandir = configuration.user_home + + # Scan for new and modified entries + + with os.scandir(scandir) as it: + for entry in it: + if not os.path.isdir(entry.path): + # Only take dirs into account + logger.debug("Skiping non-dir path: %r" % entry.path) + continue + status = __update_quota(configuration, + lustre_basepath, + lustre_setting, + entry.name, + quota_type, + gocryptfs_sock, + timestamp) + if not status: + retval = False + + # Save updated lustre quota settings + + lustre_setting['mtime'] = timestamp + status = pickle(lustre_setting, + lustre_setting_filepath, + logger) + if not status: + logger.error("Failed to save lustra quota settings: %r" + % lustre_setting_filepath) + + return retval diff --git a/mig/lib/quota.py b/mig/lib/quota.py new file mode 100644 index 000000000..a1dec8e45 --- /dev/null +++ b/mig/lib/quota.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# quota - helpers to support storage quota +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""helpers to support storage quota""" + +from mig.lib.lustrequota import update_lustre_quota + + +supported_quota_backends = ['lustre', 'lustre-gocryptfs'] + + +def update_quota(configuration): + """Update quota for users and vgrids""" + logger = configuration.logger + retval = False + + if configuration.quota_backend == 'lustre' \ + or configuration.quota_backend == 'lustre-gocryptfs': + retval = update_lustre_quota(configuration) + + return retval diff --git a/mig/server/grid_quota.py b/mig/server/grid_quota.py new file mode 120000 index 000000000..5d3c8e420 --- /dev/null +++ b/mig/server/grid_quota.py @@ -0,0 +1 @@ +../../sbin/grid_quota.py \ No newline at end of file diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py index 2f872694a..84d7d110d 100644 --- a/mig/shared/configuration.py +++ b/mig/shared/configuration.py @@ -397,6 +397,7 @@ def fix_missing(config_file, verbose=True): 'vgrid_recipes_home': '.workflow_recipes_home/', 'vgrid_history_home': '.workflow_history_home/'} quota_section = {'backend': 'lustre', + 'update_interval': 3600, 'user_limit': 1024**4, 'vgrid_limit': 1024**4} defaults = { @@ -2002,6 +2003,9 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, if config.has_option('QUOTA', 'backend'): self.quota_backend = config.get( 'QUOTA', 'backend') + if config.has_option('QUOTA', 'update_interval'): + self.quota_update_interval = config.getint( + 'QUOTA', 'update_interval') if config.has_option('QUOTA', 'user_limit'): self.quota_user_limit = config.getint( 'QUOTA', 'user_limit') diff --git a/mig/shared/install.py b/mig/shared/install.py index 92a1ab997..c9579fc60 100644 --- a/mig/shared/install.py +++ b/mig/shared/install.py @@ -533,6 +533,7 @@ def generate_confs( gdp_id_scramble='safe_hash', gdp_path_scramble='safe_encrypt', quota_backend='lustre', + quota_update_interval=3600, quota_user_limit=(1024**4), quota_vgrid_limit=(1024**4), ca_fqdn='', @@ -859,6 +860,7 @@ def _generate_confs_prepare( gdp_id_scramble, gdp_path_scramble, quota_backend, + quota_update_interval, quota_user_limit, quota_vgrid_limit, ca_fqdn, @@ -1117,6 +1119,7 @@ def _generate_confs_prepare( user_dict['__PUBLIC_ALIAS_HTTPS_LISTEN__'] = listen_clause user_dict['__STATUS_ALIAS_HTTPS_LISTEN__'] = listen_clause user_dict['__QUOTA_BACKEND__'] = quota_backend + user_dict['__QUOTA_UPDATE_INTERVAL__'] = "%s" % quota_update_interval user_dict['__QUOTA_USER_LIMIT__'] = "%s" % quota_user_limit user_dict['__QUOTA_VGRID_LIMIT__'] = "%s" % quota_vgrid_limit user_dict['__CA_FQDN__'] = ca_fqdn @@ -2343,7 +2346,6 @@ def _generate_confs_writefiles(options, user_dict, insert_list=[], cleanup_list= ("migacctexpire-template.sh.cronjob", "migacctexpire"), ("migverifyarchives-template.sh.cronjob", "migverifyarchives"), ("migstats-template.sh.cronjob", "migstats"), - ("miglustrequota-template.sh.cronjob", "miglustrequota"), ] overrides_out_name = { 'apache.initd': _override_apache_initd @@ -2522,11 +2524,11 @@ def _generate_confs_instructions(options, user_dict): /etc/cron.daily/ until the janitor service is ready to take care of those tasks. -The migcheckssl, migverifyarchives, migstats, migacctexpire and miglustrequota +The migcheckssl, migverifyarchives, migstats and migacctexpire files are cron scripts to automatically check for LetsEncrypt certificate renewal, run pending archive verification before sending a copy to tape, save -various usage stats, generate account expire stats and create/update lustre -quota. +various usage stats and generate account expire stats. + You can install them with: chmod 700 %(destination)s/migcheckssl sudo cp %(destination)s/migcheckssl /etc/cron.daily/ @@ -2536,8 +2538,6 @@ def _generate_confs_instructions(options, user_dict): sudo cp %(destination)s/migstats /etc/cron.weekly/ chmod 700 %(destination)s/migacctexpire sudo cp %(destination)s/migacctexpire /etc/cron.monthly/ -chmod 700 %(destination)s/miglustrequota -sudo cp %(destination)s/miglustrequota /etc/cron.hourly/ ''' % instructions_dict instructions_path = os.path.join( diff --git a/sbin/grid_quota.py b/sbin/grid_quota.py new file mode 100755 index 000000000..1e228566c --- /dev/null +++ b/sbin/grid_quota.py @@ -0,0 +1,144 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# grid_quota - daemon to create storage quota +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +"""Daemon that create storage quota""" + +from __future__ import absolute_import, print_function + +import os +import sys +import time +import traceback +import datetime + +from mig.lib.daemon import check_run, check_stop, interruptible_sleep, \ + register_run_handler, register_stop_handler, reset_run, stop_running +from mig.lib.quota import update_quota, supported_quota_backends +from mig.shared.conf import get_configuration_object +from mig.shared.logger import daemon_logger, register_hangup_handler + + +if __name__ == "__main__": + print( + """This is the MiG lustre quota daemon which collect storage quota + information for users and vgrids. + +Set the MIG_CONF environment to the server configuration path +unless it is available in mig/server/MiGserver.conf +""" + ) + # Force no log init since we use separate logger + configuration = get_configuration_object(skip_log=True) + + log_level = configuration.loglevel + if sys.argv[1:] and sys.argv[1] in ["debug", "info", "warning", "error"]: + log_level = sys.argv[1] + + # Use separate logger + + logger = daemon_logger("quota", + configuration.user_quota_log, + log_level) + configuration.logger = logger + + # Check if quota is enabled + + if not configuration.site_enable_quota: + msg = "Quota support is disabled in configuration!" + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + sys.exit(1) + + # Check quota backend + + if configuration.quota_backend not in supported_quota_backends: + msg = "Quota backend: %s not in supported backends: %s" \ + % (configuration.quota_backend, + ", ".join(supported_quota_backends)) + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + sys.exit(1) + + # Allow e.g. logrotate to force log re-open after rotates + register_hangup_handler(configuration) + + # Allow trigger next run on SIGCONT to main process + register_run_handler(configuration) + + # Allow clean shutdown on SIGINT only to main process + register_stop_handler(configuration) + + throttle_secs = float(configuration.quota_update_interval) + main_pid = os.getpid() + msg = "(%s) Starting quota daemon with throttle: %d secs" \ + % (main_pid, throttle_secs) + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + + throttle = False + while not check_stop(): + try: + if throttle: + interruptible_sleep(configuration, throttle_secs, + (check_run, check_stop)) + reset_run() + if check_stop(): + break + t1 = time.time() + status = update_quota(configuration) + t2 = time.time() + msg = "(%s) Updated quota in %d secs with status: %s" \ + % (os.getpid(), int(t2-t1), status) + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + throttle = True + except KeyboardInterrupt: + stop_running() + # NOTE: we can't be sure if SIGINT was sent to only main process + # so we make sure to propagate to monitor child + msg = "(%s) Interrupt requested - shutdown" \ + % os.getpid() + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + except Exception as exc: + throttle = True + msg = "(%s) Caught unexpected exception:\n%s" \ + % (os.getpid(), traceback.format_exc()) + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + + msg = "(%s) Quota daemon shutting down" % main_pid + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + + sys.exit(0) From e37279eab91c5c0d03876156305df078a116f4bc Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 15:06:44 +0100 Subject: [PATCH 26/46] Updated unit tests to match new grid quota daemon setup --- tests/fixture/confs-stdlocal/miglustrequota | 62 ------------------- .../fixture/confs-stdlocal/migrid-init.d-deb | 56 ++++++++++++++++- tests/fixture/confs-stdlocal/migrid-init.d-rh | 53 +++++++++++++++- 3 files changed, 105 insertions(+), 66 deletions(-) delete mode 100644 tests/fixture/confs-stdlocal/miglustrequota diff --git a/tests/fixture/confs-stdlocal/miglustrequota b/tests/fixture/confs-stdlocal/miglustrequota deleted file mode 100644 index 062f246bc..000000000 --- a/tests/fixture/confs-stdlocal/miglustrequota +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# -# Run lustre quota for MiG servers -# -# The script depends on a miglustrequota setup -# (please refer to mig/src/pylustrequota/README). -# -# IMPORTANT: if placed in /etc/cron.X the script filename must be -# something consisting entirely of upper and lower case letters, digits, -# underscores, and hyphens. I.e. if the script name contains e.g. a period, -# '.', it will be silently ignored! -# This is a limitation on the run-parts wrapper used by cron -# (see man run-parts for the rationale behind this). - -# By default bash silently ignores and continues on most errors but we can set -# options to e.g. catch uninitialized variables and errors as explained in: -# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ -# NOTE: 'set -eE' exits on non-zero exit codes to add safety and as recommended -# best-practice (CWE-252, CWE-248, ...), yet, in some cases it hurts more to -# exit midway, so it can be a trade-off. -set -eEuo pipefail - -# Send output to another email address -#MAILTO="root" - -MIG_CONF=/home/mig/mig/server/MiGserver.conf - -# Specify if migrid runs natively or inside containers with lustre at host. -# Value is the container manager (docker, podman, or empty string for none) -container_manager="" -container="migrid-lustre-quota" - -# Look in miglustrequota install dir first -export PATH="/usr/local/bin:${PATH}" - -if [[ $(id -u) -ne 0 ]]; then - echo "Please run $0 as root" - exit 1 -fi - -if [ -z "${container_manager}" ]; then - miglustrequota=$(which "miglustrequota.py" 2>/dev/null) - if [ ! -x "${miglustrequota}" ]; then - echo "ERROR: Missing miglustrequota.py" - exit 1 - fi - quota_cmd="${miglustrequota} -c ${MIG_CONF}" -else - check_cmd="${container_manager} container ls -a | grep -q '${container}'" - eval "$check_cmd" - ret=$? - if [ "$ret" -ne 0 ]; then - echo "ERROR: Missing ${container} container" - exit 1 - fi - quota_cmd="${container_manager} start -a ${container}" -fi - -eval "$quota_cmd" -ret=$? - -exit $ret diff --git a/tests/fixture/confs-stdlocal/migrid-init.d-deb b/tests/fixture/confs-stdlocal/migrid-init.d-deb index d6ac02101..87351aa07 100755 --- a/tests/fixture/confs-stdlocal/migrid-init.d-deb +++ b/tests/fixture/confs-stdlocal/migrid-init.d-deb @@ -43,6 +43,9 @@ if [ -z "$PYTHONPATH" ]; then else PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -62,13 +65,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -264,6 +268,18 @@ start_vmproxy() { log_end_msg 1 || true fi } +start_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Starting MiG quota daemon" ${SHORT_NAME} || true + if start-stop-daemon --start --quiet --oknodo --pidfile ${PID_FILE} --make-pidfile --user root --chuid root --background --name ${SHORT_NAME} --startas ${DAEMON_PATH} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} start_sftpsubsys() { check_enabled "sftp_subsys" || return 0 DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -292,6 +308,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -524,6 +541,19 @@ stop_vmproxy() { log_end_msg 1 || true fi } +stop_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Stopping MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --quiet --oknodo --pidfile ${PID_FILE} ; then + rm -f ${PID_FILE} + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -563,6 +593,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -735,6 +766,18 @@ reload_vmproxy() { log_end_msg 1 || true fi } +reload_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Reloading MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile ${PID_FILE} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -787,6 +830,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -891,6 +935,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} } +status_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -929,6 +980,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -940,7 +992,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') diff --git a/tests/fixture/confs-stdlocal/migrid-init.d-rh b/tests/fixture/confs-stdlocal/migrid-init.d-rh index c4bfad648..911c5bb3c 100755 --- a/tests/fixture/confs-stdlocal/migrid-init.d-rh +++ b/tests/fixture/confs-stdlocal/migrid-init.d-rh @@ -35,6 +35,7 @@ # processname: grid_notify.py # processname: grid_imnotify.py # processname: grid_vmproxy.py +# processname: grid_quota.py # processname: sshd # config: /etc/sysconfig/migrid # @@ -74,6 +75,9 @@ if [ -z "$PYTHONPATH" ]; then else export PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -93,13 +97,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -353,6 +358,21 @@ start_vmproxy() { [ $RET2 -ne 0 ] && echo "Warning: vmproxy not started." echo } +start_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Starting MiG quota daemon: $SHORT_NAME" + daemon --user root --pidfile ${PID_FILE} \ + "${DAEMON_PATH} >> ${MIG_LOG}/quota.out 2>&1 &" + fallback_save_pid "$DAEMON_PATH" "$PID_FILE" "$!" + RET2=$? + [ $RET2 -eq 0 ] && success + echo + [ $RET2 -ne 0 ] && echo "Warning: quota not started." + echo +} start_sftpsubsys() { check_enabled "sftp_subsys" || return DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -385,6 +405,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -545,6 +566,15 @@ stop_vmproxy() { killproc ${DAEMON_PATH} echo } +stop_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Shutting down MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} + echo +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -588,6 +618,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -717,6 +748,15 @@ reload_vmproxy() { killproc ${DAEMON_PATH} -HUP echo } +reload_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Reloading MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} -HUP + echo +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -773,6 +813,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -877,6 +918,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status ${DAEMON_PATH} } +status_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status ${DAEMON_PATH} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -916,6 +964,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -927,7 +976,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') From 546345ae5067c0b1548a1fb7c0517f26820be402 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 15:44:17 +0100 Subject: [PATCH 27/46] Added 'quota_update_interval' option to generateconfs --- mig/install/generateconfs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mig/install/generateconfs.py b/mig/install/generateconfs.py index 411af14b3..2299233cb 100755 --- a/mig/install/generateconfs.py +++ b/mig/install/generateconfs.py @@ -252,6 +252,7 @@ def main(argv, _generate_confs=generate_confs, _print=print): 'seafile_seafhttp_port', 'seafile_client_port', 'seafile_quota', + 'quota_update_interval', 'quota_user_limit', 'quota_vgrid_limit', 'wwwserve_max_bytes', From e6d4d6013c8fc72c3b2c4a85b34516ae0211c976 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 17:56:45 +0100 Subject: [PATCH 28/46] Addwed missing 'user_quota_log' entry to configuration --- mig/shared/configuration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py index 84d7d110d..8ff46f01d 100644 --- a/mig/shared/configuration.py +++ b/mig/shared/configuration.py @@ -1738,6 +1738,9 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, if config.has_option('GLOBAL', 'user_shared_dhparams'): self.user_shared_dhparams = config.get('GLOBAL', 'user_shared_dhparams') + if config.has_option('GLOBAL', 'user_quota_log'): + self.user_quota_log = config.get('GLOBAL', + 'user_quota_log') if config.has_option('GLOBAL', 'public_key_file'): self.public_key_file = config.get('GLOBAL', 'public_key_file') if config.has_option('GLOBAL', 'smtp_sender'): From 119ca23c8a33a7f75e9cc22eb6b9cf61683d22b1 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 19:01:18 +0100 Subject: [PATCH 29/46] Added missing 'update_interval' to MiGserver-template.conf --- mig/install/MiGserver-template.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/mig/install/MiGserver-template.conf b/mig/install/MiGserver-template.conf index 3f02388a5..568cc737f 100644 --- a/mig/install/MiGserver-template.conf +++ b/mig/install/MiGserver-template.conf @@ -550,6 +550,7 @@ default_mount_re = SSHFS-2.X-1 [QUOTA] backend = __QUOTA_BACKEND__ +update_interval = __QUOTA_UPDATE_INTERVAL__ user_limit = __QUOTA_USER_LIMIT__ vgrid_limit = __QUOTA_VGRID_LIMIT__ From 8c565469fa8cd3efd173f394b796cf3406bd929c Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 25 Nov 2025 19:19:10 +0100 Subject: [PATCH 30/46] Added 'update_interval' to MiGserver.conf fixture --- tests/fixture/confs-stdlocal/MiGserver.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fixture/confs-stdlocal/MiGserver.conf b/tests/fixture/confs-stdlocal/MiGserver.conf index 9520491ab..a9983263f 100644 --- a/tests/fixture/confs-stdlocal/MiGserver.conf +++ b/tests/fixture/confs-stdlocal/MiGserver.conf @@ -550,6 +550,7 @@ default_mount_re = SSHFS-2.X-1 [QUOTA] backend = lustre +update_interval = 3600 user_limit = 1099511627776 vgrid_limit = 1099511627776 From a276e09e906c50cc2cc3e1cabb1d2cfb171edace Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Thu, 27 Nov 2025 14:11:22 +0100 Subject: [PATCH 31/46] Minor comment corrections thanks to @jonasbardino --- mig/src/lustreclient/lustreclient/__init__.py | 4 ++-- sbin/grid_quota.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mig/src/lustreclient/lustreclient/__init__.py b/mig/src/lustreclient/lustreclient/__init__.py index 46385bde5..8723f555f 100644 --- a/mig/src/lustreclient/lustreclient/__init__.py +++ b/mig/src/lustreclient/lustreclient/__init__.py @@ -3,7 +3,7 @@ # # --- BEGIN_HEADER --- # -# __init__ - luste client python extension +# __init__ - lustre client python extension # Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. @@ -24,7 +24,7 @@ # # -- END_HEADER --- # -"""This package provide luste client functionality""" +"""This package provide lustre client functionality""" __dummy = True diff --git a/sbin/grid_quota.py b/sbin/grid_quota.py index 1e228566c..48d2d401e 100755 --- a/sbin/grid_quota.py +++ b/sbin/grid_quota.py @@ -44,7 +44,7 @@ if __name__ == "__main__": print( - """This is the MiG lustre quota daemon which collect storage quota + """This is the MiG quota daemon which collects storage quota information for users and vgrids. Set the MIG_CONF environment to the server configuration path From 391cceace78995db556138c684a369311a01a0cf Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Mon, 1 Dec 2025 13:34:49 +0100 Subject: [PATCH 32/46] Added 'lustreclient' to 'c-ext-sanity-check' ignore paths as it depends on the 'lustre' source code which we do not wan't to include in our code path. --- .github/workflows/python-c-ext-sanity-check.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-c-ext-sanity-check.yml b/.github/workflows/python-c-ext-sanity-check.yml index dec1e4843..53028b9f4 100644 --- a/.github/workflows/python-c-ext-sanity-check.yml +++ b/.github/workflows/python-c-ext-sanity-check.yml @@ -29,6 +29,7 @@ on: - 'mig/apache/**' - 'mig/bin/**' - 'mig/java-bin/**' + - 'mig/src/lustreclient/**' - '**/*.py' - '**/*.js' branches: @@ -57,6 +58,7 @@ on: - 'mig/apache/**' - 'mig/bin/**' - 'mig/java-bin/**' + - 'mig/src/lustreclient/**' - '**/*.py' - '**/*.js' branches: From 3fcbb77a4f9f477aebe81b5aea3253dec0ce1741 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 10:03:36 +0100 Subject: [PATCH 33/46] Added check for 'lustreclient' module import --- mig/lib/lustrequota.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/mig/lib/lustrequota.py b/mig/lib/lustrequota.py index cd652814e..026769074 100644 --- a/mig/lib/lustrequota.py +++ b/mig/lib/lustrequota.py @@ -38,8 +38,13 @@ from mig.shared.base import force_unicode from mig.shared.fileio import unpickle, pickle, save_json, makedirs_rec, \ make_symlink -from lustreclient.lfs import lfs_set_project_id, lfs_get_project_quota, \ - lfs_set_project_quota +try: + from lustreclient.lfs import lfs_set_project_id, lfs_get_project_quota, \ + lfs_set_project_quota +except: + lfs_set_project_id = None + lfs_get_project_quota = None + lfs_set_project_quota = None def __get_lustre_basepath(configuration, lustre_basepath=None): @@ -175,10 +180,10 @@ def __set_project_id(configuration, return -1 if currfiles == 0: break - logger.info("Skipping project id: %d" \ - % next_lustre_pid \ - + " already registered with %d files" \ - % currfiles) + logger.info("Skipping project id: %d" + % next_lustre_pid + + " already registered with %d files" + % currfiles) next_lustre_pid += 1 if next_lustre_pid == max_lustre_pid: @@ -400,6 +405,15 @@ def __update_quota(configuration, def update_lustre_quota(configuration): """Update lustre quota for users and vgrids""" logger = configuration.logger + + # Check if lustreclient module was imported correctly + + if lfs_set_project_id is None \ + or lfs_get_project_quota is None \ + or lfs_set_project_quota is None: + logger.error("Failed to import lustreclient module") + return False + retval = True timestamp = int(time.time()) From a19531377b57d2a7d971e2f01ae8b5b090fde7bb Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 10:04:00 +0100 Subject: [PATCH 34/46] Added log error message if requested 'quota' backend is unsupported --- mig/lib/quota.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mig/lib/quota.py b/mig/lib/quota.py index a1dec8e45..e93830d28 100644 --- a/mig/lib/quota.py +++ b/mig/lib/quota.py @@ -36,11 +36,14 @@ def update_quota(configuration): """Update quota for users and vgrids""" - logger = configuration.logger retval = False - + logger = configuration.logger if configuration.quota_backend == 'lustre' \ or configuration.quota_backend == 'lustre-gocryptfs': retval = update_lustre_quota(configuration) + else: + logger.error("quota_backend: %r not in supported_quota_backends: %r" + % (configuration.quota_backend, + supported_quota_backends)) return retval From 49cda2f181a815a63e8b93c38c4777098a17251c Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 13:16:47 +0100 Subject: [PATCH 35/46] Removed trailing whitespace --- mig/src/lustreclient/lustreclient/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mig/src/lustreclient/lustreclient/__init__.py b/mig/src/lustreclient/lustreclient/__init__.py index 8723f555f..78e0d2121 100644 --- a/mig/src/lustreclient/lustreclient/__init__.py +++ b/mig/src/lustreclient/lustreclient/__init__.py @@ -34,7 +34,7 @@ # All sub modules to load in case of 'from X import *' __all__ = [] - + # Collect all package information here for easy use from scripts and helpers package_name = 'Lustre Client Python extension' From 5fd857b5468474229b0f5ef6705836126f2b3971 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 13:50:02 +0100 Subject: [PATCH 36/46] Restrict 'lustreclient.lfs' import exception to 'ImportError' to satisfy pylint --- mig/lib/lustrequota.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mig/lib/lustrequota.py b/mig/lib/lustrequota.py index 026769074..3b0e8f319 100644 --- a/mig/lib/lustrequota.py +++ b/mig/lib/lustrequota.py @@ -29,6 +29,7 @@ """helpers to support lustre quota""" import os +import sys import stat import time import shlex @@ -41,7 +42,7 @@ try: from lustreclient.lfs import lfs_set_project_id, lfs_get_project_quota, \ lfs_set_project_quota -except: +except ImportError: lfs_set_project_id = None lfs_get_project_quota = None lfs_set_project_quota = None From 7a7ac273f606fcc493fe02cefd9511dc0c01c129 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 14:51:29 +0100 Subject: [PATCH 37/46] Removed absolete 'import sys' --- mig/lib/lustrequota.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mig/lib/lustrequota.py b/mig/lib/lustrequota.py index 3b0e8f319..c9e9a6a9e 100644 --- a/mig/lib/lustrequota.py +++ b/mig/lib/lustrequota.py @@ -29,7 +29,6 @@ """helpers to support lustre quota""" import os -import sys import stat import time import shlex From fbeca52e0bb22d53642d594ffa5adc682d0557ce Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 15:35:16 +0100 Subject: [PATCH 38/46] lustrequota: Added check for 'psutil' module --- mig/lib/lustrequota.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mig/lib/lustrequota.py b/mig/lib/lustrequota.py index c9e9a6a9e..d335554d3 100644 --- a/mig/lib/lustrequota.py +++ b/mig/lib/lustrequota.py @@ -33,7 +33,12 @@ import time import shlex import subprocess -import psutil + +# NOTE: we rely on psutil to resolve lustre mount point +try: + import psutil +except ImportError: + psutil = None from mig.shared.base import force_unicode from mig.shared.fileio import unpickle, pickle, save_json, makedirs_rec, \ @@ -50,6 +55,9 @@ def __get_lustre_basepath(configuration, lustre_basepath=None): """If *lustre_basepath* is provided then check it, otherwise try to resolve it""" + if psutil is None: + return None + valid_lustre_basepath = None for mount in psutil.disk_partitions(all=True): if mount.fstype == "lustre": From fd536c4751d45d5837876ca03155b6e036a4fe21 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 2 Dec 2025 15:36:01 +0100 Subject: [PATCH 39/46] Added quota unittest --- tests/test_mig_lib_quota.py | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/test_mig_lib_quota.py diff --git a/tests/test_mig_lib_quota.py b/tests/test_mig_lib_quota.py new file mode 100644 index 000000000..25dd1328a --- /dev/null +++ b/tests/test_mig_lib_quota.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# test_mig_lib_quota - unit test of the corresponding mig lib module +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""Unit tests for the migrid module pointed to in the filename""" + +from mig.lib.quota import update_quota +from tests.support import MigTestCase + + +class MigLibQouta(MigTestCase): + """Unit tests for quota related helper functions""" + + def _provide_configuration(self): + """Prepare isolated test config""" + return 'testconfig' + + def before_each(self): + """Set up test configuration and reset state before each test""" + pass + + def test_invalid_quota_backend(self): + """Test invalid quota_backend in configuration""" + self.configuration.quota_backend = "NEVERNEVER" + with self.assertLogs(level='DEBUG') as log_capture: + update_quota(self.configuration) + self.assertTrue(any("quota_backend: 'NEVERNEVER' not in supported_quota_backends" in msg + for msg in log_capture.output)) From 1b1bf91b4463c1bfa5c1d3102b980b6b5d7c5ab8 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Fri, 12 Dec 2025 09:09:23 +0100 Subject: [PATCH 40/46] python-c-ext-sanity-check: Test --- mig/src/libpam-mig/libpam_mig.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mig/src/libpam-mig/libpam_mig.c b/mig/src/libpam-mig/libpam_mig.c index b030fb340..4ca6f89a1 100644 --- a/mig/src/libpam-mig/libpam_mig.c +++ b/mig/src/libpam-mig/libpam_mig.c @@ -32,6 +32,7 @@ * */ + #include #include #include From 403412131d2cc77b554c51be0f7dd6e25520add0 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Wed, 17 Dec 2025 09:13:23 +0100 Subject: [PATCH 41/46] migrid init: Append /usr/local/(s)bin to existing PATH variable as suggested by @jonasbardino --- mig/install/migrid-init.d-deb-template | 4 ++-- mig/install/migrid-init.d-rh-template | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mig/install/migrid-init.d-deb-template b/mig/install/migrid-init.d-deb-template index 87351aa07..2d5e262fb 100755 --- a/mig/install/migrid-init.d-deb-template +++ b/mig/install/migrid-init.d-deb-template @@ -43,8 +43,8 @@ if [ -z "$PYTHONPATH" ]; then else PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi -# Make sure '/usr/local/(s)bin' is in path and force lookup order -export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# Make sure '/usr/local/(s)bin' is in PATH +export PATH="$PATH:/usr/local/sbin:/usr/local/bin" # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} diff --git a/mig/install/migrid-init.d-rh-template b/mig/install/migrid-init.d-rh-template index 911c5bb3c..cc4b3e15d 100755 --- a/mig/install/migrid-init.d-rh-template +++ b/mig/install/migrid-init.d-rh-template @@ -75,8 +75,8 @@ if [ -z "$PYTHONPATH" ]; then else export PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi -# Make sure '/usr/local/(s)bin' is in path and force lookup order -export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# Make sure '/usr/local/(s)bin' is in PATH +export PATH="$PATH:/usr/local/sbin:/usr/local/bin" # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} From 56604f35d2398980deb4e90d4a74a8d7f7f8cda0 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Wed, 17 Dec 2025 11:25:58 +0100 Subject: [PATCH 42/46] Removed 'newline' --- .github/workflows/python-c-ext-sanity-check.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python-c-ext-sanity-check.yml b/.github/workflows/python-c-ext-sanity-check.yml index f99f24e64..53028b9f4 100644 --- a/.github/workflows/python-c-ext-sanity-check.yml +++ b/.github/workflows/python-c-ext-sanity-check.yml @@ -11,7 +11,6 @@ env: PATHS_IGNORE_SPLINT: mig/src/lustreclient/.*$ on: - # Triggers the workflow on push or pull request events but only for this git branch push: paths-ignore: From b781f5ab1f47be3f024516d3271b3da6c027bf64 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Wed, 17 Dec 2025 11:29:33 +0100 Subject: [PATCH 43/46] Adjusted migrid-init fixture to new PATH setup --- tests/fixture/confs-stdlocal/migrid-init.d-deb | 4 ++-- tests/fixture/confs-stdlocal/migrid-init.d-rh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/fixture/confs-stdlocal/migrid-init.d-deb b/tests/fixture/confs-stdlocal/migrid-init.d-deb index 87351aa07..2d5e262fb 100755 --- a/tests/fixture/confs-stdlocal/migrid-init.d-deb +++ b/tests/fixture/confs-stdlocal/migrid-init.d-deb @@ -43,8 +43,8 @@ if [ -z "$PYTHONPATH" ]; then else PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi -# Make sure '/usr/local/(s)bin' is in path and force lookup order -export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# Make sure '/usr/local/(s)bin' is in PATH +export PATH="$PATH:/usr/local/sbin:/usr/local/bin" # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} diff --git a/tests/fixture/confs-stdlocal/migrid-init.d-rh b/tests/fixture/confs-stdlocal/migrid-init.d-rh index 911c5bb3c..cc4b3e15d 100755 --- a/tests/fixture/confs-stdlocal/migrid-init.d-rh +++ b/tests/fixture/confs-stdlocal/migrid-init.d-rh @@ -75,8 +75,8 @@ if [ -z "$PYTHONPATH" ]; then else export PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi -# Make sure '/usr/local/(s)bin' is in path and force lookup order -export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# Make sure '/usr/local/(s)bin' is in PATH +export PATH="$PATH:/usr/local/sbin:/usr/local/bin" # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} From 895b0731f14accdf89dc633c135a24d54212a1ad Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Wed, 17 Dec 2025 11:34:51 +0100 Subject: [PATCH 44/46] Changed grid_quota.py description as suggested by @jonasbardino --- sbin/grid_quota.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbin/grid_quota.py b/sbin/grid_quota.py index 48d2d401e..f7cf13ab8 100755 --- a/sbin/grid_quota.py +++ b/sbin/grid_quota.py @@ -3,7 +3,7 @@ # # --- BEGIN_HEADER --- # -# grid_quota - daemon to create storage quota +# grid_quota - daemon to manage storage quotas # Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. @@ -25,7 +25,7 @@ # -- END_HEADER --- # -"""Daemon that create storage quota""" +"""Daemon to manage storage quotas""" from __future__ import absolute_import, print_function From 3c24a35c1f21de59fb9eeb3ff4f1086ec9522b18 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Wed, 17 Dec 2025 11:37:03 +0100 Subject: [PATCH 45/46] python-c-ext-sanity-check: Removed test 'newline' --- mig/src/libpam-mig/libpam_mig.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mig/src/libpam-mig/libpam_mig.c b/mig/src/libpam-mig/libpam_mig.c index 4ca6f89a1..b030fb340 100644 --- a/mig/src/libpam-mig/libpam_mig.c +++ b/mig/src/libpam-mig/libpam_mig.c @@ -32,7 +32,6 @@ * */ - #include #include #include From ba8191644c8459c739a28799e6fcf589f6e3fd73 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Thu, 18 Dec 2025 15:09:15 +0100 Subject: [PATCH 46/46] migrid init: Define PATH if it doesn't exists as suggested by @jonasbardino --- mig/install/migrid-init.d-deb-template | 8 ++++++-- mig/install/migrid-init.d-rh-template | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/mig/install/migrid-init.d-deb-template b/mig/install/migrid-init.d-deb-template index 2d5e262fb..b83ed781f 100755 --- a/mig/install/migrid-init.d-deb-template +++ b/mig/install/migrid-init.d-deb-template @@ -43,8 +43,12 @@ if [ -z "$PYTHONPATH" ]; then else PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi -# Make sure '/usr/local/(s)bin' is in PATH -export PATH="$PATH:/usr/local/sbin:/usr/local/bin" +# Append '/usr/local/(s)bin' to PATH if it exists otherwise define PATH +if [ -z "$PATH" ]; then + export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +else + export PATH="$PATH:/usr/local/sbin:/usr/local/bin" +fi # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} diff --git a/mig/install/migrid-init.d-rh-template b/mig/install/migrid-init.d-rh-template index cc4b3e15d..9c838ef0e 100755 --- a/mig/install/migrid-init.d-rh-template +++ b/mig/install/migrid-init.d-rh-template @@ -75,8 +75,12 @@ if [ -z "$PYTHONPATH" ]; then else export PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi -# Make sure '/usr/local/(s)bin' is in PATH -export PATH="$PATH:/usr/local/sbin:/usr/local/bin" +# Append '/usr/local/(s)bin' to PATH if it exists otherwise define PATH +if [ -z "$PATH" ]; then + export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +else + export PATH="$PATH:/usr/local/sbin:/usr/local/bin" +fi # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run}