From 7251e7ef6d66569a7720d22a3dbaa0ed4c5fe133 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Nov 2025 22:57:57 +0000 Subject: [PATCH 1/6] Convert docstrings to Google style in utils.py and conductor.py - Updated all RST-style docstrings to Google-style format - Converted :param/:return/:raise to Args:/Returns:/Raises: sections - utils.py: 44 parameter annotations converted - conductor.py: 35 parameter annotations converted --- looper/conductor.py | 287 +++++++++++++++++++--------------- looper/utils.py | 371 +++++++++++++++++++++++++++----------------- 2 files changed, 390 insertions(+), 268 deletions(-) diff --git a/looper/conductor.py b/looper/conductor.py index cf438845..20fb7160 100644 --- a/looper/conductor.py +++ b/looper/conductor.py @@ -56,18 +56,20 @@ def _get_yaml_path(namespaces, template_key, default_name_appendix="", filename=None): - """ - Get a path to a YAML file for the sample. - - :param dict[dict]] namespaces: namespaces mapping - :param str template_key: the name of the key in 'var_templates' piface - section that points to a template to render to get the - user-provided target YAML path - :param str default_name_appendix: a string to append to insert in target - YAML file name: '{sample.sample_name}<>.yaml' - :param str filename: A filename without folders. If not provided, a - default name of sample_name.yaml will be used. - :return str: sample YAML file path + """Get a path to a YAML file for the sample. + + Args: + namespaces (dict[dict]): Namespaces mapping. + template_key (str): The name of the key in 'var_templates' piface + section that points to a template to render to get the + user-provided target YAML path. + default_name_appendix (str): A string to append to insert in target + YAML file name: '{sample.sample_name}<>.yaml'. + filename (str): A filename without folders. If not provided, a + default name of sample_name.yaml will be used. + + Returns: + str: Sample YAML file path. """ if ( VAR_TEMPL_KEY in namespaces["pipeline"] @@ -107,11 +109,16 @@ def _get_yaml_path(namespaces, template_key, default_name_appendix="", filename= def write_pipestat_config(looper_pipestat_config_path, pipestat_config_dict): - """ - This writes a combined configuration file to be passed to a PipestatManager. - :param str looper_pipestat_config_path: path to the created pipestat configuration file - :param dict pipestat_config_dict: the dict containing key value pairs to be written to the pipestat configutation - return bool + """Write a combined configuration file to be passed to a PipestatManager. + + Args: + looper_pipestat_config_path (str): Path to the created pipestat + configuration file. + pipestat_config_dict (dict): The dict containing key value pairs to be + written to the pipestat configuration. + + Returns: + bool: True if successful. """ if not os.path.exists(os.path.dirname(looper_pipestat_config_path)): @@ -130,11 +137,13 @@ def write_pipestat_config(looper_pipestat_config_path, pipestat_config_dict): def write_submission_yaml(namespaces): - """ - Save all namespaces to YAML. + """Save all namespaces to YAML. - :param dict namespaces: variable namespaces dict - :return dict: sample namespace dict + Args: + namespaces (dict): Variable namespaces dict. + + Returns: + dict: Sample namespace dict. """ path = _get_yaml_path(namespaces, SAMPLE_CWL_YAML_PATH_KEY, "_submission") my_namespaces = {} @@ -172,43 +181,43 @@ def __init__( automatic=True, collate=False, ): - """ - Create a job submission manager. + """Create a job submission manager. The most critical inputs are the pipeline interface and the pipeline key, which together determine which provide critical pipeline information like resource allocation packages and which pipeline will be overseen by this instance, respectively. - :param PipelineInterface pipeline_interface: Collection of important - data for one or more pipelines, like resource allocation packages - and option/argument specifications - :param prj: Project with which each sample being considered is - associated (what generated each sample) - :param float delay: Time (in seconds) to wait before submitting a job - once it's ready - :param str extra_args: string to pass to each job generated, - for example additional pipeline arguments - :param str extra_args_override: string to pass to each job generated, - for example additional pipeline arguments. This deactivates the - 'extra' functionality that appends strings defined in - Sample.command_extra and Project.looper.command_extra to the - command template. - :param bool ignore_flags: Whether to ignore flag files present in - the sample folder for each sample considered for submission - :param dict[str] compute_variables: A dict with variables that will be made - available to the compute package. For example, this should include - the name of the cluster partition to which job or jobs will be submitted - :param int | NoneType max_cmds: Upper bound on number of commands to - include in a single job script. - :param int | float | NoneType max_size: Upper bound on total file - size of inputs used by the commands lumped into single job script. - :param int | float | NoneType max_jobs: Upper bound on total number of jobs to - group samples for submission. - :param bool automatic: Whether the submission should be automatic once - the pool reaches capacity. - :param bool collate: Whether a collate job is to be submitted (runs on - the project level, rather that on the sample level) + Args: + pipeline_interface (PipelineInterface): Collection of important + data for one or more pipelines, like resource allocation packages + and option/argument specifications. + prj: Project with which each sample being considered is + associated (what generated each sample). + delay (float): Time (in seconds) to wait before submitting a job + once it's ready. + extra_args (str): String to pass to each job generated, + for example additional pipeline arguments. + extra_args_override (str): String to pass to each job generated, + for example additional pipeline arguments. This deactivates the + 'extra' functionality that appends strings defined in + Sample.command_extra and Project.looper.command_extra to the + command template. + ignore_flags (bool): Whether to ignore flag files present in + the sample folder for each sample considered for submission. + compute_variables (dict[str]): A dict with variables that will be made + available to the compute package. For example, this should include + the name of the cluster partition to which job or jobs will be submitted. + max_cmds (int | None): Upper bound on number of commands to + include in a single job script. + max_size (int | float | None): Upper bound on total file + size of inputs used by the commands lumped into single job script. + max_jobs (int | float | None): Upper bound on total number of jobs to + group samples for submission. + automatic (bool): Whether the submission should be automatic once + the pool reaches capacity. + collate (bool): Whether a collate job is to be submitted (runs on + the project level, rather that on the sample level). """ super(SubmissionConductor, self).__init__() @@ -279,27 +288,30 @@ def failed_samples(self): @property def num_cmd_submissions(self): - """ - Return the number of commands that this conductor has submitted. + """Return the number of commands that this conductor has submitted. - :return int: Number of commands submitted so far. + Returns: + int: Number of commands submitted so far. """ return self._num_cmds_submitted @property def num_job_submissions(self): - """ - Return the number of jobs that this conductor has submitted. + """Return the number of jobs that this conductor has submitted. - :return int: Number of jobs submitted so far. + Returns: + int: Number of jobs submitted so far. """ return self._num_good_job_submissions def is_project_submittable(self, force=False): - """ - Check whether the current project has been already submitted + """Check whether the current project has been already submitted. - :param bool frorce: whether to force the project submission (ignore status/flags) + Args: + force (bool): Whether to force the project submission (ignore status/flags). + + Returns: + bool: True if the project is submittable, False otherwise. """ psms = {} if self.prj.pipestat_configured_project: @@ -314,17 +326,20 @@ def is_project_submittable(self, force=False): return True def add_sample(self, sample, rerun=False): - """ - Add a sample for submission to this conductor. - - :param peppy.Sample sample: sample to be included with this conductor's - currently growing collection of command submissions - :param bool rerun: whether the given sample is being rerun rather than - run for the first time - :return bool: Indication of whether the given sample was added to - the current 'pool.' - :raise TypeError: If sample subtype is provided but does not extend - the base Sample class, raise a TypeError. + """Add a sample for submission to this conductor. + + Args: + sample (peppy.Sample): Sample to be included with this conductor's + currently growing collection of command submissions. + rerun (bool): Whether the given sample is being rerun rather than + run for the first time. + + Returns: + list: List of skip reasons if sample was not added. + + Raises: + TypeError: If sample subtype is provided but does not extend + the base Sample class. """ _LOGGER.debug( "Adding {} to conductor for {} to {}run".format( @@ -406,17 +421,19 @@ def add_sample(self, sample, rerun=False): return skip_reasons def submit(self, force=False): - """ - Submit one or more commands as a job. + """Submit one or more commands as a job. This call will submit the commands corresponding to the current pool of samples if and only if the argument to 'force' evaluates to a true value, or the pool of samples is full. - :param bool force: Whether submission should be done/simulated even - if this conductor's pool isn't full. - :return bool: Whether a job was submitted (or would've been if - not for dry run) + Args: + force (bool): Whether submission should be done/simulated even + if this conductor's pool isn't full. + + Returns: + bool: Whether a job was submitted (or would've been if + not for dry run). """ submitted = False @@ -479,25 +496,29 @@ def submit(self, force=False): return submitted def _is_full(self, pool, size): - """ - Determine whether it's time to submit a job for the pool of commands. + """Determine whether it's time to submit a job for the pool of commands. Instances of this class maintain a sort of 'pool' of commands that expands as each new command is added, until a time that it's deemed - 'full' and th + 'full'. - :return bool: Whether this conductor's pool of commands is 'full' and - ready for submission, as determined by its parameterization + Args: + pool: Collection of samples/commands. + size: Current total size. + + Returns: + bool: Whether this conductor's pool of commands is 'full' and + ready for submission, as determined by its parameterization. """ return self.max_cmds == len(pool) or size >= self.max_size @property def _samples(self): - """ - Return a collection of pooled samples. + """Return a collection of pooled samples. - :return Iterable[str]: collection of samples currently in the active - pool for this submission conductor + Returns: + Iterable[str]: Collection of samples currently in the active + pool for this submission conductor. """ return [s for s in self._pool] @@ -522,15 +543,20 @@ def _sample_lump_name(self, pool): return "lump{}".format(self._num_total_job_submissions + 1) def _signal_int_handler(self, signal, frame): - """ - For catching interrupt (Ctrl +C) signals. Fails gracefully. + """For catching interrupt (Ctrl +C) signals. Fails gracefully. + + Args: + signal: Signal received. + frame: Current stack frame. """ signal_type = "SIGINT" self._generic_signal_handler(signal_type) def _generic_signal_handler(self, signal_type): - """ - Function for handling both SIGTERM and SIGINT + """Function for handling both SIGTERM and SIGINT. + + Args: + signal_type (str): Type of signal received (SIGTERM or SIGINT). """ message = "Received " + signal_type + ". Failing gracefully..." _LOGGER.warning(msg=message) @@ -587,14 +613,17 @@ def pskill(proc_pid, sig=signal.SIGINT): _LOGGER.warning(msg=f"Child process {self.process_id} {note}.") def _attend_process(self, proc, sleeptime): - """ - Waits on a process for a given time to see if it finishes, returns True - if it's still running after the given time or False as soon as it - returns. + """Wait on a process for a given time to see if it finishes. - :param psutil.Process proc: Process object opened by psutil.Popen() - :param float sleeptime: Time to wait - :return bool: True if process is still running; otherwise false + Returns True if it's still running after the given time or False as + soon as it returns. + + Args: + proc (psutil.Process): Process object opened by psutil.Popen(). + sleeptime (float): Time to wait. + + Returns: + bool: True if process is still running; otherwise false. """ try: proc.wait(timeout=int(sleeptime)) @@ -607,14 +636,18 @@ def _jobname(self, pool): return "{}_{}".format(self.pl_iface.pipeline_name, self._sample_lump_name(pool)) def _build_looper_namespace(self, pool, size): - """ + """Compile a mapping of looper/submission related settings. + Compile a mapping of looper/submission related settings for use in the command templates and in submission script creation in divvy (via adapters). - :param Iterable[peppy.Sample] pool: collection of sample instances - :param float size: cumulative size of the given pool - :return yacman.YAMLConfigManager: looper/submission related settings + Args: + pool (Iterable[peppy.Sample]): Collection of sample instances. + size (float): Cumulative size of the given pool. + + Returns: + yacman.YAMLConfigManager: Looper/submission related settings. """ settings = YAMLConfigManager() settings["config_file"] = self.prj.config_file @@ -651,14 +684,18 @@ def _build_looper_namespace(self, pool, size): def _set_pipestat_namespace( self, sample_name: Optional[str] = None ) -> YAMLConfigManager: - """ + """Compile a mapping of pipestat-related settings. + Compile a mapping of pipestat-related settings for use in the command templates. Accessible via: {pipestat.attrname} - :param str sample_name: name of the sample to get the pipestat - namespace for. If not provided the pipestat namespace will - be determined based on the Project - :return yacman.YAMLConfigManager: pipestat namespace + Args: + sample_name (str): Name of the sample to get the pipestat + namespace for. If not provided the pipestat namespace will + be determined based on the Project. + + Returns: + yacman.YAMLConfigManager: Pipestat namespace. """ try: psm = self.pl_iface.psm @@ -684,12 +721,14 @@ def _set_pipestat_namespace( return YAMLConfigManager(filtered_namespace) def write_script(self, pool, size): - """ - Create the script for job submission. + """Create the script for job submission. + + Args: + pool (Iterable[peppy.Sample]): Collection of sample instances. + size (float): Cumulative size of the given pool. - :param Iterable[peppy.Sample] pool: collection of sample instances - :param float size: cumulative size of the given pool - :return str: Path to the job submission script created. + Returns: + str: Path to the job submission script created. """ # looper settings determination if self.collate: @@ -806,23 +845,27 @@ def _use_sample(flag, skips): def _exec_pre_submit(piface, namespaces): - """ - Execute pre submission hooks defined in the pipeline interface + """Execute pre submission hooks defined in the pipeline interface. + + Args: + piface (PipelineInterface): Piface, a source of pre_submit hooks to execute. + namespaces (dict[dict]): Namespaces mapping. - :param PipelineInterface piface: piface, a source of pre_submit hooks to execute - :param dict[dict[]] namespaces: namspaces mapping - :return dict[dict[]]: updated namspaces mapping + Returns: + dict[dict]: Updated namespaces mapping. """ def _update_namespaces(x, y, cmd=False): - """ + """Update namespaces mapping with new values. + Update namespaces mapping with a dictionary of the same structure, that includes just the values that need to be updated. - :param dict[dict] x: namespaces mapping - :param dict[dict] y: mapping to update namespaces with - :param bool cmd: whether the mapping to update with comes from the - command template, used for messaging + Args: + x (dict[dict]): Namespaces mapping. + y (dict[dict]): Mapping to update namespaces with. + cmd (bool): Whether the mapping to update with comes from the + command template, used for messaging. """ if not y: return diff --git a/looper/utils.py b/looper/utils.py index 78de020c..9cc57ef6 100644 --- a/looper/utils.py +++ b/looper/utils.py @@ -46,22 +46,26 @@ def fetch_flag_files(prj=None, results_folder="", flags=FLAGS): - """ - Find all flag file paths for the given project. - - :param Project | AttributeDict prj: full Project or AttributeDict with - similar metadata and access/usage pattern - :param str results_folder: path to results folder, corresponding to the - 1:1 sample:folder notion that a looper Project has. That is, this - function uses the assumption that if results_folder rather than project - is provided, the structure of the file tree rooted at results_folder is - such that any flag files to be found are not directly within rootdir but - are directly within on of its first layer of subfolders. - :param Iterable[str] | str flags: Collection of flag names or single flag - name for which to fetch files - :return Mapping[str, list[str]]: collection of filepaths associated with - particular flag for samples within the given project - :raise TypeError: if neither or both of project and rootdir are given + """Find all flag file paths for the given project. + + Args: + prj (Project | AttributeDict): Full Project or AttributeDict with + similar metadata and access/usage pattern. + results_folder (str): Path to results folder, corresponding to the + 1:1 sample:folder notion that a looper Project has. That is, this + function uses the assumption that if results_folder rather than project + is provided, the structure of the file tree rooted at results_folder is + such that any flag files to be found are not directly within rootdir but + are directly within on of its first layer of subfolders. + flags (Iterable[str] | str): Collection of flag names or single flag + name for which to fetch files. + + Returns: + Mapping[str, list[str]]: Collection of filepaths associated with + particular flag for samples within the given project. + + Raises: + TypeError: If neither or both of project and rootdir are given. """ if not (prj or results_folder) or (prj and results_folder): @@ -94,14 +98,17 @@ def fetch_flag_files(prj=None, results_folder="", flags=FLAGS): def fetch_sample_flags(prj, sample, pl_name, flag_dir=None): - """ - Find any flag files present for a sample associated with a project + """Find any flag files present for a sample associated with a project. - :param looper.Project prj: project of interest - :param peppy.Sample sample: sample object of interest - :param str pl_name: name of the pipeline for which flag(s) should be found - :return Iterable[str]: collection of flag file path(s) associated with the - given sample for the given project + Args: + prj (looper.Project): Project of interest. + sample (peppy.Sample): Sample object of interest. + pl_name (str): Name of the pipeline for which flag(s) should be found. + flag_dir: Flag directory path. + + Returns: + Iterable[str]: Collection of flag file path(s) associated with the + given sample for the given project. """ sfolder = flag_dir or sample_folder(prj=prj, sample=sample) if not os.path.isdir(sfolder): @@ -122,9 +129,14 @@ def fetch_sample_flags(prj, sample, pl_name, flag_dir=None): def get_sample_status(sample, flags): - """ - get a sample status + """Get a sample status. + Args: + sample: Sample identifier. + flags: Collection of flag file paths. + + Returns: + str or None: Status string if found, None otherwise. """ statuses = [] @@ -145,8 +157,7 @@ def get_sample_status(sample, flags): def grab_project_data(prj): - """ - From the given Project, grab Sample-independent data. + """From the given Project, grab Sample-independent data. There are some aspects of a Project of which it's beneficial for a Sample to be aware, particularly for post-hoc analysis. Since Sample objects @@ -155,8 +166,11 @@ def grab_project_data(prj): so for each Sample knowledge of Project data is limited. This method facilitates adoption of that conceptual model. - :param Project prj: Project from which to grab data - :return Mapping: Sample-independent data sections from given Project + Args: + prj (Project): Project from which to grab data. + + Returns: + Mapping: Sample-independent data sections from given Project. """ if not prj: return {} @@ -168,30 +182,36 @@ def grab_project_data(prj): def sample_folder(prj, sample): - """ - Get the path to this Project's root folder for the given Sample. + """Get the path to this Project's root folder for the given Sample. - :param AttributeDict | Project prj: project with which sample is associated - :param Mapping sample: Sample or sample data for which to get root output - folder path. - :return str: this Project's root folder for the given Sample + Args: + prj (AttributeDict | Project): Project with which sample is associated. + sample (Mapping): Sample or sample data for which to get root output + folder path. + + Returns: + str: This Project's root folder for the given Sample. """ return os.path.join(prj.results_folder, sample[prj.sample_table_index]) def get_file_for_project(prj, pipeline_name, appendix=None, directory=None): - """ - Create a path to the file for the current project. - Takes the possibility of amendment being activated at the time + """Create a path to the file for the current project. + + Takes the possibility of amendment being activated at the time. Format of the output path: {output_dir}/{directory}/{p.name}_{pipeline_name}_{active_amendments}_{appendix} - :param looper.Project prj: project object - :param str pipeline_name: name of the pipeline to get the file for - :param str appendix: the appendix of the file to create the path for, - like 'objs_summary.tsv' for objects summary file - :return str: path to the file + Args: + prj (looper.Project): Project object. + pipeline_name (str): Name of the pipeline to get the file for. + appendix (str): The appendix of the file to create the path for, + like 'objs_summary.tsv' for objects summary file. + directory (str): Directory path component. + + Returns: + str: Path to the file. """ fp = os.path.join( prj.output_dir, directory or "", f"{prj[NAME_KEY]}_{pipeline_name}" @@ -203,14 +223,17 @@ def get_file_for_project(prj, pipeline_name, appendix=None, directory=None): def get_file_for_project_old(prj, appendix): - """ - Create a path to the file for the current project. - Takes the possibility of amendment being activated at the time + """Create a path to the file for the current project. + + Takes the possibility of amendment being activated at the time. + + Args: + prj (looper.Project): Project object. + appendix (str): The appendix of the file to create the path for, + like 'objs_summary.tsv' for objects summary file. - :param looper.Project prj: project object - :param str appendix: the appendix of the file to create the path for, - like 'objs_summary.tsv' for objects summary file - :return str: path to the file + Returns: + str: Path to the file. """ fp = os.path.join(prj.output_dir, prj[NAME_KEY]) if hasattr(prj, AMENDMENTS_KEY) and getattr(prj, AMENDMENTS_KEY): @@ -220,18 +243,20 @@ def get_file_for_project_old(prj, appendix): def jinja_render_template_strictly(template, namespaces): - """ - Render a command string in the provided namespaces context. + """Render a command string in the provided namespaces context. Strictly, which means that all the requested attributes must be - available in the namespaces - - :param str template: command template do be filled in with the - variables in the provided namespaces. For example: - "prog.py --name {project.name} --len {sample.len}" - :param Mapping[Mapping[str] namespaces: context for command rendering. - Possible namespaces are: looper, project, sample, pipeline - :return str: rendered command + available in the namespaces. + + Args: + template (str): Command template to be filled in with the + variables in the provided namespaces. For example: + "prog.py --name {project.name} --len {sample.len}". + namespaces (Mapping[Mapping[str]]): Context for command rendering. + Possible namespaces are: looper, project, sample, pipeline. + + Returns: + str: Rendered command. """ def _finfun(x): @@ -260,11 +285,13 @@ def _finfun(x): def read_yaml_file(filepath): - """ - Read a YAML file + """Read a YAML file. - :param str filepath: path to the file to read - :return dict: read data + Args: + filepath (str): Path to the file to read. + + Returns: + dict: Read data. """ data = None if os.path.exists(filepath): @@ -280,18 +307,21 @@ def enrich_args_via_cfg( test_args=None, cli_modifiers=None, ): - """ - Read in a looper dotfile, pep config and set arguments. + """Read in a looper dotfile, pep config and set arguments. Priority order: CLI > dotfile/config > pep_config > parser default - :param subcommand name: the name of the command used - :param argparse.Namespace parser_args: parsed args by the original parser - :param argparse.Namespace aux_parser: parsed args by the argument parser - with defaults suppressed - :param dict test_args: dict of args used for pytesting - :param dict cli_modifiers: dict of args existing if user supplied cli args in looper config file - :return argparse.Namespace: selected argument values + Args: + subcommand_name: The name of the command used. + parser_args (argparse.Namespace): Parsed args by the original parser. + aux_parser (argparse.Namespace): Parsed args by the argument parser + with defaults suppressed. + test_args (dict): Dict of args used for pytesting. + cli_modifiers (dict): Dict of args existing if user supplied cli args + in looper config file. + + Returns: + argparse.Namespace: Selected argument values. """ # Did the user provide arguments in the PEP config? @@ -373,7 +403,8 @@ def set_single_arg(argname, default_source_namespace, result_namespace): def _get_subcommand_args(subcommand_name, parser_args): - """ + """Get the union of values for the subcommand arguments. + Get the union of values for the subcommand arguments from Project.looper, Project.looper.cli. and Project.looper.cli.all. If any are duplicated, the above is the selection priority order. @@ -382,8 +413,11 @@ def _get_subcommand_args(subcommand_name, parser_args): with '_'), which strongly relies on argument parser using default destinations. - :param argparser.Namespace parser_args: argument namespace - :return dict: mapping of argument destinations to their values + Args: + parser_args (argparser.Namespace): Argument namespace. + + Returns: + dict: Mapping of argument destinations to their values. """ args = dict() cfg = peppyProject( @@ -430,8 +464,13 @@ def _get_subcommand_args(subcommand_name, parser_args): def init_generic_pipeline(pipelinepath: Optional[str] = None): - """ - Create generic pipeline interface + """Create generic pipeline interface. + + Args: + pipelinepath (str, optional): Path to pipeline directory. + + Returns: + bool: True if successful. """ console = Console() @@ -541,11 +580,14 @@ def init_generic_pipeline(pipelinepath: Optional[str] = None): def read_looper_dotfile(): - """ - Read looper config file - :return str: path to the config file read from the dotfile - :raise MisconfigurationException: if the dotfile does not consist of the - required key pointing to the PEP + """Read looper config file. + + Returns: + str: Path to the config file read from the dotfile. + + Raises: + MisconfigurationException: If the dotfile does not consist of the + required key pointing to the PEP. """ dot_file_path = dotfile_path(must_exist=True) return read_looper_config_file(looper_config_path=dot_file_path) @@ -559,16 +601,20 @@ def initiate_looper_config( project_pipeline_interfaces: Union[List[str], str] = None, force=False, ): - """ - Initialize looper config file - - :param str looper_config_path: absolute path to the file to initialize - :param str pep_path: path to the PEP to be used in pipeline - :param str output_dir: path to the output directory - :param str|list sample_pipeline_interfaces: path or list of paths to sample pipeline interfaces - :param str|list project_pipeline_interfaces: path or list of paths to project pipeline interfaces - :param bool force: whether the existing file should be overwritten - :return bool: whether the file was initialized + """Initialize looper config file. + + Args: + looper_config_path (str): Absolute path to the file to initialize. + pep_path (str): Path to the PEP to be used in pipeline. + output_dir (str): Path to the output directory. + sample_pipeline_interfaces (str | list): Path or list of paths to + sample pipeline interfaces. + project_pipeline_interfaces (str | list): Path or list of paths to + project pipeline interfaces. + force (bool): Whether the existing file should be overwritten. + + Returns: + bool: Whether the file was initialized. """ console = Console() console.clear() @@ -624,10 +670,10 @@ def initiate_looper_config( def looper_config_tutorial(): - """ - Prompt a user through configuring a .looper.yaml file for a new project. + """Prompt a user through configuring a .looper.yaml file for a new project. - :return bool: whether the file was initialized + Returns: + bool: Whether the file was initialized. """ console = Console() @@ -743,13 +789,18 @@ def looper_config_tutorial(): def determine_pipeline_type(piface_path: str, looper_config_path: str): - """ - Read pipeline interface from disk and determine if it contains "sample_interface", "project_interface" or both + """Read pipeline interface and determine its type. + Read pipeline interface from disk and determine if it contains + "sample_interface", "project_interface" or both. - :param str piface_path: path to pipeline_interface - :param str looper_config_path: path to looper config file - :return Tuple[Union[str,None],Union[str,None]] : (pipeline type, resolved path) or (None, None) + Args: + piface_path (str): Path to pipeline_interface. + looper_config_path (str): Path to looper config file. + + Returns: + Tuple[Union[str, None], Union[str, None]]: (pipeline type, resolved path) + or (None, None). """ if piface_path is None: @@ -788,15 +839,21 @@ def determine_pipeline_type(piface_path: str, looper_config_path: str): def read_looper_config_file(looper_config_path: str) -> dict: - """ + """Read Looper config file. + Read Looper config file which includes: - PEP config (local path or pephub registry path) - looper output dir - looper pipeline interfaces - :param str looper_config_path: path to looper config path - :return dict: looper config file content - :raise MisconfigurationException: incorrect configuration. + Args: + looper_config_path (str): Path to looper config path. + + Returns: + dict: Looper config file content. + + Raises: + MisconfigurationException: Incorrect configuration. """ return_dict = {} @@ -891,16 +948,20 @@ def read_looper_config_file(looper_config_path: str) -> dict: def dotfile_path(directory=os.getcwd(), must_exist=False): - """ - Get the path to the looper dotfile + """Get the path to the looper dotfile. If file existence is forced this function will look for it in - the directory parents + the directory parents. + + Args: + directory (str): Directory path to start the search in. + must_exist (bool): Whether the file must exist. - :param str directory: directory path to start the search in - :param bool must_exist: whether the file must exist - :return str: path to the dotfile - :raise OSError: if the file does not exist + Returns: + str: Path to the dotfile. + + Raises: + OSError: If the file does not exist. """ cur_dir = directory if not must_exist: @@ -919,8 +980,13 @@ def dotfile_path(directory=os.getcwd(), must_exist=False): def is_PEP_file_type(input_string: str) -> bool: - """ - Determines if the provided path is actually a file type that Looper can use for loading PEP + """Determines if the provided path is a file type that Looper can use for loading PEP. + + Args: + input_string (str): Path to check. + + Returns: + bool: True if the path is a valid PEP file type. """ PEP_FILE_TYPES = ["yaml", "csv"] @@ -930,10 +996,13 @@ def is_PEP_file_type(input_string: str) -> bool: def is_pephub_registry_path(input_string: str) -> bool: - """ - Check if input is a registry path to pephub - :param str input_string: path to the PEP (or registry path) - :return bool: True if input is a registry path + """Check if input is a registry path to pephub. + + Args: + input_string (str): Path to the PEP (or registry path). + + Returns: + bool: True if input is a registry path. """ try: registry_path = RegistryPath(**parse_registry_path(input_string)) @@ -997,11 +1066,14 @@ def to_range(self) -> Iterable[int]: @classmethod def from_string(cls, s: str, upper_bound: int) -> "IntRange": - """ - Create an instance from a string, e.g. command-line argument. + """Create an instance from a string, e.g. command-line argument. + + Args: + s (str): The string to parse as an interval. + upper_bound (int): The default upper bound. - :param str s: The string to parse as an interval - :param int upper_bound: the default upper bound + Returns: + IntRange: New instance created from the string. """ if upper_bound < 1: raise NatIntervalException(f"Upper bound must be positive: {upper_bound}") @@ -1035,18 +1107,20 @@ def from_string(cls, s: str, upper_bound: int) -> "IntRange": def desired_samples_range_limited(arg: str, num_samples: int) -> Iterable[int]: - """ - Create a contiguous interval of natural numbers. Used for _positive_ selection of samples. + """Create a contiguous interval of natural numbers for positive selection of samples. Interpret given arg as upper bound (1-based) if it's a single value, but take the minimum of that and the given number of samples. If arg is parseable as a range, use that. - :param str arg: CLI specification of a range of samples to use, or as the greatest - 1-based index of a sample to include - :param int num_samples: what to use as the upper bound on the 1-based index interval - if the given arg isn't a range but rather a single value. - :return: an iterable of 1-based indices into samples to select + Args: + arg (str): CLI specification of a range of samples to use, or as the greatest + 1-based index of a sample to include. + num_samples (int): What to use as the upper bound on the 1-based index interval + if the given arg isn't a range but rather a single value. + + Returns: + Iterable[int]: An iterable of 1-based indices into samples to select. """ try: upper_bound = min(int(arg), num_samples) @@ -1059,13 +1133,15 @@ def desired_samples_range_limited(arg: str, num_samples: int) -> Iterable[int]: def desired_samples_range_skipped(arg: str, num_samples: int) -> Iterable[int]: - """ - Create a contiguous interval of natural numbers. Used for _negative_ selection of samples. + """Create a contiguous interval of natural numbers for negative selection of samples. - :param str arg: CLI specification of a range of samples to use, or as the lowest - 1-based index of a sample to skip - :param int num_samples: highest 1-based index of samples to include - :return: an iterable of 1-based indices into samples to select + Args: + arg (str): CLI specification of a range of samples to use, or as the lowest + 1-based index of a sample to skip. + num_samples (int): Highest 1-based index of samples to include. + + Returns: + Iterable[int]: An iterable of 1-based indices into samples to select. """ try: lower_bound = int(arg) @@ -1082,14 +1158,17 @@ def desired_samples_range_skipped(arg: str, num_samples: int) -> Iterable[int]: def write_submit_script(fp, content, data): - """ - Write a submission script for divvy by populating a template with data. - :param str fp: Path to the file to which to create/write submissions script. - :param str content: Template for submission script, defining keys that - will be filled by given data - :param Mapping data: a "pool" from which values are available to replace - keys in the template - :return str: Path to the submission script + """Write a submission script for divvy by populating a template with data. + + Args: + fp (str): Path to the file to which to create/write submissions script. + content (str): Template for submission script, defining keys that + will be filled by given data. + data (Mapping): A "pool" from which values are available to replace + keys in the template. + + Returns: + str: Path to the submission script. """ for k, v in data.items(): @@ -1117,10 +1196,10 @@ def write_submit_script(fp, content, data): def inspect_looper_config_file(looper_config_dict) -> None: - """ - Inspects looper config by printing it to terminal. - param dict looper_config_dict: dict representing looper_config + """Inspects looper config by printing it to terminal. + Args: + looper_config_dict (dict): Dict representing looper_config. """ # Simply print this to terminal print("LOOPER INSPECT") From 58d1dc591a3bfc499cfa3434872de0cdb62c964e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Nov 2025 22:59:35 +0000 Subject: [PATCH 2/6] Convert docstrings to Google style in exceptions.py, const.py, and plugins.py - exceptions.py: 1 docstring converted - const.py: 1 docstring converted - plugins.py: 3 docstrings converted - All RST-style :param/:return annotations updated to Args:/Returns: format --- looper/const.py | 16 ++++++++++------ looper/exceptions.py | 6 +++--- looper/plugins.py | 30 ++++++++++++++++++------------ 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/looper/const.py b/looper/const.py index bfa51309..d60d052e 100644 --- a/looper/const.py +++ b/looper/const.py @@ -108,13 +108,17 @@ def _get_apperance_dict(type, templ=APPEARANCE_BY_FLAG): - """ - Based on the type of the HTML element provided construct the appearence - mapping using the template + """Construct the appearance mapping using the template. + + Based on the type of the HTML element provided construct the appearance + mapping using the template. + + Args: + type (str): Type of HTML element to populate template with. + templ (dict): Appearance template to populate. - :param dict templ: appearance template to populate - :param str type: type of HTML element to populate template with - :return dict: populated appearance template + Returns: + dict: Populated appearance template. """ from copy import deepcopy diff --git a/looper/exceptions.py b/looper/exceptions.py index 62b9e041..c3fd631b 100644 --- a/looper/exceptions.py +++ b/looper/exceptions.py @@ -90,10 +90,10 @@ class PipelineInterfaceConfigError(LooperError): """Error with PipelineInterface config data during construction.""" def __init__(self, context): - """ - For exception context, provide message or collection of missing sections. + """For exception context, provide message or collection of missing sections. - :param str | Iterable[str] context: + Args: + context (str | Iterable[str]): Message or collection of missing sections. """ if not isinstance(context, str) and isinstance(context, Iterable): context = "Missing section(s): {}".format(", ".join(context)) diff --git a/looper/plugins.py b/looper/plugins.py index d4cc5265..37859db2 100644 --- a/looper/plugins.py +++ b/looper/plugins.py @@ -11,16 +11,18 @@ def write_sample_yaml_prj(namespaces): - """ - Plugin: saves sample representation with project reference to YAML. + """Plugin: saves sample representation with project reference to YAML. This plugin can be parametrized by providing the path value/template in 'pipeline.var_templates.sample_yaml_prj_path'. This needs to be a complete and absolute path to the file where sample YAML representation is to be stored. - :param dict namespaces: variable namespaces dict - :return dict: sample namespace dict + Args: + namespaces (dict): Variable namespaces dict. + + Returns: + dict: Sample namespace dict. """ sample = namespaces["sample"] sample.to_yaml( @@ -68,8 +70,7 @@ def load_template(pipeline): def write_sample_yaml_cwl(namespaces): - """ - Plugin: Produce a cwl-compatible yaml representation of the sample + """Plugin: Produce a cwl-compatible yaml representation of the sample. Also adds the 'cwl_yaml' attribute to sample objects, which points to the file produced. @@ -79,8 +80,11 @@ def write_sample_yaml_cwl(namespaces): absolute path to the file where sample YAML representation is to be stored. - :param dict namespaces: variable namespaces dict - :return dict: updated variable namespaces dict + Args: + namespaces (dict): Variable namespaces dict. + + Returns: + dict: Updated variable namespaces dict. """ from eido import read_schema from ubiquerg import is_url @@ -145,16 +149,18 @@ def _get_schema_source( def write_sample_yaml(namespaces): - """ - Plugin: saves sample representation to YAML. + """Plugin: saves sample representation to YAML. This plugin can be parametrized by providing the path value/template in 'pipeline.var_templates.sample_yaml_path'. This needs to be a complete and absolute path to the file where sample YAML representation is to be stored. - :param dict namespaces: variable namespaces dict - :return dict: sample namespace dict + Args: + namespaces (dict): Variable namespaces dict. + + Returns: + dict: Sample namespace dict. """ sample = namespaces["sample"] sample["sample_yaml_path"] = _get_yaml_path( From 28713ef9deca933e7f2ef8461095b720deaae0b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Nov 2025 23:01:18 +0000 Subject: [PATCH 3/6] Convert docstrings to Google style in parser_types.py and command_models - parser_types.py: 2 docstrings converted - command_models/arguments.py: 1 docstring converted - command_models/commands.py: 2 docstrings converted - All RST-style annotations updated to Google-style format --- looper/command_models/arguments.py | 18 ++++++++--------- looper/command_models/commands.py | 28 ++++++++++++++++---------- looper/parser_types.py | 32 +++++++++++++++++------------- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py index 68c32977..04c11d5c 100644 --- a/looper/command_models/arguments.py +++ b/looper/command_models/arguments.py @@ -11,8 +11,7 @@ class Argument(pydantic.fields.FieldInfo): - """ - CLI argument / flag definition + """CLI argument / flag definition. This class is designed to define CLI arguments or flags. It leverages Pydantic for data validation and serves as a source of truth for multiple @@ -24,13 +23,14 @@ class Argument(pydantic.fields.FieldInfo): so we instead subclass `FieldInfo` directly and validate it in the constructor. - :param str name: argument name, e.g. "ignore-args" - :param Any default: a tuple of the form (type, default_value). If the - default value is `...` (Ellipsis), then the argument is required. - :param str description: argument description, which will appear as the - help text for this argument - :param dict kwargs: additional keyword arguments supported by - `FieldInfo`. These are passed along as they are. + Args: + name (str): Argument name, e.g. "ignore-args". + default (Any): A tuple of the form (type, default_value). If the + default value is `...` (Ellipsis), then the argument is required. + description (str): Argument description, which will appear as the + help text for this argument. + kwargs (dict): Additional keyword arguments supported by + `FieldInfo`. These are passed along as they are. """ def __init__( diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py index 69312f0d..d88c8232 100644 --- a/looper/command_models/commands.py +++ b/looper/command_models/commands.py @@ -14,12 +14,12 @@ @dataclass class Command: - """ - Representation of a command + """Representation of a command. - :param str name: command name - :param str description: command description - :param list[Argument] arguments: list of arguments supported by this command + Args: + name (str): Command name. + description (str): Command description. + arguments (list[Argument]): List of arguments supported by this command. """ name: str @@ -242,13 +242,19 @@ def create_model(self) -> Type[pydantic.BaseModel]: def add_short_arguments( parser: ArgumentParser, argument_enums: Type[ArgumentEnum] ) -> ArgumentParser: - """ - This function takes a parser object created under pydantic argparse and adds the short arguments AFTER the initial creation. - This is a workaround as pydantic-argparse does not currently support this during initial parser creation. + """Add short arguments to parser after initial creation. + + This function takes a parser object created under pydantic argparse and adds + the short arguments AFTER the initial creation. This is a workaround as + pydantic-argparse does not currently support this during initial parser creation. + + Args: + parser (ArgumentParser): Parser before adding short arguments. + argument_enums (Type[ArgumentEnum]): Enumeration of arguments that contain + names and aliases. - :param ArgumentParser parser: parser before adding short arguments - :param Type[ArgumentEnum] argument_enums: enumeration of arguments that contain names and aliases - :return ArgumentParser parser: parser after short arguments have been added + Returns: + ArgumentParser: Parser after short arguments have been added. """ for cmd in parser._subcommands.choices.keys(): diff --git a/looper/parser_types.py b/looper/parser_types.py index 98404965..97c8b6f9 100644 --- a/looper/parser_types.py +++ b/looper/parser_types.py @@ -29,14 +29,16 @@ def fun(x=None, caravel_data=caravel_data, caravel=caravel): def html_checkbox(caravel=False, checked=False): - """ - Create argument for type parameter on argparse.ArgumentParser.add_argument. + """Create argument for type parameter on argparse.ArgumentParser.add_argument. + + Args: + caravel (bool): Whether this is being used in the caravel context. + checked (bool): Whether to add a particular key-value entry to a + collection used by caravel. - :param bool caravel: whether this is being used in the caravel context - :param bool checked: whether to add a particular key-value entry to a - collection used by caravel - :return callable: argument to the type parameter of an - argparse.ArgumentParser's add_argument method. + Returns: + callable: Argument to the type parameter of an + argparse.ArgumentParser's add_argument method. """ caravel_data = YAMLConfigManager({"element_type": "checkbox", "element_args": {}}) if checked: @@ -49,14 +51,16 @@ def fun(x=None, caravel_data=caravel_data, caravel=caravel): def html_select(choices, caravel=False): - """ - Create argument for type parameter on argparse.ArgumentParser.add_argument. + """Create argument for type parameter on argparse.ArgumentParser.add_argument. + + Args: + choices (list[object]): Collection of valid argument provisions via + to a particular CLI option. + caravel (bool): Whether this is being used in the caravel context. - :param list[object] choices: collection of valid argument provisions via - to a particular CLI option - :param bool caravel: whether this is being used in the caravel context - :return callable: argument to the type parameter of an - argparse.ArgumentParser's add_argument method. + Returns: + callable: Argument to the type parameter of an + argparse.ArgumentParser's add_argument method. """ if not isinstance(choices, list): raise TypeError( From b843f3952275907677f8af81958c33367b98e3e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Nov 2025 23:11:31 +0000 Subject: [PATCH 4/6] Convert docstrings to Google style in remaining core files - divvy.py: 16 docstrings converted - looper.py: 11 docstrings converted - pipeline_interface.py: 10 docstrings converted - project.py: 24 docstrings converted - processed_project.py: 5 docstrings converted - cli_pydantic.py: 1 docstring converted - cli_divvy.py: 1 docstring converted All RST-style docstrings (:param/:return/:raise) have been converted to Google-style format (Args:/Returns:/Raises:) with proper formatting and capitalization. --- looper/cli_divvy.py | 6 +- looper/cli_pydantic.py | 21 +-- looper/divvy.py | 149 ++++++++++--------- looper/looper.py | 94 ++++++------ looper/pipeline_interface.py | 116 ++++++++------- looper/processed_project.py | 88 +++++++----- looper/project.py | 267 ++++++++++++++++++----------------- 7 files changed, 395 insertions(+), 346 deletions(-) diff --git a/looper/cli_divvy.py b/looper/cli_divvy.py index 1fa98b69..0305e40e 100644 --- a/looper/cli_divvy.py +++ b/looper/cli_divvy.py @@ -12,10 +12,10 @@ def build_argparser(): - """ - Builds argument parser. + """Builds argument parser. - :return argparse.ArgumentParser + Returns: + argparse.ArgumentParser: The argument parser. """ banner = ( diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py index fb06b687..37452a26 100644 --- a/looper/cli_pydantic.py +++ b/looper/cli_pydantic.py @@ -380,18 +380,23 @@ def main_cli() -> None: def _proc_resources_spec(args): - """ - Process CLI-sources compute setting specification. There are two sources - of compute settings in the CLI alone: + """Process CLI-sources compute setting specification. + + There are two sources of compute settings in the CLI alone: * YAML file (--settings argument) * itemized compute settings (--compute argument) - The itemized compute specification is given priority + The itemized compute specification is given priority. + + Args: + args (argparse.Namespace): Arguments namespace. + + Returns: + Mapping[str, str]: Binding between resource setting name and value. - :param argparse.Namespace: arguments namespace - :return Mapping[str, str]: binding between resource setting name and value - :raise ValueError: if interpretation of the given specification as encoding - of key-value pairs fails + Raises: + ValueError: If interpretation of the given specification as encoding + of key-value pairs fails. """ spec = getattr(args, "compute", None) settings = args.settings diff --git a/looper/divvy.py b/looper/divvy.py index 84e66ed7..031cae69 100644 --- a/looper/divvy.py +++ b/looper/divvy.py @@ -26,8 +26,7 @@ class ComputingConfiguration(YAMLConfigManager): - """ - Represents computing configuration objects. + """Represents computing configuration objects. The ComputingConfiguration class provides a computing configuration object that is an *in memory* representation of a `divvy` computing configuration @@ -35,10 +34,11 @@ class ComputingConfiguration(YAMLConfigManager): and retrieve computing configuration files, and use these values to populate job submission script templates. - :param str | Iterable[(str, object)] | Mapping[str, object] entries: config - Collection of key-value pairs. - :param str filepath: YAML file specifying computing package data. (the - `DIVCFG` file) + Args: + entries (str | Iterable[(str, object)] | Mapping[str, object]): Config + collection of key-value pairs. + filepath (str): YAML file specifying computing package data (the + `DIVCFG` file). """ def __init__( @@ -75,20 +75,20 @@ def write(self, filename=None): @property def compute_env_var(self): - """ - Environment variable through which to access compute settings. + """Environment variable through which to access compute settings. - :return list[str]: names of candidate environment variables, for which - value may be path to compute settings file; first found is used. + Returns: + list[str]: Names of candidate environment variables, for which + value may be path to compute settings file; first found is used. """ return COMPUTE_SETTINGS_VARNAME @property def default_config_file(self): - """ - Path to default compute environment settings file. + """Path to default compute environment settings file. - :return str: Path to default compute settings file + Returns: + str: Path to default compute settings file. """ return DEFAULT_CONFIG_FILEPATH @@ -96,20 +96,20 @@ def default_config_file(self): # it will get treated as a PathExAttMap treats all properties, which # is that it will turn any double-slashes into single slashes. def template(self): - """ - Get the currently active submission template. + """Get the currently active submission template. - :return str: submission script content template for current state + Returns: + str: Submission script content template for current state. """ with open(self.compute["submission_template"], "r") as f: return f.read() @property def templates_folder(self): - """ - Path to folder with default submission templates. + """Path to folder with default submission templates. - :return str: path to folder with default submission templates + Returns: + str: Path to folder with default submission templates. """ if self.filepath: return os.path.join(os.path.dirname(self.filepath), "divvy_templates") @@ -119,16 +119,18 @@ def templates_folder(self): ) def activate_package(self, package_name): - """ - Activates a compute package. + """Activates a compute package. This copies the computing attributes from the configuration file into the `compute` attribute, where the class stores current compute settings. - :param str package_name: name for non-resource compute bundle, - the name of a subsection in an environment configuration file - :return bool: success flag for attempt to establish compute settings + Args: + package_name (str): Name for non-resource compute bundle, + the name of a subsection in an environment configuration file. + + Returns: + bool: Success flag for attempt to establish compute settings. """ # Hope that environment & environment compute are present. @@ -193,20 +195,22 @@ def activate_package(self, package_name): return False def clean_start(self, package_name): - """ - Clear current active settings and then activate the given package. + """Clear current active settings and then activate the given package. + + Args: + package_name (str): Name of the resource package to activate. - :param str package_name: name of the resource package to activate - :return bool: success flag + Returns: + bool: Success flag. """ self.reset_active_settings() return self.activate_package(package_name) def get_active_package(self) -> YAMLConfigManager: - """ - Returns settings for the currently active compute package + """Returns settings for the currently active compute package. - :return YAMLConfigManager: data defining the active compute package + Returns: + YAMLConfigManager: Data defining the active compute package. """ return self.compute @@ -216,46 +220,46 @@ def compute_packages(self): return self["compute_packages"] def list_compute_packages(self): - """ - Returns a list of available compute packages. + """Returns a list of available compute packages. - :return set[str]: names of available compute packages + Returns: + set[str]: Names of available compute packages. """ return set(self["compute_packages"].keys()) def reset_active_settings(self): - """ - Clear out current compute settings. + """Clear out current compute settings. - :return bool: success flag + Returns: + bool: Success flag. """ self.compute = YAMLConfigManager() return True def update_packages(self, config_file): - """ - Parse data from divvy configuration file. + """Parse data from divvy configuration file. Given a divvy configuration file, this function will update (not overwrite) existing compute packages with existing values. It does not affect any currently active settings. - :param str config_file: path to file with new divvy configuration data + Args: + config_file (str): Path to file with new divvy configuration data. """ entries = load_yaml(config_file) self.update(entries) return True def get_adapters(self) -> YAMLConfigManager: - """ - Get current adapters, if defined. + """Get current adapters, if defined. Adapters are sourced from the 'adapters' section in the root of the divvy configuration file and updated with an active compute package-specific set of adapters, if any defined in 'adapters' section under currently active compute package. - :return YAMLConfigManager: current adapters mapping + Returns: + YAMLConfigManager: Current adapters mapping. """ adapters = YAMLConfigManager() if "adapters" in self and self["adapters"] is not None: @@ -284,26 +288,31 @@ def submit(self, output_path, extra_vars=None): os.system(submission_command) def write_script(self, output_path, extra_vars=None): - """ - Given currently active settings, populate the active template to write a - submission script. Additionally use the current adapters to adjust - the select of the provided variables - - :param str output_path: Path to file to write as submission script - :param Iterable[Mapping] extra_vars: A list of Dict objects with - key-value pairs with which to populate template fields. These will - override any values in the currently active compute package. - :return str: Path to the submission script file + """Given currently active settings, populate the active template to write a submission script. + + Additionally use the current adapters to adjust the select of the + provided variables. + + Args: + output_path (str): Path to file to write as submission script. + extra_vars (Iterable[Mapping]): A list of Dict objects with + key-value pairs with which to populate template fields. These will + override any values in the currently active compute package. + + Returns: + str: Path to the submission script file. """ def _get_from_dict(map, attrs): - """ - Get value from a possibly mapping using a list of its attributes + """Get value from a possibly mapping using a list of its attributes. + + Args: + map (collections.Mapping): Mapping to retrieve values from. + attrs (Iterable[str]): A list of attributes. - :param collections.Mapping map: mapping to retrieve values from - :param Iterable[str] attrs: a list of attributes - :return: value found in the the requested attribute or - None if one of the keys does not exist + Returns: + Value found in the the requested attribute or None if one of the + keys does not exist. """ for a in attrs: try: @@ -372,16 +381,18 @@ def _handle_missing_env_attrs(self, config_file, when_missing): def select_divvy_config(filepath): - """ - Selects the divvy config file path to load. + """Selects the divvy config file path to load. This uses a priority ordering to first choose a config file path if it's given, but if not, then look in a priority list of environment variables and choose the first available file path to return. If none of these options succeed, the default config path will be returned. - :param str | NoneType filepath: direct file path specification - :return str: path to the config file to read + Args: + filepath (str | NoneType): Direct file path specification. + + Returns: + str: Path to the config file to read. """ divcfg = select_config( config_filepath=filepath, @@ -395,13 +406,13 @@ def select_divvy_config(filepath): def divvy_init(config_path, template_config_path): - """ - Initialize a genome config file. + """Initialize a genome config file. - :param str config_path: path to divvy configuration file to - create/initialize - :param str template_config_path: path to divvy configuration file to - copy FROM + Args: + config_path (str): Path to divvy configuration file to + create/initialize. + template_config_path (str): Path to divvy configuration file to + copy FROM. """ if not config_path: _LOGGER.error("You must specify a file path to initialize.") diff --git a/looper/looper.py b/looper/looper.py index fa0fd124..de101553 100755 --- a/looper/looper.py +++ b/looper/looper.py @@ -74,10 +74,10 @@ class Executor(object): __metaclass__ = abc.ABCMeta def __init__(self, prj): - """ - The Project defines the instance; establish an iteration counter. + """The Project defines the instance; establish an iteration counter. - :param Project prj: Project with which to work/operate on + Args: + prj (Project): Project with which to work/operate on. """ super(Executor, self).__init__() self.prj = prj @@ -91,10 +91,10 @@ def __call__(self, *args, **kwargs): class Checker(Executor): def __call__(self, args): - """ - Check Project status, using pipestat. + """Check Project status, using pipestat. - :param argparse.Namespace: arguments provided to the command + Args: + args (argparse.Namespace): Arguments provided to the command. """ # aggregate pipeline status data @@ -201,11 +201,11 @@ class Cleaner(Executor): """Remove all intermediate files (defined by pypiper clean scripts).""" def __call__(self, args, preview_flag=True): - """ - Execute the file cleaning process. + """Execute the file cleaning process. - :param argparse.Namespace args: command-line options and arguments - :param bool preview_flag: whether to halt before actually removing files + Args: + args (argparse.Namespace): Command-line options and arguments. + preview_flag (bool): Whether to halt before actually removing files. """ self.counter.show(name=self.prj.name, type="project") for sample in self.prj.samples: @@ -262,11 +262,11 @@ class Destroyer(Executor): """Destroyer of files and folders associated with Project's Samples""" def __call__(self, args, preview_flag=True): - """ - Completely remove all output produced by any pipelines. + """Completely remove all output produced by any pipelines. - :param argparse.Namespace args: command-line options and arguments - :param bool preview_flag: whether to halt before actually removing files + Args: + args (argparse.Namespace): Command-line options and arguments. + preview_flag (bool): Whether to halt before actually removing files. """ use_pipestat = ( @@ -328,21 +328,19 @@ class Collator(Executor): """Submitter for project-level pipelines""" def __init__(self, prj): - """ - Initializes an instance + """Initializes an instance. - :param Project prj: Project with which to work/operate on + Args: + prj (Project): Project with which to work/operate on. """ super(Executor, self).__init__() self.prj = prj def __call__(self, args, **compute_kwargs): - """ - Matches collators by protocols, creates submission scripts - and submits them + """Matches collators by protocols, creates submission scripts and submits them. - :param argparse.Namespace args: parsed command-line options and - arguments, recognized by looper + Args: + args (argparse.Namespace): Parsed command-line options and arguments, recognized by looper. """ jobs = 0 self.debug = {} @@ -389,15 +387,12 @@ class Runner(Executor): """The true submitter of pipelines""" def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs): - """ - Do the Sample submission. - - :param argparse.Namespace args: parsed command-line options and - arguments, recognized by looper - :param list remaining_args: command-line options and arguments not - recognized by looper, germane to samples/pipelines - :param bool rerun: whether the given sample is being rerun rather than - run for the first time + """Do the Sample submission. + + Args: + args (argparse.Namespace): Parsed command-line options and arguments, recognized by looper. + remaining_args (list): Command-line options and arguments not recognized by looper, germane to samples/pipelines. + rerun (bool): Whether the given sample is being rerun rather than run for the first time. """ self.debug = {} # initialize empty dict for return values max_cmds = sum(list(map(len, self.prj._samples_by_interface.values()))) @@ -649,9 +644,10 @@ def __call__(self, args): class Tabulator(Executor): - """Project/Sample statistics and table output generator + """Project/Sample statistics and table output generator. - :return list[str|any] results: list containing output file paths of stats and objects + Returns: + list[str|any]: List containing output file paths of stats and objects. """ def __call__(self, args): @@ -684,13 +680,11 @@ def _create_failure_message(reason, samples): def _remove_or_dry_run(paths, dry_run=False): - """ - Remove file or directory or just inform what would be removed in - case of dry run + """Remove file or directory or just inform what would be removed in case of dry run. - :param list|str paths: list of paths to files/dirs to be removed - :param bool dry_run: logical indicating whether the files should remain - untouched and message printed + Args: + paths (list|str): List of paths to files/dirs to be removed. + dry_run (bool): Logical indicating whether the files should remain untouched and message printed. """ paths = paths if isinstance(paths, list) else [paths] for path in paths: @@ -770,11 +764,10 @@ def destroy_summary(prj, dry_run=False, project_level=False): class LooperCounter(object): - """ - Count samples as you loop through them, and create text for the - subcommand logging status messages. + """Count samples as you loop through them, and create text for the subcommand logging status messages. - :param int total: number of jobs to process + Args: + total (int): Number of jobs to process. """ def __init__(self, total): @@ -782,17 +775,18 @@ def __init__(self, total): self.total = total def show(self, name, type="sample", pipeline_name=None): - """ - Display sample counts status for a particular protocol type. + """Display sample counts status for a particular protocol type. The counts are running vs. total for the protocol within the Project, and as a side-effect of the call, the running count is incremented. - :param str name: name of the sample - :param str type: the name of the level of entity being displayed, - either project or sample - :param str pipeline_name: name of the pipeline - :return str: message suitable for logging a status update + Args: + name (str): Name of the sample. + type (str): The name of the level of entity being displayed, either project or sample. + pipeline_name (str): Name of the pipeline. + + Returns: + str: Message suitable for logging a status update. """ self.count += 1 return _submission_status_text( diff --git a/looper/pipeline_interface.py b/looper/pipeline_interface.py index ee76f790..bf023961 100644 --- a/looper/pipeline_interface.py +++ b/looper/pipeline_interface.py @@ -40,15 +40,15 @@ @peputil.copy class PipelineInterface(YAMLConfigManager): """ - This class parses, holds, and returns information for a yaml file that - specifies how to interact with each individual pipeline. This - includes both resources to request for cluster job submission, as well as - arguments to be passed from the sample annotation metadata to the pipeline - - :param str | Mapping config: path to file from which to parse - configuration data, or pre-parsed configuration data. - :param str pipeline_type: type of the pipeline, - must be either 'sample' or 'project'. + This class parses, holds, and returns information for a yaml file that specifies how to interact with each individual pipeline. + + This includes both resources to request for cluster job submission, as well as + arguments to be passed from the sample annotation metadata to the pipeline. + + Args: + config (str | Mapping): Path to file from which to parse configuration data, + or pre-parsed configuration data. + pipeline_type (str): Type of the pipeline, must be either 'sample' or 'project'. """ def __init__(self, config, pipeline_type=None): @@ -78,7 +78,8 @@ def render_var_templates(self, namespaces): """ Render path templates under 'var_templates' in this pipeline interface. - :param dict namespaces: namespaces to use for rendering + Args: + namespaces (dict): Namespaces to use for rendering. """ try: curr_data = self[VAR_TEMPL_KEY] @@ -99,8 +100,11 @@ def get_pipeline_schemas(self, schema_key=INPUT_SCHEMA_KEY): """ Get path to the pipeline schema. - :param str schema_key: where to look for schemas in the pipeline iface - :return str: absolute path to the pipeline schema file + Args: + schema_key (str): Where to look for schemas in the pipeline iface. + + Returns: + str: Absolute path to the pipeline schema file. """ schema_source = None if schema_key in self: @@ -119,15 +123,19 @@ def choose_resource_package(self, namespaces, file_size): """ Select resource bundle for given input file size to given pipeline. - :param float file_size: Size of input data (in gigabytes). - :param Mapping[Mapping[str]] namespaces: namespaced variables to pass - as a context for fluid attributes command rendering - :return MutableMapping: resource bundle appropriate for given pipeline, - for given input file size - :raises ValueError: if indicated file size is negative, or if the - file size value specified for any resource package is negative - :raises InvalidResourceSpecificationException: if no default - resource package specification is provided + Args: + file_size (float): Size of input data (in gigabytes). + namespaces (Mapping[Mapping[str]]): Namespaced variables to pass as a context + for fluid attributes command rendering. + + Returns: + MutableMapping: Resource bundle appropriate for given pipeline, for given input file size. + + Raises: + ValueError: If indicated file size is negative, or if the file size value + specified for any resource package is negative. + InvalidResourceSpecificationException: If no default resource package + specification is provided. """ def _file_size_ante(name, data): @@ -157,12 +165,13 @@ def _notify(msg): def _load_dynamic_vars(pipeline): """ - Render command string (jinja2 template), execute it in a subprocess - and return its result (JSON object) as a dict + Render command string (jinja2 template), execute it in a subprocess and return its result (JSON object) as a dict. - :param Mapping pipeline: pipeline dict - :return Mapping: a dict with attributes returned in the JSON - by called command + Args: + pipeline (Mapping): Pipeline dict. + + Returns: + Mapping: A dict with attributes returned in the JSON by called command. """ def _log_raise_latest(): @@ -209,11 +218,14 @@ def _log_raise_latest(): def _load_size_dep_vars(piface): """ - Read the resources from a TSV provided in the pipeline interface + Read the resources from a TSV provided in the pipeline interface. + + Args: + piface (looper.PipelineInterface): Currently processed piface. + section (str): Section of pipeline interface to process. - :param looper.PipelineInterface piface: currently processed piface - :param str section: section of pipeline interface to process - :return pandas.DataFrame: resources + Returns: + pandas.DataFrame: Resources. """ df = None if COMPUTE_KEY in piface and SIZE_DEP_VARS_KEY in piface[COMPUTE_KEY]: @@ -291,20 +303,23 @@ def _load_size_dep_vars(piface): def _expand_paths(self, keys): """ - Expand paths defined in the pipeline interface file + Expand paths defined in the pipeline interface file. - :param list keys: list of keys resembling the nested structure to get - to the pipeline interface attributre to expand + Args: + keys (list): List of keys resembling the nested structure to get to the + pipeline interface attribute to expand. """ def _get_from_dict(map, attrs): """ - Get value from a possibly nested mapping using a list of its attributes + Get value from a possibly nested mapping using a list of its attributes. - :param collections.Mapping map: mapping to retrieve values from - :param Iterable[str] attrs: a list of attributes - :return: value found in the the requested attribute or - None if one of the keys does not exist + Args: + map (collections.Mapping): Mapping to retrieve values from. + attrs (Iterable[str]): A list of attributes. + + Returns: + Value found in the requested attribute or None if one of the keys does not exist. """ for a in attrs: try: @@ -315,13 +330,15 @@ def _get_from_dict(map, attrs): def _set_in_dict(map, attrs, val): """ - Set value in a mapping, creating a possibly nested structure + Set value in a mapping, creating a possibly nested structure. + + Args: + map (collections.Mapping): Mapping to retrieve values from. + attrs (Iterable[str]): A list of attributes. + val: Value to set. - :param collections.Mapping map: mapping to retrieve values from - :param Iterable[str] attrs: a list of attributes - :param val: value to set - :return: value found in the the requested attribute or - None if one of the keys does not exist + Returns: + Value found in the requested attribute or None if one of the keys does not exist. """ for a in attrs: if a == attrs[-1]: @@ -355,12 +372,13 @@ def _set_in_dict(map, attrs, val): def _validate(self, schema_src, exclude_case=False, flavor="generic"): """ - Generic function to validate the object against a schema + Generic function to validate the object against a schema. - :param str schema_src: schema source to validate against, URL or path - :param bool exclude_case: whether to exclude validated objects - from the error. Useful when used ith large projects - :param str flavor: type of the pipeline schema to use + Args: + schema_src (str): Schema source to validate against, URL or path. + exclude_case (bool): Whether to exclude validated objects from the error. + Useful when used with large projects. + flavor (str): Type of the pipeline schema to use. """ schema_source = schema_src.format(flavor) for schema in read_schema(schema_source): diff --git a/looper/processed_project.py b/looper/processed_project.py index 9222dadc..16cc0775 100644 --- a/looper/processed_project.py +++ b/looper/processed_project.py @@ -140,26 +140,31 @@ def _get_path_sect_keys(mapping, keys=[PATH_KEY]): - """ - Get names of subsections in a mapping that contain collection of keys + """Get names of subsections in a mapping that contain collection of keys. + + Args: + mapping (Mapping): Schema subsection to search for paths. + keys (Iterable[str]): Collection of keys to check for. - :param Mapping mapping: schema subsection to search for paths - :param Iterable[str] keys: collection of keys to check for - :return Iterable[str]: collection of keys to path-like sections + Returns: + Iterable[str]: Collection of keys to path-like sections. """ return [k for k, v in mapping.items() if bool(set(keys) & set(mapping[k]))] def _populate_paths(object, schema, check_exist): - """ - Populate path-like object attributes with other object attributes - based on a defined template, e.g. '/Users/x/test_{name}/{genome}_file.txt' - - :param Mapping object: object with attributes to populate path template with - :param dict schema: schema with path attributes defined, e.g. - output of read_schema function - :param bool check_exist: whether the paths should be check for existence - :return Mapping: object with path templates populated + """Populate path-like object attributes with other object attributes. + + Based on a defined template, e.g. '/Users/x/test_{name}/{genome}_file.txt' + + Args: + object (Mapping): Object with attributes to populate path template with. + schema (dict): Schema with path attributes defined, e.g. + output of read_schema function. + check_exist (bool): Whether the paths should be check for existence. + + Returns: + Mapping: Object with path templates populated. """ if PROP_KEY not in schema: raise EidoSchemaInvalidError("Schema is missing properties section.") @@ -189,15 +194,18 @@ def _populate_paths(object, schema, check_exist): def populate_sample_paths(sample, schema, check_exist=False): - """ - Populate path-like Sample attributes with other object attributes - based on a defined template, e.g. '/Users/x/test_{name}/{genome}_file.txt' - - :param peppy.Sample sample: sample to populate paths in - :param Iterable[dict] schema: schema with path attributes defined, e.g. - output of read_schema function - :param bool check_exist: whether the paths should be check for existence - :return Mapping: Sample with path templates populated + """Populate path-like Sample attributes with other object attributes. + + Based on a defined template, e.g. '/Users/x/test_{name}/{genome}_file.txt' + + Args: + sample (peppy.Sample): Sample to populate paths in. + schema (Iterable[dict]): Schema with path attributes defined, e.g. + output of read_schema function. + check_exist (bool): Whether the paths should be check for existence. + + Returns: + Mapping: Sample with path templates populated. """ if not isinstance(sample, Sample): raise TypeError("Can only populate paths in peppy.Sample objects") @@ -207,15 +215,18 @@ def populate_sample_paths(sample, schema, check_exist=False): def populate_project_paths(project, schema, check_exist=False): - """ - Populate path-like Project attributes with other object attributes - based on a defined template, e.g. '/Users/x/test_{name}/{genome}_file.txt' - - :param peppy.Project project: project to populate paths in - :param dict schema: schema with path attributes defined, e.g. - output of read_schema function - :param bool check_exist: whether the paths should be check for existence - :return Mapping: Project with path templates populated + """Populate path-like Project attributes with other object attributes. + + Based on a defined template, e.g. '/Users/x/test_{name}/{genome}_file.txt' + + Args: + project (peppy.Project): Project to populate paths in. + schema (dict): Schema with path attributes defined, e.g. + output of read_schema function. + check_exist (bool): Whether the paths should be check for existence. + + Returns: + Mapping: Project with path templates populated. """ if not isinstance(project, Project): raise TypeError("Can only populate paths in peppy.Project objects") @@ -223,13 +234,14 @@ def populate_project_paths(project, schema, check_exist=False): def get_project_outputs(project, schema): - """ - Get project level outputs with path-like attributes populated with - project attributes + """Get project level outputs with path-like attributes populated with project attributes. + + Args: + project (peppy.Project): Project to get outputs for. + schema (Iterable[dict]): Schema to source the outputs from. - :param peppy.Project project: - :param Iterable[dict] schema: - :return yacman.YAMLConfigManager: mapping with populated path-like attributes + Returns: + yacman.YAMLConfigManager: Mapping with populated path-like attributes. """ from yacman import YAMLConfigManager diff --git a/looper/project.py b/looper/project.py index 18fb1a4b..9a854370 100644 --- a/looper/project.py +++ b/looper/project.py @@ -123,18 +123,18 @@ def __exit__(self, *args): class Project(peppyProject): - """ - Looper-specific Project. - - :param str cfg: path to configuration file with data from - which Project is to be built - :param Iterable[str] amendments: name indicating amendment to use, optional - :param str divcfg_path: path to an environment configuration YAML file - specifying compute settings. - :param bool permissive: Whether a error should be thrown if - a sample input file(s) do not exist or cannot be open. - :param str compute_env_file: Environment configuration YAML file specifying - compute settings. + """Looper-specific Project. + + Args: + cfg (str): Path to configuration file with data from which Project is + to be built. + amendments (Iterable[str]): Name indicating amendment to use, optional. + divcfg_path (str): Path to an environment configuration YAML file + specifying compute settings. + permissive (bool): Whether a error should be thrown if a sample input + file(s) do not exist or cannot be open. + compute_env_file (str): Environment configuration YAML file specifying + compute settings. """ def __init__(self, cfg=None, amendments=None, divcfg_path=None, **kwargs): @@ -184,28 +184,28 @@ def __init__(self, cfg=None, amendments=None, divcfg_path=None, **kwargs): @property def piface_key(self): - """ - Name of the pipeline interface attribute for this project + """Name of the pipeline interface attribute for this project. - :return str: name of the pipeline interface attribute + Returns: + str: Name of the pipeline interface attribute. """ return self._extra_cli_or_cfg(PIFACE_KEY_SELECTOR) or PIPELINE_INTERFACES_KEY @property def selected_compute_package(self): - """ - Compute package name specified in object constructor + """Compute package name specified in object constructor. - :return str: compute package name + Returns: + str: Compute package name. """ return self._extra_cli_or_cfg(COMPUTE_PACKAGE_KEY) @property def cli_pifaces(self): - """ - Collection of pipeline interface sources specified in object constructor + """Collection of pipeline interface sources specified in object constructor. - :return list[str]: collection of pipeline interface sources + Returns: + list[str]: Collection of pipeline interface sources. """ x = self._extra_cli_or_cfg(self.piface_key) return ( @@ -216,22 +216,22 @@ def cli_pifaces(self): @property def output_dir(self): - """ - Output directory for the project, specified in object constructor + """Output directory for the project, specified in object constructor. - :return str: path to the output directory + Returns: + str: Path to the output directory. """ return self._extra_cli_or_cfg(OUTDIR_KEY, strict=True) def _extra_cli_or_cfg(self, attr_name, strict=False): - """ - Get attribute value provided in kwargs in object constructor of from - looper section in the configuration file + """Get attribute value provided in kwargs in object constructor of from looper section in the configuration file. + + Args: + attr_name (str): Name of the attribute to get value for. + strict (bool): Whether a non-existent attribute is exceptional. - :param str attr_name: name of the attribute to get value for - :param bool strict: whether a non-existent attribute is exceptional - :raise MisconfigurationException: in strict mode, when no attribute - found + Raises: + MisconfigurationException: In strict mode, when no attribute found. """ try: result = self[EXTRA_KEY][attr_name] @@ -257,31 +257,34 @@ def _extra_cli_or_cfg(self, attr_name, strict=False): @property def results_folder(self): - """ - Path to the results folder for the project + """Path to the results folder for the project. - :return str: path to the results folder in the output folder + Returns: + str: Path to the results folder in the output folder. """ return self._out_subdir_path(RESULTS_SUBDIR_KEY, default="results_pipeline") @property def submission_folder(self): - """ - Path to the submission folder for the project + """Path to the submission folder for the project. - :return str: path to the submission in the output folder + Returns: + str: Path to the submission in the output folder. """ return self._out_subdir_path(SUBMISSION_SUBDIR_KEY, default="submission") def _out_subdir_path(self, key: str, default: str) -> str: - """ - Create a system path relative to the project output directory. + """Create a system path relative to the project output directory. + The values for the names of the subdirectories are sourced from kwargs passed to the object constructor. - :param str key: name of the attribute mapped to the value of interest - :param str default: if key not specified, a default to use - :return str: path to the folder + Args: + key (str): Name of the attribute mapped to the value of interest. + default (str): If key not specified, a default to use. + + Returns: + str: Path to the folder. """ parent = getattr(self, OUTDIR_KEY) child = getattr(self[EXTRA_KEY], key, default) or default @@ -307,11 +310,12 @@ def make_project_dirs(self): @cached_property def project_pipeline_interface_sources(self): - """ - Get a list of all valid project-level pipeline interface sources - associated with this project. Sources that are file paths are expanded + """Get a list of all valid project-level pipeline interface sources associated with this project. - :return list[str]: collection of valid pipeline interface sources: + Sources that are file paths are expanded. + + Returns: + list[str]: Collection of valid pipeline interface sources. """ return ( [self._resolve_path_with_cfg(src) for src in self.cli_pifaces] @@ -321,15 +325,14 @@ def project_pipeline_interface_sources(self): @cached_property def project_pipeline_interfaces(self): - """ - Flat list of all valid project-level interface objects associated - with this Project + """Flat list of all valid project-level interface objects associated with this Project. Note that only valid pipeline interfaces will show up in the result (ones that exist on disk/remotely and validate successfully - against the schema) + against the schema). - :return list[looper.PipelineInterface]: list of pipeline interfaces + Returns: + list[looper.PipelineInterface]: List of pipeline interfaces. """ return [ PipelineInterface(pi, pipeline_type=PipelineLevel.PROJECT.value) @@ -338,59 +341,62 @@ def project_pipeline_interfaces(self): @cached_property def pipeline_interfaces(self): - """ - Flat list of all valid interface objects associated with this Project + """Flat list of all valid interface objects associated with this Project. Note that only valid pipeline interfaces will show up in the result (ones that exist on disk/remotely and validate successfully - against the schema) + against the schema). - :return list[looper.PipelineInterface]: list of pipeline interfaces + Returns: + list[looper.PipelineInterface]: List of pipeline interfaces. """ return [pi for ifaces in self._interfaces_by_sample.values() for pi in ifaces] @cached_property def pipeline_interface_sources(self): - """ - Get a list of all valid pipeline interface sources associated - with this project. Sources that are file paths are expanded + """Get a list of all valid pipeline interface sources associated with this project. - :return list[str]: collection of valid pipeline interface sources + Sources that are file paths are expanded. + + Returns: + list[str]: Collection of valid pipeline interface sources. """ return self._samples_by_interface.keys() @cached_property def pipestat_configured(self): - """ - Whether pipestat configuration is complete for all sample pipelines + """Whether pipestat configuration is complete for all sample pipelines. - :return bool: whether pipestat configuration is complete + Returns: + bool: Whether pipestat configuration is complete. """ return self._check_if_pipestat_configured() @cached_property def pipestat_configured_project(self): - """ - Whether pipestat configuration is complete for all project pipelines + """Whether pipestat configuration is complete for all project pipelines. - :return bool: whether pipestat configuration is complete + Returns: + bool: Whether pipestat configuration is complete. """ return self._check_if_pipestat_configured( pipeline_type=PipelineLevel.PROJECT.value ) def get_sample_piface(self, sample_name): - """ - Get a list of pipeline interfaces associated with the specified sample. + """Get a list of pipeline interfaces associated with the specified sample. Note that only valid pipeline interfaces will show up in the result (ones that exist on disk/remotely and validate successfully - against the schema) + against the schema). - :param str sample_name: name of the sample to retrieve list of - pipeline interfaces for - :return list[looper.PipelineInterface]: collection of valid - pipeline interfaces associated with selected sample + Args: + sample_name (str): Name of the sample to retrieve list of pipeline + interfaces for. + + Returns: + list[looper.PipelineInterface]: Collection of valid pipeline + interfaces associated with selected sample. """ try: return self._interfaces_by_sample[sample_name] @@ -399,13 +405,14 @@ def get_sample_piface(self, sample_name): @staticmethod def get_schemas(pifaces, schema_key=INPUT_SCHEMA_KEY): - """ - Get the list of unique schema paths for a list of pipeline interfaces + """Get the list of unique schema paths for a list of pipeline interfaces. - :param str | Iterable[str] pifaces: pipeline interfaces to search - schemas for - :param str schema_key: where to look for schemas in the piface - :return Iterable[str]: unique list of schema file paths + Args: + pifaces (str | Iterable[str]): Pipeline interfaces to search schemas for. + schema_key (str): Where to look for schemas in the piface. + + Returns: + Iterable[str]: Unique list of schema file paths. """ if isinstance(pifaces, str): pifaces = [pifaces] @@ -625,16 +632,16 @@ def populate_pipeline_outputs(self): populate_project_paths(self, read_schema(schema)[0]) def _get_linked_pifaces(self): - """ - Get linked sample pipeline interfaces by project pipeline interface. + """Get linked sample pipeline interfaces by project pipeline interface. These are indicated in project pipeline interface by 'linked_pipeline_interfaces' key. If a project pipeline interface - does not have such key defined, an empty list is returned for that - pipeline interface. + does not have such key defined, an empty list is returned for that + pipeline interface. - :return dict[list[str]]: mapping of sample pipeline interfaces - by project pipeline interfaces + Returns: + dict[list[str]]: Mapping of sample pipeline interfaces by project + pipeline interfaces. """ def _process_linked_piface(p, piface, prj_piface): @@ -662,11 +669,11 @@ def _process_linked_piface(p, piface, prj_piface): return linked_pifaces def _piface_by_samples(self): - """ - Create a mapping of all defined interfaces in this Project by samples. + """Create a mapping of all defined interfaces in this Project by samples. - :return dict[str, list[PipelineInterface]]: a collection of pipeline - interfaces keyed by sample name + Returns: + dict[str, list[PipelineInterface]]: A collection of pipeline + interfaces keyed by sample name. """ pifaces_by_sample = {} for source, sample_names in self._samples_by_interface.items(): @@ -680,20 +687,22 @@ def _piface_by_samples(self): return pifaces_by_sample def _omit_from_repr(self, k, cls): - """ - Exclude the interfaces from representation. + """Exclude the interfaces from representation. - :param str k: key of item to consider for omission - :param type cls: placeholder to comply with superclass signature + Args: + k (str): Key of item to consider for omission. + cls (type): Placeholder to comply with superclass signature. """ return super(Project, self)._omit_from_repr(k, cls) or k == "interfaces" def _resolve_path_with_cfg(self, pth): - """ - Expand provided path and make it absolute using project config path + """Expand provided path and make it absolute using project config path. + + Args: + pth (str): Path, possibly including env vars and/or relative. - :param str pth: path, possibly including env vars and/or relative - :return str: absolute path + Returns: + str: Absolute path. """ if pth is None: return @@ -704,13 +713,13 @@ def _resolve_path_with_cfg(self, pth): return pth def _samples_by_piface(self, piface_key): - """ - Create a collection of all samples with valid pipeline interfaces + """Create a collection of all samples with valid pipeline interfaces. + + Args: + piface_key (str): Name of the attribute that holds pipeline interfaces. - :param str piface_key: name of the attribute that holds pipeline - interfaces - :return list[str]: a collection of samples keyed by pipeline interface - source + Returns: + list[str]: A collection of samples keyed by pipeline interface source. """ samples_by_piface = {} msgs = set() @@ -746,10 +755,10 @@ def _samples_by_piface(self, piface_key): return samples_by_piface def set_sample_piface(self, sample_piface: Union[List[str], str]) -> NoReturn: - """ - Add sample pipeline interfaces variable to object + """Add sample pipeline interfaces variable to object. - :param list | str sample_piface: sample pipeline interface + Args: + sample_piface (list | str): Sample pipeline interface. """ self.config.setdefault("sample_modifiers", {}) self.config["sample_modifiers"].setdefault("append", {}) @@ -766,37 +775,37 @@ def fetch_samples( selector_flag=None, exclusion_flag=None, ): - """ - Collect samples of particular protocol(s). + """Collect samples of particular protocol(s). Protocols can't be both positively selected for and negatively selected against. That is, it makes no sense and is not allowed to specify both selector_include and selector_exclude protocols. On the - other hand, if - neither is provided, all of the Project's Samples are returned. + other hand, if neither is provided, all of the Project's Samples are returned. If selector_include is specified, Samples without a protocol will be - excluded, - but if selector_exclude is specified, protocol-less Samples will be + excluded, but if selector_exclude is specified, protocol-less Samples will be included. - :param Project prj: the Project with Samples to fetch - :param str selector_attribute: name of attribute on which to base the - fetch - :param Iterable[str] | str selector_include: protocol(s) of interest; - if specified, a Sample must - :param Iterable[str] | str selector_exclude: protocol(s) to include - :param Iterable[str] | str selector_flag: flag to select on, e.g. FAILED, COMPLETED - :param Iterable[str] | str exclusion_flag: flag to exclude on, e.g. FAILED, COMPLETED - :return list[Sample]: Collection of this Project's samples with - protocol that either matches one of those in selector_include, - or either - lacks a protocol or does not match one of those in selector_exclude - :raise TypeError: if both selector_include and selector_exclude - protocols are - specified; TypeError since it's basically providing two arguments - when only one is accepted, so remain consistent with vanilla - Python2; - also possible if name of attribute for selection isn't a string + Args: + prj (Project): The Project with Samples to fetch. + selector_attribute (str): Name of attribute on which to base the fetch. + selector_include (Iterable[str] | str): Protocol(s) of interest; if + specified, a Sample must. + selector_exclude (Iterable[str] | str): Protocol(s) to include. + selector_flag (Iterable[str] | str): Flag to select on, e.g. FAILED, + COMPLETED. + exclusion_flag (Iterable[str] | str): Flag to exclude on, e.g. FAILED, + COMPLETED. + + Returns: + list[Sample]: Collection of this Project's samples with protocol that + either matches one of those in selector_include, or either lacks a + protocol or does not match one of those in selector_exclude. + + Raises: + TypeError: If both selector_include and selector_exclude protocols are + specified; TypeError since it's basically providing two arguments + when only one is accepted, so remain consistent with vanilla Python2; + also possible if name of attribute for selection isn't a string. """ kept_samples = prj.samples From 90a2e6234cc850942f125cd7a67392c130fcfad7 Mon Sep 17 00:00:00 2001 From: Nathan Sheffield Date: Wed, 5 Nov 2025 20:01:12 -0500 Subject: [PATCH 5/6] Update looper/utils.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- looper/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/looper/utils.py b/looper/utils.py index 9cc57ef6..e119947d 100644 --- a/looper/utils.py +++ b/looper/utils.py @@ -56,7 +56,7 @@ def fetch_flag_files(prj=None, results_folder="", flags=FLAGS): function uses the assumption that if results_folder rather than project is provided, the structure of the file tree rooted at results_folder is such that any flag files to be found are not directly within rootdir but - are directly within on of its first layer of subfolders. + are directly within one of its first layer of subfolders. flags (Iterable[str] | str): Collection of flag names or single flag name for which to fetch files. From e518d64b66f76643ebd45479e8ba9ad6fa610ee3 Mon Sep 17 00:00:00 2001 From: Nathan Sheffield Date: Wed, 5 Nov 2025 20:01:39 -0500 Subject: [PATCH 6/6] Update looper/project.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- looper/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/looper/project.py b/looper/project.py index 9a854370..60c476c1 100644 --- a/looper/project.py +++ b/looper/project.py @@ -224,7 +224,7 @@ def output_dir(self): return self._extra_cli_or_cfg(OUTDIR_KEY, strict=True) def _extra_cli_or_cfg(self, attr_name, strict=False): - """Get attribute value provided in kwargs in object constructor of from looper section in the configuration file. + """Get attribute value provided in kwargs in object constructor or from looper section in the configuration file. Args: attr_name (str): Name of the attribute to get value for.