From 609752a70adf568f9d5a74e1b6d2846083fa883e Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Tue, 18 Feb 2025 09:48:41 -0800 Subject: [PATCH 1/2] NextFlow improvements: unique job names, full file list, skip template file --- .../labkey/nextflow/NextFlowController.java | 28 +++++++++++++++---- .../pipeline/NextFlowPipelineJob.java | 12 ++++---- .../nextflow/pipeline/NextFlowRunTask.java | 26 ++++++++++++----- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/nextflow/src/org/labkey/nextflow/NextFlowController.java b/nextflow/src/org/labkey/nextflow/NextFlowController.java index 69e1297d..520cbc51 100644 --- a/nextflow/src/org/labkey/nextflow/NextFlowController.java +++ b/nextflow/src/org/labkey/nextflow/NextFlowController.java @@ -14,6 +14,7 @@ import org.labkey.api.data.PropertyStore; import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineProvider; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineStatusUrls; import org.labkey.api.pipeline.browse.PipelinePathForm; @@ -38,11 +39,13 @@ import org.labkey.api.view.UnauthorizedException; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.nextflow.pipeline.NextFlowPipelineJob; +import org.labkey.nextflow.pipeline.NextFlowProtocol; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; import java.io.File; +import java.nio.file.Files; import java.util.Arrays; import java.util.List; @@ -262,24 +265,39 @@ public void validateCommand(AnalyzeForm o, Errors errors) @Override public ModelAndView getView(AnalyzeForm o, boolean b, BindException errors) { + List selectedFiles = o.getValidatedFiles(getContainer(), false); + if (selectedFiles.isEmpty()) + { + return new HtmlView(HtmlString.of("Couldn't find input file(s)")); + } + // NextFlow operates on the full directory so show the list to the user, regardless of what they selected + // from the file listing + File inputDir = selectedFiles.get(0).getParentFile(); + + File[] inputFiles = inputDir.listFiles(new PipelineProvider.FileTypesEntryFilter(NextFlowProtocol.INPUT_TYPES)); + if (inputFiles == null || inputFiles.length == 0) + { + return new HtmlView(HtmlString.of("Couldn't find input file(s)")); + } + NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); if (config.getNextFlowConfigFilePath() != null) { File configDir = new File(config.getNextFlowConfigFilePath()); if (configDir.isDirectory()) { - File[] files = configDir.listFiles(); - if (files != null && files.length > 0) + File[] configFiles = configDir.listFiles(); + if (configFiles != null && configFiles.length > 0) { - List configFiles = Arrays.asList(files); return new HtmlView("NextFlow Runner", DIV( FORM(at(method, "POST"), INPUT(at(hidden, true, name, "launch", value, true)), Arrays.stream(o.getFile()).map(f -> INPUT(at(hidden, true, name, "file", value, f))).toList(), "Files: ", - UL(Arrays.stream(o.getFile()).map(DOM::LI)), + UL(Arrays.stream(inputFiles).map(File::getName).map(DOM::LI)), "Config: ", - new Select.SelectBuilder().name("configFile").addOptions(configFiles.stream().filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(), + new Select.SelectBuilder().name("configFile").addOptions(Arrays.stream(configFiles).filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(), + DOM.BR(), new Button.ButtonBuilder("Start NextFlow").submit(true).build()))); } } diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java index 6f31dafa..8a3cd0c0 100644 --- a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java @@ -59,24 +59,26 @@ public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, Path super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getFileName().toString(), config, inputFiles, false, false); this.config = config; setLogFile(log); - LOG.info("NextFlow job queued: {}", getJsonJobInfo()); + LOG.info("NextFlow job queued: {}", getJsonJobInfo(null)); } - protected JSONObject getJsonJobInfo() + protected JSONObject getJsonJobInfo(Long invocationCount) { JSONObject result = new JSONObject(); result.put("user", getUser().getEmail()); result.put("container", getContainer().getPath()); result.put("filePath", getLogFilePath().getParent().toString()); - result.put("runName", getNextFlowRunName()); + result.put("runName", getNextFlowRunName(invocationCount)); result.put("configFile", getConfig().getFileName().toString()); return result; } - protected String getNextFlowRunName() + protected String getNextFlowRunName(Long invocationCount) { PipelineStatusFile file = PipelineService.get().getStatusFile(getJobGUID()); - return file == null ? "Unknown" : ("LabKeyJob" + file.getRowId()); + String result = file == null ? "Unknown" : ("LabKeyJob" + file.getRowId()); + result += invocationCount == null ? "" : ("_" + invocationCount); + return result; } @Override diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java index 35670120..2749796f 100644 --- a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java @@ -2,6 +2,9 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSequence; +import org.labkey.api.data.DbSequenceManager; import org.labkey.api.exp.XarFormatException; import org.labkey.api.pipeline.AbstractTaskFactory; import org.labkey.api.pipeline.AbstractTaskFactorySettings; @@ -37,6 +40,8 @@ public class NextFlowRunTask extends WorkDirectoryTask public static final String ACTION_NAME = "NextFlow"; + private static final DbSequence INVOCATION_SEQUENCE = DbSequenceManager.get(ContainerManager.getRoot(), NextFlowRunTask.class.getName()); + public NextFlowRunTask(Factory factory, PipelineJob job) { super(factory, job); @@ -46,7 +51,12 @@ public NextFlowRunTask(Factory factory, PipelineJob job) public @NotNull RecordedActionSet run() throws PipelineJobException { Logger log = getJob().getLogger(); - NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo()); + + // NextFlow requires a unique job name for every execution. Increment a counter to append as a suffix to + // ensure uniqueness + long invocationCount = INVOCATION_SEQUENCE.next(); + INVOCATION_SEQUENCE.sync(); + NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo(invocationCount)); SecurityManager.TransformSession session = null; boolean success = false; @@ -73,10 +83,10 @@ public NextFlowRunTask(Factory factory, PipelineJob job) File dir = getJob().getLogFile().getParentFile(); getJob().runSubProcess(secretsPB, dir); - ProcessBuilder executionPB = new ProcessBuilder(getArgs()); + ProcessBuilder executionPB = new ProcessBuilder(getArgs(invocationCount)); getJob().runSubProcess(executionPB, dir); log.info("Job Finished"); - NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo()); + NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount)); RecordedAction action = new RecordedAction(ACTION_NAME); for (Path inputFile : getJob().getInputFilePaths()) @@ -100,14 +110,16 @@ public NextFlowRunTask(Factory factory, PipelineJob job) } if (!success) { - NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo()); + NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount)); } } } private void addOutputs(RecordedAction action, Path path, Logger log) throws IOException { - if (Files.isRegularFile(path)) + // Skip results.sky.zip files - it's the template document. We want the file output doc that includes + // the replicate analysis + if (Files.isRegularFile(path) && !path.endsWith("results.sky.zip")) { action.addOutput(path.toFile(), "Output", false); if (path.toString().toLowerCase().endsWith(".sky.zip")) @@ -164,7 +176,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException } - private @NotNull List getArgs() throws PipelineJobException + private @NotNull List getArgs(long invocationCount) throws PipelineJobException { NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); Path configFile = getJob().getConfig(); @@ -189,7 +201,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException args.add("-c"); args.add(configFile.toAbsolutePath().toString()); args.add("-name"); - args.add(getJob().getNextFlowRunName()); + args.add(getJob().getNextFlowRunName(invocationCount)); return args; } From 73d10802509e926fed84c1f6c27908f500612d81 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Tue, 18 Feb 2025 10:08:55 -0800 Subject: [PATCH 2/2] Code cleanup --- nextflow/src/org/labkey/nextflow/NextFlowController.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nextflow/src/org/labkey/nextflow/NextFlowController.java b/nextflow/src/org/labkey/nextflow/NextFlowController.java index 520cbc51..ea758886 100644 --- a/nextflow/src/org/labkey/nextflow/NextFlowController.java +++ b/nextflow/src/org/labkey/nextflow/NextFlowController.java @@ -3,7 +3,6 @@ import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; import org.labkey.api.action.ApiResponse; import org.labkey.api.action.ApiSimpleResponse; import org.labkey.api.action.FormViewAction; @@ -32,7 +31,6 @@ import org.labkey.api.util.Path; import org.labkey.api.util.URLHelper; import org.labkey.api.util.element.Select; -import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.HtmlView; import org.labkey.api.view.JspView; import org.labkey.api.view.NavTree; @@ -45,7 +43,6 @@ import org.springframework.web.servlet.ModelAndView; import java.io.File; -import java.nio.file.Files; import java.util.Arrays; import java.util.List; @@ -67,8 +64,6 @@ public class NextFlowController extends SpringActionController private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(NextFlowController.class); public static final String NAME = "nextflow"; - private static final Logger LOG = LogHelper.getLogger(NextFlowController.class, NAME); - public NextFlowController() { setActionResolver(_actionResolver);