Skip to content

NextFlow improvements: unique job names, full file list, skip template file #508

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
labkey-jeckels merged 2 commits into from
Feb 18, 2025
Merged

NextFlow improvements: unique job names, full file list, skip template file #508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
609752a
NextFlow improvements: unique job names, full file list, skip templat…
labkey-jeckels Feb 18, 2025
73d1080
Code cleanup
labkey-jeckels Feb 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 22 additions & 9 deletions nextflow/src/org/labkey/nextflow/NextFlowController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.labkey.api.action.ApiResponse;
import org.labkey.api.action.ApiSimpleResponse;
import org.labkey.api.action.FormViewAction;
Expand All @@ -14,6 +13,7 @@
import org.labkey.api.data.PropertyStore;
import org.labkey.api.pipeline.PipeRoot;
import org.labkey.api.pipeline.PipelineJob;
import org.labkey.api.pipeline.PipelineProvider;
import org.labkey.api.pipeline.PipelineService;
import org.labkey.api.pipeline.PipelineStatusUrls;
import org.labkey.api.pipeline.browse.PipelinePathForm;
Expand All @@ -31,13 +31,13 @@
import org.labkey.api.util.Path;
import org.labkey.api.util.URLHelper;
import org.labkey.api.util.element.Select;
import org.labkey.api.util.logging.LogHelper;
import org.labkey.api.view.HtmlView;
import org.labkey.api.view.JspView;
import org.labkey.api.view.NavTree;
import org.labkey.api.view.UnauthorizedException;
import org.labkey.api.view.ViewBackgroundInfo;
import org.labkey.nextflow.pipeline.NextFlowPipelineJob;
import org.labkey.nextflow.pipeline.NextFlowProtocol;
import org.springframework.validation.BindException;
import org.springframework.validation.Errors;
import org.springframework.web.servlet.ModelAndView;
Expand All @@ -64,8 +64,6 @@ public class NextFlowController extends SpringActionController
private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(NextFlowController.class);
public static final String NAME = "nextflow";

private static final Logger LOG = LogHelper.getLogger(NextFlowController.class, NAME);

public NextFlowController()
{
setActionResolver(_actionResolver);
Expand Down Expand Up @@ -262,24 +260,39 @@ public void validateCommand(AnalyzeForm o, Errors errors)
@Override
public ModelAndView getView(AnalyzeForm o, boolean b, BindException errors)
{
List<File> selectedFiles = o.getValidatedFiles(getContainer(), false);
if (selectedFiles.isEmpty())
{
return new HtmlView(HtmlString.of("Couldn't find input file(s)"));
}
// NextFlow operates on the full directory so show the list to the user, regardless of what they selected
// from the file listing
File inputDir = selectedFiles.get(0).getParentFile();

File[] inputFiles = inputDir.listFiles(new PipelineProvider.FileTypesEntryFilter(NextFlowProtocol.INPUT_TYPES));
if (inputFiles == null || inputFiles.length == 0)
{
return new HtmlView(HtmlString.of("Couldn't find input file(s)"));
}

NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
if (config.getNextFlowConfigFilePath() != null)
{
File configDir = new File(config.getNextFlowConfigFilePath());
if (configDir.isDirectory())
{
File[] files = configDir.listFiles();
if (files != null && files.length > 0)
File[] configFiles = configDir.listFiles();
if (configFiles != null && configFiles.length > 0)
{
List<File> configFiles = Arrays.asList(files);
return new HtmlView("NextFlow Runner", DIV(
FORM(at(method, "POST"),
INPUT(at(hidden, true, name, "launch", value, true)),
Arrays.stream(o.getFile()).map(f -> INPUT(at(hidden, true, name, "file", value, f))).toList(),
"Files: ",
UL(Arrays.stream(o.getFile()).map(DOM::LI)),
UL(Arrays.stream(inputFiles).map(File::getName).map(DOM::LI)),
"Config: ",
new Select.SelectBuilder().name("configFile").addOptions(configFiles.stream().filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(),
new Select.SelectBuilder().name("configFile").addOptions(Arrays.stream(configFiles).filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(),
DOM.BR(),
new Button.ButtonBuilder("Start NextFlow").submit(true).build())));
}
}
Expand Down
12 changes: 7 additions & 5 deletions nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,24 +59,26 @@ public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, Path
super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getFileName().toString(), config, inputFiles, false, false);
this.config = config;
setLogFile(log);
LOG.info("NextFlow job queued: {}", getJsonJobInfo());
LOG.info("NextFlow job queued: {}", getJsonJobInfo(null));
}

protected JSONObject getJsonJobInfo()
protected JSONObject getJsonJobInfo(Long invocationCount)
{
JSONObject result = new JSONObject();
result.put("user", getUser().getEmail());
result.put("container", getContainer().getPath());
result.put("filePath", getLogFilePath().getParent().toString());
result.put("runName", getNextFlowRunName());
result.put("runName", getNextFlowRunName(invocationCount));
result.put("configFile", getConfig().getFileName().toString());
return result;
}

protected String getNextFlowRunName()
protected String getNextFlowRunName(Long invocationCount)
{
PipelineStatusFile file = PipelineService.get().getStatusFile(getJobGUID());
return file == null ? "Unknown" : ("LabKeyJob" + file.getRowId());
String result = file == null ? "Unknown" : ("LabKeyJob" + file.getRowId());
result += invocationCount == null ? "" : ("_" + invocationCount);
return result;
}

@Override
Expand Down
26 changes: 19 additions & 7 deletions nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.labkey.api.data.ContainerManager;
import org.labkey.api.data.DbSequence;
import org.labkey.api.data.DbSequenceManager;
import org.labkey.api.exp.XarFormatException;
import org.labkey.api.pipeline.AbstractTaskFactory;
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
Expand Down Expand Up @@ -37,6 +40,8 @@ public class NextFlowRunTask extends WorkDirectoryTask<NextFlowRunTask.Factory>

public static final String ACTION_NAME = "NextFlow";

private static final DbSequence INVOCATION_SEQUENCE = DbSequenceManager.get(ContainerManager.getRoot(), NextFlowRunTask.class.getName());

public NextFlowRunTask(Factory factory, PipelineJob job)
{
super(factory, job);
Expand All @@ -46,7 +51,12 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
public @NotNull RecordedActionSet run() throws PipelineJobException
{
Logger log = getJob().getLogger();
NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo());

// NextFlow requires a unique job name for every execution. Increment a counter to append as a suffix to
// ensure uniqueness
long invocationCount = INVOCATION_SEQUENCE.next();
INVOCATION_SEQUENCE.sync();
NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo(invocationCount));

SecurityManager.TransformSession session = null;
boolean success = false;
Expand All @@ -73,10 +83,10 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
File dir = getJob().getLogFile().getParentFile();
getJob().runSubProcess(secretsPB, dir);

ProcessBuilder executionPB = new ProcessBuilder(getArgs());
ProcessBuilder executionPB = new ProcessBuilder(getArgs(invocationCount));
getJob().runSubProcess(executionPB, dir);
log.info("Job Finished");
NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo());
NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount));

RecordedAction action = new RecordedAction(ACTION_NAME);
for (Path inputFile : getJob().getInputFilePaths())
Expand All @@ -100,14 +110,16 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
}
if (!success)
{
NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo());
NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
}
}
}

private void addOutputs(RecordedAction action, Path path, Logger log) throws IOException
{
if (Files.isRegularFile(path))
// Skip results.sky.zip files - it's the template document. We want the file output doc that includes
// the replicate analysis
if (Files.isRegularFile(path) && !path.endsWith("results.sky.zip"))
{
action.addOutput(path.toFile(), "Output", false);
if (path.toString().toLowerCase().endsWith(".sky.zip"))
Expand Down Expand Up @@ -164,7 +176,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
}


private @NotNull List<String> getArgs() throws PipelineJobException
private @NotNull List<String> getArgs(long invocationCount) throws PipelineJobException
{
NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
Path configFile = getJob().getConfig();
Expand All @@ -189,7 +201,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
args.add("-c");
args.add(configFile.toAbsolutePath().toString());
args.add("-name");
args.add(getJob().getNextFlowRunName());
args.add(getJob().getNextFlowRunName(invocationCount));
return args;
}

Expand Down