diff --git a/nextflow/build.gradle b/nextflow/build.gradle index 01f789da..bb072afd 100644 --- a/nextflow/build.gradle +++ b/nextflow/build.gradle @@ -6,5 +6,10 @@ plugins { dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "pipeline"), depProjectConfig: "published", depExtension: "module") -} + compileOnly "org.projectlombok:lombok:${lombokVersion}" + annotationProcessor "org.projectlombok:lombok:${lombokVersion}" + + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "experiment"), depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "pipeline"), depProjectConfig: "published", depExtension: "module") +} diff --git a/nextflow/resources/views/begin.html b/nextflow/resources/views/begin.html deleted file mode 100644 index 9ba623a2..00000000 --- a/nextflow/resources/views/begin.html +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/nextflow/resources/views/nextFlowConfiguration.html b/nextflow/resources/views/nextFlowConfiguration.html deleted file mode 100644 index fd2a895c..00000000 --- a/nextflow/resources/views/nextFlowConfiguration.html +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - -
NextFlow Config File Path
AWS Account Name
AWS Identity
AWS S3 Bucket Path
AWS Credential
- - - - - diff --git a/nextflow/resources/views/nextFlowConfiguration.view.xml b/nextflow/resources/views/nextFlowConfiguration.view.xml deleted file mode 100644 index 8a589d3c..00000000 --- a/nextflow/resources/views/nextFlowConfiguration.view.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - \ No newline at end of file diff --git a/nextflow/src/org/labkey/nextflow/NextFlowConfiguration.java b/nextflow/src/org/labkey/nextflow/NextFlowConfiguration.java new file mode 100644 index 00000000..e8480095 --- /dev/null +++ b/nextflow/src/org/labkey/nextflow/NextFlowConfiguration.java @@ -0,0 +1,71 @@ +package org.labkey.nextflow; + +public class NextFlowConfiguration +{ + private String nextFlowConfigFilePath; + private String accountName; + private String identity; + private String s3BucketPath; + private String credential; + private String apiKey; + + public String getNextFlowConfigFilePath() + { + return nextFlowConfigFilePath; + } + + public void setNextFlowConfigFilePath(String nextFlowConfigFilePath) + { + this.nextFlowConfigFilePath = nextFlowConfigFilePath; + } + + public String getAccountName() + { + return accountName; + } + + public void setAccountName(String accountName) + { + this.accountName = accountName; + } + + public String getIdentity() + { + return identity; + } + + public void setIdentity(String identity) + { + this.identity = identity; + } + + public String getS3BucketPath() + { + return s3BucketPath; + } + + public void setS3BucketPath(String s3BucketPath) + { + this.s3BucketPath = s3BucketPath; + } + + public String getCredential() + { + return credential; + } + + public void setCredential(String credential) + { + this.credential = credential; + } + + public String getApiKey() + { + return apiKey; + } + + public void setApiKey(String apiKey) + { + this.apiKey = apiKey; + } +} diff --git a/nextflow/src/org/labkey/nextflow/NextFlowController.java b/nextflow/src/org/labkey/nextflow/NextFlowController.java index f22af232..b30bd5d5 100644 --- a/nextflow/src/org/labkey/nextflow/NextFlowController.java +++ b/nextflow/src/org/labkey/nextflow/NextFlowController.java @@ -1,47 +1,60 @@ package org.labkey.nextflow; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.labkey.api.action.ApiResponse; import org.labkey.api.action.ApiSimpleResponse; import org.labkey.api.action.FormViewAction; import org.labkey.api.action.MutatingApiAction; -import org.labkey.api.action.ReadOnlyApiAction; import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AdminUrls; import org.labkey.api.data.PropertyManager; import org.labkey.api.data.PropertyStore; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleHtmlView; -import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineStatusUrls; +import org.labkey.api.pipeline.browse.PipelinePathForm; import org.labkey.api.security.AdminConsoleAction; import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.SecurityManager; import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.SiteAdminPermission; import org.labkey.api.util.Button; +import org.labkey.api.util.DOM; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.HtmlString; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; import org.labkey.api.util.URLHelper; +import org.labkey.api.util.element.Select; import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; import org.labkey.api.view.HtmlView; +import org.labkey.api.view.JspView; import org.labkey.api.view.NavTree; +import org.labkey.api.view.UnauthorizedException; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.nextflow.pipeline.NextFlowPipelineJob; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; -import java.util.HashSet; -import java.util.Set; +import java.io.File; +import java.util.Arrays; +import java.util.List; +import static org.labkey.api.util.DOM.Attribute.checked; +import static org.labkey.api.util.DOM.Attribute.hidden; import static org.labkey.api.util.DOM.Attribute.method; +import static org.labkey.api.util.DOM.Attribute.name; +import static org.labkey.api.util.DOM.Attribute.type; +import static org.labkey.api.util.DOM.Attribute.value; import static org.labkey.api.util.DOM.DIV; +import static org.labkey.api.util.DOM.INPUT; import static org.labkey.api.util.DOM.LK.FORM; -import static org.labkey.api.util.DOM.P; +import static org.labkey.api.util.DOM.UL; import static org.labkey.api.util.DOM.at; import static org.labkey.nextflow.NextFlowManager.NEXTFLOW_CONFIG; @@ -49,7 +62,6 @@ public class NextFlowController extends SpringActionController { private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(NextFlowController.class); public static final String NAME = "nextflow"; - private static final String IS_NEXTFLOW_ENABLED = "enabled"; private static final Logger LOG = LogHelper.getLogger(NextFlowController.class, NAME); @@ -58,22 +70,16 @@ public NextFlowController() setActionResolver(_actionResolver); } - @RequiresPermission(AdminPermission.class) - public class GetNextFlowConfigurationAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object form, BindException errors) throws Exception - { - return new ApiSimpleResponse("config", PropertyManager.getEncryptedStore().getProperties(NEXTFLOW_CONFIG)); - } - } - - @RequiresPermission(AdminPermission.class) - public class DeleteNextFlowConfigurationAction extends MutatingApiAction + @RequiresPermission(SiteAdminPermission.class) + public static class DeleteNextFlowConfigurationAction extends MutatingApiAction { @Override - public ApiResponse execute(Object form, BindException errors) throws Exception + public ApiResponse execute(Object form, BindException errors) { + if (!getContainer().isRoot()) + { + throw new UnauthorizedException(); + } PropertyStore store = PropertyManager.getEncryptedStore(); store.deletePropertySet(NEXTFLOW_CONFIG); return new ApiSimpleResponse("success", true); @@ -93,208 +99,229 @@ public void validateCommand(NextFlowConfiguration target, Errors errors) } @Override - public ModelAndView getView(NextFlowConfiguration nextFlowConfiguration, boolean reshow, BindException errors) throws Exception + public ModelAndView getView(NextFlowConfiguration newConfig, boolean reshow, BindException errors) { - return ModuleHtmlView.get(ModuleLoader.getInstance().getModule(NextFlowModule.class), "nextFlowConfiguration"); + NextFlowConfiguration existingConfig = NextFlowManager.get().getConfiguration(); + if (existingConfig != null) + { + if (StringUtils.isEmpty(newConfig.getNextFlowConfigFilePath())) + { + newConfig.setNextFlowConfigFilePath(existingConfig.getNextFlowConfigFilePath()); + } + if (StringUtils.isEmpty(newConfig.getAccountName())) + { + newConfig.setAccountName(existingConfig.getAccountName()); + } + if (StringUtils.isEmpty(newConfig.getIdentity())) + { + newConfig.setIdentity(existingConfig.getIdentity()); + } + if (StringUtils.isEmpty(newConfig.getCredential())) + { + newConfig.setCredential(existingConfig.getCredential()); + } + if (StringUtils.isEmpty(newConfig.getS3BucketPath())) + { + newConfig.setS3BucketPath(existingConfig.getS3BucketPath()); + } + if (StringUtils.isEmpty(newConfig.getApiKey())) + { + newConfig.setApiKey(existingConfig.getApiKey()); + } + } + + return new JspView<>("/org/labkey/nextflow/nextFlowConfiguration.jsp", newConfig, errors); } @Override - public boolean handlePost(NextFlowConfiguration nextFlowConfiguration, BindException errors) throws Exception + public boolean handlePost(NextFlowConfiguration newConfig, BindException errors) { - NextFlowManager.get().addConfiguration(nextFlowConfiguration, errors); + NextFlowConfiguration existingConfig = NextFlowManager.get().getConfiguration(); + if (existingConfig != null) + { + if (StringUtils.isEmpty(newConfig.getApiKey())) + { + newConfig.setApiKey(existingConfig.getApiKey()); + } + if (StringUtils.isEmpty(newConfig.getCredential())) + { + newConfig.setCredential(existingConfig.getCredential()); + } + } + NextFlowManager.get().saveConfig(newConfig, errors); return !errors.hasErrors(); } @Override public URLHelper getSuccessURL(NextFlowConfiguration nextFlowConfiguration) { - return getContainer().getStartURL(getUser()); + return PageFlowUtil.urlProvider(AdminUrls.class).getAdminConsoleURL(); } @Override public void addNavTrail(NavTree root) { - + root.addChild("Admin Console", PageFlowUtil.urlProvider(AdminUrls.class).getAdminConsoleURL()); + root.addChild("Configure NextFlow"); } } - public static class NextFlowConfiguration + public static class EnabledForm { - private String nextFlowConfigFilePath; - private String accountName; - private String identity; - private String s3BucketPath; - private String credential; - - public String getNextFlowConfigFilePath() - { - return nextFlowConfigFilePath; - } - - public void setNextFlowConfigFilePath(String nextFlowConfigFilePath) - { - this.nextFlowConfigFilePath = nextFlowConfigFilePath; - } - - public String getAccountName() - { - return accountName; - } - - public void setAccountName(String accountName) - { - this.accountName = accountName; - } - - public String getIdentity() - { - return identity; - } - - public void setIdentity(String identity) - { - this.identity = identity; - } - - public String getS3BucketPath() - { - return s3BucketPath; - } - - public void setS3BucketPath(String s3BucketPath) - { - this.s3BucketPath = s3BucketPath; - } + Boolean _enabled; - public String getCredential() + public Boolean getEnabled() { - return credential; + return _enabled; } - public void setCredential(String credential) + public void setEnabled(Boolean enabled) { - this.credential = credential; + _enabled = enabled; } } - @RequiresPermission(SiteAdminPermission.class) - public static class NextFlowEnableAction extends FormViewAction + @RequiresPermission(ReadPermission.class) + public static class BeginAction extends FormViewAction { - @Override - public void validateCommand(Object target, Errors errors) + public void validateCommand(EnabledForm target, Errors errors) { } @Override - public ModelAndView getView(Object form, boolean reshow, BindException errors) throws Exception + public ModelAndView getView(EnabledForm form, boolean reshow, BindException errors) { - PropertyStore store = PropertyManager.getNormalStore(); - PropertyManager.PropertyMap map = store.getProperties(NextFlowManager.NEXTFLOW_ENABLE); - String btnTxt = "Enable NextFlow"; - // check if nextflow is enabled - if (Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED))) + if (getUser().hasSiteAdminPermission()) { - btnTxt = "Disable NextFlow"; + Boolean status = NextFlowManager.get().getEnabledState(getContainer()); + boolean inheritedStatus = NextFlowManager.get().isEnabled(getContainer().getParent()); + + return new HtmlView("Enable or Disable NextFlow", + FORM(at(method, "POST"), + DIV(INPUT(at(type, "radio", name, "enabled", value, Boolean.TRUE.toString(), (status == Boolean.TRUE ? checked : null), null)), + "Enabled"), + DIV(INPUT(at(type, "radio", name, "enabled", value, Boolean.FALSE.toString(), (status == Boolean.FALSE ? checked : null), null)), + "Disabled"), + DIV(INPUT(at(type, "radio", name, "enabled", value, "", (status == null ? checked : null), null)), + getContainer().isRoot() ? + "Unset" : + "Inherited from " + getContainer().getParent().getPath() + " (currently " + (inheritedStatus ? "enabled" : "disabled") + ")"), + new Button.ButtonBuilder("Save").submit(true).build(), " ", + new Button.ButtonBuilder("Cancel").href(getContainer().getStartURL(getUser())).build())); } else { - btnTxt = "Enable NextFlow"; + return new HtmlView("NextFlow Integration Status", + DIV("NextFlow integration is " + (NextFlowManager.get().isEnabled(getContainer()) ? "enabled" : "disabled") + " in this " + (getContainer().isProject() ? "project" : "folder") + ".") + ); } - - return new HtmlView("Enable/Disable Nextflow", DIV( P("NextFlow is currently " + (Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED)) ? "enabled" : "disabled")), - FORM(at(method, "POST"), - new Button.ButtonBuilder(btnTxt).submit(true).build()))); } @Override - public boolean handlePost(Object form, BindException errors) throws Exception + public boolean handlePost(EnabledForm form, BindException errors) { - PropertyStore store = PropertyManager.getNormalStore(); - PropertyManager.WritablePropertyMap map = store.getWritableProperties(NextFlowManager.NEXTFLOW_ENABLE, true); - if (map.isEmpty()) - { - map.put(IS_NEXTFLOW_ENABLED, Boolean.TRUE.toString()); - } - else - { - if (Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED))) - { - map.put(IS_NEXTFLOW_ENABLED, Boolean.FALSE.toString()); - } - else - { - map.put(IS_NEXTFLOW_ENABLED, Boolean.TRUE.toString()); - } - } - map.save(); + NextFlowManager.get().saveEnabledState(getContainer(), form.getEnabled()); return true; } @Override public void addNavTrail(NavTree root) { - + root.addChild("NextFlow Integration Status"); } @Override - public URLHelper getSuccessURL(Object o) + public URLHelper getSuccessURL(EnabledForm o) { return getContainer().getStartURL(getUser()); } } + @Getter @Setter + public static class AnalyzeForm extends PipelinePathForm + { + private boolean launch = false; + private String configFile; + } + @RequiresPermission(AdminOperationsPermission.class) - public class NextFlowRunAction extends FormViewAction + public class NextFlowRunAction extends FormViewAction { - private ActionURL _successURL; @Override - public void validateCommand(Object o, Errors errors) + public void validateCommand(AnalyzeForm o, Errors errors) { - PropertyStore store = PropertyManager.getNormalStore(); - PropertyManager.PropertyMap map = store.getProperties(NextFlowManager.NEXTFLOW_ENABLE); - if (!Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED))) + if (!NextFlowManager.get().isEnabled(getContainer())) { errors.reject(ERROR_MSG, "NextFlow is not enabled"); } } @Override - public ModelAndView getView(Object o, boolean b, BindException errors) throws Exception + public ModelAndView getView(AnalyzeForm o, boolean b, BindException errors) { - return new HtmlView("NextFlow Runner", DIV("Run NextFlow Pipeline", - FORM(at(method, "POST"), - new Button.ButtonBuilder("Start NextFlow").submit(true).build()))); + NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); + if (config.getNextFlowConfigFilePath() != null) + { + File configDir = new File(config.getNextFlowConfigFilePath()); + if (configDir.isDirectory()) + { + File[] files = configDir.listFiles(); + if (files != null && files.length > 0) + { + List configFiles = Arrays.asList(files); + return new HtmlView("NextFlow Runner", DIV( + FORM(at(method, "POST"), + INPUT(at(hidden, true, name, "launch", value, true)), + Arrays.stream(o.getFile()).map(f -> INPUT(at(hidden, true, name, "file", value, f))).toList(), + "Files: ", + UL(Arrays.stream(o.getFile()).map(DOM::LI)), + "Config: ", + new Select.SelectBuilder().name("configFile").addOptions(configFiles.stream().filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(), + new Button.ButtonBuilder("Start NextFlow").submit(true).build()))); + } + } + } + return new HtmlView(HtmlString.of("Couldn't find NextFlow config file(s)")); } @Override - public boolean handlePost(Object o, BindException errors) throws Exception + public boolean handlePost(AnalyzeForm form, BindException errors) throws Exception { - // check if nextflow is enabled - PropertyStore store = PropertyManager.getNormalStore(); - PropertyManager.PropertyMap map = store.getProperties(NextFlowManager.NEXTFLOW_ENABLE); - if (map == null || !Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED))) + if (!form.isLaunch()) { - errors.reject(ERROR_MSG, "NextFlow is not enabled"); return false; } - try (SecurityManager.TransformSession session = SecurityManager.createTransformSession(getViewContext())) + NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); + File configDir = new File(config.getNextFlowConfigFilePath()); + File configFile = FileUtil.appendPath(configDir, Path.parse(form.getConfigFile())); + if (!configFile.exists()) + { + errors.reject(ERROR_MSG, "Config file does not exist"); + } + else { - // TODO: pass the apiKey to Nextflow job - String apiKey = session.getApiKey(); - ViewBackgroundInfo info = getViewBackgroundInfo(); - PipeRoot root = PipelineService.get().findPipelineRoot(info.getContainer()); - PipelineJob job = new NextFlowPipelineJob(info, root, apiKey); - PipelineService.get().queueJob(job); + List inputFiles = form.getValidatedFiles(getContainer()); + if (inputFiles.isEmpty()) + { + errors.reject(ERROR_MSG, "No input files"); + } + else + { + ViewBackgroundInfo info = getViewBackgroundInfo(); + PipeRoot root = PipelineService.get().findPipelineRoot(info.getContainer()); + PipelineJob job = NextFlowPipelineJob.create(info, root, configFile.toPath(), inputFiles.stream().map(File::toPath).toList()); + PipelineService.get().queueJob(job); + } } return !errors.hasErrors(); } @Override - public URLHelper getSuccessURL(Object o) + public URLHelper getSuccessURL(AnalyzeForm o) { return PageFlowUtil.urlProvider(PipelineStatusUrls.class).urlBegin(getContainer()); } diff --git a/nextflow/src/org/labkey/nextflow/NextFlowManager.java b/nextflow/src/org/labkey/nextflow/NextFlowManager.java index 23684b32..e560e413 100644 --- a/nextflow/src/org/labkey/nextflow/NextFlowManager.java +++ b/nextflow/src/org/labkey/nextflow/NextFlowManager.java @@ -1,12 +1,15 @@ package org.labkey.nextflow; import org.apache.commons.lang3.StringUtils; +import org.labkey.api.data.Container; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.DbScope; import org.labkey.api.data.PropertyManager; -import org.labkey.api.data.PropertyStore; import org.springframework.validation.BindException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; @@ -15,21 +18,18 @@ public class NextFlowManager { public static final String NEXTFLOW_CONFIG = "nextflow-config"; - public static final String NEXTFLOW_ENABLE = "nextflow-enable"; + private static final String NEXTFLOW_ENABLE_PROP_CATEGORY = "nextflow-enable"; private static final String NEXTFLOW_ACCOUNT_NAME = "accountName"; private static final String NEXTFLOW_CONFIG_FILE_PATH = "nextFlowConfigFilePath"; private static final String NEXTFLOW_IDENTITY = "identity"; private static final String NEXTFLOW_CREDENTIAL = "credential"; private static final String NEXTFLOW_S3_BUCKET_PATH = "s3BucketPath"; + private static final String NEXTFLOW_API_KEY = "apiKey"; - private static final NextFlowManager _instance = new NextFlowManager(); - - // Normal store is used for enabled/disabled module - private static final PropertyStore _normalStore = PropertyManager.getNormalStore(); + private static final String IS_NEXTFLOW_ENABLED = "enabled"; - // Encrypted store is used for aws settings & nextflow file configuration - private static final PropertyStore _encryptedStore = PropertyManager.getEncryptedStore(); + private static final NextFlowManager _instance = new NextFlowManager(); private NextFlowManager() { @@ -42,48 +42,89 @@ public static NextFlowManager get() } - private void checkArgs(String nextFlowConfigFilePath, String name, String identity, String credential,String s3BucketPath, BindException errors) + private void checkArgs(NextFlowConfiguration config, BindException errors) { - if (StringUtils.isEmpty(nextFlowConfigFilePath)) + if (StringUtils.isEmpty(config.getNextFlowConfigFilePath())) errors.rejectValue("nextFlowConfigFilePath", ERROR_MSG, "NextFlow config file path is required"); - if (StringUtils.isEmpty(name)) - errors.rejectValue("name", ERROR_MSG, "AWS account name is required"); - - if (StringUtils.isEmpty(identity)) - errors.rejectValue("identity", ERROR_MSG, "AWS identity is required"); + Path configPath = Paths.get(config.getNextFlowConfigFilePath()); + if (!Files.isDirectory(configPath)) + { + errors.rejectValue("nextFlowConfigFilePath", ERROR_MSG, "NextFlow config file path must be a directory"); + } - if (StringUtils.isEmpty(credential)) - errors.rejectValue("credential", ERROR_MSG, "AWS credential is required"); + // Not yet used +// if (StringUtils.isEmpty(config.getAccountName())) +// errors.rejectValue("accountName", ERROR_MSG, "AWS account name is required"); +// if (StringUtils.isEmpty(config.getIdentity())) +// errors.rejectValue("identity", ERROR_MSG, "AWS identity is required"); +// if (StringUtils.isEmpty(config.getCredential())) +// errors.rejectValue("credential", ERROR_MSG, "AWS credential is required"); + if (StringUtils.isEmpty(config.getS3BucketPath())) + errors.rejectValue("credential", ERROR_MSG, "S3 bucket path is required"); } - public NextFlowController.NextFlowConfiguration getConfiguration() + public NextFlowConfiguration getConfiguration() { - PropertyManager.PropertyMap props = _encryptedStore.getWritableProperties(NEXTFLOW_CONFIG, false); + PropertyManager.PropertyMap props = PropertyManager.getEncryptedStore().getWritableProperties(NEXTFLOW_CONFIG, false); if (props != null) { - NextFlowController.NextFlowConfiguration configuration = new NextFlowController.NextFlowConfiguration(); + NextFlowConfiguration configuration = new NextFlowConfiguration(); configuration.setAccountName(props.get(NEXTFLOW_ACCOUNT_NAME)); configuration.setNextFlowConfigFilePath(props.get(NEXTFLOW_CONFIG_FILE_PATH)); configuration.setIdentity(props.get(NEXTFLOW_IDENTITY)); configuration.setCredential(props.get(NEXTFLOW_CREDENTIAL)); configuration.setS3BucketPath(props.get(NEXTFLOW_S3_BUCKET_PATH)); + configuration.setApiKey(props.get(NEXTFLOW_API_KEY)); return configuration; } return null; } - public void addConfiguration(NextFlowController.NextFlowConfiguration configuration, BindException errors) + /** + * Checks in the specified container and traverses up the container tree to determine if NextFlow integration + * is enabled directly or in a parent container. + */ + public boolean isEnabled(Container c) + { + do + { + PropertyManager.PropertyMap map = PropertyManager.getProperties(c, NEXTFLOW_ENABLE_PROP_CATEGORY); + if (map.containsKey(IS_NEXTFLOW_ENABLED)) + { + return Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED)); + } + c = c.getParent(); + } + while (c != null); + + return false; + } + + /** + * @return configured state for the container (or null if not configured there), for whether NextFlow is enabled + */ + public Boolean getEnabledState(Container c) + { + PropertyManager.PropertyMap map = PropertyManager.getProperties(c, NEXTFLOW_ENABLE_PROP_CATEGORY); + if (map.containsKey(IS_NEXTFLOW_ENABLED)) + { + return Boolean.parseBoolean(map.get(IS_NEXTFLOW_ENABLED)); + } + return null; + } + + public void saveConfig(NextFlowConfiguration configuration, BindException errors) { - checkArgs(configuration.getNextFlowConfigFilePath(), configuration.getAccountName(), configuration.getIdentity(), configuration.getCredential(), configuration.getS3BucketPath(), errors); + checkArgs(configuration, errors); if (!errors.hasErrors()) saveConfiguration(configuration); } - private void saveConfiguration( NextFlowController.NextFlowConfiguration configuration) + private void saveConfiguration( NextFlowConfiguration configuration) { try (DbScope.Transaction tx = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) { @@ -93,8 +134,9 @@ private void saveConfiguration( NextFlowController.NextFlowConfiguration configu properties.put(NEXTFLOW_CREDENTIAL, configuration.getCredential()); properties.put(NEXTFLOW_S3_BUCKET_PATH, configuration.getS3BucketPath()); properties.put(NEXTFLOW_ACCOUNT_NAME, configuration.getAccountName()); + properties.put(NEXTFLOW_API_KEY, configuration.getApiKey()); - PropertyManager.WritablePropertyMap props = _encryptedStore.getWritableProperties(NEXTFLOW_CONFIG, true); + PropertyManager.WritablePropertyMap props = PropertyManager.getEncryptedStore().getWritableProperties(NEXTFLOW_CONFIG, true); props.clear(); props.putAll(properties); props.save(); @@ -103,4 +145,17 @@ private void saveConfiguration( NextFlowController.NextFlowConfiguration configu } } + public void saveEnabledState(Container container, Boolean enabled) + { + PropertyManager.WritablePropertyMap map = PropertyManager.getWritableProperties(container, NEXTFLOW_ENABLE_PROP_CATEGORY, true); + if (enabled == null) + { + map.delete(); + } + else + { + map.put(IS_NEXTFLOW_ENABLED, enabled.toString()); + map.save(); + } + } } diff --git a/nextflow/src/org/labkey/nextflow/NextFlowModule.java b/nextflow/src/org/labkey/nextflow/NextFlowModule.java index a962764d..46853d27 100644 --- a/nextflow/src/org/labkey/nextflow/NextFlowModule.java +++ b/nextflow/src/org/labkey/nextflow/NextFlowModule.java @@ -1,14 +1,15 @@ package org.labkey.nextflow; import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.SpringModule; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.security.permissions.AdminPermission; import org.labkey.api.settings.AdminConsole; import org.labkey.api.view.ActionURL; import org.labkey.api.view.WebPartFactory; +import org.labkey.nextflow.pipeline.NextFlowPipelineProvider; import java.util.Collection; import java.util.List; @@ -20,6 +21,8 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) { ActionURL adminUrl = new ActionURL(NextFlowController.NextFlowConfigurationAction.class, ContainerManager.getRoot()); AdminConsole.addLink(AdminConsole.SettingsLinkType.Configuration, "NextFlow Configuration", adminUrl, AdminPermission.class); + + PipelineService.get().registerPipelineProvider(new NextFlowPipelineProvider(this)); } @Override diff --git a/nextflow/src/org/labkey/nextflow/nextFlowConfiguration.jsp b/nextflow/src/org/labkey/nextflow/nextFlowConfiguration.jsp new file mode 100644 index 00000000..f2f738b6 --- /dev/null +++ b/nextflow/src/org/labkey/nextflow/nextFlowConfiguration.jsp @@ -0,0 +1,59 @@ +<%@ taglib prefix="labkey" uri="http://www.labkey.org/taglib" %> +<%@ page import="org.labkey.api.view.HttpView" %> +<%@ page import="org.labkey.nextflow.NextFlowConfiguration" %> +<%@ page import="org.labkey.api.util.Button" %> +<%@ page import="org.labkey.api.util.PageFlowUtil" %> +<%@ page import="org.labkey.api.admin.AdminUrls" %> +<%@ page import="org.labkey.api.security.permissions.AdminOperationsPermission" %> +<%@ page extends="org.labkey.api.jsp.JspBase" %> +<% + NextFlowConfiguration form = (NextFlowConfiguration) HttpView.currentModel(); + boolean hasAdminOpsPerms = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); +%> + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ <%= new Button.ButtonBuilder("Save").submit(true).primary(true).enabled(hasAdminOpsPerms) %> + <%= new Button.ButtonBuilder("Delete").onClick("deleteConfig()").enabled(hasAdminOpsPerms) %> + <%= new Button.ButtonBuilder("Cancel").href(PageFlowUtil.urlProvider(AdminUrls.class).getAdminConsoleURL()) %> + + + diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java index 42c2882f..a09f7571 100644 --- a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java @@ -1,54 +1,120 @@ package org.labkey.nextflow.pipeline; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.files.FileContentService; +import org.labkey.api.pipeline.ParamParser; import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobService; import org.labkey.api.pipeline.TaskId; import org.labkey.api.pipeline.TaskPipeline; +import org.labkey.api.pipeline.file.AbstractFileAnalysisJob; import org.labkey.api.util.FileUtil; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.UnexpectedException; +import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ViewBackgroundInfo; +import java.io.BufferedWriter; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; -public class NextFlowPipelineJob extends PipelineJob +@Getter +public class NextFlowPipelineJob extends AbstractFileAnalysisJob { - private String _apiKey; - // For serialization + private Path config; + + @SuppressWarnings("unused") // For serialization protected NextFlowPipelineJob() {} - public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, String apiKey) + public static NextFlowPipelineJob create(ViewBackgroundInfo info, @NotNull PipeRoot root, Path templateConfig, List inputFiles) throws IOException { - super(null, info, root); - this._apiKey = apiKey; - setLogFile(new File(String.valueOf(root.getLogDirectory()), FileUtil.makeFileNameWithTimestamp("NextFlowPipelineJob", "log")).toPath()); + Path parentDir = inputFiles.get(0).getParent(); + + String jobName = FileUtil.makeFileNameWithTimestamp("NextFlow"); + Path jobDir = parentDir.resolve(jobName); + Path log = jobDir.resolve(jobName + ".log"); + FileUtil.createDirectory(jobDir); + + Path config = createConfig(templateConfig, parentDir, jobDir, info.getContainer()); + + return new NextFlowPipelineJob(info, root, config, inputFiles, log); } - public String getApiKey() + public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, Path config, List inputFiles, Path log) throws IOException { - return _apiKey; + super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getFileName().toString(), config, inputFiles, false, false); + this.config = config; + setLogFile(log); } @Override - public URLHelper getStatusHref() + public ParamParser getInputParameters() { - return null; + return PipelineJobService.get().createParamParser(); + } + + /** Take the template config file and substitute in the values for this job */ + private static Path createConfig(Path configTemplate, Path parentDir, Path jobDir, Container container) throws IOException + { + String template; + try (InputStream in = Files.newInputStream(configTemplate)) + { + template = PageFlowUtil.getStreamContentsAsString(in); + } + + String webdavUrl = FileContentService.get().getWebDavUrl(parentDir, container, FileContentService.PathType.full).toString(); + webdavUrl = StringUtils.stripEnd(webdavUrl, "/"); + + String substitutedContent = template.replace("${quant_spectra_dir}", "quant_spectra_dir = '" + webdavUrl + "'"); + + Path substitutedFile = jobDir.resolve(configTemplate.getFileName()); + try (BufferedWriter writer = Files.newBufferedWriter(substitutedFile)) + { + writer.write(substitutedContent); + } + return substitutedFile; } @Override public String getDescription() { - return "NextFlow Job"; + return "NextFlow analysis using " + config.getFileName() + " of " + getInputFilePaths().size() + " files"; + } + + @Override + public TaskPipeline getTaskPipeline() + { + return PipelineJobService.get().getTaskPipeline(getTaskPipelineId()); } @Override - public TaskPipeline getTaskPipeline() + public TaskId getTaskPipelineId() { - return PipelineJobService.get().getTaskPipeline(new TaskId(NextFlowPipelineJob.class)); + return new TaskId(NextFlowPipelineJob.class); } + + @Override + public AbstractFileAnalysisJob createSingleFileJob(File file) + { + throw new UnsupportedOperationException(); + } + + @Override + public File findInputFile(String name) + { + throw new UnsupportedOperationException(); + } + + @Override + public File findOutputFile(String name) + { + return null; + } + } diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineProvider.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineProvider.java new file mode 100644 index 00000000..30ec70ce --- /dev/null +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineProvider.java @@ -0,0 +1,47 @@ +package org.labkey.nextflow.pipeline; + +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineDirectory; +import org.labkey.api.pipeline.PipelineProvider; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.view.ViewContext; +import org.labkey.nextflow.NextFlowController; +import org.labkey.nextflow.NextFlowManager; +import org.labkey.nextflow.NextFlowModule; + +public class NextFlowPipelineProvider extends PipelineProvider +{ + + public static final String NAME = "NextFlow"; + + public NextFlowPipelineProvider(NextFlowModule owningModule) + { + super(NAME, owningModule); + } + + @Override + public boolean isShowActionsIfModuleInactive() + { + // We rely on a setting that folder admins can't control to determine if NextFlow is available + return true; + } + + @Override + public void updateFileProperties(ViewContext context, PipeRoot pr, PipelineDirectory directory, boolean includeAll) + { + if (!context.getContainer().hasPermission(context.getUser(), InsertPermission.class)) + return; + if (!NextFlowManager.get().isEnabled(context.getContainer())) + return; + + String actionId = createActionId(NextFlowController.NextFlowRunAction.class, "Analyze with NextFlow"); + addAction(actionId, + NextFlowController.NextFlowRunAction.class, + "Analyze with NextFlow", + directory, + directory.listPaths(new FileTypesEntryFilter(NextFlowProtocol.INPUT_TYPES)), + true, + true, + includeAll); + } +} diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowProtocol.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowProtocol.java new file mode 100644 index 00000000..a2d7270c --- /dev/null +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowProtocol.java @@ -0,0 +1,69 @@ +package org.labkey.nextflow.pipeline; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.file.AbstractFileAnalysisProtocol; +import org.labkey.api.pipeline.file.AbstractFileAnalysisProtocolFactory; +import org.labkey.api.util.FileType; +import org.labkey.api.view.ViewBackgroundInfo; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +public class NextFlowProtocol extends AbstractFileAnalysisProtocol +{ + public static final List INPUT_TYPES = List.of( + new FileType(".RAW"), + new FileType(".mzML")); + + public NextFlowProtocol() + { + super("NextFlow", null, null); + } + + @Override + public List getInputTypes() + { + return INPUT_TYPES; + } + + @Override + public void setXml(String xml) + { + // No-op since NextFlow doesn't use XML + } + + @Override + public AbstractFileAnalysisProtocolFactory getFactory() + { + return new AbstractFileAnalysisProtocolFactory<>() + { + @Override + public NextFlowProtocol createProtocolInstance(String name, String description, String xml) + { + return new NextFlowProtocol(); + } + + @Override + public Path getDefaultParametersFile(PipeRoot root) + { + return null; + } + + @Override + public String getName() + { + return "NextFlow"; + } + }; + } + + @Override + public NextFlowPipelineJob createPipelineJob(ViewBackgroundInfo info, PipeRoot root, List filesInput, File fileParameters, @Nullable Map variableMap) throws IOException + { + throw new UnsupportedOperationException(); + } +} diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java index a75312a3..8954963e 100644 --- a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java +++ b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java @@ -2,46 +2,170 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; -import org.labkey.api.files.FileContentService; import org.labkey.api.pipeline.AbstractTaskFactory; import org.labkey.api.pipeline.AbstractTaskFactorySettings; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.pipeline.PipelineJobService; +import org.labkey.api.pipeline.RecordedAction; import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.pipeline.WorkDirectoryTask; +import org.labkey.api.security.SecurityManager; import org.labkey.api.util.FileType; -import org.labkey.api.util.FileUtil; -import org.labkey.nextflow.NextFlowController; +import org.labkey.nextflow.NextFlowConfiguration; import org.labkey.nextflow.NextFlowManager; +import java.io.BufferedReader; import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Stream; -public class NextFlowRunTask extends PipelineJob.Task +public class NextFlowRunTask extends WorkDirectoryTask { + public static final String SPECTRA_INPUT_ROLE = "Spectra"; + + public static final String ACTION_NAME = "NextFlow"; + public NextFlowRunTask(Factory factory, PipelineJob job) { super(factory, job); } + + @Override public @NotNull RecordedActionSet run() throws PipelineJobException { Logger log = getJob().getLogger(); - NextFlowController.NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); - String nextFlowConfigFilePath = config.getNextFlowConfigFilePath(); - String s3BucketPath = config.getS3BucketPath(); - String s3Path = "s3://" + s3BucketPath; - String apiKey = getJob().getApiKey(); - ProcessBuilder pb = new ProcessBuilder( "nextflow" , "secrets", "set", "PANORAMA_API_KEY", apiKey); - log.info("Job Started"); - File dir = FileContentService.get().getDefaultRootInfo(getJob().getContainer()).getPath().toFile(); - getJob().runSubProcess(pb, dir); - pb.command("nextflow" , "run", "-resume", "-r", "main", "-profile", "aws", "mriffle/nf-skyline-dia-ms", "-bucket-dir", s3Path, "-c", nextFlowConfigFilePath); - getJob().runSubProcess(pb, dir); - log.info("Job Finished"); - return new RecordedActionSet(); + + SecurityManager.TransformSession session = null; + + try + { + NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); + if (config == null) + { + throw new PipelineJobException("No NextFlow configuration found"); + } + + // Use the configured API key if set + String apiKey = config.getApiKey(); + if (apiKey == null) + { + session = SecurityManager.createTransformSession(getJob().getUser()); + apiKey = session.getApiKey(); + } + + // Need to pass to the main process directly in the future to allow concurrent execution for different users + ProcessBuilder secretsPB = new ProcessBuilder("nextflow", "secrets", "set", "PANORAMA_API_KEY", apiKey); + log.info("Job Started"); + File dir = getJob().getLogFile().getParentFile(); + getJob().runSubProcess(secretsPB, dir); + + ProcessBuilder executionPB = new ProcessBuilder(getArgs()); + getJob().runSubProcess(executionPB, dir); + log.info("Job Finished"); + + RecordedAction action = new RecordedAction(ACTION_NAME); + for (Path inputFile : getJob().getInputFilePaths()) + { + action.addInput(inputFile.toFile(), SPECTRA_INPUT_ROLE); + } + addOutputs(action, getJob().getLogFilePath().getParent().resolve("reports")); + return new RecordedActionSet(action); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + finally + { + if (session != null) + { + session.close(); + } + } + } + + private void addOutputs(RecordedAction action, Path path) throws IOException + { + if (Files.isRegularFile(path)) + { + action.addOutput(path.toFile(), "Output", false); + } + else if (Files.isDirectory(path)) + { + try (Stream listing = Files.list(path)) + { + for (Path child : listing.toList()) + { + addOutputs(action, child); + } + } + } + } + + private boolean hasAwsSection(Path configFile) throws PipelineJobException + { + try (InputStream in = Files.newInputStream(configFile); + InputStreamReader isReader = new InputStreamReader(in, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(isReader)) + { + String line; + while ((line = reader.readLine()) != null) + { + line = line.trim(); + // Ignore comments + if (!line.startsWith("//")) + { + if (line.startsWith("aws")) + { + return true; + } + } + } + return false; + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + } + + + private @NotNull List getArgs() throws PipelineJobException + { + NextFlowConfiguration config = NextFlowManager.get().getConfiguration(); + Path configFile = getJob().getConfig(); + + boolean aws = hasAwsSection(configFile); + + List args = new ArrayList<>(Arrays.asList("nextflow", "run", "-resume", "-r", "main")); + if (aws) + { + args.add("-profile"); + args.add("aws"); + } + args.add("mriffle/nf-skyline-dia-ms"); + if (aws) + { + String s3BucketPath = config.getS3BucketPath(); + String s3Path = "s3://" + s3BucketPath; + + args.add("-bucket-dir"); + args.add(s3Path); + } + args.add("-c"); + args.add(configFile.toAbsolutePath().toString()); + return args; } @Override @@ -58,7 +182,7 @@ public Factory() } @Override - public PipelineJob.Task createTask(PipelineJob job) + public NextFlowRunTask createTask(PipelineJob job) { return new NextFlowRunTask(this, job); } @@ -72,7 +196,7 @@ public List getInputTypes() @Override public List getProtocolActionNames() { - return Collections.emptyList(); + return List.of(ACTION_NAME); } @Override diff --git a/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml b/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml index 3cb859bd..eee32d08 100644 --- a/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml +++ b/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml @@ -19,10 +19,21 @@ org.labkey.nextflow.pipeline.NextFlowRunTask + + + + + + + + + + org.labkey.api.exp.pipeline.XarGeneratorId +