diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index caab88879..dd9bde018 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,5 +1,8 @@ package org.breedinginsight.brapi.v2.services; +import com.github.filosganga.geogson.model.Coordinates; +import com.github.filosganga.geogson.model.positions.SinglePosition; +import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; @@ -465,6 +468,26 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + + // Lat, Long, Elevation + Coordinates coordinates = extractCoordinates(ou); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getLat())) + .ifPresent(lat -> row.put(ExperimentObservation.Columns.LAT, lat)); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getLon())) + .ifPresent(lon -> row.put(ExperimentObservation.Columns.LONG, lon)); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getAlt())) + .ifPresent(elevation -> row.put(ExperimentObservation.Columns.ELEVATION, elevation)); + + // RTK + JsonObject additionalInfo = ou.getAdditionalInfo(); + String rtk = ( additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.RTK) ==null ) + ? null + : additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString(); + row.put(ExperimentObservation.Columns.RTK, rtk); + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); @@ -484,11 +507,13 @@ private Map createExportRow( .findFirst(); blockLevel.ifPresent(brAPIObservationUnitLevelRelationship -> row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); - if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null && - ou.getObservationUnitPosition().getPositionCoordinateY() != null) { + + //Row and Column + if ( ou.getObservationUnitPosition() != null ) { row.put(ExperimentObservation.Columns.ROW, ou.getObservationUnitPosition().getPositionCoordinateX()); row.put(ExperimentObservation.Columns.COLUMN, ou.getObservationUnitPosition().getPositionCoordinateY()); } + if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor()); } else { @@ -499,7 +524,25 @@ private Map createExportRow( return row; } - + private String doubleToString(double val){ + return Double.isNaN(val) ? null : String.valueOf( val ); + } + private Coordinates extractCoordinates(BrAPIObservationUnit ou){ + Coordinates coordinates = null; + if ( ou.getObservationUnitPosition()!=null + && ou.getObservationUnitPosition().getGeoCoordinates()!=null + && ou.getObservationUnitPosition().getGeoCoordinates().getGeometry()!=null + && ou.getObservationUnitPosition().getGeoCoordinates().getGeometry().positions()!=null + ) + { + Object o = ou.getObservationUnitPosition().getGeoCoordinates().getGeometry().positions(); + if (o instanceof SinglePosition){ + SinglePosition sp = (SinglePosition)o; + coordinates= sp.coordinates(); + } + } + return coordinates; + } private void addObsVarColumns( List columns, diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index 5c9ac0ca1..69dcb3e98 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,7 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.services.FileImportService; @@ -214,16 +214,22 @@ public HttpResponse>> getSystemMappings(@Nu @Produces(MediaType.APPLICATION_JSON) @AddMetadata @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) throws Exception { - List workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + List workflows = null; + try { + workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } List metadataStatus = new ArrayList<>(); metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(workflows)); + Response> response = new Response(metadata, new DataResponse<>(workflows)); return HttpResponse.ok(response); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index 053f1dc3c..eb2e2ec50 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -163,7 +163,7 @@ public HttpResponse> previewData(@PathVariable UUID pro @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable UUID workflowId, @PathVariable UUID uploadId) { + @PathVariable String workflowId, @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); @@ -189,7 +189,7 @@ public HttpResponse> previewData(@PathVariable UUID pro @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable UUID workflowId, @PathVariable UUID uploadId, + @PathVariable String workflowId, @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java deleted file mode 100644 index 1b60bda45..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.daos; - -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; -import org.jooq.Configuration; -import org.jooq.DSLContext; - -import javax.inject.Inject; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -import static org.breedinginsight.dao.db.Tables.*; - -public class ImportMappingWorkflowDAO extends ImporterMappingWorkflowDao { - - private DSLContext dsl; - - @Inject - public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { - super(config); - this.dsl = dsl; - } - - /** - * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. They are ordered by - * position for proper ordering on the front end. - * - * @param mappingId The UUID of the mapping to retrieve the workflows for. - * @return A list of ImportMappingWorkflow objects. - */ - public List getWorkflowsByImportMappingId(UUID mappingId) { - return dsl.select() - .from(IMPORTER_MAPPING_WORKFLOW) - .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) - .orderBy(IMPORTER_MAPPING_WORKFLOW.POSITION.asc()) - .fetch(ImportMappingWorkflow::parseSQLRecord); - } - - /** - * Retrieves a workflow by its ID. - * - * @param workflowId The ID of the workflow to retrieve. - * @return An Optional containing the ImportMappingWorkflow if found, otherwise an empty Optional. - */ - public Optional getWorkflowById(UUID workflowId) { - return Optional.ofNullable(fetchOneById(workflowId)) - .map(ImportMappingWorkflow::new); - } - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index d06454c30..4f6971b3c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -18,10 +18,14 @@ package org.breedinginsight.brapps.importer.model.imports; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; + +import java.util.List; public interface BrAPIImportService { String getImportTypeId(); BrAPIImport getImportClass(); + List getWorkflows() throws Exception; default String getInvalidIntegerMsg(String columnName) { return String.format("Column name \"%s\" must be integer type, but non-integer type provided.", columnName); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java new file mode 100644 index 000000000..6cfffe73c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -0,0 +1,96 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; + +@Singleton +@Slf4j +public abstract class DomainImportService implements BrAPIImportService { + + // TODO: delete processor fields once WorkflowNavigator is used + private final Provider experimentProcessorProvider; + private final Provider processorManagerProvider; + private final Workflow workflowNavigator; + + + public DomainImportService(Provider experimentProcessorProvider, + Provider processorManagerProvider, + Workflow workflowNavigator) + { + this.experimentProcessorProvider = experimentProcessorProvider; + this.processorManagerProvider = processorManagerProvider; + this.workflowNavigator = workflowNavigator; + } + + @Override + public String getMissingColumnMsg(String columnName) { + return "Column heading does not match template or ontology"; + } + @Override + public List getWorkflows() throws Exception{ + return workflowNavigator.getWorkflows(); + } + + @Override + public ImportPreviewResponse process(ImportServiceContext context) + throws Exception { + + Optional.ofNullable(context.getWorkflowId()) + .filter(workflowId -> !workflowId.isEmpty()) + .ifPresent(workflowId -> log.info("Workflow: " + workflowId)); + + // TODO: return results from WorkflowNavigator once processing logic is in separate workflows + // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflowId())) { + Optional result = workflowNavigator.process(context); + + // Throw any exceptions caught during workflow processing + if (result.flatMap(ImportWorkflowResult::getCaughtException).isPresent()) { + throw result.flatMap(ImportWorkflowResult::getCaughtException).get(); + } + + return result.flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + + } else { + return processorManagerProvider.get().process(context.getBrAPIImports(), + List.of(experimentProcessorProvider.get()), + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); + } + } +} + diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index 45393f67c..90e8915f9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -19,10 +19,10 @@ import lombok.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import tech.tablesaw.api.Table; + import java.util.List; @Getter @@ -32,7 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { - private Workflow workflow; + private String workflowId; private List brAPIImports; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 616b6fa58..a9b6c62ac 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -18,39 +18,28 @@ package org.breedinginsight.brapps.importer.model.imports.experimentObservation; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.imports.DomainImportService; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; -import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import tech.tablesaw.api.Table; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; -import java.util.HashMap; -import java.util.List; @Singleton @Slf4j -public class ExperimentImportService implements BrAPIImportService { +public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - private final Provider experimentProcessorProvider; - private final Provider processorManagerProvider; - + // TODO: delete processor fields once WorkflowNavigator is used @Inject - public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider) + public ExperimentImportService(Provider experimentProcessorProvider, + Provider processorManagerProvider, + ExperimentWorkflowNavigator workflowNavigator) { - this.experimentProcessorProvider = experimentProcessorProvider; - this.processorManagerProvider = processorManagerProvider; + super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); } @Override @@ -63,47 +52,5 @@ public String getImportTypeId() { return IMPORT_TYPE_ID; } - @Override - public String getMissingColumnMsg(String columnName) { - return "Column heading does not match template or ontology"; - } - - @Override - public ImportPreviewResponse process(ImportServiceContext context) - throws Exception { - - ImportPreviewResponse response = null; - List processors = List.of(experimentProcessorProvider.get()); - - if (context.getWorkflow() != null) { - log.info("Workflow: " + context.getWorkflow().getName()); - - // TODO: change when workflows selection is ready - if (context.getWorkflow().getName().equals("CreateNewExperimentWorkflow")) { - ImportContext importContext = ImportContext.builder() - .importRows(context.getBrAPIImports()) - .user(context.getUser()) - .data(context.getData()) - .commit(context.isCommit()) - .upload(context.getUpload()) - .program(context.getProgram()) - .build(); - - return context.getWorkflow().process(importContext); - } - } - - // TODO: change to calling workflow process instead of processor manager - // other workflows besides create will pass through to old flow - response = processorManagerProvider.get().process(context.getBrAPIImports(), - processors, - context.getData(), - context.getProgram(), - context.getUpload(), - context.getUser(), - context.isCommit()); - return response; - - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 579c34a31..d69de11dd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -321,11 +321,11 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( try { double lat = Double.parseDouble(getLatitude()); double lon = Double.parseDouble(getLongitude()); - Point geoPoint = Point.from(lat, lon); + Point geoPoint = Point.from(lon, lat); if (getElevation() != null) { double elevation = Double.parseDouble(getElevation()); - geoPoint = Point.from(lat, lon, elevation); // geoPoint.withAlt(elevation) did not work + geoPoint = Point.from(lon, lat, elevation); // geoPoint.withAlt(elevation) did not work } BrApiGeoJSON coords = BrApiGeoJSON.builder() diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index ce52f483f..0caebe65e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -33,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; @Singleton @@ -57,6 +59,11 @@ public GermplasmImport getImportClass() { return new GermplasmImport(); } + @Override + public List getWorkflows() { + return new ArrayList<>(); + } + @Override public String getImportTypeId() { return IMPORT_TYPE_ID; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 2c8e8b7b5..eb7328ecf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.SampleSubmissionProcessor; @@ -33,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; @Singleton @@ -59,6 +61,11 @@ public BrAPIImport getImportClass() { return new SampleSubmissionImport(); } + @Override + public List getWorkflows() { + return new ArrayList<>(); + } + @Override public ImportPreviewResponse process(ImportServiceContext context) throws Exception { List processors = List.of(sampleProcessorProvider.get()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java new file mode 100644 index 000000000..c34794bab --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface ExperimentWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java new file mode 100644 index 000000000..f26342d3b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface GermplasmWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java index f9205cb87..a7b6f7dc3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -20,6 +20,7 @@ import lombok.*; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; @@ -39,10 +40,19 @@ public class ImportContext { private UUID workflowId; private ImportUpload upload; private List importRows; - // TODO: move this out potentially - //private Map mappedBrAPIImport; private Table data; private Program program; private User user; private boolean commit; + + public static ImportContext from(ImportServiceContext importServiceContext) { + return ImportContext.builder() + .program(importServiceContext.getProgram()) + .user(importServiceContext.getUser()) + .commit(importServiceContext.isCommit()) + .data(importServiceContext.getData()) + .importRows(importServiceContext.getBrAPIImports()) + .upload(importServiceContext.getUpload()) + .build(); + } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java deleted file mode 100644 index 8dc5800bc..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.model.workflow; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; -import lombok.experimental.Accessors; -import lombok.experimental.SuperBuilder; - -import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; -import org.jooq.Record; - -import static org.breedinginsight.dao.db.tables.ImporterMappingWorkflowTable.IMPORTER_MAPPING_WORKFLOW; - -@Getter -@Setter -@Accessors(chain=true) -@ToString -@NoArgsConstructor -@SuperBuilder() -public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { - - - public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { - this.setId(importMappingWorkflowEntity.getId()); - this.setName(importMappingWorkflowEntity.getName()); - this.setBean(importMappingWorkflowEntity.getBean()); - this.setPosition(importMappingWorkflowEntity.getPosition()); - } - public static ImportMappingWorkflow parseSQLRecord(Record record) { - - return ImportMappingWorkflow.builder() - .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) - .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) - .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) - .position(record.getValue(IMPORTER_MAPPING_WORKFLOW.POSITION)) - .build(); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java new file mode 100644 index 000000000..30dbac06a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -0,0 +1,31 @@ +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +public class ImportWorkflow { + private String id; + private String name; + private int order; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java new file mode 100644 index 000000000..f59b83b55 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -0,0 +1,34 @@ +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; + +import java.util.Optional; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +public class ImportWorkflowResult { + private ImportWorkflow workflow; + private Optional importPreviewResponse; + private Optional caughtException; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index 677f35469..fc29774f0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -28,9 +28,5 @@ @ToString @NoArgsConstructor public class ProcessedData { - // TODO: remove this, already in ImportPreviewResponse - //private Map statistics; - // TODO private Map mappedBrAPIImport; - private ImportPreviewResponse importPreviewResponse; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java new file mode 100644 index 000000000..19be9bdcc --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface SampleSubmissionWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 5d8cbe66e..17f48ee57 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -1,38 +1,17 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.model.workflow; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; - -public interface Workflow { +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; - /** - * Processes the given import context and returns the processed data. - * - * @param context the import context containing the necessary data for processing - * @return the processed data - */ - ImportPreviewResponse process(ImportContext context) throws Exception; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; - /** - * Retrieves the name of the Workflow for logging display purposes. - * - * @return the name of the Workflow - */ - String getName(); +@FunctionalInterface +public interface Workflow extends Ordered { + Optional process(ImportServiceContext context); + default List getWorkflows() { + // Default implementation for getWorkflows method + return new ArrayList<>(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index ef6c5f884..e24cfeef2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.micronaut.http.HttpStatus; -import io.micronaut.http.annotation.PathVariable; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.multipart.CompletedFileUpload; import io.micronaut.http.server.exceptions.InternalServerException; @@ -33,7 +32,6 @@ import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingProgramDAO; -import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; @@ -43,12 +41,9 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; -import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramService; @@ -87,14 +82,12 @@ public class FileImportService { private final ImportDAO importDAO; private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; - private final ImportMappingWorkflowDAO importMappingWorkflowDAO; - private final WorkflowFactory workflowFactory; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - ImportMappingWorkflowDAO importMappingWorkflowDAO, WorkflowFactory workflowFactory, UserService userService) { + UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -106,8 +99,6 @@ public class FileImportService { this.dsl = dsl; this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; - this.importMappingWorkflowDAO = importMappingWorkflowDAO; - this.workflowFactory = workflowFactory; } public List getAllImportTypeConfigs() { @@ -333,7 +324,7 @@ public ImportResponse uploadData(UUID programId, UUID mappingId, AuthenticatedUs return response; } - public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws + public ImportResponse updateUpload(UUID programId, UUID uploadId, String workflow, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws DoesNotExistException, UnprocessableEntityException, AuthorizationException { Program program = validateRequest(programId, actingUser); @@ -383,7 +374,7 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowI } else { brAPIImportList = mappingManager.map(mappingConfig, data); } - processFile(workflowId, brAPIImportList, data, program, upload, user, commit, importService, actingUser); + processFile(workflow, brAPIImportList, data, program, upload, user, commit, importService, actingUser); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -429,20 +420,14 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(UUID workflowId, List finalBrAPIImportList, Table data, Program program, + private void processFile(String workflowId, List finalBrAPIImportList, Table data, Program program, ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - Workflow workflow = null; - if (workflowId != null) { - Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); - workflow = optionalWorkflow.orElse(null); - } - ImportServiceContext context = ImportServiceContext.builder() - .workflow(workflow) + .workflowId(workflowId) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) @@ -586,8 +571,12 @@ public List getSystemMappingByName(String name) { return importMappings; } - public List getWorkflowsForSystemMapping(UUID mappingId) { - return importMappingWorkflowDAO.getWorkflowsByImportMappingId(mappingId); + public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException, Exception { + ImportMapping mappingConfig = importMappingDAO.getMapping(mappingId) + .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); + BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) + .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); + return importService.getWorkflows(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java index cb93daf48..8d3ed8146 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java @@ -32,13 +32,7 @@ @Singleton public class FileMappingUtil { - public static final String EXPERIMENT_TEMPLATE_NAME = "ExperimentsTemplateMap"; - private FileImportService fileImportService; - - - @Inject - public FileMappingUtil(FileImportService fileImportService) { - this.fileImportService = fileImportService; + public FileMappingUtil() { } // Returns a list of integers to identify the target row of the relationship. -1 if no relationship was found diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f27d89338..95e196c5b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1244,7 +1244,8 @@ private void fetchOrCreateObservationPIO(Program program, } if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(key, value, column, rowNum)){ + // Update observation value only if it is changed and new value is not blank. + if (!isObservationMatched(key, value, column, rowNum) && StringUtils.isNotBlank(value)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java new file mode 100644 index 000000000..6acd4aad5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -0,0 +1,87 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; + +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +@Primary +@Singleton +public class ExperimentWorkflowNavigator implements ExperimentWorkflow { + private final List workflows; + + public ExperimentWorkflowNavigator(List workflows) { + this.workflows = workflows; + } + + /** + * Process the import service context by executing a series of workflows in order + * + * This method iterates over the list of workflows provided, executing each workflow's process method + * with the given import service context. It then filters out empty results and returns the first non-empty result. + * + * @param context The import service context containing the data to be processed + * @return An Optional containing the first non-empty ImportWorkflowResult from the executed workflows, or an empty Optional if no non-empty result is found + */ + @Override + public Optional process(ImportServiceContext context) { + /** + * Have each workflow in order process the context, returning the first non-empty result + */ + return workflows.stream() + .map(workflow->workflow.process(context)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } + + /** + * Retrieves a list of ImportWorkflow objects containing metadata about each workflow that processed the import context. + * + * @return List of ImportWorkflow objects with workflow metadata + */ + public List getWorkflows() { + List workflowSummaryList = workflows.stream() + .map(workflow -> workflow.process(null)) // Process each workflow with a null context + .filter(Optional::isPresent) // Filter out any workflows that do not return a result + .map(Optional::get) // Extract the result from Optional + .map(result -> result.getWorkflow()) // Retrieve the workflow metadata + .collect(Collectors.toList()); // Collect the workflow metadata into a list + + // Set the order field for each workflow based on its position in the list + for (int i = 0; i < workflowSummaryList.size(); i++) { + workflowSummaryList.get(i).setOrder(i); // Set the order for each workflow + } + + return workflowSummaryList; // Return the list of workflow metadata + } + + public enum Workflow { + NEW_OBSERVATION("new-experiment","Create new experiment"), + APPEND_OVERWRITE("append-dataset", "Append experimental dataset"); + + private String id; + private String name; + + Workflow(String id, String name) { + + this.id = id; + this.name = name; + } + + public String getId() { + return id; + } + public String getName() { return name; } + + public boolean isEqual(String value) { + return Optional.ofNullable(id.equals(value)).orElse(false); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 124a3a0f9..ea5c388cb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -1,51 +1,54 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; -import javax.inject.Named; +import javax.inject.Singleton; +import java.util.Optional; -/** - * This class represents a workflow for appending and overwriting phenotypes. The bean name must match the appropriate - * bean column value in the import_mapping_workflow db table - */ +@Getter +@Singleton +public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; -@Prototype -@Named("AppendOverwritePhenotypesWorkflow") -public class AppendOverwritePhenotypesWorkflow implements Workflow { - @Override - public ImportPreviewResponse process(ImportContext context) { - // TODO - return null; + public AppendOverwritePhenotypesWorkflow(){ + this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; + } + + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending or overwriting observation data + if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; } - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ @Override - public String getName() { - return "AppendOverwritePhenotypesWorkflow"; + public int getOrder() { + return 2; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java index cd6842476..c81e265cd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java @@ -23,7 +23,8 @@ import java.util.List; import java.util.Map; -// TODO: move to common higher level location +// TODO: move to common higher level location, could be used by both append and create workflows so being located +// in the create namespace won't make sense if we decide to do that in the future. @Getter @Setter @Builder diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 343b4475d..d0a4ca975 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -55,16 +55,21 @@ import java.util.List; import java.util.Map; -/** - * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column - * value in the import_mapping_workflow db table - */ +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; -@Prototype -@Slf4j -@Named("CreateNewExperimentWorkflow") -public class CreateNewExperimentWorkflow implements Workflow { +import javax.inject.Singleton; +import java.util.Optional; +@Slf4j +@Getter +@Singleton +public class CreateNewExperimentWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; private final PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep; private final PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep; private final CommitPendingImportObjectsStep commitPendingImportObjectsStep; @@ -85,27 +90,26 @@ public CreateNewExperimentWorkflow(PopulateExistingPendingImportObjectsStep popu this.validatePendingImportObjectsStep = validatePendingImportObjectsStep; this.statusService = statusService; this.experimentPhenotypeService = experimentPhenotypeService; + this.workflow = ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION; } - @Override - public ImportPreviewResponse process(ImportContext context) throws Exception { + private ImportPreviewResponse runWorkflow(ImportContext context) throws Exception { ImportUpload upload = context.getUpload(); boolean commit = context.isCommit(); List importRows = context.getImportRows(); + ProcessedData processedData = new ProcessedData(); // Make sure the file does not contain obs unit ids before proceeding if (containsObsUnitIDs(context)) { - // TODO: get file name - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Error detected in file, " + - upload.getUploadFileName() + ". ObsUnitIDs are detected. Import cannot proceed"); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "ObsUnitIDs are detected"); } statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); - ProcessedData processedData = populateNewPendingImportObjectsStep.process(processContext, phenotypeData); + populateNewPendingImportObjectsStep.process(processContext, phenotypeData); ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); // short circuit if there were validation errors @@ -136,14 +140,54 @@ public ImportPreviewResponse process(ImportContext context) throws Exception { } /** - * Retrieves the name of the workflow. This is used for logging display purposes. + * Process the import service context and returns an Optional ImportWorkflowResult. * - * @return the name of the workflow + * @param context The import service context to be processed. If null, then it skips processing but returns the result with no-preview. + * @return An Optional ImportWorkflowResult which contains the workflow and import preview response (if available). + * If the context is null, it returns the result with no-preview. */ + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) + .build(); + + // No-preview result + ImportWorkflowResult workflowResult = ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .caughtException(Optional.empty()) + .build(); + + // Skip this workflow unless creating a new experiment + if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return Optional.of(workflowResult); + } + + // TODO: unify usage of single import context type throughout + ImportContext importContext = ImportContext.from(context); + + // Start processing the import... + ImportPreviewResponse response; + try { + response = runWorkflow(importContext); + workflowResult.setImportPreviewResponse(Optional.of(response)); + } catch(Exception e) { + workflowResult.setCaughtException(Optional.of(e)); + } + + return Optional.of(workflowResult); + } @Override - public String getName() { - return "CreateNewExperimentWorkflow"; + public int getOrder() { + return 1; } // TODO: move to shared area diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index a471b84f3..f20c8a1a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -73,9 +73,7 @@ @Slf4j public class PopulateNewPendingImportObjectsStep { - private final ExperimentValidateService experimentValidateService; private final ExperimentSeasonService experimentSeasonService; - private final BrAPIObservationDAO brAPIObservationDAO; private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; private final Gson gson; @@ -84,31 +82,28 @@ public class PopulateNewPendingImportObjectsStep { private String BRAPI_REFERENCE_SOURCE; @Inject - public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentValidateService, - ExperimentSeasonService experimentSeasonService, - BrAPIObservationDAO brAPIObservationDAO, + public PopulateNewPendingImportObjectsStep(ExperimentSeasonService experimentSeasonService, BrAPIObservationUnitDAO brAPIObservationUnitDAO, DSLContext dsl) { - this.experimentValidateService = experimentValidateService; this.experimentSeasonService = experimentSeasonService; - this.brAPIObservationDAO = brAPIObservationDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.dsl = dsl; this.gson = new JSON().getGson(); } - public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) + /** + * TODO: in the future returning ProcessedData rather than modifying in-place would be preferrable. + * + * @param context (modified in-place) + * @param phenotypeData + * @return + * @throws MissingRequiredInfoException + * @throws UnprocessableEntityException + * @throws ApiException + */ + public void process(ProcessContext context, ProcessedPhenotypeData phenotypeData) throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { - - Table data = context.getImportContext().getData(); - ImportUpload upload = context.getImportContext().getUpload(); - ImportContext importContext = context.getImportContext(); - populatePendingImportObjects(context, phenotypeData); - - - // TODO: implement - return new ProcessedData(); } @@ -573,7 +568,9 @@ private void fetchOrCreateObservationPIO(ProcessedPhenotypeData phenotypeData, key = getImportObservationHash(importRow, variableName); if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(phenotypeData, pendingData, key, value, column, rowNum)){ + // NOTE: BI-2128 change added after refactor branch + // Update observation value only if it is changed and new value is not blank. + if (!isObservationMatched(phenotypeData, pendingData, key, value, column, rowNum) && StringUtils.isNotBlank(value)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java deleted file mode 100644 index 05269964f..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; - -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; - -import javax.inject.Named; - -/** - * This class represents a workflow for adding new environments to an existing experiment. The bean name must match - * the appropriate bean column value in the import_mapping_workflow db table - */ - -@Prototype -@Named("CreateNewEnvironmentWorkflow") -public class CreateNewEnvironmentWorkflow implements Workflow { - @Override - public ImportPreviewResponse process(ImportContext context) { - // TODO - return null; - } - - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ - @Override - public String getName() { - return "CreateNewEnvironmentWorkflow"; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java deleted file mode 100644 index 17d6fe64e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.workflow; - -import io.micronaut.context.ApplicationContext; -import io.micronaut.context.annotation.Factory; -import io.micronaut.context.exceptions.NoSuchBeanException; -import io.micronaut.inject.qualifiers.Qualifiers; -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; - -import javax.inject.Inject; -import java.util.Optional; -import java.util.UUID; - -@Factory -@Slf4j -public class WorkflowFactory { - - private final ImportMappingWorkflowDAO importMappingWorkflowDAO; - private final ApplicationContext applicationContext; - - @Inject - public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, - ApplicationContext applicationContext) { - this.importMappingWorkflowDAO = importMappingWorkflowDAO; - this.applicationContext = applicationContext; - } - - /** - * Produces the appropriate workflow instance based on the import context - * - * @param context the import context - * @return an Optional containing the workflow if id is not null, otherwise an empty Optional - * - * @throws IllegalStateException - * @throws NoSuchBeanException - */ - public Optional getWorkflow(UUID workflowId) { - - if (workflowId != null) { - // construct workflow from db record - Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); - - ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { - String msg = "Must have record in db for workflowId"; - log.error(msg); - return new IllegalStateException(msg); - }); - - // newer versions of micronaut have fancier ways to do this using annotations with provider but as - // far as I can tell it's not available in 2.5 - Workflow workflow; - try { - workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); - } catch (NoSuchBeanException e) { - log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); - throw e; - } - - return Optional.of(workflow); - } - - return Optional.empty(); - } -} diff --git a/src/main/resources/db/migration/V1.22.0__germplasm_restore_accession_number_field.sql b/src/main/resources/db/migration/V1.22.0__germplasm_restore_accession_number_field.sql new file mode 100644 index 000000000..31530da81 --- /dev/null +++ b/src/main/resources/db/migration/V1.22.0__germplasm_restore_accession_number_field.sql @@ -0,0 +1,21 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +update importer_mapping +set mapping = '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent GID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent GID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "b910adfe-a474-47a0-8410-514578898436", "value": {"fileFieldName": "Synonyms"}, "objectId": "synonyms"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "bc09c6e1-866f-45c3-a285-a25859e8c982", "value": {"fileFieldName": "GID"}, "objectId": "germplasmAccessionNumber"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', + file = '[{"Germplasm Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2"}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "2", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "1", "Synonyms": "test1;test2"}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "7", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "2", "Synonyms": "test1;test2"}, {"Name": "BITest BI003", "Source": "Ithaca Nursery", "Entry No": "8", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "7", "Female Parent Entry No": "2", "Synonyms": "test1;test2"}, {"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "3", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2"}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "640e8b58-1b1c-44a6-91a6-85b2b773376b", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2"}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "5", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent GID": "", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "2", "Synonyms": "test1;test2"}]' +where name = 'GermplasmTemplateMap'; \ No newline at end of file diff --git a/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql deleted file mode 100644 index 2c9d4d547..000000000 --- a/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql +++ /dev/null @@ -1,51 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - Table maps workflows to import mappings and provides required configuration options - - mapping_id - link to importer_mapping that this provides a workflow for - name - name that will be displayed on front end - bean - must match @Named("") annotation on Workflow class - position - for ordering records explicitly, wanted for front end default option and order - */ -CREATE TABLE importer_mapping_workflow -( - like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, - mapping_id UUID NOT NULL, - name TEXT NOT NULL, - bean TEXT NOT NULL, - position INTEGER NOT NULL -); - -ALTER TABLE importer_mapping_workflow - ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); - -DO -$$ -DECLARE - exp_mapping_id UUID; -BEGIN - exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); - -INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) -VALUES - (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), - (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), - (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); -END -$$; \ No newline at end of file diff --git a/src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql b/src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql new file mode 100644 index 000000000..7cd4be61e --- /dev/null +++ b/src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql @@ -0,0 +1,18 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +update breeding_method set name = 'Unknown', category = 'Increase' where abbreviation = 'UNK'; \ No newline at end of file diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 50c54602e..4fafa232e 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+737 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/72bf86b660d7fcb7b0b0d3479a8fdc72f64e1a5f +version=v0.10.0+756 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a960d556180bd80ec3ca543555cee1b912fad236 diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 6bc203d64..fbb02435e 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -24,6 +24,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; @@ -77,6 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static io.micronaut.http.HttpRequest.GET; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.*; @@ -85,7 +87,6 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentFileImportTest extends BrAPITest { - private static final String OVERWRITE = "overwrite"; private FannyPack securityFp; private String mappingId; @@ -143,9 +144,11 @@ public class ExperimentFileImportTest extends BrAPITest { private BrAPISeasonDAO seasonDAO; private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) - (json, type, context) -> OffsetDateTime.parse(json.getAsString())) - .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) - .create(); + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) + .create(); + + private String newExperimentWorkflowId; @BeforeAll public void setup() { @@ -154,7 +157,27 @@ public void setup() { mappingId = (String) setupObjects.get("mappingId"); testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); + newExperimentWorkflowId = getNewExperimentWorkflowId(); + } + + /** + * TODO: assumes new workflow is first in list, doesn't look at position property, would be more robust to + * look at that instead of assuming order + * @return + */ + public String getNewExperimentWorkflowId() { + // GET /import/mappings{?importName} + Flowable> call = client.exchange( + GET("/import/mappings/"+mappingId+"/workflows").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + + return JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); } /* @@ -194,16 +217,20 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String workflowId = "new-experiment"; + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + // TODO: remove this + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -252,16 +279,18 @@ public void importNewExpMultiNewEnvSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(2, previewRows.size()); JsonObject firstRow = previewRows.get(0).getAsJsonObject(); @@ -299,7 +328,8 @@ public void importExistingExpAndEnvErrorMessage() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map dupExp = new HashMap<>(); dupExp.put(Columns.GERMPLASM_GID, "1"); @@ -316,16 +346,17 @@ public void importExistingExpAndEnvErrorMessage() { dupExp.put(Columns.ROW, "1"); dupExp.put(Columns.COLUMN, "1"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, expResult.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + expResult); + assertTrue(expResult.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); } @Test @@ -348,9 +379,10 @@ public void importNewEnvNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -382,43 +414,53 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit, newExperimentWorkflowId); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit, newExperimentWorkflowId); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @Test @@ -442,7 +484,8 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -492,7 +535,9 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -530,7 +575,8 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); } @ParameterizedTest @@ -555,7 +601,8 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -595,7 +642,9 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -618,21 +667,24 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + //importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -822,6 +874,81 @@ public void importNewObservationDataByObsUnitId(boolean commit) { } } + /* + Scenario: + - an experiment was created with observations + - an overwrite operation is attempted with blank observation values + - verify blank observation values do not overwrite original values + */ + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { + List traits = importTestUtils.createTraits(1); + Program program = createProgram("Overwrite Attempt With Blank Obs"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "1"); // Valid observation value. + + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + + // Fetch the ObsUnitId to use in the overwrite upload. + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + assertRowSaved(newExp, program, traits); + + Map newObsVar = new HashMap<>(); + newObsVar.put(Columns.GERMPLASM_GID, "1"); + newObsVar.put(Columns.TEST_CHECK, "T"); + newObsVar.put(Columns.EXP_TITLE, "Test Exp"); + newObsVar.put(Columns.EXP_UNIT, "Plot"); + newObsVar.put(Columns.EXP_TYPE, "Phenotyping"); + newObsVar.put(Columns.ENV, "New Env"); + newObsVar.put(Columns.ENV_LOCATION, "Location A"); + newObsVar.put(Columns.ENV_YEAR, "2023"); + newObsVar.put(Columns.EXP_UNIT_ID, "a-1"); + newObsVar.put(Columns.REP_NUM, "1"); + newObsVar.put(Columns.BLOCK_NUM, "1"); + newObsVar.put(Columns.ROW, "1"); + newObsVar.put(Columns.COLUMN, "1"); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); // Indicates this is an overwrite. + newObsVar.put(traits.get(0).getObservationVariableName(), ""); // Empty string should be no op. + + Map requestBody = new HashMap<>(); + requestBody.put("overwrite", "true"); + requestBody.put("overwriteReason", "testing"); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), requestBody, commit, client, program, mappingId); + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + // Verify that the overwrite attempt with blank observation value did not overwrite the original value. + assertRowSaved(newExp, program, traits); + if(commit) { + assertRowSaved(newExp, program, traits); + } else { + assertValidPreviewRow(newExp, row, program, traits); + } + } @ParameterizedTest @ValueSource(booleans = {true, false}) @@ -948,6 +1075,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { - a new experiment is created after the first experiment - verify the second experiment gets created successfully */ + //TODO: this one @Test @SneakyThrows public void importSecondExpAfterFirstExpWithObs() { @@ -969,7 +1097,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + //JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -997,7 +1126,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + //JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -1086,15 +1216,15 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { /* Scenario: - - an experiment was created with observations - - do a second upload with additional observations for the experiment. Make sure the original observations are blank values - - verify the second set of observations get uploaded successfully + - Create an experiment with valid observations. + - Upload a second file with (1) a blank observation, (2) a changed valid observation, and (3) a new observation for the experiment. + - Verify that (1) the blank observation makes no change, (2) the changed observation is overwritten, and (3) new observations are appended to the experiment. */ @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { - List traits = importTestUtils.createTraits(2); + List traits = importTestUtils.createTraits(3); Program program = createProgram("Exp with additional Uploads (blank) "+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); @@ -1110,7 +1240,9 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(Columns.BLOCK_NUM, "1"); newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - newExp.put(traits.get(0).getObservationVariableName(), "1"); + String originalValue = "1"; // Convenience variable, this value is reused. + newExp.put(traits.get(0).getObservationVariableName(), originalValue); + newExp.put(traits.get(1).getObservationVariableName(), "2"); importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); @@ -1140,10 +1272,15 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); - newObservation.put(traits.get(0).getObservationVariableName(), ""); - newObservation.put(traits.get(1).getObservationVariableName(), "2"); + newObservation.put(traits.get(0).getObservationVariableName(), ""); // This blank value should not overwrite. + newObservation.put(traits.get(1).getObservationVariableName(), "3"); // This valid value should overwrite. + newObservation.put(traits.get(2).getObservationVariableName(), "4"); // This valid new observation should be appended. - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + Map requestBody = new HashMap<>(); + requestBody.put("overwrite", "true"); + requestBody.put("overwriteReason", "testing"); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), requestBody, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1154,6 +1291,9 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + // The blank value should not have produced an overwrite. + // This change is to make the "expected" value correct. + newObservation.put(traits.get(0).getObservationVariableName(), originalValue); if(commit) { assertRowSaved(newObservation, program, traits); } else { @@ -1253,10 +1393,10 @@ private Map assertRowSaved(Map expected, Program assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { assertEquals(expected.get(Columns.TEST_CHECK), - ou.getObservationUnitPosition() - .getEntryType() - .name() - .substring(0, 1)); + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); } assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); @@ -1349,10 +1489,10 @@ private Map assertValidPreviewRow(Map expected, if(traits != null) { assertNotNull(actual.get("observations")); observations = StreamSupport.stream(actual.getAsJsonArray("observations") - .spliterator(), false) - .map(obs -> gson.fromJson(obs.getAsJsonObject() - .getAsJsonObject("brAPIObject"), BrAPIObservation.class)) - .collect(Collectors.toList()); + .spliterator(), false) + .map(obs -> gson.fromJson(obs.getAsJsonObject() + .getAsJsonObject("brAPIObject"), BrAPIObservation.class)) + .collect(Collectors.toList()); ret.put("observations", observations); } @@ -1360,10 +1500,10 @@ private Map assertValidPreviewRow(Map expected, assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { assertEquals(expected.get(Columns.TEST_CHECK), - ou.getObservationUnitPosition() - .getEntryType() - .name() - .substring(0, 1)); + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); } assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); @@ -1434,8 +1574,8 @@ private String yearToSeasonDbId(String year, UUID programId) throws ApiException for (BrAPISeason season : seasons) { if (null == season.getSeasonName() || season.getSeasonName() - .isBlank() || season.getSeasonName() - .equals(year)) { + .isBlank() || season.getSeasonName() + .equals(year)) { return season.getSeasonDbId(); } } @@ -1446,17 +1586,17 @@ private String yearToSeasonDbId(String year, UUID programId) throws ApiException private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { SpeciesEntity validSpecies = speciesDAO.findAll().get(0); SpeciesRequest speciesRequest = SpeciesRequest.builder() - .commonName(validSpecies.getCommonName()) - .id(validSpecies.getId()) - .build(); + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); ProgramRequest programRequest1 = ProgramRequest.builder() - .name(name) - .abbreviation(abbv) - .documentationUrl("localhost:8080") - .objective("To test things") - .species(speciesRequest) - .key(key) - .build(); + .name(name) + .abbreviation(abbv) + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key(key) + .build(); TestUtils.insertAndFetchTestProgram(gson, client, programRequest1); @@ -1525,6 +1665,33 @@ private JsonObject uploadAndVerifyFailure(Program program, File file, String exp JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + private JsonObject uploadAndVerifyWorkflowFailure(Program program, File file, String expectedColumnError, boolean commit, String workflowId) throws InterruptedException, IOException { + + //Flowable> call = importTestUtils.uploadDataFile(file, null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(file, null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject result = JsonParser.parseString(upload).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); assertEquals(1, rowErrors.size()); JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 12b79ac15..f5dd37f51 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -97,6 +97,38 @@ public Flowable> uploadDataFile(File file, Map> uploadWorkflowDataFile(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) { + + MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); + + // Upload file + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", program.getId(), mappingId); + Flowable> call = client.exchange( + POST(uploadUrl, requestBody) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + String importId = result.get("importId").getAsString(); + + // Process data + String url = String.format("/programs/%s/import/mappings/%s/workflows/%s/data/%s/%s", program.getId(), mappingId, workflowId, importId, commit ? "commit" : "preview"); + Flowable> processCall = client.exchange( + PUT(url, userData) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + return processCall; + + } + public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = client.exchange( GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) @@ -123,16 +155,16 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, // Species Species validSpecies = speciesService.getAll().get(0); SpeciesRequest speciesRequest = SpeciesRequest.builder() - .commonName(validSpecies.getCommonName()) - .id(validSpecies.getId()) - .build(); + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); // Insert program ProgramRequest program = ProgramRequest.builder() - .name("Test Program") - .species(speciesRequest) - .key("TEST") - .build(); + .name("Test Program") + .species(speciesRequest) + .key("TEST") + .build(); Program validProgram = this.insertAndFetchTestProgram(program, client, gson); // Get import @@ -141,18 +173,18 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, ); HttpResponse response = call.blockingFirst(); String mappingId = JsonParser.parseString(response.body()).getAsJsonObject() - .getAsJsonObject("result") - .getAsJsonArray("data") - .get(0).getAsJsonObject().get("id").getAsString(); + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); BiUserEntity testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); return Map.of("program", validProgram, - "mappingId", mappingId, - "testUser", testUser, - "securityFp", securityFp); + "mappingId", mappingId, + "testUser", testUser, + "securityFp", securityFp); } @@ -170,6 +202,43 @@ public JsonObject uploadAndFetch(File file, Map userData, Boolea return result; } + public JsonObject uploadAndFetchWorkflow(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } + + public JsonObject uploadAndFetchWorkflowNoStatusCheck(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + return result; + } + public List createTraits(int numToCreate) { List traits = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) {