Merge pull request #109 from oesteban/enh/smriprep-fasttrack

oesteban · web-flow · commit f2df371ced33 · 2020-05-06T09:42:41.000-07:00
ENH: Port the new *anatomical fast-track* from *fMRIPrep*
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -126,8 +126,8 @@ jobs:
     steps:
       - restore_cache:
           keys:
-            - data-v2-{{ .Revision }}
-            - data-v2-
+            - data-thp-v1-{{ .Revision }}
+            - data-thp-v1-
       - run:
           name: Get test data from THP002
           command: |
@@ -151,6 +151,14 @@ jobs:
             else
               echo "FreeSurfer derivatives of THP002 were cached"
             fi
+      - save_cache:
+         key: data-thp-v1-{{ .Revision }}-{{ epoch }}
+         paths:
+            - /tmp/data/THP002
+      - restore_cache:
+          keys:
+            - data-ds001771-v1-{{ .Revision }}
+            - data-ds001771-v1-
       - run:
           name: Get test data (ds001771)
           command: |
@@ -163,17 +171,20 @@ jobs:
               echo "Dataset ds001771_sub-36 was cached"
             fi
       - run:
-          name: Get FreeSurfer derivatives for ds001771_sub-36
+          name: Get anatomical derivatives for ds001771
           command: |
-            if [[ ! -d /tmp/data/ds001771/derivatives/freesurfer-6.0.1 ]]; then
+            if [[ ! -d /tmp/data/ds001771/derivatives ]]; then
               mkdir -p /tmp/data/ds001771/derivatives
               wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
-                -O ds001771_sub-36_derivatives_freesurfer.tar.gz "https://files.osf.io/v1/resources/8k95s/providers/osfstorage/5e21e74fedceab00ad82e047"
-              tar xvzf ds001771_sub-36_derivatives_freesurfer.tar.gz -C /tmp/data/ds001771/derivatives
-              mv /tmp/data/ds001771/derivatives/freesurfer /tmp/data/ds001771/derivatives/freesurfer-6.0.1
+                -O ds001771-derivs.tar.gz "https://files.osf.io/v1/resources/8k95s/providers/osfstorage/5eb251fba2199500215d1dc3"
+              tar xvzf ds001771-derivs.tar.gz -C /tmp/data/ds001771/
             else
-              echo "FreeSurfer derivatives of ds001771 were cached"
+              echo "Anatomical derivatives of ds001771 were cached"
             fi
+      - save_cache:
+         key: data-ds001771-v1-{{ .Revision }}-{{ epoch }}
+         paths:
+            - /tmp/data/ds001771
       - run:
           name: Store FreeSurfer license file
           command: |
@@ -192,10 +203,6 @@ jobs:
           paths:
             - fslicense
             - config/nipype.cfg
-      - save_cache:
-         key: data-v2-{{ .Revision }}-{{ epoch }}
-         paths:
-            - /tmp/data
 
   THP002:
     machine:
@@ -222,7 +229,7 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - data-v2-{{ .Revision }}
+            - data-thp-v1-{{ .Revision }}
       - restore_cache:
           keys:
             - build-v1-{{ .Branch }}-{{ epoch }}
@@ -329,7 +336,7 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - data-v2-{{ .Revision }}
+            - data-ds001771-v1-{{ .Revision }}
       - restore_cache:
           keys:
             - build-v1-{{ .Branch }}-{{ epoch }}
@@ -347,41 +354,11 @@ jobs:
           command: |
             docker pull localhost:5000/dmriprep
             docker tag localhost:5000/dmriprep nipreps/dmriprep:latest
-      - restore_cache:
-          keys:
-            - ds001771-anat-v00-{{ .Branch }}-{{ .Revision }}
-            - ds001771-anat-v00-{{ .Branch }}
-            - ds001771-anat-v00-master
-            - ds001771-anat-v00-
-      - run:
-          name: Run anatomical workflow on ds001771
-          no_output_timeout: 2h
-          command: |
-            mkdir -p /tmp/ds001771/{work,derivatives}
-            docker run -e FS_LICENSE=$FS_LICENSE --rm \
-                -v /tmp/data/ds001771:/data \
-                -v /tmp/ds001771/derivatives:/out \
-                -v /tmp/fslicense/license.txt:/tmp/fslicense/license.txt:ro \
-                -v /tmp/config/nipype.cfg:/home/dmriprep/.nipype/nipype.cfg \
-                -v /tmp/ds001771/work:/work \
-                --user $(id -u):$(id -g) \
-                nipreps/dmriprep:latest /data /out participant -vv \
-                --fs-subjects-dir /data/derivatives/freesurfer-6.0.1 --sloppy --anat-only \
-                --notrack --skip-bids-validation -w /work --omp-nthreads 2 --nprocs 2
-      - run:
-          name: Clean-up after anatomical run
-          command: |
-            rm -rf /tmp/ds001771/work/dmriprep_wf/fsdir*
-            rm -rf /tmp/ds001771/work/reportlets
-          when: on_success
-      - save_cache:
-         key: ds001771-anat-v00-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
-         paths:
-            - /tmp/ds001771/work
       - run:
           name: Run full diffusion workflow on ds001771
           no_output_timeout: 2h
           command: |
+            mkdir -p /tmp/ds001771/work /tmp/ds001771/derivatives
             docker run -e FS_LICENSE=$FS_LICENSE --rm \
                 -v /tmp/data/ds001771:/data \
                 -v /tmp/ds001771/derivatives:/out \
@@ -390,8 +367,10 @@ jobs:
                 -v /tmp/ds001771/work:/work \
                 --user $(id -u):$(id -g) \
                 nipreps/dmriprep:latest /data /out participant -vv \
-                --fs-subjects-dir /data/derivatives/freesurfer-6.0.1 --sloppy \
-                --notrack --skip-bids-validation -w /work --omp-nthreads 2 --nprocs 2
+                -w /work --omp-nthreads 2 --nprocs 2 \
+                --notrack --skip-bids-validation --sloppy \
+                --fs-subjects-dir /data/derivatives/freesurfer-6.0.1 \
+                --anat-derivatives /data/derivatives/smriprep-0.6.0
       - store_artifacts:
           path: /tmp/ds001771/derivatives/dmriprep
       - run:
diff --git a/dmriprep/cli/parser.py b/dmriprep/cli/parser.py
@@ -114,6 +114,11 @@ def _bids_filter(value):
         "{<suffix>:{<entity>:<filter>,...},...} "
         "(https://github.com/bids-standard/pybids/blob/master/bids/layout/config/bids.json)",
     )
+    g_bids.add_argument(
+        "--anat-derivatives", action='store', metavar="PATH", type=PathExists,
+        help="Reuse the anatomical derivatives from another fMRIPrep run or calculated "
+             "with an alternative processing tool (NOT RECOMMENDED)."
+    )
 
     g_perfm = parser.add_argument_group("Options to handle performance")
     g_perfm.add_argument(
diff --git a/dmriprep/cli/run.py b/dmriprep/cli/run.py
@@ -101,10 +101,8 @@ def main():
             config.loggers.workflow.log(
                 25,
                 "Works derived from this dMRIPrep execution should "
-                "include the following boilerplate:\n\n%s",
-                (
-                    config.execution.output_dir / "dmriprep" / "logs" / "CITATION.md"
-                ).read_text(),
+                "include the following boilerplate: "
+                f"{config.execution.output_dir / 'dmriprep' / 'logs' / 'CITATION.md'}."
             )
 
         if config.workflow.run_reconall:
diff --git a/dmriprep/config/__init__.py b/dmriprep/config/__init__.py
@@ -324,6 +324,8 @@ def init(cls):
 class execution(_Config):
     """Configure run-level settings."""
 
+    anat_derivatives = None
+    """A path where anatomical derivatives are found to fast-track *sMRIPrep*."""
     bids_dir = None
     """An existing path to the dataset, which must be BIDS-compliant."""
     bids_description_hash = None
@@ -371,6 +373,7 @@ class execution(_Config):
     _layout = None
 
     _paths = (
+        "anat_derivatives",
         "bids_dir",
         "fs_license_file",
         "fs_subjects_dir",
diff --git a/dmriprep/workflows/base.py b/dmriprep/workflows/base.py
@@ -194,10 +194,22 @@ def init_single_subject_wf(subject_id):
                             desc="about", keep_dtype=True),
         name="ds_report_about", run_without_submitting=True)
 
+    anat_derivatives = config.execution.anat_derivatives
+    if anat_derivatives:
+        from smriprep.utils.bids import collect_derivatives
+        std_spaces = spaces.get_spaces(nonstandard=False, dim=(3,))
+        anat_derivatives = collect_derivatives(
+            anat_derivatives.absolute(),
+            subject_id,
+            std_spaces,
+            config.workflow.run_reconall,
+        )
+
     # Preprocessing of T1w (includes registration to MNI)
     anat_preproc_wf = init_anat_preproc_wf(
         bids_root=str(config.execution.bids_dir),
         debug=config.execution.debug is True,
+        existing_derivatives=anat_derivatives,
         freesurfer=config.workflow.run_reconall,
         hires=config.workflow.hires,
         longitudinal=config.workflow.longitudinal,
diff --git a/setup.cfg b/setup.cfg
@@ -26,7 +26,7 @@ install_requires =
     nipype ~= 1.4
     niworkflows >=1.2.0rc4,<1.3
     numpy
-    pybids >=0.9.4
+    pybids >=0.10.2
     pyyaml
     sdcflows >= 1.2.3
     smriprep >=0.6.0rc4,<0.7
