diff --git a/CHANGELOG.md b/CHANGELOG.md index d18981b5..b19a52b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,14 @@ ## MAJOR CHANGES +* Update Matching Modalities task to OpenProblems v2 results (PR #328). + * Migrated the result scaling from R to JavaScript to allow dynamically updating the results (PR #332). ## MINOR CHANGES +* Update Mathing Modalities task name to Match Modalities (PR #328). + * Improve Equations visualisation (PR #329). ## BUG FIXES diff --git a/results/match_modalities/data/dataset_info.json b/results/match_modalities/data/dataset_info.json new file mode 100644 index 00000000..856b1994 --- /dev/null +++ b/results/match_modalities/data/dataset_info.json @@ -0,0 +1,26 @@ +[ + { + "task_id": "match_modalities", + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "dataset_name": "sci-CAR Mouse Kidney", + "dataset_summary": "sci-CAR profiles of 11k mouse kidney cells", + "data_reference": "cao2018joint", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089" + }, + { + "task_id": "match_modalities", + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "dataset_name": "CITE-Seq CBMC", + "dataset_summary": "CITE-seq profiles of 8k Cord Blood Mononuclear Cells", + "data_reference": "stoeckius2017simultaneous", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE100866" + }, + { + "task_id": "match_modalities", + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "dataset_name": "sci-CAR Cell Lines", + "dataset_summary": "sci-CAR profiles of 5k cell line cells (HEK293T, NIH/3T3, A549) across three treatment conditions (DEX 0h, 1h and 3h)", + "data_reference": "cao2018joint", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089" + } +] diff --git a/results/match_modalities/data/method_info.json b/results/match_modalities/data/method_info.json new file mode 100644 index 00000000..fb4867df --- /dev/null +++ b/results/match_modalities/data/method_info.json @@ -0,0 +1,74 @@ +[ + { + "task_id": "match_modalities", + "method_id": "random_features", + "method_name": "Random Features", + "method_summary": "Randomly permutated features", + "is_baseline": true, + "paper_reference": null, + "code_url": null, + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/control_methods/random_features/config.vsh.yaml", + "code_version": null, + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" + }, + { + "task_id": "match_modalities", + "method_id": "true_features", + "method_name": "True Features", + "method_summary": "A 1 to 1 mapping of features between modalities", + "is_baseline": true, + "paper_reference": null, + "code_url": null, + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/control_methods/true_features/config.vsh.yaml", + "code_version": null, + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" + }, + { + "task_id": "match_modalities", + "method_id": "scot", + "method_name": "Single Cell Optimal Transport", + "method_summary": "Run Single Cell Optimal Transport", + "is_baseline": false, + "paper_reference": "Demetci2020scot", + "code_url": "https://github.com/rsinghlab/SCOT", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/methods/scot/config.vsh.yaml", + "code_version": null, + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" + }, + { + "task_id": "match_modalities", + "method_id": "harmonic_alignment", + "method_name": "Harmonic Alignment", + "method_summary": "Harmonic Alignment", + "is_baseline": false, + "paper_reference": "stanley2020harmonic", + "code_url": "https://github.com/KrishnaswamyLab/harmonic-alignment", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/methods/harmonic_alignment/config.vsh.yaml", + "code_version": null, + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" + }, + { + "task_id": "match_modalities", + "method_id": "fastmnn", + "method_name": "fastMNN", + "method_summary": "A simpler version of the original mnnCorrect algorithm.", + "is_baseline": false, + "paper_reference": "haghverdi2018batch", + "code_url": "https://github.com/LTLA/batchelor", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/methods/fastmnn/config.vsh.yaml", + "code_version": null, + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" + }, + { + "task_id": "match_modalities", + "method_id": "procrustes", + "method_name": "Procrustes", + "method_summary": "\"Procrustes superimposition embeds cellular data from each modality into a common space.\"\n", + "is_baseline": false, + "paper_reference": "gower1975generalized", + "code_url": "https://github.com/scipy/scipy", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/methods/procrustes/config.vsh.yaml", + "code_version": null, + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" + } +] diff --git a/results/match_modalities/data/metric_execution_info.json b/results/match_modalities/data/metric_execution_info.json new file mode 100644 index 00000000..c7d6912e --- /dev/null +++ b/results/match_modalities/data/metric_execution_info.json @@ -0,0 +1,506 @@ +[ + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 12.7, + "cpu_pct": 138.4, + "peak_memory_mb": 3892, + "disk_read_mb": 265, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 10.6, + "cpu_pct": 216, + "peak_memory_mb": 5735, + "disk_read_mb": 276, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 14.1, + "cpu_pct": 118.8, + "peak_memory_mb": 3789, + "disk_read_mb": 264, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 12.4, + "cpu_pct": 135.8, + "peak_memory_mb": 3892, + "disk_read_mb": 264, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 16.1, + "cpu_pct": 100.1, + "peak_memory_mb": 3892, + "disk_read_mb": 264, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 24, + "cpu_pct": 113.3, + "peak_memory_mb": 3789, + "disk_read_mb": 210, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 5.3, + "cpu_pct": 333.8, + "peak_memory_mb": 5530, + "disk_read_mb": 251, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 5.1, + "cpu_pct": 311, + "peak_memory_mb": 5530, + "disk_read_mb": 250, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 4.9, + "cpu_pct": 329.1, + "peak_memory_mb": 5530, + "disk_read_mb": 250, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 5.6, + "cpu_pct": 241.6, + "peak_memory_mb": 5530, + "disk_read_mb": 247, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 9.1, + "cpu_pct": 269.3, + "peak_memory_mb": 5530, + "disk_read_mb": 250, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 6.8, + "cpu_pct": 196, + "peak_memory_mb": 5325, + "disk_read_mb": 199, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 13.3, + "cpu_pct": 351.9, + "peak_memory_mb": 6144, + "disk_read_mb": 444, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 13.9, + "cpu_pct": 282.9, + "peak_memory_mb": 6144, + "disk_read_mb": 443, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 13.7, + "cpu_pct": 323.7, + "peak_memory_mb": 6144, + "disk_read_mb": 443, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 12.5, + "cpu_pct": 367.7, + "peak_memory_mb": 6144, + "disk_read_mb": 435, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 16.1, + "cpu_pct": 302.7, + "peak_memory_mb": 6144, + "disk_read_mb": 443, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 35.6, + "cpu_pct": 207.9, + "peak_memory_mb": 5837, + "disk_read_mb": 346, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 2.9, + "cpu_pct": 200.4, + "peak_memory_mb": 3380, + "disk_read_mb": 256, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 207.2, + "peak_memory_mb": 3380, + "disk_read_mb": 267, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 2.8, + "cpu_pct": 219, + "peak_memory_mb": 3380, + "disk_read_mb": 256, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 3.8, + "cpu_pct": 164.2, + "peak_memory_mb": 3482, + "disk_read_mb": 255, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 285.8, + "peak_memory_mb": 3482, + "disk_read_mb": 256, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 1.9, + "cpu_pct": 331.6, + "peak_memory_mb": 1639, + "disk_read_mb": 201, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 3.3, + "cpu_pct": 212.3, + "peak_memory_mb": 3482, + "disk_read_mb": 242, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 4.1, + "cpu_pct": 171.3, + "peak_memory_mb": 3380, + "disk_read_mb": 241, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 2.8, + "cpu_pct": 271.1, + "peak_memory_mb": 3380, + "disk_read_mb": 241, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 285.3, + "peak_memory_mb": 3482, + "disk_read_mb": 238, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 4.8, + "cpu_pct": 108.8, + "peak_memory_mb": 3482, + "disk_read_mb": 241, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 3.4, + "cpu_pct": 179.2, + "peak_memory_mb": 3380, + "disk_read_mb": 190, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 5.1, + "cpu_pct": 154.3, + "peak_memory_mb": 3687, + "disk_read_mb": 435, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 5.9, + "cpu_pct": 135, + "peak_memory_mb": 3687, + "disk_read_mb": 434, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 4.4, + "cpu_pct": 147.8, + "peak_memory_mb": 3687, + "disk_read_mb": 434, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 3.7, + "cpu_pct": 239.6, + "peak_memory_mb": 3789, + "disk_read_mb": 426, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 4.1, + "cpu_pct": 181, + "peak_memory_mb": 3687, + "disk_read_mb": 434, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 249.9, + "peak_memory_mb": 3175, + "disk_read_mb": 337, + "disk_write_mb": 1 + } + } +] diff --git a/results/match_modalities/data/metric_info.json b/results/match_modalities/data/metric_info.json new file mode 100644 index 00000000..0a73d15d --- /dev/null +++ b/results/match_modalities/data/metric_info.json @@ -0,0 +1,24 @@ +[ + { + "task_id": "match_modalities", + "metric_id": "knn_auc", + "metric_name": "kNN Area Under the Curve", + "metric_summary": "Let $f(i) \\in F$ be the scRNA-seq measurement of cell $i$, and $g(i) \\in G$ be the scATAC- seq measurement of cell $i$. kNN-AUC calculates the average percentage overlap of neighborhoods of $f(i)$ in $F$ with neighborhoods of $g(i)$ in $G$. Higher is better.\n", + "paper_reference": "lance2022multimodal", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/metrics/knn_auc/config.vsh.yaml", + "code_version": null, + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb", + "maximize": true + }, + { + "task_id": "match_modalities", + "metric_id": "mse", + "metric_name": "Mean Squared Error", + "metric_summary": "Mean squared error (MSE) is the average distance between each pair of matched observations of the same cell in the learned latent space. Lower is better.\n", + "paper_reference": "lance2022multimodal", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/metrics/mse/config.vsh.yaml", + "code_version": null, + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb", + "maximize": false + } +] diff --git a/results/match_modalities/data/quality_control.json b/results/match_modalities/data/quality_control.json new file mode 100644 index 00000000..5fc3e5b2 --- /dev/null +++ b/results/match_modalities/data/quality_control.json @@ -0,0 +1,602 @@ +[ + { + "task_id": "match_modalities", + "category": "Task info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_id' should be defined\n Task id: match_modalities\n Field: task_id\n" + }, + { + "task_id": "match_modalities", + "category": "Task info", + "name": "Pct 'task_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_name' should be defined\n Task id: match_modalities\n Field: task_name\n" + }, + { + "task_id": "match_modalities", + "category": "Task info", + "name": "Pct 'task_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_summary' should be defined\n Task id: match_modalities\n Field: task_summary\n" + }, + { + "task_id": "match_modalities", + "category": "Task info", + "name": "Pct 'task_description' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_description' should be defined\n Task id: match_modalities\n Field: task_description\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'task_id' should be defined\n Task id: match_modalities\n Field: task_id\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'commit_sha' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'commit_sha' should be defined\n Task id: match_modalities\n Field: commit_sha\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'method_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'method_id' should be defined\n Task id: match_modalities\n Field: method_id\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'method_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'method_name' should be defined\n Task id: match_modalities\n Field: method_name\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'method_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'method_summary' should be defined\n Task id: match_modalities\n Field: method_summary\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'paper_reference' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'paper_reference' should be defined\n Task id: match_modalities\n Field: paper_reference\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'is_baseline' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'is_baseline' should be defined\n Task id: match_modalities\n Field: is_baseline\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'task_id' should be defined\n Task id: match_modalities\n Field: task_id\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'commit_sha' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'commit_sha' should be defined\n Task id: match_modalities\n Field: commit_sha\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'metric_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'metric_id' should be defined\n Task id: match_modalities\n Field: metric_id\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'metric_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'metric_name' should be defined\n Task id: match_modalities\n Field: metric_name\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'metric_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'metric_summary' should be defined\n Task id: match_modalities\n Field: metric_summary\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'paper_reference' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'paper_reference' should be defined\n Task id: match_modalities\n Field: paper_reference\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'maximize' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'maximize' should be defined\n Task id: match_modalities\n Field: maximize\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'task_id' should be defined\n Task id: match_modalities\n Field: task_id\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'dataset_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'dataset_id' should be defined\n Task id: match_modalities\n Field: dataset_id\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'dataset_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'dataset_name' should be defined\n Task id: match_modalities\n Field: dataset_name\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'dataset_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'dataset_summary' should be defined\n Task id: match_modalities\n Field: dataset_summary\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'data_reference' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'data_reference' should be defined\n Task id: match_modalities\n Field: data_reference\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'data_url' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'data_url' should be defined\n Task id: match_modalities\n Field: data_url\n" + }, + { + "task_id": "match_modalities", + "category": "Raw data", + "name": "Number of results", + "value": 18, + "severity": 0, + "severity_value": 0.0, + "code": "len(results) == len(method_info) * len(metric_info) * len(dataset_info)", + "message": "Number of results should be equal to #methods × #metrics × #datasets.\n Task id: match_modalities\n Number of results: 18\n Number of methods: 6\n Number of metrics: 2\n Number of datasets: 3\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Metric 'knn_auc' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n Metric id: knn_auc\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Metric 'mse' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n Metric id: mse\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'random_features' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: random_features\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'true_features' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: true_features\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'scot' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: scot\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'harmonic_alignment' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: harmonic_alignment\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'fastmnn' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: fastmnn\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'procrustes' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: procrustes\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Dataset 'openproblems_v1_multimodal/scicar_mouse_kidney' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/scicar_mouse_kidney\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Dataset 'openproblems_v1_multimodal/citeseq_cbmc' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/citeseq_cbmc\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Dataset 'openproblems_v1_multimodal/scicar_cell_lines' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/scicar_cell_lines\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score random_features knn_auc", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Worst score: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score random_features knn_auc", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score true_features knn_auc", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Worst score: 1%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score true_features knn_auc", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Best score: 1%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score scot knn_auc", + "value": -0.0248, + "severity": 0, + "severity_value": 0.0248, + "code": "worst_score >= -1", + "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Worst score: -0.0248%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score scot knn_auc", + "value": 0.0528, + "severity": 0, + "severity_value": 0.0264, + "code": "best_score <= 2", + "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Best score: 0.0528%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score harmonic_alignment knn_auc", + "value": -0.0052, + "severity": 0, + "severity_value": 0.0052, + "code": "worst_score >= -1", + "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Worst score: -0.0052%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score harmonic_alignment knn_auc", + "value": 0.0201, + "severity": 0, + "severity_value": 0.01005, + "code": "best_score <= 2", + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Best score: 0.0201%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score fastmnn knn_auc", + "value": -0.0147, + "severity": 0, + "severity_value": 0.0147, + "code": "worst_score >= -1", + "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Worst score: -0.0147%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score fastmnn knn_auc", + "value": 0.0274, + "severity": 0, + "severity_value": 0.0137, + "code": "best_score <= 2", + "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Best score: 0.0274%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score procrustes knn_auc", + "value": -0.011, + "severity": 0, + "severity_value": 0.011, + "code": "worst_score >= -1", + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: -0.011%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score procrustes knn_auc", + "value": 0.0432, + "severity": 0, + "severity_value": 0.0216, + "code": "best_score <= 2", + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 0.0432%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score random_features mse", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Worst score: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score random_features mse", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Best score: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score true_features mse", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Worst score: 1%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score true_features mse", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Best score: 1%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score scot mse", + "value": -0.2218, + "severity": 0, + "severity_value": 0.2218, + "code": "worst_score >= -1", + "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Worst score: -0.2218%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score scot mse", + "value": 0.1505, + "severity": 0, + "severity_value": 0.07525, + "code": "best_score <= 2", + "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Best score: 0.1505%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score harmonic_alignment mse", + "value": -0.0029, + "severity": 0, + "severity_value": 0.0029, + "code": "worst_score >= -1", + "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Worst score: -0.0029%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score harmonic_alignment mse", + "value": 0.0143, + "severity": 0, + "severity_value": 0.00715, + "code": "best_score <= 2", + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Best score: 0.0143%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score fastmnn mse", + "value": -0.158, + "severity": 0, + "severity_value": 0.158, + "code": "worst_score >= -1", + "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Worst score: -0.158%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score fastmnn mse", + "value": 0.0514, + "severity": 0, + "severity_value": 0.0257, + "code": "best_score <= 2", + "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Best score: 0.0514%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score procrustes mse", + "value": -0.0095, + "severity": 0, + "severity_value": 0.0095, + "code": "worst_score >= -1", + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: -0.0095%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score procrustes mse", + "value": 0.009, + "severity": 0, + "severity_value": 0.0045, + "code": "best_score <= 2", + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 0.009%\n" + } +] \ No newline at end of file diff --git a/results/match_modalities/data/results.json b/results/match_modalities/data/results.json new file mode 100644 index 00000000..32e068d4 --- /dev/null +++ b/results/match_modalities/data/results.json @@ -0,0 +1,416 @@ +[ + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "fastmnn", + "metric_values": { + "knn_auc": 0.0362, + "mse": 1.175 + }, + "scaled_scores": { + "knn_auc": -0.0147, + "mse": -0.158 + }, + "mean_score": 0, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 16.5, + "cpu_pct": 146.3, + "peak_memory_mb": 4096, + "disk_read_mb": 238, + "disk_write_mb": 58 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "harmonic_alignment", + "metric_values": { + "knn_auc": 0.0692, + "mse": 1.0002 + }, + "scaled_scores": { + "knn_auc": 0.0201, + "mse": 0.0143 + }, + "mean_score": 0.0172, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 64, + "cpu_pct": 530.4, + "peak_memory_mb": 3687, + "disk_read_mb": 208, + "disk_write_mb": 69 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "procrustes", + "metric_values": { + "knn_auc": 0.0397, + "mse": 1.0056 + }, + "scaled_scores": { + "knn_auc": -0.011, + "mse": 0.009 + }, + "mean_score": 0.0045, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 6.4, + "cpu_pct": 282.3, + "peak_memory_mb": 3380, + "disk_read_mb": 199, + "disk_write_mb": 58 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "random_features", + "metric_values": { + "knn_auc": 0.0502, + "mse": 1.0147 + }, + "scaled_scores": { + "knn_auc": 0, + "mse": 0 + }, + "mean_score": 0, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 6.3, + "cpu_pct": 173.6, + "peak_memory_mb": 3380, + "disk_read_mb": 198, + "disk_write_mb": 57 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "scot", + "metric_values": { + "knn_auc": 0.0266, + "mse": 1.2398 + }, + "scaled_scores": { + "knn_auc": -0.0248, + "mse": -0.2218 + }, + "mean_score": 0, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 3174, + "cpu_pct": 2926.3, + "peak_memory_mb": 12493, + "disk_read_mb": 219, + "disk_write_mb": 58 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "true_features", + "metric_values": { + "knn_auc": 1, + "mse": 0 + }, + "scaled_scores": { + "knn_auc": 1, + "mse": 1 + }, + "mean_score": 1, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 2.3, + "cpu_pct": 322.2, + "peak_memory_mb": 3482, + "disk_read_mb": 385, + "disk_write_mb": 3 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "fastmnn", + "metric_values": { + "knn_auc": 0.0621, + "mse": 1.0819 + }, + "scaled_scores": { + "knn_auc": 0.0128, + "mse": -0.0811 + }, + "mean_score": 0.0064, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 22.7, + "cpu_pct": 122.3, + "peak_memory_mb": 4199, + "disk_read_mb": 217, + "disk_write_mb": 65 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "harmonic_alignment", + "metric_values": { + "knn_auc": 0.045, + "mse": 1.0001 + }, + "scaled_scores": { + "knn_auc": -0.0052, + "mse": 0.0007 + }, + "mean_score": 0.0004, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 47.4, + "cpu_pct": 833.6, + "peak_memory_mb": 5735, + "disk_read_mb": 187, + "disk_write_mb": 64 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "procrustes", + "metric_values": { + "knn_auc": 0.0802, + "mse": 1.0103 + }, + "scaled_scores": { + "knn_auc": 0.0317, + "mse": -0.0095 + }, + "mean_score": 0.0159, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 6.4, + "cpu_pct": 246, + "peak_memory_mb": 3380, + "disk_read_mb": 178, + "disk_write_mb": 64 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "random_features", + "metric_values": { + "knn_auc": 0.05, + "mse": 1.0008 + }, + "scaled_scores": { + "knn_auc": 0, + "mse": 0 + }, + "mean_score": 0, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 6.1, + "cpu_pct": 186.7, + "peak_memory_mb": 3380, + "disk_read_mb": 178, + "disk_write_mb": 61 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "scot", + "metric_values": { + "knn_auc": 0.1002, + "mse": 0.8502 + }, + "scaled_scores": { + "knn_auc": 0.0528, + "mse": 0.1505 + }, + "mean_score": 0.1017, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 445, + "cpu_pct": 3047.9, + "peak_memory_mb": 8090, + "disk_read_mb": 198, + "disk_write_mb": 64 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "true_features", + "metric_values": { + "knn_auc": 1, + "mse": 0 + }, + "scaled_scores": { + "knn_auc": 1, + "mse": 1 + }, + "mean_score": 1, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 279.2, + "peak_memory_mb": 3482, + "disk_read_mb": 343, + "disk_write_mb": 13 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "fastmnn", + "metric_values": { + "knn_auc": 0.076, + "mse": 0.9458 + }, + "scaled_scores": { + "knn_auc": 0.0274, + "mse": 0.0514 + }, + "mean_score": 0.0394, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 43.6, + "cpu_pct": 116.6, + "peak_memory_mb": 4506, + "disk_read_mb": 356, + "disk_write_mb": 119 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "harmonic_alignment", + "metric_values": { + "knn_auc": 0.0492, + "mse": 1 + }, + "scaled_scores": { + "knn_auc": -0.0008, + "mse": -0.0029 + }, + "mean_score": 0, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 172, + "cpu_pct": 558.4, + "peak_memory_mb": 4404, + "disk_read_mb": 326, + "disk_write_mb": 118 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "procrustes", + "metric_values": { + "knn_auc": 0.091, + "mse": 1.0008 + }, + "scaled_scores": { + "knn_auc": 0.0432, + "mse": -0.0038 + }, + "mean_score": 0.0216, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 9.7, + "cpu_pct": 200.8, + "peak_memory_mb": 3584, + "disk_read_mb": 317, + "disk_write_mb": 118 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "random_features", + "metric_values": { + "knn_auc": 0.05, + "mse": 0.9971 + }, + "scaled_scores": { + "knn_auc": 0, + "mse": 0 + }, + "mean_score": 0, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 10.3, + "cpu_pct": 141.7, + "peak_memory_mb": 3482, + "disk_read_mb": 316, + "disk_write_mb": 110 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "scot", + "metric_values": { + "knn_auc": 0.0606, + "mse": 0.992 + }, + "scaled_scores": { + "knn_auc": 0.0112, + "mse": 0.0051 + }, + "mean_score": 0.0081, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 5181, + "cpu_pct": 2619.9, + "peak_memory_mb": 17101, + "disk_read_mb": 337, + "disk_write_mb": 118 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "true_features", + "metric_values": { + "knn_auc": 1, + "mse": 0 + }, + "scaled_scores": { + "knn_auc": 1, + "mse": 1 + }, + "mean_score": 1, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 3.6, + "cpu_pct": 202.3, + "peak_memory_mb": 3789, + "disk_read_mb": 621, + "disk_write_mb": 21 + }, + "task_id": "match_modalities" + } +] diff --git a/results/match_modalities/data/state.yaml b/results/match_modalities/data/state.yaml new file mode 100644 index 00000000..abbb0fc1 --- /dev/null +++ b/results/match_modalities/data/state.yaml @@ -0,0 +1,9 @@ +id: process +output_scores: !file results.json +output_method_info: !file method_info.json +output_metric_info: !file metric_info.json +output_dataset_info: !file dataset_info.json +output_task_info: !file task_info.json +output_qc: !file quality_control.json +output_metric_execution_info: !file metric_execution_info.json + diff --git a/results/match_modalities/data/task_info.json b/results/match_modalities/data/task_info.json new file mode 100644 index 00000000..67ea05ad --- /dev/null +++ b/results/match_modalities/data/task_info.json @@ -0,0 +1,8 @@ +{ + "task_id": "match_modalities", + "commit_sha": null, + "task_name": "Match Modalities", + "task_summary": "Match cells across datasets of the same set of samples on different technologies / modalities.\n", + "task_description": "Cellular function is regulated by the complex interplay of different types of biological\nmolecules (DNA, RNA, proteins, etc.), which determine the state of a cell. Several\nrecently described technologies allow for simultaneous measurement of different aspects\nof cellular state. For example, sci-CAR [@cao2018joint]\njointly profiles RNA expression and chromatin accessibility on the same cell and\nCITE-seq [@stoeckius2017simultaneous] measures\nsurface protein abundance and RNA expression from each cell. These technologies enable\nus to better understand cellular function, however datasets are still rare and there are\ntradeoffs that these measurements make for to profile multiple modalities.\n\nJoint methods can be more expensive or lower throughput or more noisy than measuring a\nsingle modality at a time. Therefore it is useful to develop methods that are capable\nof integrating measurements of the same biological system but obtained using different\ntechnologies on different cells.\n\n\nIn this task, the goal is to learn a latent space where cells profiled by different\ntechnologies in different modalities are matched if they have the same state. We use\njointly profiled data as ground truth so that we can evaluate when the observations\nfrom the same cell acquired using different modalities are similar. A perfect result\nhas each of the paired observations sharing the same coordinates in the latent space.\nA method that can achieve this would be able to match datasets across modalities to\nenable multimodal cellular analysis from separately measured profiles.\n", + "repo": "openproblems-bio/openproblems-v2" +} diff --git a/results/matching_modalities/index.qmd b/results/match_modalities/index.qmd similarity index 58% rename from results/matching_modalities/index.qmd rename to results/match_modalities/index.qmd index 3cc89097..4ce4415f 100644 --- a/results/matching_modalities/index.qmd +++ b/results/match_modalities/index.qmd @@ -1,6 +1,7 @@ --- -title: "Multimodal Data Integration" -subtitle: "Alignment of cellular profiles from two different modalities" +title: "Match Modalities" +subtitle: "Match cells across datasets of the same set of samples on different technologies / modalities. +" image: thumbnail.svg page-layout: full css: ../_include/task_template.css @@ -12,7 +13,7 @@ toc: false ```{r} #| include: false -params <- list(data_dir = "results/matching_modalities/data") +params <- list(data_dir = "results/match_modalities/data") params <- list(data_dir = "./data") ``` diff --git a/results/matching_modalities/thumbnail.svg b/results/match_modalities/thumbnail.svg similarity index 100% rename from results/matching_modalities/thumbnail.svg rename to results/match_modalities/thumbnail.svg diff --git a/results/matching_modalities/data/dataset_info.json b/results/matching_modalities/data/dataset_info.json deleted file mode 100644 index a87fa11d..00000000 --- a/results/matching_modalities/data/dataset_info.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - { - "dataset_name": "CITE-seq Cord Blood Mononuclear Cells", - "image": "openproblems", - "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE100866", - "data_reference": "stoeckius2017simultaneous", - "dataset_summary": "8k cord blood mononuclear cells sequenced by CITEseq, a multimodal addition to the 10x scRNA-seq platform that allows simultaneous measurement of RNA and protein.", - "task_id": "matching_modalities", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "citeseq_cbmc", - "source_dataset_id": "openproblems_v1/citeseq_cbmc", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/datasets/citeseq.py" - }, - { - "dataset_name": "sciCAR Cell Lines", - "image": "openproblems", - "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089", - "data_reference": "cao2018joint", - "dataset_summary": "5k cells from a time-series of dexamethasone treatment sequenced by sci-CAR, a combinatorial indexing-based co-assay that jointly profiles chromatin accessibility and mRNA.", - "task_id": "matching_modalities", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "scicar_cell_lines", - "source_dataset_id": "openproblems_v1/scicar_cell_lines", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/datasets/scicar.py" - }, - { - "dataset_name": "sciCAR Mouse Kidney", - "image": "openproblems", - "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089", - "data_reference": "cao2018joint", - "dataset_summary": "11k cells from adult mouse kidney sequenced by sci-CAR, a combinatorial indexing-based co-assay that jointly profiles chromatin accessibility and mRNA.", - "task_id": "matching_modalities", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "scicar_mouse_kidney", - "source_dataset_id": "openproblems_v1/scicar_mouse_kidney", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/datasets/scicar.py" - } -] \ No newline at end of file diff --git a/results/matching_modalities/data/method_info.json b/results/matching_modalities/data/method_info.json deleted file mode 100644 index a72e21ec..00000000 --- a/results/matching_modalities/data/method_info.json +++ /dev/null @@ -1,107 +0,0 @@ -[ - { - "method_name": "Harmonic Alignment (log scran)", - "method_summary": "Harmonic alignment embeds cellular data from each modality into a common space by computing a mapping between the 100-dimensional diffusion maps of each modality. This mapping is computed by computing an isometric transformation of the eigenmaps, and concatenating the resulting diffusion maps together into a joint 200-dimensional space. This joint diffusion map space is used as output for the task.", - "paper_name": "Harmonic Alignment", - "paper_reference": "stanley2020harmonic", - "paper_year": 2020, - "code_url": "https://github.com/KrishnaswamyLab/harmonic-alignment", - "image": "openproblems-r-extras", - "is_baseline": false, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "harmonic_alignment_log_scran_pooling", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/harmonic_alignment.py" - }, - { - "method_name": "Harmonic Alignment (sqrt CP10k)", - "method_summary": "Harmonic alignment embeds cellular data from each modality into a common space by computing a mapping between the 100-dimensional diffusion maps of each modality. This mapping is computed by computing an isometric transformation of the eigenmaps, and concatenating the resulting diffusion maps together into a joint 200-dimensional space. This joint diffusion map space is used as output for the task.", - "paper_name": "Harmonic Alignment", - "paper_reference": "stanley2020harmonic", - "paper_year": 2020, - "code_url": "https://github.com/KrishnaswamyLab/harmonic-alignment", - "image": "openproblems-python-extras", - "is_baseline": false, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "harmonic_alignment_sqrt_cp10k", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/harmonic_alignment.py" - }, - { - "method_name": "Mutual Nearest Neighbors (log CP10k)", - "method_summary": "Mutual nearest neighbors (MNN) embeds cellular data from each modality into a common space by computing a mapping between modality-specific 100-dimensional SVD embeddings. The embeddings are integrated using the FastMNN version of the MNN algorithm, which generates an embedding of the second modality mapped to the SVD space of the first. This corrected joint SVD space is used as output for the task.", - "paper_name": "Batch effects in single-cell RNA-sequencing data are corrected by matching mutual nearest neighbors", - "paper_reference": "haghverdi2018batch", - "paper_year": 2018, - "code_url": "https://github.com/LTLA/batchelor", - "image": "openproblems-r-extras", - "is_baseline": false, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "mnn_log_cp10k", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/mnn.py" - }, - { - "method_name": "Mutual Nearest Neighbors (log scran)", - "method_summary": "Mutual nearest neighbors (MNN) embeds cellular data from each modality into a common space by computing a mapping between modality-specific 100-dimensional SVD embeddings. The embeddings are integrated using the FastMNN version of the MNN algorithm, which generates an embedding of the second modality mapped to the SVD space of the first. This corrected joint SVD space is used as output for the task.", - "paper_name": "Batch effects in single-cell RNA-sequencing data are corrected by matching mutual nearest neighbors", - "paper_reference": "haghverdi2018batch", - "paper_year": 2018, - "code_url": "https://github.com/LTLA/batchelor", - "image": "openproblems-r-extras", - "is_baseline": false, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "mnn_log_scran_pooling", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/mnn.py" - }, - { - "method_name": "Procrustes superimposition", - "method_summary": "Procrustes superimposition embeds cellular data from each modality into a common space by aligning the 100-dimensional SVD embeddings to one another by using an isomorphic transformation that minimizes the root mean squared distance between points. The unmodified SVD embedding and the transformed second modality are used as output for the task.", - "paper_name": "Generalized Procrustes analysis", - "paper_reference": "gower1975generalized", - "paper_year": 1975, - "code_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.procrustes.html", - "image": "openproblems", - "is_baseline": false, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "procrustes", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/procrustes.py" - }, - { - "method_name": "Random Features", - "method_summary": "20-dimensional SVD is computed on the first modality, and is then randomly permuted twice, once for use as the output for each modality, producing random features with no correlation between modalities.", - "paper_name": "Open Problems for Single Cell Analysis", - "paper_reference": "openproblems", - "paper_year": 2022, - "code_url": "https://github.com/openproblems-bio/openproblems", - "image": "openproblems", - "is_baseline": true, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "random_features", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/baseline.py" - }, - { - "method_name": "True Features", - "method_summary": "20-dimensional SVD is computed on the first modality, and this same embedding is used as output for both modalities, producing perfectly aligned features from each modality.", - "paper_name": "Open Problems for Single Cell Analysis", - "paper_reference": "openproblems", - "paper_year": 2022, - "code_url": "https://github.com/openproblems-bio/openproblems", - "image": "openproblems", - "is_baseline": true, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "true_features", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/baseline.py" - } -] \ No newline at end of file diff --git a/results/matching_modalities/data/metric_info.json b/results/matching_modalities/data/metric_info.json deleted file mode 100644 index d85fad52..00000000 --- a/results/matching_modalities/data/metric_info.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "metric_name": "kNN Area Under the Curve", - "metric_summary": "Let $f(i) \u2208 F$ be the scRNA-seq measurement of cell $i$, and $g(i) \u2208 G$ be the scATAC- seq measurement of cell $i$. kNN-AUC calculates the average percentage overlap of neighborhoods of $f(i)$ in $F$ with neighborhoods of $g(i)$ in $G$. Higher is better.", - "paper_reference": "stanley2020harmonic", - "maximize": true, - "image": "openproblems", - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "metric_id": "knn_auc", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/metrics/knn_auc.py" - }, - { - "metric_name": "Mean squared error", - "metric_summary": "Mean squared error (MSE) is the average distance between each pair of matched observations of the same cell in the learned latent space. Lower is better.", - "paper_reference": "lance2022multimodal", - "maximize": false, - "image": "openproblems", - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "metric_id": "mse", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/metrics/mse.py" - } -] \ No newline at end of file diff --git a/results/matching_modalities/data/quality_control.json b/results/matching_modalities/data/quality_control.json deleted file mode 100644 index 477ae09f..00000000 --- a/results/matching_modalities/data/quality_control.json +++ /dev/null @@ -1,662 +0,0 @@ -[ - { - "task_id": "matching_modalities", - "category": "Task info", - "name": "Pct 'task_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_id' should be defined\n Task id: matching_modalities\n Field: task_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Task info", - "name": "Pct 'commit_sha' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'commit_sha' should be defined\n Task id: matching_modalities\n Field: commit_sha\n" - }, - { - "task_id": "matching_modalities", - "category": "Task info", - "name": "Pct 'task_name' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_name' should be defined\n Task id: matching_modalities\n Field: task_name\n" - }, - { - "task_id": "matching_modalities", - "category": "Task info", - "name": "Pct 'task_summary' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_summary' should be defined\n Task id: matching_modalities\n Field: task_summary\n" - }, - { - "task_id": "matching_modalities", - "category": "Task info", - "name": "Pct 'task_description' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_description' should be defined\n Task id: matching_modalities\n Field: task_description\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'task_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'task_id' should be defined\n Task id: matching_modalities\n Field: task_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'commit_sha' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'commit_sha' should be defined\n Task id: matching_modalities\n Field: commit_sha\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'method_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'method_id' should be defined\n Task id: matching_modalities\n Field: method_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'method_name' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'method_name' should be defined\n Task id: matching_modalities\n Field: method_name\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'method_summary' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'method_summary' should be defined\n Task id: matching_modalities\n Field: method_summary\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'paper_reference' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'paper_reference' should be defined\n Task id: matching_modalities\n Field: paper_reference\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'is_baseline' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'is_baseline' should be defined\n Task id: matching_modalities\n Field: is_baseline\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'task_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'task_id' should be defined\n Task id: matching_modalities\n Field: task_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'commit_sha' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'commit_sha' should be defined\n Task id: matching_modalities\n Field: commit_sha\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'metric_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'metric_id' should be defined\n Task id: matching_modalities\n Field: metric_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'metric_name' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'metric_name' should be defined\n Task id: matching_modalities\n Field: metric_name\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'metric_summary' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'metric_summary' should be defined\n Task id: matching_modalities\n Field: metric_summary\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'paper_reference' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'paper_reference' should be defined\n Task id: matching_modalities\n Field: paper_reference\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'maximize' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'maximize' should be defined\n Task id: matching_modalities\n Field: maximize\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'task_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'task_id' should be defined\n Task id: matching_modalities\n Field: task_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'commit_sha' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'commit_sha' should be defined\n Task id: matching_modalities\n Field: commit_sha\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'dataset_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'dataset_id' should be defined\n Task id: matching_modalities\n Field: dataset_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'dataset_name' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'dataset_name' should be defined\n Task id: matching_modalities\n Field: dataset_name\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'dataset_summary' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'dataset_summary' should be defined\n Task id: matching_modalities\n Field: dataset_summary\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'data_reference' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'data_reference' should be defined\n Task id: matching_modalities\n Field: data_reference\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw data", - "name": "Number of results", - "value": 21, - "severity": 0, - "severity_value": 0.0, - "code": "len(results) == len(method_info) * len(metric_info) * len(dataset_info)", - "message": "Number of results should be equal to #methods × #metrics × #datasets.\n Task id: matching_modalities\n Number of results: 21\n Number of methods: 7\n Number of metrics: 2\n Number of datasets: 3\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Metric 'knn_auc' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n Metric id: knn_auc\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Metric 'mse' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n Metric id: mse\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'harmonic_alignment_log_scran_pooling' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: harmonic_alignment_log_scran_pooling\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'harmonic_alignment_sqrt_cp10k' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: harmonic_alignment_sqrt_cp10k\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'mnn_log_cp10k' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: mnn_log_cp10k\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'mnn_log_scran_pooling' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: mnn_log_scran_pooling\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'procrustes' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: procrustes\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'random_features' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: random_features\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'true_features' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: true_features\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Dataset 'citeseq_cbmc' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n dataset id: citeseq_cbmc\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Dataset 'scicar_cell_lines' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n dataset id: scicar_cell_lines\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Dataset 'scicar_mouse_kidney' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n dataset id: scicar_mouse_kidney\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score harmonic_alignment_log_scran_pooling knn_auc", - "value": 0.013650181246076077, - "severity": 0, - "severity_value": -0.013650181246076077, - "code": "worst_score >= -1", - "message": "Method harmonic_alignment_log_scran_pooling performs much worse than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_log_scran_pooling\n Metric id: knn_auc\n Worst score: 0.013650181246076077%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score harmonic_alignment_log_scran_pooling knn_auc", - "value": 0.02693737142214102, - "severity": 0, - "severity_value": 0.01346868571107051, - "code": "best_score <= 2", - "message": "Method harmonic_alignment_log_scran_pooling performs a lot better than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_log_scran_pooling\n Metric id: knn_auc\n Best score: 0.02693737142214102%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score harmonic_alignment_sqrt_cp10k knn_auc", - "value": -0.007718136452076273, - "severity": 0, - "severity_value": 0.007718136452076273, - "code": "worst_score >= -1", - "message": "Method harmonic_alignment_sqrt_cp10k performs much worse than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_sqrt_cp10k\n Metric id: knn_auc\n Worst score: -0.007718136452076273%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score harmonic_alignment_sqrt_cp10k knn_auc", - "value": 0.008048361486095518, - "severity": 0, - "severity_value": 0.004024180743047759, - "code": "best_score <= 2", - "message": "Method harmonic_alignment_sqrt_cp10k performs a lot better than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_sqrt_cp10k\n Metric id: knn_auc\n Best score: 0.008048361486095518%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score mnn_log_cp10k knn_auc", - "value": -0.045073857496674405, - "severity": 0, - "severity_value": 0.045073857496674405, - "code": "worst_score >= -1", - "message": "Method mnn_log_cp10k performs much worse than baselines.\n Task id: matching_modalities\n Method id: mnn_log_cp10k\n Metric id: knn_auc\n Worst score: -0.045073857496674405%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score mnn_log_cp10k knn_auc", - "value": 0.13574548653413956, - "severity": 0, - "severity_value": 0.06787274326706978, - "code": "best_score <= 2", - "message": "Method mnn_log_cp10k performs a lot better than baselines.\n Task id: matching_modalities\n Method id: mnn_log_cp10k\n Metric id: knn_auc\n Best score: 0.13574548653413956%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score mnn_log_scran_pooling knn_auc", - "value": -0.01880946613770593, - "severity": 0, - "severity_value": 0.01880946613770593, - "code": "worst_score >= -1", - "message": "Method mnn_log_scran_pooling performs much worse than baselines.\n Task id: matching_modalities\n Method id: mnn_log_scran_pooling\n Metric id: knn_auc\n Worst score: -0.01880946613770593%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score mnn_log_scran_pooling knn_auc", - "value": 0.06533138842384231, - "severity": 0, - "severity_value": 0.032665694211921156, - "code": "best_score <= 2", - "message": "Method mnn_log_scran_pooling performs a lot better than baselines.\n Task id: matching_modalities\n Method id: mnn_log_scran_pooling\n Metric id: knn_auc\n Best score: 0.06533138842384231%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score procrustes knn_auc", - "value": 0.05705698937347893, - "severity": 0, - "severity_value": -0.05705698937347893, - "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: matching_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: 0.05705698937347893%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score procrustes knn_auc", - "value": 0.3108150404865038, - "severity": 0, - "severity_value": 0.1554075202432519, - "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: matching_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 0.3108150404865038%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score random_features knn_auc", - "value": 0.0, - "severity": 0, - "severity_value": -0.0, - "code": "worst_score >= -1", - "message": "Method random_features performs much worse than baselines.\n Task id: matching_modalities\n Method id: random_features\n Metric id: knn_auc\n Worst score: 0.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score random_features knn_auc", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: matching_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 0.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score true_features knn_auc", - "value": 1.0, - "severity": 0, - "severity_value": -1.0, - "code": "worst_score >= -1", - "message": "Method true_features performs much worse than baselines.\n Task id: matching_modalities\n Method id: true_features\n Metric id: knn_auc\n Worst score: 1.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score true_features knn_auc", - "value": 1.0, - "severity": 0, - "severity_value": 0.5, - "code": "best_score <= 2", - "message": "Method true_features performs a lot better than baselines.\n Task id: matching_modalities\n Method id: true_features\n Metric id: knn_auc\n Best score: 1.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score harmonic_alignment_log_scran_pooling mse", - "value": -0.006160570533860987, - "severity": 0, - "severity_value": 0.006160570533860987, - "code": "worst_score >= -1", - "message": "Method harmonic_alignment_log_scran_pooling performs much worse than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_log_scran_pooling\n Metric id: mse\n Worst score: -0.006160570533860987%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score harmonic_alignment_log_scran_pooling mse", - "value": 0.01539225216374196, - "severity": 0, - "severity_value": 0.00769612608187098, - "code": "best_score <= 2", - "message": "Method harmonic_alignment_log_scran_pooling performs a lot better than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_log_scran_pooling\n Metric id: mse\n Best score: 0.01539225216374196%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score harmonic_alignment_sqrt_cp10k mse", - "value": -0.006538532325927271, - "severity": 0, - "severity_value": 0.006538532325927271, - "code": "worst_score >= -1", - "message": "Method harmonic_alignment_sqrt_cp10k performs much worse than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_sqrt_cp10k\n Metric id: mse\n Worst score: -0.006538532325927271%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score harmonic_alignment_sqrt_cp10k mse", - "value": 0.016096476784697722, - "severity": 0, - "severity_value": 0.008048238392348861, - "code": "best_score <= 2", - "message": "Method harmonic_alignment_sqrt_cp10k performs a lot better than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_sqrt_cp10k\n Metric id: mse\n Best score: 0.016096476784697722%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score mnn_log_cp10k mse", - "value": -0.1744304443286302, - "severity": 0, - "severity_value": 0.1744304443286302, - "code": "worst_score >= -1", - "message": "Method mnn_log_cp10k performs much worse than baselines.\n Task id: matching_modalities\n Method id: mnn_log_cp10k\n Metric id: mse\n Worst score: -0.1744304443286302%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score mnn_log_cp10k mse", - "value": 0.32117421366700694, - "severity": 0, - "severity_value": 0.16058710683350347, - "code": "best_score <= 2", - "message": "Method mnn_log_cp10k performs a lot better than baselines.\n Task id: matching_modalities\n Method id: mnn_log_cp10k\n Metric id: mse\n Best score: 0.32117421366700694%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score mnn_log_scran_pooling mse", - "value": -0.02618801427707118, - "severity": 0, - "severity_value": 0.02618801427707118, - "code": "worst_score >= -1", - "message": "Method mnn_log_scran_pooling performs much worse than baselines.\n Task id: matching_modalities\n Method id: mnn_log_scran_pooling\n Metric id: mse\n Worst score: -0.02618801427707118%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score mnn_log_scran_pooling mse", - "value": 0.2803566688566542, - "severity": 0, - "severity_value": 0.1401783344283271, - "code": "best_score <= 2", - "message": "Method mnn_log_scran_pooling performs a lot better than baselines.\n Task id: matching_modalities\n Method id: mnn_log_scran_pooling\n Metric id: mse\n Best score: 0.2803566688566542%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score procrustes mse", - "value": 0.07392331619971682, - "severity": 0, - "severity_value": -0.07392331619971682, - "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: matching_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: 0.07392331619971682%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score procrustes mse", - "value": 0.5607972733684712, - "severity": 0, - "severity_value": 0.2803986366842356, - "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: matching_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 0.5607972733684712%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score random_features mse", - "value": 0.0, - "severity": 0, - "severity_value": -0.0, - "code": "worst_score >= -1", - "message": "Method random_features performs much worse than baselines.\n Task id: matching_modalities\n Method id: random_features\n Metric id: mse\n Worst score: 0.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score random_features mse", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: matching_modalities\n Method id: random_features\n Metric id: mse\n Best score: 0.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score true_features mse", - "value": 1.0, - "severity": 0, - "severity_value": -1.0, - "code": "worst_score >= -1", - "message": "Method true_features performs much worse than baselines.\n Task id: matching_modalities\n Method id: true_features\n Metric id: mse\n Worst score: 1.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score true_features mse", - "value": 1.0, - "severity": 0, - "severity_value": 0.5, - "code": "best_score <= 2", - "message": "Method true_features performs a lot better than baselines.\n Task id: matching_modalities\n Method id: true_features\n Metric id: mse\n Best score: 1.0%\n" - } -] \ No newline at end of file diff --git a/results/matching_modalities/data/results.json b/results/matching_modalities/data/results.json deleted file mode 100644 index defcf083..00000000 --- a/results/matching_modalities/data/results.json +++ /dev/null @@ -1,506 +0,0 @@ -[ - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_features", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.309", - "code_version": "0.7.0", - "resources": { - "duration_sec": 220.0, - "cpu_pct": 73.6, - "peak_memory_mb": 515.7, - "disk_read_mb": 272.6, - "disk_write_mb": 273.0 - }, - "metric_values": { - "knn_auc": 0.6208419863517854, - "mse": 0.0 - }, - "scaled_scores": { - "knn_auc": 1.0, - "mse": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "procrustes", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.293", - "code_version": "1.9.3", - "resources": { - "duration_sec": 279.0, - "cpu_pct": 143.1, - "peak_memory_mb": 963.6, - "disk_read_mb": 272.6, - "disk_write_mb": 319.3 - }, - "metric_values": { - "knn_auc": 0.09674569704735692, - "mse": 0.9387379116135324 - }, - "scaled_scores": { - "knn_auc": 0.08183270172346102, - "mse": 0.07392331619971682 - }, - "mean_score": 0.07787800896158892 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_features", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.293", - "code_version": "0.7.0", - "resources": { - "duration_sec": 319.0, - "cpu_pct": 32.4, - "peak_memory_mb": 518.4, - "disk_read_mb": 272.6, - "disk_write_mb": 273.0 - }, - "metric_values": { - "knn_auc": 0.05003502090203344, - "mse": 1.0136719 - }, - "scaled_scores": { - "knn_auc": 0.0, - "mse": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_sqrt_cp10k", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.533", - "code_version": "0.0", - "resources": { - "duration_sec": 429.0, - "cpu_pct": 193.3, - "peak_memory_mb": 1300.0, - "disk_read_mb": 272.9, - "disk_write_mb": 319.2 - }, - "metric_values": { - "knn_auc": 0.05462908169875428, - "mse": 1.000013512131858 - }, - "scaled_scores": { - "knn_auc": 0.008048361486095518, - "mse": 0.013474170358418691 - }, - "mean_score": 0.010761265922257105 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_cp10k", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:51.859", - "code_version": "1.14.1", - "resources": { - "duration_sec": 490.0, - "cpu_pct": 99.6, - "peak_memory_mb": 2200.0, - "disk_read_mb": 317.4, - "disk_write_mb": 423.0 - }, - "metric_values": { - "knn_auc": 0.08589789433030783, - "mse": 1.0547959275709011 - }, - "scaled_scores": { - "knn_auc": 0.06282837386193632, - "mse": -0.040569367238946885 - }, - "mean_score": 0.011129503311494718 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_scran_pooling", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.025", - "code_version": "1.14.1", - "resources": { - "duration_sec": 750.0, - "cpu_pct": 99.4, - "peak_memory_mb": 4200.0, - "disk_read_mb": 320.3, - "disk_write_mb": 423.1 - }, - "metric_values": { - "knn_auc": 0.07631922843774054, - "mse": 1.038986769000114 - }, - "scaled_scores": { - "knn_auc": 0.046047454089837826, - "mse": -0.024973434698262853 - }, - "mean_score": 0.010537009695787487 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_log_scran_pooling", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.149", - "code_version": "0.0", - "resources": { - "duration_sec": 859.0, - "cpu_pct": 120.2, - "peak_memory_mb": 4200.0, - "disk_read_mb": 318.6, - "disk_write_mb": 364.2 - }, - "metric_values": { - "knn_auc": 0.06541106014069863, - "mse": 1.000107517902145 - }, - "scaled_scores": { - "knn_auc": 0.02693737142214102, - "mse": 0.01338143249098156 - }, - "mean_score": 0.02015940195656129 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "procrustes", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:21.934", - "code_version": "1.9.3", - "resources": { - "duration_sec": 340.0, - "cpu_pct": 230.3, - "peak_memory_mb": 692.5, - "disk_read_mb": 210.8, - "disk_write_mb": 234.2 - }, - "metric_values": { - "knn_auc": 0.08257473591291212, - "mse": 0.8323309514022647 - }, - "scaled_scores": { - "knn_auc": 0.05705698937347893, - "mse": 0.1806911435619616 - }, - "mean_score": 0.11887406646772025 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_features", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:22.007", - "code_version": "0.7.0", - "resources": { - "duration_sec": 430.0, - "cpu_pct": 38.6, - "peak_memory_mb": 460.7, - "disk_read_mb": 210.8, - "disk_write_mb": 221.7 - }, - "metric_values": { - "knn_auc": 0.05042255760201557, - "mse": 1.015894 - }, - "scaled_scores": { - "knn_auc": 0.0, - "mse": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_features", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:22.020", - "code_version": "0.7.0", - "resources": { - "duration_sec": 430.0, - "cpu_pct": 35.1, - "peak_memory_mb": 453.9, - "disk_read_mb": 210.8, - "disk_write_mb": 221.7 - }, - "metric_values": { - "knn_auc": 0.6139324564583585, - "mse": 0.0 - }, - "scaled_scores": { - "knn_auc": 1.0, - "mse": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_sqrt_cp10k", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:21.923", - "code_version": "0.0", - "resources": { - "duration_sec": 701.0, - "cpu_pct": 225.3, - "peak_memory_mb": 1023.9, - "disk_read_mb": 211.1, - "disk_write_mb": 234.1 - }, - "metric_values": { - "knn_auc": 0.046073311310546615, - "mse": 0.9995416858132864 - }, - "scaled_scores": { - "knn_auc": -0.007718136452076273, - "mse": 0.016096476784697722 - }, - "mean_score": 0.004189170166310725 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_scran_pooling", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:21.988", - "code_version": "1.14.1", - "resources": { - "duration_sec": 701.0, - "cpu_pct": 94.5, - "peak_memory_mb": 3300.0, - "disk_read_mb": 258.5, - "disk_write_mb": 301.0 - }, - "metric_values": { - "knn_auc": 0.0872374416848794, - "mse": 0.7310813422485382 - }, - "scaled_scores": { - "knn_auc": 0.06533138842384231, - "mse": 0.2803566688566542 - }, - "mean_score": 0.17284402864024828 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_cp10k", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:22.048", - "code_version": "1.14.1", - "resources": { - "duration_sec": 750.0, - "cpu_pct": 91.8, - "peak_memory_mb": 2000.0, - "disk_read_mb": 255.6, - "disk_write_mb": 301.0 - }, - "metric_values": { - "knn_auc": 0.12691648298907363, - "mse": 0.6896150433809697 - }, - "scaled_scores": { - "knn_auc": 0.13574548653413956, - "mse": 0.32117421366700694 - }, - "mean_score": 0.22845985010057324 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_log_scran_pooling", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:21.897", - "code_version": "0.0", - "resources": { - "duration_sec": 1013.0, - "cpu_pct": 151.3, - "peak_memory_mb": 3300.0, - "disk_read_mb": 256.899999, - "disk_write_mb": 290.5 - }, - "metric_values": { - "knn_auc": 0.06031383406703828, - "mse": 1.0002571033803676 - }, - "scaled_scores": { - "knn_auc": 0.017552977303677005, - "mse": 0.01539225216374196 - }, - "mean_score": 0.016472614733709482 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_features", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:02.058", - "code_version": "0.7.0", - "resources": { - "duration_sec": 300.0, - "cpu_pct": 62.5, - "peak_memory_mb": 487.4, - "disk_read_mb": 232.2, - "disk_write_mb": 277.4 - }, - "metric_values": { - "knn_auc": 0.5117302105076631, - "mse": 0.0 - }, - "scaled_scores": { - "knn_auc": 1.0, - "mse": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_features", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:02.040", - "code_version": "0.7.0", - "resources": { - "duration_sec": 331.0, - "cpu_pct": 64.9, - "peak_memory_mb": 488.1, - "disk_read_mb": 232.2, - "disk_write_mb": 277.4 - }, - "metric_values": { - "knn_auc": 0.05000744227734146, - "mse": 0.99353725 - }, - "scaled_scores": { - "knn_auc": 0.0, - "mse": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "procrustes", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:02.120", - "code_version": "1.9.3", - "resources": { - "duration_sec": 331.0, - "cpu_pct": 65.2, - "peak_memory_mb": 487.7, - "disk_read_mb": 232.2, - "disk_write_mb": 278.6 - }, - "metric_values": { - "knn_auc": 0.1935178231783895, - "mse": 0.436364269209991 - }, - "scaled_scores": { - "knn_auc": 0.3108150404865038, - "mse": 0.5607972733684712 - }, - "mean_score": 0.4358061569274875 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_cp10k", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:02.166", - "code_version": "1.14.1", - "resources": { - "duration_sec": 490.0, - "cpu_pct": 86.6, - "peak_memory_mb": 2000.0, - "disk_read_mb": 277.1, - "disk_write_mb": 369.5 - }, - "metric_values": { - "knn_auc": 0.029195816019157918, - "mse": 1.1668403939745453 - }, - "scaled_scores": { - "knn_auc": -0.045073857496674405, - "mse": -0.1744304443286302 - }, - "mean_score": -0.10975215091265231 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_sqrt_cp10k", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:01.988", - "code_version": "0.0", - "resources": { - "duration_sec": 650.0, - "cpu_pct": 198.6, - "peak_memory_mb": 646.4, - "disk_read_mb": 232.5, - "disk_write_mb": 290.1 - }, - "metric_values": { - "knn_auc": 0.05061583797299569, - "mse": 1.0000335254261379 - }, - "scaled_scores": { - "knn_auc": 0.0013176644894209325, - "mse": -0.006538532325927271 - }, - "mean_score": -0.0026104339182531694 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_scran_pooling", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:01.878", - "code_version": "1.14.1", - "resources": { - "duration_sec": 720.0, - "cpu_pct": 99.7, - "peak_memory_mb": 4200.0, - "disk_read_mb": 279.9, - "disk_write_mb": 369.6 - }, - "metric_values": { - "knn_auc": 0.04132268350330538, - "mse": 1.019556017687802 - }, - "scaled_scores": { - "knn_auc": -0.01880946613770593, - "mse": -0.02618801427707118 - }, - "mean_score": -0.022498740207388554 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_log_scran_pooling", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:01.959", - "code_version": "0.0", - "resources": { - "duration_sec": 1054.0, - "cpu_pct": 140.0, - "peak_memory_mb": 4200.0, - "disk_read_mb": 278.2, - "disk_write_mb": 379.9 - }, - "metric_values": { - "knn_auc": 0.05631004174912533, - "mse": 0.9996580063066434 - }, - "scaled_scores": { - "knn_auc": 0.013650181246076077, - "mse": -0.006160570533860987 - }, - "mean_score": 0.003744805356107545 - } -] \ No newline at end of file diff --git a/results/matching_modalities/data/task_info.json b/results/matching_modalities/data/task_info.json deleted file mode 100644 index b5b25ab2..00000000 --- a/results/matching_modalities/data/task_info.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "task_id": "matching_modalities", - "commit_sha": "d77a94914531882aca05653a4c6efc495c397fc6", - "task_name": "Multimodal Data Integration", - "task_summary": "Alignment of cellular profiles from two different modalities", - "task_description": "\nCellular function is regulated by the complex interplay of different types of biological\nmolecules (DNA, RNA, proteins, etc.), which determine the state of a cell. Several\nrecently described technologies allow for simultaneous measurement of different aspects\nof cellular state. For example, [sci-CAR](https://openproblems.bio/bibliography#cao2018joint)\njointly profiles RNA expression and chromatin accessibility on the same cell and\n[CITE-seq](https://openproblems.bio/bibliography#stoeckius2017simultaneous) measures\nsurface protein abundance and RNA expression from each cell. These technologies enable\nus to better understand cellular function, however datasets are still rare and there are\ntradeoffs that these measurements make for to profile multiple modalities.\n\nJoint methods can be more expensive or lower throughput or more noisy than measuring a\nsingle modality at a time. Therefore it is useful to develop methods that are capable\nof integrating measurements of the same biological system but obtained using different\ntechnologies on different cells.\n\nHere the goal is to learn a latent space where cells profiled by different technologies in\ndifferent modalities are matched if they have the same state. We use jointly profiled\ndata as ground truth so that we can evaluate when the observations from the same cell\nacquired using different modalities are similar. A perfect result has each of the paired\nobservations sharing the same coordinates in the latent space.\n\n", - "repo": "openproblems-bio/openproblems" -} \ No newline at end of file diff --git a/results/matching_modalities/index.markdown_strict_files/figure-markdown_strict/raw_results-1.png b/results/matching_modalities/index.markdown_strict_files/figure-markdown_strict/raw_results-1.png deleted file mode 100644 index 0e9b7d58..00000000 Binary files a/results/matching_modalities/index.markdown_strict_files/figure-markdown_strict/raw_results-1.png and /dev/null differ diff --git a/results/matching_modalities/index.markdown_strict_files/figure-markdown_strict/summary-1.png b/results/matching_modalities/index.markdown_strict_files/figure-markdown_strict/summary-1.png deleted file mode 100644 index 1509d4e1..00000000 Binary files a/results/matching_modalities/index.markdown_strict_files/figure-markdown_strict/summary-1.png and /dev/null differ