From af2b60d5379fd9e864869a0f89f82857890a1589 Mon Sep 17 00:00:00 2001 From: Kai Waldrant Date: Tue, 5 Mar 2024 16:10:42 +0100 Subject: [PATCH 1/7] Update match_mod results to v2 --- .../match_modalities/data/dataset_info.json | 26 + .../match_modalities/data/method_info.json | 74 ++ .../match_modalities/data/metric_info.json | 24 + .../data/quality_control.json | 602 ++++++++++++++++ results/match_modalities/data/results.json | 416 +++++++++++ results/match_modalities/data/state.yaml | 8 + results/match_modalities/data/task_info.json | 8 + .../index.qmd | 7 +- .../thumbnail.svg | 0 .../data/dataset_info.json | 38 - .../matching_modalities/data/method_info.json | 107 --- .../matching_modalities/data/metric_info.json | 24 - .../data/quality_control.json | 662 ------------------ results/matching_modalities/data/results.json | 506 ------------- .../matching_modalities/data/task_info.json | 8 - .../figure-markdown_strict/raw_results-1.png | Bin 83478 -> 0 bytes .../figure-markdown_strict/summary-1.png | Bin 100238 -> 0 bytes 17 files changed, 1162 insertions(+), 1348 deletions(-) create mode 100644 results/match_modalities/data/dataset_info.json create mode 100644 results/match_modalities/data/method_info.json create mode 100644 results/match_modalities/data/metric_info.json create mode 100644 results/match_modalities/data/quality_control.json create mode 100644 results/match_modalities/data/results.json create mode 100644 results/match_modalities/data/state.yaml create mode 100644 results/match_modalities/data/task_info.json rename results/{matching_modalities => match_modalities}/index.qmd (58%) rename results/{matching_modalities => match_modalities}/thumbnail.svg (100%) delete mode 100644 results/matching_modalities/data/dataset_info.json delete mode 100644 results/matching_modalities/data/method_info.json delete mode 100644 results/matching_modalities/data/metric_info.json delete mode 100644 results/matching_modalities/data/quality_control.json delete mode 100644 results/matching_modalities/data/results.json delete mode 100644 results/matching_modalities/data/task_info.json delete mode 100644 results/matching_modalities/index.markdown_strict_files/figure-markdown_strict/raw_results-1.png delete mode 100644 results/matching_modalities/index.markdown_strict_files/figure-markdown_strict/summary-1.png diff --git a/results/match_modalities/data/dataset_info.json b/results/match_modalities/data/dataset_info.json new file mode 100644 index 00000000..7b505e5f --- /dev/null +++ b/results/match_modalities/data/dataset_info.json @@ -0,0 +1,26 @@ +[ + { + "task_id": "match_modalities", + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "dataset_name": "sci-CAR Cell Lines", + "dataset_summary": "sci-CAR profiles of 5k cell line cells (HEK293T, NIH/3T3, A549) across three treatment conditions (DEX 0h, 1h and 3h)", + "data_reference": "cao2018joint", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089" + }, + { + "task_id": "match_modalities", + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "dataset_name": "sci-CAR Mouse Kidney", + "dataset_summary": "sci-CAR profiles of 11k mouse kidney cells", + "data_reference": "cao2018joint", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089" + }, + { + "task_id": "match_modalities", + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "dataset_name": "CITE-Seq CBMC", + "dataset_summary": "CITE-seq profiles of 8k Cord Blood Mononuclear Cells", + "data_reference": "stoeckius2017simultaneous", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE100866" + } +] diff --git a/results/match_modalities/data/method_info.json b/results/match_modalities/data/method_info.json new file mode 100644 index 00000000..c6635657 --- /dev/null +++ b/results/match_modalities/data/method_info.json @@ -0,0 +1,74 @@ +[ + { + "task_id": "match_modalities", + "method_id": "random_features", + "method_name": "Random Features", + "method_summary": "Randomly permutated features", + "is_baseline": true, + "paper_reference": null, + "code_url": null, + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/control_methods/random_features/config.vsh.yaml", + "code_version": null, + "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + }, + { + "task_id": "match_modalities", + "method_id": "true_features", + "method_name": "True Features", + "method_summary": "A 1 to 1 mapping of features between modalities", + "is_baseline": true, + "paper_reference": null, + "code_url": null, + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/control_methods/true_features/config.vsh.yaml", + "code_version": null, + "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + }, + { + "task_id": "match_modalities", + "method_id": "scot", + "method_name": "Single Cell Optimal Transport", + "method_summary": "Run Single Cell Optimal Transport", + "is_baseline": false, + "paper_reference": "Demetci2020scot", + "code_url": "https://github.com/rsinghlab/SCOT", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/methods/scot/config.vsh.yaml", + "code_version": null, + "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + }, + { + "task_id": "match_modalities", + "method_id": "harmonic_alignment", + "method_name": "Harmonic Alignment", + "method_summary": "Harmonic Alignment", + "is_baseline": false, + "paper_reference": "stanley2020harmonic", + "code_url": "https://github.com/KrishnaswamyLab/harmonic-alignment", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/methods/harmonic_alignment/config.vsh.yaml", + "code_version": null, + "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + }, + { + "task_id": "match_modalities", + "method_id": "fastmnn", + "method_name": "fastMNN", + "method_summary": "A simpler version of the original mnnCorrect algorithm.", + "is_baseline": false, + "paper_reference": "haghverdi2018batch", + "code_url": "https://github.com/LTLA/batchelor", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/methods/fastmnn/config.vsh.yaml", + "code_version": null, + "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + }, + { + "task_id": "match_modalities", + "method_id": "procrustes", + "method_name": "Procrustes", + "method_summary": "\"Procrustes superimposition embeds cellular data from each modality into a common space.\"\n", + "is_baseline": false, + "paper_reference": "gower1975generalized", + "code_url": "https://github.com/scipy/scipy", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/methods/procrustes/config.vsh.yaml", + "code_version": null, + "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + } +] diff --git a/results/match_modalities/data/metric_info.json b/results/match_modalities/data/metric_info.json new file mode 100644 index 00000000..648dbcc2 --- /dev/null +++ b/results/match_modalities/data/metric_info.json @@ -0,0 +1,24 @@ +[ + { + "task_id": "match_modalities", + "metric_id": "knn_auc", + "metric_name": "KNN AUC", + "metric_summary": "\"Compute the kNN Area Under the Curve\"\n", + "paper_reference": "", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/metrics/knn_auc/config.vsh.yaml", + "code_version": null, + "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32", + "maximize": true + }, + { + "task_id": "match_modalities", + "metric_id": "mse", + "metric_name": "Mean Squared Error", + "metric_summary": "The mean squared error (MSE) is a measure of the quality of an estimator. It is always non-negative, and values closer to zero are better.\n", + "paper_reference": "", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/metrics/mse/config.vsh.yaml", + "code_version": null, + "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32", + "maximize": true + } +] diff --git a/results/match_modalities/data/quality_control.json b/results/match_modalities/data/quality_control.json new file mode 100644 index 00000000..a1d48c37 --- /dev/null +++ b/results/match_modalities/data/quality_control.json @@ -0,0 +1,602 @@ +[ + { + "task_id": "match_modalities", + "category": "Task info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_id' should be defined\n Task id: match_modalities\n Field: task_id\n" + }, + { + "task_id": "match_modalities", + "category": "Task info", + "name": "Pct 'task_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_name' should be defined\n Task id: match_modalities\n Field: task_name\n" + }, + { + "task_id": "match_modalities", + "category": "Task info", + "name": "Pct 'task_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_summary' should be defined\n Task id: match_modalities\n Field: task_summary\n" + }, + { + "task_id": "match_modalities", + "category": "Task info", + "name": "Pct 'task_description' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_description' should be defined\n Task id: match_modalities\n Field: task_description\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'task_id' should be defined\n Task id: match_modalities\n Field: task_id\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'commit_sha' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'commit_sha' should be defined\n Task id: match_modalities\n Field: commit_sha\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'method_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'method_id' should be defined\n Task id: match_modalities\n Field: method_id\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'method_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'method_name' should be defined\n Task id: match_modalities\n Field: method_name\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'method_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'method_summary' should be defined\n Task id: match_modalities\n Field: method_summary\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'paper_reference' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'paper_reference' should be defined\n Task id: match_modalities\n Field: paper_reference\n" + }, + { + "task_id": "match_modalities", + "category": "Method info", + "name": "Pct 'is_baseline' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'is_baseline' should be defined\n Task id: match_modalities\n Field: is_baseline\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'task_id' should be defined\n Task id: match_modalities\n Field: task_id\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'commit_sha' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'commit_sha' should be defined\n Task id: match_modalities\n Field: commit_sha\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'metric_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'metric_id' should be defined\n Task id: match_modalities\n Field: metric_id\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'metric_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'metric_name' should be defined\n Task id: match_modalities\n Field: metric_name\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'metric_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'metric_summary' should be defined\n Task id: match_modalities\n Field: metric_summary\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'paper_reference' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'paper_reference' should be defined\n Task id: match_modalities\n Field: paper_reference\n" + }, + { + "task_id": "match_modalities", + "category": "Metric info", + "name": "Pct 'maximize' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'maximize' should be defined\n Task id: match_modalities\n Field: maximize\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'task_id' should be defined\n Task id: match_modalities\n Field: task_id\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'dataset_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'dataset_id' should be defined\n Task id: match_modalities\n Field: dataset_id\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'dataset_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'dataset_name' should be defined\n Task id: match_modalities\n Field: dataset_name\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'dataset_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'dataset_summary' should be defined\n Task id: match_modalities\n Field: dataset_summary\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'data_reference' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'data_reference' should be defined\n Task id: match_modalities\n Field: data_reference\n" + }, + { + "task_id": "match_modalities", + "category": "Dataset info", + "name": "Pct 'data_url' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'data_url' should be defined\n Task id: match_modalities\n Field: data_url\n" + }, + { + "task_id": "match_modalities", + "category": "Raw data", + "name": "Number of results", + "value": 18, + "severity": 0, + "severity_value": 0.0, + "code": "len(results) == len(method_info) * len(metric_info) * len(dataset_info)", + "message": "Number of results should be equal to #methods × #metrics × #datasets.\n Task id: match_modalities\n Number of results: 18\n Number of methods: 6\n Number of metrics: 2\n Number of datasets: 3\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Metric 'knn_auc' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n Metric id: knn_auc\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Metric 'mse' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n Metric id: mse\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'random_features' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: random_features\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'true_features' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: true_features\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'scot' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: scot\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'harmonic_alignment' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: harmonic_alignment\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'fastmnn' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: fastmnn\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Method 'procrustes' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n method id: procrustes\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Dataset 'openproblems_v1_multimodal/scicar_cell_lines' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/scicar_cell_lines\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Dataset 'openproblems_v1_multimodal/scicar_mouse_kidney' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/scicar_mouse_kidney\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Raw results", + "name": "Dataset 'openproblems_v1_multimodal/citeseq_cbmc' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/citeseq_cbmc\n Percentage missing: 0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score random_features knn_auc", + "value": 0.0069, + "severity": 0, + "severity_value": -0.0069, + "code": "worst_score >= -1", + "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Worst score: 0.0069%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score random_features knn_auc", + "value": 0.0457, + "severity": 0, + "severity_value": 0.02285, + "code": "best_score <= 2", + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 0.0457%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score true_features knn_auc", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Worst score: 1%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score true_features knn_auc", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Best score: 1%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score scot knn_auc", + "value": 0.034, + "severity": 0, + "severity_value": -0.034, + "code": "worst_score >= -1", + "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Worst score: 0.034%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score scot knn_auc", + "value": 0.0541, + "severity": 0, + "severity_value": 0.02705, + "code": "best_score <= 2", + "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Best score: 0.0541%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score harmonic_alignment knn_auc", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Worst score: 0.0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score harmonic_alignment knn_auc", + "value": 0.0467, + "severity": 0, + "severity_value": 0.02335, + "code": "best_score <= 2", + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Best score: 0.0467%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score fastmnn knn_auc", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Worst score: 0.0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score fastmnn knn_auc", + "value": 0.0343, + "severity": 0, + "severity_value": 0.01715, + "code": "best_score <= 2", + "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Best score: 0.0343%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score procrustes knn_auc", + "value": 0.0663, + "severity": 0, + "severity_value": -0.0663, + "code": "worst_score >= -1", + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: 0.0663%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score procrustes knn_auc", + "value": 0.2744, + "severity": 0, + "severity_value": 0.1372, + "code": "best_score <= 2", + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 0.2744%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score random_features mse", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Worst score: 0.0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score random_features mse", + "value": 1.0, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Best score: 1.0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score true_features mse", + "value": 0.0941, + "severity": 0, + "severity_value": -0.0941, + "code": "worst_score >= -1", + "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Worst score: 0.0941%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score true_features mse", + "value": 0.5746, + "severity": 0, + "severity_value": 0.2873, + "code": "best_score <= 2", + "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Best score: 0.5746%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score scot mse", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Worst score: 0.0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score scot mse", + "value": 1.0, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Best score: 1.0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score harmonic_alignment mse", + "value": 0.0267, + "severity": 0, + "severity_value": -0.0267, + "code": "worst_score >= -1", + "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Worst score: 0.0267%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score harmonic_alignment mse", + "value": 0.9548, + "severity": 0, + "severity_value": 0.4774, + "code": "best_score <= 2", + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Best score: 0.9548%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score fastmnn mse", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Worst score: 0.0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score fastmnn mse", + "value": 1.0, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Best score: 1.0%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Worst score procrustes mse", + "value": 0.0486, + "severity": 0, + "severity_value": -0.0486, + "code": "worst_score >= -1", + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: 0.0486%\n" + }, + { + "task_id": "match_modalities", + "category": "Scaling", + "name": "Best score procrustes mse", + "value": 0.9763, + "severity": 0, + "severity_value": 0.48815, + "code": "best_score <= 2", + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 0.9763%\n" + } +] \ No newline at end of file diff --git a/results/match_modalities/data/results.json b/results/match_modalities/data/results.json new file mode 100644 index 00000000..9ed8c2e9 --- /dev/null +++ b/results/match_modalities/data/results.json @@ -0,0 +1,416 @@ +[ + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "fastmnn", + "metric_values": { + "knn_auc": 0.0383, + "mse": 1.1745 + }, + "scaled_scores": { + "knn_auc": 0.0343, + "mse": 0.6988 + }, + "mean_score": 0.3666, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 43.2, + "cpu_pct": 69, + "peak_memory_mb": 6554, + "disk_read_mb": 238, + "disk_write_mb": 58 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "harmonic_alignment", + "metric_values": { + "knn_auc": 0.0041, + "mse": 1 + }, + "scaled_scores": { + "knn_auc": 0, + "mse": 0.0267 + }, + "mean_score": 0.0133, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 98, + "cpu_pct": 958.3, + "peak_memory_mb": 5940, + "disk_read_mb": 208, + "disk_write_mb": 69 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "procrustes", + "metric_values": { + "knn_auc": 0.1126, + "mse": 1.0056 + }, + "scaled_scores": { + "knn_auc": 0.1089, + "mse": 0.0486 + }, + "mean_score": 0.0787, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 13.3, + "cpu_pct": 175.7, + "peak_memory_mb": 5735, + "disk_read_mb": 199, + "disk_write_mb": 58 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "random_features", + "metric_values": { + "knn_auc": 0.0496, + "mse": 0.993 + }, + "scaled_scores": { + "knn_auc": 0.0457, + "mse": 0 + }, + "mean_score": 0.0228, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 23.6, + "cpu_pct": 57.7, + "peak_memory_mb": 5632, + "disk_read_mb": 198, + "disk_write_mb": 57 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "scot", + "metric_values": { + "knn_auc": 0.058, + "mse": 1.2526 + }, + "scaled_scores": { + "knn_auc": 0.0541, + "mse": 1 + }, + "mean_score": 0.5271, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 3133, + "cpu_pct": 2889.1, + "peak_memory_mb": 12493, + "disk_read_mb": 219, + "disk_write_mb": 58 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "method_id": "true_features", + "metric_values": { + "knn_auc": 1, + "mse": 1.0453 + }, + "scaled_scores": { + "knn_auc": 1, + "mse": 0.2013 + }, + "mean_score": 0.6007, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 18.2, + "cpu_pct": 55.4, + "peak_memory_mb": 2765, + "disk_read_mb": 198, + "disk_write_mb": 3 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "fastmnn", + "metric_values": { + "knn_auc": 0.0138, + "mse": 1.0755 + }, + "scaled_scores": { + "knn_auc": 0.0075, + "mse": 1 + }, + "mean_score": 0.5037, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 24.5, + "cpu_pct": 147.7, + "peak_memory_mb": 6656, + "disk_read_mb": 217, + "disk_write_mb": 65 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "harmonic_alignment", + "metric_values": { + "knn_auc": 0.0064, + "mse": 1 + }, + "scaled_scores": { + "knn_auc": 0, + "mse": 0.6653 + }, + "mean_score": 0.3327, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 49.1, + "cpu_pct": 812.4, + "peak_memory_mb": 5223, + "disk_read_mb": 187, + "disk_write_mb": 64 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "procrustes", + "metric_values": { + "knn_auc": 0.0723, + "mse": 1.0115 + }, + "scaled_scores": { + "knn_auc": 0.0663, + "mse": 0.7163 + }, + "mean_score": 0.3913, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 6.2, + "cpu_pct": 268.5, + "peak_memory_mb": 3380, + "disk_read_mb": 178, + "disk_write_mb": 64 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "random_features", + "metric_values": { + "knn_auc": 0.049, + "mse": 1.0008 + }, + "scaled_scores": { + "knn_auc": 0.0429, + "mse": 0.6689 + }, + "mean_score": 0.3559, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 15.9, + "cpu_pct": 74.7, + "peak_memory_mb": 5632, + "disk_read_mb": 177, + "disk_write_mb": 61 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "scot", + "metric_values": { + "knn_auc": 0.0517, + "mse": 0.8498 + }, + "scaled_scores": { + "knn_auc": 0.0457, + "mse": 0 + }, + "mean_score": 0.0228, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 630, + "cpu_pct": 2797.2, + "peak_memory_mb": 7783, + "disk_read_mb": 198, + "disk_write_mb": 64 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "method_id": "true_features", + "metric_values": { + "knn_auc": 1, + "mse": 0.9795 + }, + "scaled_scores": { + "knn_auc": 1, + "mse": 0.5746 + }, + "mean_score": 0.7873, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 13.9, + "cpu_pct": 57.7, + "peak_memory_mb": 5428, + "disk_read_mb": 177, + "disk_write_mb": 14 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "fastmnn", + "metric_values": { + "knn_auc": 0.0432, + "mse": 0.9461 + }, + "scaled_scores": { + "knn_auc": 0, + "mse": 0 + }, + "mean_score": 0, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 78, + "cpu_pct": 82.2, + "peak_memory_mb": 6964, + "disk_read_mb": 356, + "disk_write_mb": 119 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "harmonic_alignment", + "metric_values": { + "knn_auc": 0.0879, + "mse": 0.9999 + }, + "scaled_scores": { + "knn_auc": 0.0467, + "mse": 0.9548 + }, + "mean_score": 0.5008, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 251, + "cpu_pct": 778.1, + "peak_memory_mb": 6759, + "disk_read_mb": 326, + "disk_write_mb": 118 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "procrustes", + "metric_values": { + "knn_auc": 0.3057, + "mse": 1.0011 + }, + "scaled_scores": { + "knn_auc": 0.2744, + "mse": 0.9763 + }, + "mean_score": 0.6253, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 9.8, + "cpu_pct": 206.7, + "peak_memory_mb": 3584, + "disk_read_mb": 317, + "disk_write_mb": 118 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "random_features", + "metric_values": { + "knn_auc": 0.0498, + "mse": 1.0025 + }, + "scaled_scores": { + "knn_auc": 0.0069, + "mse": 1 + }, + "mean_score": 0.5035, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 9.1, + "cpu_pct": 158, + "peak_memory_mb": 3482, + "disk_read_mb": 316, + "disk_write_mb": 110 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "scot", + "metric_values": { + "knn_auc": 0.0758, + "mse": 0.9948 + }, + "scaled_scores": { + "knn_auc": 0.034, + "mse": 0.8637 + }, + "mean_score": 0.4488, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 5053, + "cpu_pct": 2626.5, + "peak_memory_mb": 17101, + "disk_read_mb": 336, + "disk_write_mb": 118 + }, + "task_id": "match_modalities" + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "method_id": "true_features", + "metric_values": { + "knn_auc": 1, + "mse": 0.9514 + }, + "scaled_scores": { + "knn_auc": 1, + "mse": 0.0941 + }, + "mean_score": 0.5471, + "normalization_id": "log_cp10k", + "resources": { + "exit_code": 0, + "duration_sec": 5.2, + "cpu_pct": 215.3, + "peak_memory_mb": 5837, + "disk_read_mb": 316, + "disk_write_mb": 25 + }, + "task_id": "match_modalities" + } +] diff --git a/results/match_modalities/data/state.yaml b/results/match_modalities/data/state.yaml new file mode 100644 index 00000000..9cc4fc6d --- /dev/null +++ b/results/match_modalities/data/state.yaml @@ -0,0 +1,8 @@ +id: process +output_scores: !file results.json +output_method_info: !file method_info.json +output_metric_info: !file metric_info.json +output_dataset_info: !file dataset_info.json +output_task_info: !file task_info.json +output_qc: !file quality_control.json + diff --git a/results/match_modalities/data/task_info.json b/results/match_modalities/data/task_info.json new file mode 100644 index 00000000..67ea05ad --- /dev/null +++ b/results/match_modalities/data/task_info.json @@ -0,0 +1,8 @@ +{ + "task_id": "match_modalities", + "commit_sha": null, + "task_name": "Match Modalities", + "task_summary": "Match cells across datasets of the same set of samples on different technologies / modalities.\n", + "task_description": "Cellular function is regulated by the complex interplay of different types of biological\nmolecules (DNA, RNA, proteins, etc.), which determine the state of a cell. Several\nrecently described technologies allow for simultaneous measurement of different aspects\nof cellular state. For example, sci-CAR [@cao2018joint]\njointly profiles RNA expression and chromatin accessibility on the same cell and\nCITE-seq [@stoeckius2017simultaneous] measures\nsurface protein abundance and RNA expression from each cell. These technologies enable\nus to better understand cellular function, however datasets are still rare and there are\ntradeoffs that these measurements make for to profile multiple modalities.\n\nJoint methods can be more expensive or lower throughput or more noisy than measuring a\nsingle modality at a time. Therefore it is useful to develop methods that are capable\nof integrating measurements of the same biological system but obtained using different\ntechnologies on different cells.\n\n\nIn this task, the goal is to learn a latent space where cells profiled by different\ntechnologies in different modalities are matched if they have the same state. We use\njointly profiled data as ground truth so that we can evaluate when the observations\nfrom the same cell acquired using different modalities are similar. A perfect result\nhas each of the paired observations sharing the same coordinates in the latent space.\nA method that can achieve this would be able to match datasets across modalities to\nenable multimodal cellular analysis from separately measured profiles.\n", + "repo": "openproblems-bio/openproblems-v2" +} diff --git a/results/matching_modalities/index.qmd b/results/match_modalities/index.qmd similarity index 58% rename from results/matching_modalities/index.qmd rename to results/match_modalities/index.qmd index 3cc89097..4ce4415f 100644 --- a/results/matching_modalities/index.qmd +++ b/results/match_modalities/index.qmd @@ -1,6 +1,7 @@ --- -title: "Multimodal Data Integration" -subtitle: "Alignment of cellular profiles from two different modalities" +title: "Match Modalities" +subtitle: "Match cells across datasets of the same set of samples on different technologies / modalities. +" image: thumbnail.svg page-layout: full css: ../_include/task_template.css @@ -12,7 +13,7 @@ toc: false ```{r} #| include: false -params <- list(data_dir = "results/matching_modalities/data") +params <- list(data_dir = "results/match_modalities/data") params <- list(data_dir = "./data") ``` diff --git a/results/matching_modalities/thumbnail.svg b/results/match_modalities/thumbnail.svg similarity index 100% rename from results/matching_modalities/thumbnail.svg rename to results/match_modalities/thumbnail.svg diff --git a/results/matching_modalities/data/dataset_info.json b/results/matching_modalities/data/dataset_info.json deleted file mode 100644 index a87fa11d..00000000 --- a/results/matching_modalities/data/dataset_info.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - { - "dataset_name": "CITE-seq Cord Blood Mononuclear Cells", - "image": "openproblems", - "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE100866", - "data_reference": "stoeckius2017simultaneous", - "dataset_summary": "8k cord blood mononuclear cells sequenced by CITEseq, a multimodal addition to the 10x scRNA-seq platform that allows simultaneous measurement of RNA and protein.", - "task_id": "matching_modalities", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "citeseq_cbmc", - "source_dataset_id": "openproblems_v1/citeseq_cbmc", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/datasets/citeseq.py" - }, - { - "dataset_name": "sciCAR Cell Lines", - "image": "openproblems", - "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089", - "data_reference": "cao2018joint", - "dataset_summary": "5k cells from a time-series of dexamethasone treatment sequenced by sci-CAR, a combinatorial indexing-based co-assay that jointly profiles chromatin accessibility and mRNA.", - "task_id": "matching_modalities", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "scicar_cell_lines", - "source_dataset_id": "openproblems_v1/scicar_cell_lines", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/datasets/scicar.py" - }, - { - "dataset_name": "sciCAR Mouse Kidney", - "image": "openproblems", - "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089", - "data_reference": "cao2018joint", - "dataset_summary": "11k cells from adult mouse kidney sequenced by sci-CAR, a combinatorial indexing-based co-assay that jointly profiles chromatin accessibility and mRNA.", - "task_id": "matching_modalities", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "scicar_mouse_kidney", - "source_dataset_id": "openproblems_v1/scicar_mouse_kidney", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/datasets/scicar.py" - } -] \ No newline at end of file diff --git a/results/matching_modalities/data/method_info.json b/results/matching_modalities/data/method_info.json deleted file mode 100644 index a72e21ec..00000000 --- a/results/matching_modalities/data/method_info.json +++ /dev/null @@ -1,107 +0,0 @@ -[ - { - "method_name": "Harmonic Alignment (log scran)", - "method_summary": "Harmonic alignment embeds cellular data from each modality into a common space by computing a mapping between the 100-dimensional diffusion maps of each modality. This mapping is computed by computing an isometric transformation of the eigenmaps, and concatenating the resulting diffusion maps together into a joint 200-dimensional space. This joint diffusion map space is used as output for the task.", - "paper_name": "Harmonic Alignment", - "paper_reference": "stanley2020harmonic", - "paper_year": 2020, - "code_url": "https://github.com/KrishnaswamyLab/harmonic-alignment", - "image": "openproblems-r-extras", - "is_baseline": false, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "harmonic_alignment_log_scran_pooling", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/harmonic_alignment.py" - }, - { - "method_name": "Harmonic Alignment (sqrt CP10k)", - "method_summary": "Harmonic alignment embeds cellular data from each modality into a common space by computing a mapping between the 100-dimensional diffusion maps of each modality. This mapping is computed by computing an isometric transformation of the eigenmaps, and concatenating the resulting diffusion maps together into a joint 200-dimensional space. This joint diffusion map space is used as output for the task.", - "paper_name": "Harmonic Alignment", - "paper_reference": "stanley2020harmonic", - "paper_year": 2020, - "code_url": "https://github.com/KrishnaswamyLab/harmonic-alignment", - "image": "openproblems-python-extras", - "is_baseline": false, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "harmonic_alignment_sqrt_cp10k", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/harmonic_alignment.py" - }, - { - "method_name": "Mutual Nearest Neighbors (log CP10k)", - "method_summary": "Mutual nearest neighbors (MNN) embeds cellular data from each modality into a common space by computing a mapping between modality-specific 100-dimensional SVD embeddings. The embeddings are integrated using the FastMNN version of the MNN algorithm, which generates an embedding of the second modality mapped to the SVD space of the first. This corrected joint SVD space is used as output for the task.", - "paper_name": "Batch effects in single-cell RNA-sequencing data are corrected by matching mutual nearest neighbors", - "paper_reference": "haghverdi2018batch", - "paper_year": 2018, - "code_url": "https://github.com/LTLA/batchelor", - "image": "openproblems-r-extras", - "is_baseline": false, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "mnn_log_cp10k", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/mnn.py" - }, - { - "method_name": "Mutual Nearest Neighbors (log scran)", - "method_summary": "Mutual nearest neighbors (MNN) embeds cellular data from each modality into a common space by computing a mapping between modality-specific 100-dimensional SVD embeddings. The embeddings are integrated using the FastMNN version of the MNN algorithm, which generates an embedding of the second modality mapped to the SVD space of the first. This corrected joint SVD space is used as output for the task.", - "paper_name": "Batch effects in single-cell RNA-sequencing data are corrected by matching mutual nearest neighbors", - "paper_reference": "haghverdi2018batch", - "paper_year": 2018, - "code_url": "https://github.com/LTLA/batchelor", - "image": "openproblems-r-extras", - "is_baseline": false, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "mnn_log_scran_pooling", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/mnn.py" - }, - { - "method_name": "Procrustes superimposition", - "method_summary": "Procrustes superimposition embeds cellular data from each modality into a common space by aligning the 100-dimensional SVD embeddings to one another by using an isomorphic transformation that minimizes the root mean squared distance between points. The unmodified SVD embedding and the transformed second modality are used as output for the task.", - "paper_name": "Generalized Procrustes analysis", - "paper_reference": "gower1975generalized", - "paper_year": 1975, - "code_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.procrustes.html", - "image": "openproblems", - "is_baseline": false, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "procrustes", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/procrustes.py" - }, - { - "method_name": "Random Features", - "method_summary": "20-dimensional SVD is computed on the first modality, and is then randomly permuted twice, once for use as the output for each modality, producing random features with no correlation between modalities.", - "paper_name": "Open Problems for Single Cell Analysis", - "paper_reference": "openproblems", - "paper_year": 2022, - "code_url": "https://github.com/openproblems-bio/openproblems", - "image": "openproblems", - "is_baseline": true, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "random_features", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/baseline.py" - }, - { - "method_name": "True Features", - "method_summary": "20-dimensional SVD is computed on the first modality, and this same embedding is used as output for both modalities, producing perfectly aligned features from each modality.", - "paper_name": "Open Problems for Single Cell Analysis", - "paper_reference": "openproblems", - "paper_year": 2022, - "code_url": "https://github.com/openproblems-bio/openproblems", - "image": "openproblems", - "is_baseline": true, - "code_version": null, - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "true_features", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/methods/baseline.py" - } -] \ No newline at end of file diff --git a/results/matching_modalities/data/metric_info.json b/results/matching_modalities/data/metric_info.json deleted file mode 100644 index d85fad52..00000000 --- a/results/matching_modalities/data/metric_info.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "metric_name": "kNN Area Under the Curve", - "metric_summary": "Let $f(i) \u2208 F$ be the scRNA-seq measurement of cell $i$, and $g(i) \u2208 G$ be the scATAC- seq measurement of cell $i$. kNN-AUC calculates the average percentage overlap of neighborhoods of $f(i)$ in $F$ with neighborhoods of $g(i)$ in $G$. Higher is better.", - "paper_reference": "stanley2020harmonic", - "maximize": true, - "image": "openproblems", - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "metric_id": "knn_auc", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/metrics/knn_auc.py" - }, - { - "metric_name": "Mean squared error", - "metric_summary": "Mean squared error (MSE) is the average distance between each pair of matched observations of the same cell in the learned latent space. Lower is better.", - "paper_reference": "lance2022multimodal", - "maximize": false, - "image": "openproblems", - "task_id": "matching_modalities", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "metric_id": "mse", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/matching_modalities/metrics/mse.py" - } -] \ No newline at end of file diff --git a/results/matching_modalities/data/quality_control.json b/results/matching_modalities/data/quality_control.json deleted file mode 100644 index 477ae09f..00000000 --- a/results/matching_modalities/data/quality_control.json +++ /dev/null @@ -1,662 +0,0 @@ -[ - { - "task_id": "matching_modalities", - "category": "Task info", - "name": "Pct 'task_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_id' should be defined\n Task id: matching_modalities\n Field: task_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Task info", - "name": "Pct 'commit_sha' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'commit_sha' should be defined\n Task id: matching_modalities\n Field: commit_sha\n" - }, - { - "task_id": "matching_modalities", - "category": "Task info", - "name": "Pct 'task_name' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_name' should be defined\n Task id: matching_modalities\n Field: task_name\n" - }, - { - "task_id": "matching_modalities", - "category": "Task info", - "name": "Pct 'task_summary' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_summary' should be defined\n Task id: matching_modalities\n Field: task_summary\n" - }, - { - "task_id": "matching_modalities", - "category": "Task info", - "name": "Pct 'task_description' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_description' should be defined\n Task id: matching_modalities\n Field: task_description\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'task_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'task_id' should be defined\n Task id: matching_modalities\n Field: task_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'commit_sha' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'commit_sha' should be defined\n Task id: matching_modalities\n Field: commit_sha\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'method_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'method_id' should be defined\n Task id: matching_modalities\n Field: method_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'method_name' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'method_name' should be defined\n Task id: matching_modalities\n Field: method_name\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'method_summary' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'method_summary' should be defined\n Task id: matching_modalities\n Field: method_summary\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'paper_reference' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'paper_reference' should be defined\n Task id: matching_modalities\n Field: paper_reference\n" - }, - { - "task_id": "matching_modalities", - "category": "Method info", - "name": "Pct 'is_baseline' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'is_baseline' should be defined\n Task id: matching_modalities\n Field: is_baseline\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'task_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'task_id' should be defined\n Task id: matching_modalities\n Field: task_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'commit_sha' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'commit_sha' should be defined\n Task id: matching_modalities\n Field: commit_sha\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'metric_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'metric_id' should be defined\n Task id: matching_modalities\n Field: metric_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'metric_name' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'metric_name' should be defined\n Task id: matching_modalities\n Field: metric_name\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'metric_summary' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'metric_summary' should be defined\n Task id: matching_modalities\n Field: metric_summary\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'paper_reference' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'paper_reference' should be defined\n Task id: matching_modalities\n Field: paper_reference\n" - }, - { - "task_id": "matching_modalities", - "category": "Metric info", - "name": "Pct 'maximize' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'maximize' should be defined\n Task id: matching_modalities\n Field: maximize\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'task_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'task_id' should be defined\n Task id: matching_modalities\n Field: task_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'commit_sha' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'commit_sha' should be defined\n Task id: matching_modalities\n Field: commit_sha\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'dataset_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'dataset_id' should be defined\n Task id: matching_modalities\n Field: dataset_id\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'dataset_name' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'dataset_name' should be defined\n Task id: matching_modalities\n Field: dataset_name\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'dataset_summary' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'dataset_summary' should be defined\n Task id: matching_modalities\n Field: dataset_summary\n" - }, - { - "task_id": "matching_modalities", - "category": "Dataset info", - "name": "Pct 'data_reference' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'data_reference' should be defined\n Task id: matching_modalities\n Field: data_reference\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw data", - "name": "Number of results", - "value": 21, - "severity": 0, - "severity_value": 0.0, - "code": "len(results) == len(method_info) * len(metric_info) * len(dataset_info)", - "message": "Number of results should be equal to #methods × #metrics × #datasets.\n Task id: matching_modalities\n Number of results: 21\n Number of methods: 7\n Number of metrics: 2\n Number of datasets: 3\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Metric 'knn_auc' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n Metric id: knn_auc\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Metric 'mse' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n Metric id: mse\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'harmonic_alignment_log_scran_pooling' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: harmonic_alignment_log_scran_pooling\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'harmonic_alignment_sqrt_cp10k' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: harmonic_alignment_sqrt_cp10k\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'mnn_log_cp10k' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: mnn_log_cp10k\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'mnn_log_scran_pooling' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: mnn_log_scran_pooling\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'procrustes' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: procrustes\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'random_features' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: random_features\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Method 'true_features' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n method id: true_features\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Dataset 'citeseq_cbmc' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n dataset id: citeseq_cbmc\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Dataset 'scicar_cell_lines' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n dataset id: scicar_cell_lines\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Raw results", - "name": "Dataset 'scicar_mouse_kidney' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: matching_modalities\n dataset id: scicar_mouse_kidney\n Percentage missing: 0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score harmonic_alignment_log_scran_pooling knn_auc", - "value": 0.013650181246076077, - "severity": 0, - "severity_value": -0.013650181246076077, - "code": "worst_score >= -1", - "message": "Method harmonic_alignment_log_scran_pooling performs much worse than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_log_scran_pooling\n Metric id: knn_auc\n Worst score: 0.013650181246076077%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score harmonic_alignment_log_scran_pooling knn_auc", - "value": 0.02693737142214102, - "severity": 0, - "severity_value": 0.01346868571107051, - "code": "best_score <= 2", - "message": "Method harmonic_alignment_log_scran_pooling performs a lot better than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_log_scran_pooling\n Metric id: knn_auc\n Best score: 0.02693737142214102%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score harmonic_alignment_sqrt_cp10k knn_auc", - "value": -0.007718136452076273, - "severity": 0, - "severity_value": 0.007718136452076273, - "code": "worst_score >= -1", - "message": "Method harmonic_alignment_sqrt_cp10k performs much worse than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_sqrt_cp10k\n Metric id: knn_auc\n Worst score: -0.007718136452076273%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score harmonic_alignment_sqrt_cp10k knn_auc", - "value": 0.008048361486095518, - "severity": 0, - "severity_value": 0.004024180743047759, - "code": "best_score <= 2", - "message": "Method harmonic_alignment_sqrt_cp10k performs a lot better than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_sqrt_cp10k\n Metric id: knn_auc\n Best score: 0.008048361486095518%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score mnn_log_cp10k knn_auc", - "value": -0.045073857496674405, - "severity": 0, - "severity_value": 0.045073857496674405, - "code": "worst_score >= -1", - "message": "Method mnn_log_cp10k performs much worse than baselines.\n Task id: matching_modalities\n Method id: mnn_log_cp10k\n Metric id: knn_auc\n Worst score: -0.045073857496674405%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score mnn_log_cp10k knn_auc", - "value": 0.13574548653413956, - "severity": 0, - "severity_value": 0.06787274326706978, - "code": "best_score <= 2", - "message": "Method mnn_log_cp10k performs a lot better than baselines.\n Task id: matching_modalities\n Method id: mnn_log_cp10k\n Metric id: knn_auc\n Best score: 0.13574548653413956%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score mnn_log_scran_pooling knn_auc", - "value": -0.01880946613770593, - "severity": 0, - "severity_value": 0.01880946613770593, - "code": "worst_score >= -1", - "message": "Method mnn_log_scran_pooling performs much worse than baselines.\n Task id: matching_modalities\n Method id: mnn_log_scran_pooling\n Metric id: knn_auc\n Worst score: -0.01880946613770593%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score mnn_log_scran_pooling knn_auc", - "value": 0.06533138842384231, - "severity": 0, - "severity_value": 0.032665694211921156, - "code": "best_score <= 2", - "message": "Method mnn_log_scran_pooling performs a lot better than baselines.\n Task id: matching_modalities\n Method id: mnn_log_scran_pooling\n Metric id: knn_auc\n Best score: 0.06533138842384231%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score procrustes knn_auc", - "value": 0.05705698937347893, - "severity": 0, - "severity_value": -0.05705698937347893, - "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: matching_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: 0.05705698937347893%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score procrustes knn_auc", - "value": 0.3108150404865038, - "severity": 0, - "severity_value": 0.1554075202432519, - "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: matching_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 0.3108150404865038%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score random_features knn_auc", - "value": 0.0, - "severity": 0, - "severity_value": -0.0, - "code": "worst_score >= -1", - "message": "Method random_features performs much worse than baselines.\n Task id: matching_modalities\n Method id: random_features\n Metric id: knn_auc\n Worst score: 0.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score random_features knn_auc", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: matching_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 0.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score true_features knn_auc", - "value": 1.0, - "severity": 0, - "severity_value": -1.0, - "code": "worst_score >= -1", - "message": "Method true_features performs much worse than baselines.\n Task id: matching_modalities\n Method id: true_features\n Metric id: knn_auc\n Worst score: 1.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score true_features knn_auc", - "value": 1.0, - "severity": 0, - "severity_value": 0.5, - "code": "best_score <= 2", - "message": "Method true_features performs a lot better than baselines.\n Task id: matching_modalities\n Method id: true_features\n Metric id: knn_auc\n Best score: 1.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score harmonic_alignment_log_scran_pooling mse", - "value": -0.006160570533860987, - "severity": 0, - "severity_value": 0.006160570533860987, - "code": "worst_score >= -1", - "message": "Method harmonic_alignment_log_scran_pooling performs much worse than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_log_scran_pooling\n Metric id: mse\n Worst score: -0.006160570533860987%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score harmonic_alignment_log_scran_pooling mse", - "value": 0.01539225216374196, - "severity": 0, - "severity_value": 0.00769612608187098, - "code": "best_score <= 2", - "message": "Method harmonic_alignment_log_scran_pooling performs a lot better than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_log_scran_pooling\n Metric id: mse\n Best score: 0.01539225216374196%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score harmonic_alignment_sqrt_cp10k mse", - "value": -0.006538532325927271, - "severity": 0, - "severity_value": 0.006538532325927271, - "code": "worst_score >= -1", - "message": "Method harmonic_alignment_sqrt_cp10k performs much worse than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_sqrt_cp10k\n Metric id: mse\n Worst score: -0.006538532325927271%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score harmonic_alignment_sqrt_cp10k mse", - "value": 0.016096476784697722, - "severity": 0, - "severity_value": 0.008048238392348861, - "code": "best_score <= 2", - "message": "Method harmonic_alignment_sqrt_cp10k performs a lot better than baselines.\n Task id: matching_modalities\n Method id: harmonic_alignment_sqrt_cp10k\n Metric id: mse\n Best score: 0.016096476784697722%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score mnn_log_cp10k mse", - "value": -0.1744304443286302, - "severity": 0, - "severity_value": 0.1744304443286302, - "code": "worst_score >= -1", - "message": "Method mnn_log_cp10k performs much worse than baselines.\n Task id: matching_modalities\n Method id: mnn_log_cp10k\n Metric id: mse\n Worst score: -0.1744304443286302%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score mnn_log_cp10k mse", - "value": 0.32117421366700694, - "severity": 0, - "severity_value": 0.16058710683350347, - "code": "best_score <= 2", - "message": "Method mnn_log_cp10k performs a lot better than baselines.\n Task id: matching_modalities\n Method id: mnn_log_cp10k\n Metric id: mse\n Best score: 0.32117421366700694%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score mnn_log_scran_pooling mse", - "value": -0.02618801427707118, - "severity": 0, - "severity_value": 0.02618801427707118, - "code": "worst_score >= -1", - "message": "Method mnn_log_scran_pooling performs much worse than baselines.\n Task id: matching_modalities\n Method id: mnn_log_scran_pooling\n Metric id: mse\n Worst score: -0.02618801427707118%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score mnn_log_scran_pooling mse", - "value": 0.2803566688566542, - "severity": 0, - "severity_value": 0.1401783344283271, - "code": "best_score <= 2", - "message": "Method mnn_log_scran_pooling performs a lot better than baselines.\n Task id: matching_modalities\n Method id: mnn_log_scran_pooling\n Metric id: mse\n Best score: 0.2803566688566542%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score procrustes mse", - "value": 0.07392331619971682, - "severity": 0, - "severity_value": -0.07392331619971682, - "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: matching_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: 0.07392331619971682%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score procrustes mse", - "value": 0.5607972733684712, - "severity": 0, - "severity_value": 0.2803986366842356, - "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: matching_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 0.5607972733684712%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score random_features mse", - "value": 0.0, - "severity": 0, - "severity_value": -0.0, - "code": "worst_score >= -1", - "message": "Method random_features performs much worse than baselines.\n Task id: matching_modalities\n Method id: random_features\n Metric id: mse\n Worst score: 0.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score random_features mse", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: matching_modalities\n Method id: random_features\n Metric id: mse\n Best score: 0.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Worst score true_features mse", - "value": 1.0, - "severity": 0, - "severity_value": -1.0, - "code": "worst_score >= -1", - "message": "Method true_features performs much worse than baselines.\n Task id: matching_modalities\n Method id: true_features\n Metric id: mse\n Worst score: 1.0%\n" - }, - { - "task_id": "matching_modalities", - "category": "Scaling", - "name": "Best score true_features mse", - "value": 1.0, - "severity": 0, - "severity_value": 0.5, - "code": "best_score <= 2", - "message": "Method true_features performs a lot better than baselines.\n Task id: matching_modalities\n Method id: true_features\n Metric id: mse\n Best score: 1.0%\n" - } -] \ No newline at end of file diff --git a/results/matching_modalities/data/results.json b/results/matching_modalities/data/results.json deleted file mode 100644 index defcf083..00000000 --- a/results/matching_modalities/data/results.json +++ /dev/null @@ -1,506 +0,0 @@ -[ - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_features", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.309", - "code_version": "0.7.0", - "resources": { - "duration_sec": 220.0, - "cpu_pct": 73.6, - "peak_memory_mb": 515.7, - "disk_read_mb": 272.6, - "disk_write_mb": 273.0 - }, - "metric_values": { - "knn_auc": 0.6208419863517854, - "mse": 0.0 - }, - "scaled_scores": { - "knn_auc": 1.0, - "mse": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "procrustes", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.293", - "code_version": "1.9.3", - "resources": { - "duration_sec": 279.0, - "cpu_pct": 143.1, - "peak_memory_mb": 963.6, - "disk_read_mb": 272.6, - "disk_write_mb": 319.3 - }, - "metric_values": { - "knn_auc": 0.09674569704735692, - "mse": 0.9387379116135324 - }, - "scaled_scores": { - "knn_auc": 0.08183270172346102, - "mse": 0.07392331619971682 - }, - "mean_score": 0.07787800896158892 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_features", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.293", - "code_version": "0.7.0", - "resources": { - "duration_sec": 319.0, - "cpu_pct": 32.4, - "peak_memory_mb": 518.4, - "disk_read_mb": 272.6, - "disk_write_mb": 273.0 - }, - "metric_values": { - "knn_auc": 0.05003502090203344, - "mse": 1.0136719 - }, - "scaled_scores": { - "knn_auc": 0.0, - "mse": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_sqrt_cp10k", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.533", - "code_version": "0.0", - "resources": { - "duration_sec": 429.0, - "cpu_pct": 193.3, - "peak_memory_mb": 1300.0, - "disk_read_mb": 272.9, - "disk_write_mb": 319.2 - }, - "metric_values": { - "knn_auc": 0.05462908169875428, - "mse": 1.000013512131858 - }, - "scaled_scores": { - "knn_auc": 0.008048361486095518, - "mse": 0.013474170358418691 - }, - "mean_score": 0.010761265922257105 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_cp10k", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:51.859", - "code_version": "1.14.1", - "resources": { - "duration_sec": 490.0, - "cpu_pct": 99.6, - "peak_memory_mb": 2200.0, - "disk_read_mb": 317.4, - "disk_write_mb": 423.0 - }, - "metric_values": { - "knn_auc": 0.08589789433030783, - "mse": 1.0547959275709011 - }, - "scaled_scores": { - "knn_auc": 0.06282837386193632, - "mse": -0.040569367238946885 - }, - "mean_score": 0.011129503311494718 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_scran_pooling", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.025", - "code_version": "1.14.1", - "resources": { - "duration_sec": 750.0, - "cpu_pct": 99.4, - "peak_memory_mb": 4200.0, - "disk_read_mb": 320.3, - "disk_write_mb": 423.1 - }, - "metric_values": { - "knn_auc": 0.07631922843774054, - "mse": 1.038986769000114 - }, - "scaled_scores": { - "knn_auc": 0.046047454089837826, - "mse": -0.024973434698262853 - }, - "mean_score": 0.010537009695787487 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_log_scran_pooling", - "dataset_id": "scicar_mouse_kidney", - "submission_time": "2023-02-21 17:57:32.149", - "code_version": "0.0", - "resources": { - "duration_sec": 859.0, - "cpu_pct": 120.2, - "peak_memory_mb": 4200.0, - "disk_read_mb": 318.6, - "disk_write_mb": 364.2 - }, - "metric_values": { - "knn_auc": 0.06541106014069863, - "mse": 1.000107517902145 - }, - "scaled_scores": { - "knn_auc": 0.02693737142214102, - "mse": 0.01338143249098156 - }, - "mean_score": 0.02015940195656129 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "procrustes", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:21.934", - "code_version": "1.9.3", - "resources": { - "duration_sec": 340.0, - "cpu_pct": 230.3, - "peak_memory_mb": 692.5, - "disk_read_mb": 210.8, - "disk_write_mb": 234.2 - }, - "metric_values": { - "knn_auc": 0.08257473591291212, - "mse": 0.8323309514022647 - }, - "scaled_scores": { - "knn_auc": 0.05705698937347893, - "mse": 0.1806911435619616 - }, - "mean_score": 0.11887406646772025 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_features", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:22.007", - "code_version": "0.7.0", - "resources": { - "duration_sec": 430.0, - "cpu_pct": 38.6, - "peak_memory_mb": 460.7, - "disk_read_mb": 210.8, - "disk_write_mb": 221.7 - }, - "metric_values": { - "knn_auc": 0.05042255760201557, - "mse": 1.015894 - }, - "scaled_scores": { - "knn_auc": 0.0, - "mse": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_features", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:22.020", - "code_version": "0.7.0", - "resources": { - "duration_sec": 430.0, - "cpu_pct": 35.1, - "peak_memory_mb": 453.9, - "disk_read_mb": 210.8, - "disk_write_mb": 221.7 - }, - "metric_values": { - "knn_auc": 0.6139324564583585, - "mse": 0.0 - }, - "scaled_scores": { - "knn_auc": 1.0, - "mse": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_sqrt_cp10k", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:21.923", - "code_version": "0.0", - "resources": { - "duration_sec": 701.0, - "cpu_pct": 225.3, - "peak_memory_mb": 1023.9, - "disk_read_mb": 211.1, - "disk_write_mb": 234.1 - }, - "metric_values": { - "knn_auc": 0.046073311310546615, - "mse": 0.9995416858132864 - }, - "scaled_scores": { - "knn_auc": -0.007718136452076273, - "mse": 0.016096476784697722 - }, - "mean_score": 0.004189170166310725 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_scran_pooling", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:21.988", - "code_version": "1.14.1", - "resources": { - "duration_sec": 701.0, - "cpu_pct": 94.5, - "peak_memory_mb": 3300.0, - "disk_read_mb": 258.5, - "disk_write_mb": 301.0 - }, - "metric_values": { - "knn_auc": 0.0872374416848794, - "mse": 0.7310813422485382 - }, - "scaled_scores": { - "knn_auc": 0.06533138842384231, - "mse": 0.2803566688566542 - }, - "mean_score": 0.17284402864024828 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_cp10k", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:22.048", - "code_version": "1.14.1", - "resources": { - "duration_sec": 750.0, - "cpu_pct": 91.8, - "peak_memory_mb": 2000.0, - "disk_read_mb": 255.6, - "disk_write_mb": 301.0 - }, - "metric_values": { - "knn_auc": 0.12691648298907363, - "mse": 0.6896150433809697 - }, - "scaled_scores": { - "knn_auc": 0.13574548653413956, - "mse": 0.32117421366700694 - }, - "mean_score": 0.22845985010057324 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_log_scran_pooling", - "dataset_id": "scicar_cell_lines", - "submission_time": "2023-02-21 17:58:21.897", - "code_version": "0.0", - "resources": { - "duration_sec": 1013.0, - "cpu_pct": 151.3, - "peak_memory_mb": 3300.0, - "disk_read_mb": 256.899999, - "disk_write_mb": 290.5 - }, - "metric_values": { - "knn_auc": 0.06031383406703828, - "mse": 1.0002571033803676 - }, - "scaled_scores": { - "knn_auc": 0.017552977303677005, - "mse": 0.01539225216374196 - }, - "mean_score": 0.016472614733709482 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_features", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:02.058", - "code_version": "0.7.0", - "resources": { - "duration_sec": 300.0, - "cpu_pct": 62.5, - "peak_memory_mb": 487.4, - "disk_read_mb": 232.2, - "disk_write_mb": 277.4 - }, - "metric_values": { - "knn_auc": 0.5117302105076631, - "mse": 0.0 - }, - "scaled_scores": { - "knn_auc": 1.0, - "mse": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_features", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:02.040", - "code_version": "0.7.0", - "resources": { - "duration_sec": 331.0, - "cpu_pct": 64.9, - "peak_memory_mb": 488.1, - "disk_read_mb": 232.2, - "disk_write_mb": 277.4 - }, - "metric_values": { - "knn_auc": 0.05000744227734146, - "mse": 0.99353725 - }, - "scaled_scores": { - "knn_auc": 0.0, - "mse": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "procrustes", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:02.120", - "code_version": "1.9.3", - "resources": { - "duration_sec": 331.0, - "cpu_pct": 65.2, - "peak_memory_mb": 487.7, - "disk_read_mb": 232.2, - "disk_write_mb": 278.6 - }, - "metric_values": { - "knn_auc": 0.1935178231783895, - "mse": 0.436364269209991 - }, - "scaled_scores": { - "knn_auc": 0.3108150404865038, - "mse": 0.5607972733684712 - }, - "mean_score": 0.4358061569274875 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_cp10k", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:02.166", - "code_version": "1.14.1", - "resources": { - "duration_sec": 490.0, - "cpu_pct": 86.6, - "peak_memory_mb": 2000.0, - "disk_read_mb": 277.1, - "disk_write_mb": 369.5 - }, - "metric_values": { - "knn_auc": 0.029195816019157918, - "mse": 1.1668403939745453 - }, - "scaled_scores": { - "knn_auc": -0.045073857496674405, - "mse": -0.1744304443286302 - }, - "mean_score": -0.10975215091265231 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_sqrt_cp10k", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:01.988", - "code_version": "0.0", - "resources": { - "duration_sec": 650.0, - "cpu_pct": 198.6, - "peak_memory_mb": 646.4, - "disk_read_mb": 232.5, - "disk_write_mb": 290.1 - }, - "metric_values": { - "knn_auc": 0.05061583797299569, - "mse": 1.0000335254261379 - }, - "scaled_scores": { - "knn_auc": 0.0013176644894209325, - "mse": -0.006538532325927271 - }, - "mean_score": -0.0026104339182531694 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mnn_log_scran_pooling", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:01.878", - "code_version": "1.14.1", - "resources": { - "duration_sec": 720.0, - "cpu_pct": 99.7, - "peak_memory_mb": 4200.0, - "disk_read_mb": 279.9, - "disk_write_mb": 369.6 - }, - "metric_values": { - "knn_auc": 0.04132268350330538, - "mse": 1.019556017687802 - }, - "scaled_scores": { - "knn_auc": -0.01880946613770593, - "mse": -0.02618801427707118 - }, - "mean_score": -0.022498740207388554 - }, - { - "task_id": "matching_modalities", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "harmonic_alignment_log_scran_pooling", - "dataset_id": "citeseq_cbmc", - "submission_time": "2023-02-21 18:02:01.959", - "code_version": "0.0", - "resources": { - "duration_sec": 1054.0, - "cpu_pct": 140.0, - "peak_memory_mb": 4200.0, - "disk_read_mb": 278.2, - "disk_write_mb": 379.9 - }, - "metric_values": { - "knn_auc": 0.05631004174912533, - "mse": 0.9996580063066434 - }, - "scaled_scores": { - "knn_auc": 0.013650181246076077, - "mse": -0.006160570533860987 - }, - "mean_score": 0.003744805356107545 - } -] \ No newline at end of file diff --git a/results/matching_modalities/data/task_info.json b/results/matching_modalities/data/task_info.json deleted file mode 100644 index b5b25ab2..00000000 --- a/results/matching_modalities/data/task_info.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "task_id": "matching_modalities", - "commit_sha": "d77a94914531882aca05653a4c6efc495c397fc6", - "task_name": "Multimodal Data Integration", - "task_summary": "Alignment of cellular profiles from two different modalities", - "task_description": "\nCellular function is regulated by the complex interplay of different types of biological\nmolecules (DNA, RNA, proteins, etc.), which determine the state of a cell. Several\nrecently described technologies allow for simultaneous measurement of different aspects\nof cellular state. For example, [sci-CAR](https://openproblems.bio/bibliography#cao2018joint)\njointly profiles RNA expression and chromatin accessibility on the same cell and\n[CITE-seq](https://openproblems.bio/bibliography#stoeckius2017simultaneous) measures\nsurface protein abundance and RNA expression from each cell. These technologies enable\nus to better understand cellular function, however datasets are still rare and there are\ntradeoffs that these measurements make for to profile multiple modalities.\n\nJoint methods can be more expensive or lower throughput or more noisy than measuring a\nsingle modality at a time. Therefore it is useful to develop methods that are capable\nof integrating measurements of the same biological system but obtained using different\ntechnologies on different cells.\n\nHere the goal is to learn a latent space where cells profiled by different technologies in\ndifferent modalities are matched if they have the same state. We use jointly profiled\ndata as ground truth so that we can evaluate when the observations from the same cell\nacquired using different modalities are similar. A perfect result has each of the paired\nobservations sharing the same coordinates in the latent space.\n\n", - "repo": "openproblems-bio/openproblems" -} \ No newline at end of file diff --git a/results/matching_modalities/index.markdown_strict_files/figure-markdown_strict/raw_results-1.png b/results/matching_modalities/index.markdown_strict_files/figure-markdown_strict/raw_results-1.png deleted file mode 100644 index 0e9b7d58a989bedd3cc9289aa45b2dd15d268cf0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 83478 zcmce8by!t>*Cr+)5{iHlB7z_xNOy`LjdY`Qcef89C<4;bV1RT>gMf&DbazR2=Yd%p zpYNNQ>zhAju4`uY^}aku4riaef3eoR?)zT*jl8Tl_BG;bXlQ8I63;{x(9qC*(9kZX zU|fXX>?-ZUPc%%MXX^H7Xk-D%|1X5jZW^MY-9wWQ6;^UdSQ&HmQ0haSZQ2`4QPZnS zunckD^U1#d731lxZ21h0N8i@|iR$+<-!(4z$1y`K>z_);0hPB3!WW)@eS7cMUBV}W ztQ79`RMLCDy}Z4(5J!r#B7p4vU34E-I;y6;6DVK|q_noQYT$bei4ax7TVc@zD8 z3I9mONV)LuKLmu)u>XCzfqBv7-=7R_WWiw#^bGe!G+T2 zs*hFY@@{Hs>S-=AM`&o)8ae$F575Y7;ESC98x8IM*FFE2;r!0MRI!|U+$`s%SD<#H zjRDtl^yq&*73bpx?_YlZBmhU9A^Pjruf@e|6r!%Ku9lXolXbpuw6?a=)6w0hq>PA& zXsEB}v+ip&9h^ZLa$yAFYd5Mx1&$VGV{}78LcFH@i6X;r1rA=c#j)qr94);i zpszWY#@heg96>W--@$+5#*MMKen;wv#V;{B!xfHMV^*qSS8+%WvUE{qbH8KPxbv#E zB)z;&>GP^s41VEtj0kM~qJ%%+?hwSq!BG)oAZ!g{swuOb`57HeNFi!q@TWIJalK-h zu|#8RtR60>^U>es=!d5V*T7VJV&~#=z}ms5|M)gvo*Qa+V1H$Z)#G4esv#hm&vh#+ zuVy&^rEi;&`}7+cx5ul+vI$(aBbKEl#l=+A)Hvhgau5@HJy7Dlw-U?iytXn_6cyByCUZPPepY5V^6lF$e}L6c!d5 z_sA{})SfyX*X7loR#Z0K5FzDt`fXMca+|r{e5MYTjIptCWxOZ^o$uo3A;iai_*1Ar zW|v8V?0obi&AqiTUb{cP_Ux;=M-&&Qi*s#w5@E?cgoBIXhmNp|VZ@tczj*iX` zPv=pNKc29#u)4Z>wfsZ4R-UI=IPjmzgfxYW)4kH#wghfPjLqpFCZ5FhJ^P9>MO|yl z1yuzF7UT8=81Q}Ts?DFA6COSN{Tx=KWxVTEMd$$vddnLd8#=Vt;Y4GN*N|8_nXtF; zDHN6tA0_3?a$YS5ad4JuUJ|=`H2Jb^xz_f42NM%h?VoTdD>&cp-_lW%5B$DnWVo)D z3{=?p!Lr_;zl4r+OQg52@9?VzoX{ZzVi}A9x6I-(DIuZL@oFg+#kkvY&g<7#1^X4X zw6t_|?vjukZ#3Y-fYC1&;nnLEIB-pvH#^D5$e5dF9qZZI72-{IramcTaF?a0r9FiC zg1_r^(Dq#_cLDlg^aSpFeMHZDEsexP;`X7p3LoJffjV-@k^Ah9-pmH|k?h_(^^_ zLalofm6^d7`bO!U6`eAhDRFX7rRUF|D=Y8jd85n^w`O7D zYTLx%mfElUjRG~CupL_&D}T@0Rd3W|>G0FH6DP{<2MC@B=cv>k&c&;0**5Mh_RRl| zjYv*D8`KZQhP7ExusJVyc9^SO>4#T)(8v@X5z%I}?YTe3X*pc1dooQV6~ibaA#sPp z;=}xfbm`cwk%?24Xb6E`hp?V^u{lRK_C~Ci9V*<{t2rqt8<_TQz_J*vqxL(WMmi>@ zyqcQvF#~m52(59_)G@7&8Esm9&w$&0Y3zU&vz7> zfb-5mS9ElA?r%8W1qN<7^257aynk23!Kis&wG)Kk+}{a;-lq^ACYRmdeH*900Ar)v zILY-VBw|?exC?Qb1sQLfPSGfp_il#!Cu6vr#S?Kpd@g zrp3j@CM6mW0M|uqh;wps7-OU`ZV(gGNrcz73L9%Ns+yadKYNCeS939eq4m+DM{&20g_{?OAmB=f;7CrlvBTvW-oCEmocBpz~U! zl$1YSTyumnGxI59I_rejv89(+4IrMX8$y?aIHhA*9&mCFAXk_LPGYODY&q+`i6UZsr)c7F{d!aCbRO8fH5^Sb;@`-{bj+BmTWqni1UzP z+>V2HOi%35k=sR+%l8Oh{~b&;wEqc7R5lGb^bzM*yi$ja(M$^eG{) zG#bAM1S-_q<>{aiu~0%0lZ6KmXVB1$e=}Rg8Fo`qh<5s1IL*DX5OFZ@AB*Py5PSX? z9Cm)0{~NG2b<_C*n$P%W9-Dg$mFs?Z-G7?zv-V8PBA*wUNNef9rK7tk5=ftS>k>K) zVaoc;D{N>x^x@5)1stAvF1$Cf*l~V@{Bh@eMrLM>cUWKl+`5L3z#BATq2$eGn&eU2 zCNhXGUuv=bqI98{&VMcmC3kgq+fDy?!*M$Av6eqFTcI&!CvoYc=(At3q!I$^G{YMH z0-|q27IT(-1jM4@?UyBiosUh0B5Sjd)7a!9fc-LQ`JSKt81cX>RQ*dL5ex~1FKJon zvZ}K*7*2 z|2v9a6(TnR#TFf`h6-mFx;}A^IYi6_ptb82+#(_(3I3Uqk^-Rwa4q1xQwPGJ*}(#R zu}{n#91tgI7#VZv(+?p1J&>qat61iv;pF7Z)2Zg+;c*)-Hcgrdz#)T}Yi(iiE;N*! z&&9^VBAaErvvUKY>FPwSE=E{#^2F#UpIPr$h?(>Qa4p^4-GGPzjrrC+*835Nvn<$G zcf&+W9B1fkH#%SajFYN0H(YOE9j9W zF*>fZQIKy(03N$*W%aekbFOzwF*$~2Yn68!70DzEpqRW5TLHlO<8h9=e6s6X8>=K@ z(i!dSoJ>jx54Sr*?WceF;+kZBytf)e#AIb@=@=@V#CPaidz>hnzI<|v++*XFMvfJ^ z5R-NV8!zt&nUJcI5`#|F!LKmNr2Xk(^SnfU_iuO~Q&V3qc7Iv>$9a7m;!+e^MOD>F zMtTy!D%*+292|SOm1{g_C);C`iR7dgo?2Pt*PhMn9C{Blhn#s)KcLEZoiq^jWazBu z2fgR%L|BW|>e*rG-p`Sa&q^t{?+u%dt=+$LSmo-AlV+)!(RChkl#q~U*_zMMEInPd z_Kq-gt@p$HBo=;R1yHYNmvY?0d59Z z!eKG^f+aS|W2?Dcu8V*peiiU{UUoJq{Qk2<#6P~ip?CQ!jN0O0dSKz|*wkIQ>N}O_ zJg#1#*SOS|W!#bEc{m#r@cQ*N3P~xc^~!aDphgzAodqaVGWwQIPEG*a^KbsZ2lPkI z>tvydGp>ny+0Ea6MzwzU`O(RxCZgoU3Ou3KsM*~5b)MMC;aa)e?Dyf<#~5!c@M?+U zR)&!9#!ilwEA!9VX^0?Y_y*ZUlavlfA+2p~o+|}SA|fJwc&tXh)!>RM*J~;r7Gcp; zvpszHEVjS5x7hpaR8`0I0o~GZO5LKv>2~s2q~T$-rd9dTQdU%5EUPi2PL<2d&tR!& zhVrx1V}Cr3c$wI}@0W0;KRgzWb%J_r8~~8NzyFmhR~iCv1^Z|#7NCfVii$EdHT6a< z>nhVl;pugMmT;>+ZNL>At?@jD#Gay(rLU(K$$OiSkPv?xC1AH+bDZ0#cR0)B?b?9N z`H&K+eIVUVxXvefpUjZwRqw@VmL7TDG#YA)5^xgNKCAT$vQJGJ6FzrU073^$Hxhv{9c=J&(Ee@tA)nW z(lXp(*g98;Vs(NZlfr;PR=Kd%@X;WBD491$))xD09V`~7%CU)(6e%r6qS~vMk{hPp z8^(&$ZE%y~%;Hy$wa*Y7MAoYFDZG+9oytgQGb%#gn=vdMF;w4kCh*~exezRyjn<5` zQQ3sC^*|l%e);kxp2ISZ$1!uh%$ThT*6`c6Z~grI)QgPD(Us}?9+i$3x^B-|*P{9@ zIK-bm_1tdb1e^;+ykmNYNFW?Ae>vE*S(oT$~yx{=8w)qNArDnb=dR;A)dn zQ0Rt&26eQYXANlyR=!3K2QRPtXqjyUwanY#VE(?GNHpzoi2(e#_;`1x4f@=&`%Fw7 z<{MBD1ra{FhKUJqf4$C=&zw|3qP@=7AFpCiP3A?+aKWJ3bH%Zt%Mxao*Fvzdcx6cZ z{6xFdu}moZI;p8mQK$z{zjrS>tf_Egr~wd_{6^w5yk4y{tJ7sspvFKbrKuTj*F|rl zQhX*>=r0}^k*0W88_-IM=#%%g-gs^?J&P8Wme`n>&$B8wI_M5U8CqL(%}O*@55*WS zQ-8^kQB(VHjz(9Vf*SApO=5~AcR~VH$k=hKKL5RU7_b@?80YMtmGG7>&QRrOwaA@ zi^M)H28H&P{~YU}c=+%aQOBY-V@x;;{SA8;G=JFr`AGexw9rtw8#)BN!UccN@SLmg!w&BF_9 za*vPv0Bw|Lv{4cqtjs4{lL=^Hq>?|r_dWT=Wfxa2<4sS@9akFJ1QWrp*`Bh}(kp*n zx0JB6v#0Z=T#)zlaDlRw#lnV$dbs3OY8rYt?9fCR@Jo$vmcpLOnl~9_f)|`j&S9Yu}HL`^YV4){_Je4wIe6ywG?j1 zKIuKQ$<9@ioOCS$9-p&btY_eSN#aw!X2?$o+X#*PzR%U8s-Hi9!qmF4>z+d0?2iX? zm^k{FmR16zz<$1c#HzB*o;h*ILcne3D_;GJ$7O#JcIom!=P?A<~V>l z8mL{X5(#LSa^T^QfWkJ!MfL!aKCBon!En&Y)fAOZC`;vufs+1rnh ztCTwf9~WjgWG+07jE=6Z_k`#E)2B~&S0*<|s&(8)tXI}go>-a02%EDFn;RJ$84wW= z&<4$EXlPu+!lJx?AMhfSPF3k=E5swsJiB4Ot8yQSosx{^`fAEAecva=2{8UK>YJjj zPI&Z*pslXXt-3}NOS5*X_jFLx2rSsC|P9g7&cHVWqy+QrD zy1JIezAROM!7#Ck?0NFjp9~Tz~ zkO$el20zvmAFh+{Xf&;^dBj1UkBM=f{Qhcx+L z0vE!-V6i@4tyyYCAfg4xG0}SQlS-C#&EZ^nlH1a1=>#mYg|E*)WveXDwZ&s$VL9@Q z0mJ!K8VeUU8RUj|_*1|ua>)Xoohia?ZEZm9<>us&u^PQI&KybIZs)azz}~^P{To=I z_u=6R8X8Ak;^gdXY%aO#H47hufVTVwiy9p;IN;a2k6#GM$h5&6jSLSnRxI4#PxEeJ zSxw}1HZnGDIzJ|W=)>jP_zqP1JF%I=`DF6DcWbOCYIc_gfFByMuC3NY7Z(Z>5n)&T zeGC5Q6mP@xG=KjmgRQM?K|ui^3b9YmO~XezJM-aFAPxZKIQ!Dl!h&sJAm~Tr>*DN9 zz^E%6H*PWTEG1w$_(N(RvVgE=FZkW+N?SP96?F8<21Dn5Qc}{Dk!n39y4^C&C0l&I z9$E_(FFa(N6J%hyCUXKExH;2Qi`pAGgS{AeBUlV!k06Up7ndUJ8xe!r25iWGlV8gV zGqa_$WA8J$#j4bUp$;#?n>V+h^vD5x1kC-~2vYYHr+?sewtD*1zstk}*zYl4XQ+dZ z*K5~nOdj7}E1ypq&~eX!O_853e@7`pDS}3BkFOD6so=q+??2}UL=VgUY<1tO*ltTu zyLcTfRwLEKu_pop2D$C$%80a_HzzUVH4;0WJc6ktB(G~j7FSTvE79=BGb-X16x4l_ zDkUiy8B0q`OBXDbPljHRhVt?u3~DrBp`FkNNkbbmh<6ZogW+;p`WF=}Pvf%>mtjinNSqGDohvSmw|SwQCSdT{PTF+1ThgZ-&XO-n1g zXOV@KwWp_NWZi(>aj6%E6e@<5gUuQErk3K|g!?QkcAL|Ua2Jn&<^K9Le`FQ#12MCn zACQz&8DyV`z%Nk_P-o99XSpjTnT)WXKSRbL}RRUrbYcL>KIgaK;e4hf`SeNafC_6Zo! zaqLr09OdBR;sPcd$k1&?I8iVu_m&brCl9wjl>`)TcH%$Z0!`R&t|2eqnX%;D+KwE1 z#Jy%EpiU*OKNyVi{j{!-p#&EQbb0pIuMZzRa^xa~oM=>3UQ^=*$Pr4IynU4a)O zs$?0e-{o@wR2dF;)b*=S3zP>S(%!#+pOm!LmwLrB%(&B{Xed&acx)Iq-?I~DK634j zKn^K#IqmGgVEcGAnw7v%o3IfB5nu^U5k4laSaE@DWLU2G5I~@JP9N&huAwWVa%YiOT9$Pm%(A(Z#~Z7_ugw0i`k>B zwsq6Du#hFD^?~0rAR}>+Lrzi>V0BJrX1L+dq60t*6(uFSC+LwlzH?AGC|aThQumH8 z++>5>o&0UupVPBGLgR<$JW8RU2;^nw{zg5?LPM{aBal0_Swb(u5@eh8ZioajGTm5} zTXAw%R*u3G3I7}!dCTxItN|E6cJ@B{*SVRQk}#RFvML!$UO7jk(|6eDNiJ0s&E=B+ zpf|y}b`22h3Ob^M>K2UDdWX&09ai2-kplpMwnGl1A!+~m)g^Q{_HAW76W=#t6D&Vq-(L zFtN9{mymd4d|g{zJrpEugCg1qDH!h^R57+;*Rfiz9CzDUAvj^_e-AwlDO1`M|c3O}E$GF`Y#V zVRV%H4=(_6YQgi{LYGZ1`uI`gkgvp?#8*^Q^xbV{AZ1d^%F1eLbWZS|0Br++tf{f_ zFn<)yY3}%fWccSkvk@jUBS9hLEL`Zo0izxHf)p0tM(r0E60+7mmk<_)pd+^UHW-+U z3<(V25EazaZW9v^*Xl3ez}|JVAHbb1A;0%`jJ{)mlAqE6;IXPo>yQr?2D*6Q+qcB5 zM$+2a+K^=cCm&F}4hanW_3IbBvD9YQ!KfY`mG)b*Y;*>NHg=<^((&r?g>yn(HVD~e z^UA8Knhc=d!eL-w5E_aP2N{{RrlvU?F=RqNJP@G~_Uhfc-@ks{h9sG%yh0){jQc+2 z4su_e`pC4^$VAV=rO$5f?Ch}e#ZgK8&3F(2AXcf`t%6($v0m)yQ$r&or>xq5&|=)U zDD4Z{jH*PToi97H2bS51lWrsDLvzE=W4(apIRd3;WM9k6moK}zx~lD@CBmVmo}QV> z%E}5Q7tAew3Q^?n@DNJSJ9qBDUscj+oagvkTL?7yIXN=i`a}iin|cx}m*UCF)RYLu*q!3zOYMd*8l8E+GBWlC z)hI-5&>Y@!VVRkmYielt;en11D4>SRUn56WMuz?N^|jj?(S{I5G&M=c$&NH$CbSc4dbL32NjjDk^Cn;c;_2T2U1;;k*am{wq0LLq^Ga0CnO z!-v}e4P7rgM=>SBhx1JSUGPF@--9&Kgmh=uPJs!93e>}m(NIumUjxu%W}g{sNVrR0 zbBHjoRv-zGF)Hkh;&!46%FDS~SXij2exx^^2k}WJWQ7T26clt>a9~k5@Mr_RwQY7Pqls<6DJV`*FXlQV__|FH?p8Cdx`dC-Z+(Wl|0+P>Vb( zi=#RwCJX-=*rN9CsbuW}VGL_FBO~KSCvrL;dpsmogqoU}!J;nE$N>^gK)`FF#&Zn& z-*jotR=r`&3?=E0z#G`u*o}>iySuw07(tC7Qig$){O;X52yd){2o;Qm=6MCqcTWg7 zIGDotwls%s6H7~e;KRDT;l!svj3kn#Lgw5tS+f!RK2C7w4v3x4iiRxMh~wkp{P1A- zfU<;;xwyFaAD4(2lXmD8?29Cu4cLiq-1@U_SMj+4g{G;gnJ8;~-0TFDm{qDm#wc?x zl?2pYYc@lPT*T9DUr5$|GO%3erxBt0<1vLpuTWTF~bigJuZj0-iDxr}}(kivm zhO>ZU1#CM^P|3lH=rB+A+*6124|;u(a-Lwu*0yi$sRjncn2qmk?(T`%lV<_K2p<8^ z>qrC|76 zi3g6KP$bZGThy&0IqSmFJ>WBwxf~YW4_P>sYn?KF-d9#p@j5-62ZX)0F$^5#Bds#| z#bSEDgQcvzXeQm77j^$23H^sr5y7on4>QQyaUc#3-R?|ecShZ7@$y+K`3>F+oguP%jCt>u zpAh_etVpKlSif}d5&$a6vXwiW0e*g8`^@%9%f}hw&HV5Ta|pjMHGO||=~(_~g<3)Z z*1d;^hoj>zU~o&0c)X7_H3EWyf-t0%l%G(XImb0pwwJzKC|*I9D%uV+gp7+}a2+s< z5M91mjzqSXlW9~qWV9JoLh0wVUpYZb3AQZ60>Z$cRc<$%=e3(vq--r@<5}9YInm*l zA@%^1!tZ2L;dq>EiQX$svXyJwH`%pw)Jsrk_ETEVU9()u08(?+RszG8#>*7-l=r3V z1JV+_Ph-a)oIU?fFbw(r;Y0bW%S%fTi|&w+#K*>Fu+7?#aWAMHXB^jHQGEVv(6$qs zDg5L_5n(gIyCRN;?O@UM@hL25T>hQTmMHpCtFcU#ELK+509fE_;qif96cp8EP6f5k+!LXTN zo6IMBdnqU=9Ge2?GE{1<3(!qgwjJu5i16^cUI$YU@h48^IQh9uU%s4jp71ngBi7f~ zx3{ySmi{nlWvi*6umu#VuFI5vjFw$dl`diOIjL zO0CB#lra!aew6>KN){BP>S}6;e@yn`0u+egRTIil>4@iKqM%4ccV_M6>i3vFF+!a& zwu(HfSJwJ-)U+L^dm_nSk!731Yw}GaXRZHEI>7ZO|AM!bwM09s@yUZ7{mq$oyu}4C zpPj16cw~QIE2%`k{?M$1hnpMV0Ibf4=07%E2I?HK(om8^>bawA*zG;wGUi-fIC|PH z$R=1V>DheCJL+$gF9paz#_Mrl1M(F3=1N~STs8chpKr6f)Ca4)<@aw7*o>g&QArds zSVLI{A$EVlyLR3Z^6Wsi`Zg5Om=uww$WkmZ>-+dL2!ehkdyX*}r@(Rp8+NT+Xcx6V zQ45Skx-3f9f~Iv#)4Ce|mTq<$R<6LwMgxcw$X;%{P~qMU^WZ$&5(Unm7^X8AOpC{W z3?RCK8U*=?UNtv6H&=xk>}=u~%)qw6M_ZKPET&+V; z;J+#%jP$5Y12#k!5T`uBhs9|<&gZ^ov7hUVq_C~Ic}dw@cL zD{5BSdKM6^Y_b49BwV0~*K5yCk9L=#>YZy(%=-O2s}E9l%RGDySyFCxwloG10DlOa zmv)bULOicE#|3;~?;n6Q3-}Na@756arSU!+?31(7qO8RBZ)dp~$t+`JZ1h?2m0lis zC^UCY69L1%9z7G293u~vfx7z7@eArsUc0(| zWDq%#DG=}h!tOVCk;W`Zt*UqXfo%I}!7)EOi#pw}RlN#%vIJHZj8J5BwA{aZz>`1~Arz+W*-^|* zAx(y8#LyW2nI_3aL_L)h0C7YFtMl3@)Vym+y%rqcBGAZD5j^}8vGxfuJyy&oxrdMo z*1+^*;!{6lHB?Um6Dj%Gf%vQ*kijY{{ZDZDi><~g%yY{jO{4`GTbP;M6!8lPD6OUA z;~NFW2%ireTnuXYDdpv^kO`CM>#DPLYrSCZ$aMOKhleHP%+5^AdR3Gi1R5O=CTl+Q zz*jt5K1oXY$?2Yyj;%_Mi(_2&!f% zUG=TA;BdbZ1M5jVgcOjUPH7@I9_Nobv$GjePs*3Nimir?mo3O+ZGN^TScG`f; zvi$?|rwGdRiCS-Q&G96@ak&^Y|HG)Op%kpQ>{ueyiY7H9#jbnlyj4 zM}n3<9W$Z5aOZp+bwTb4aN(~Ly0?5}MZZCm*!rbPp>W9Je^f&6ZaKQ8MUvvjdx|GjKS64O%v=jn27sm?KST}VR4>iqh*x-ZS5@)l zZGv|ejDJJ+;9*Yl3|IzJ;mUaR$k^=)`be2opkR)(B1b8>xCZ&V!S~{QdNMq$fuwiS zfsUI8RBemi5v}2oaY;8wZ+V?C=JzucgR5I^_9H&V>Xn}APF zzRs8t6zPrYaPXSluygwiy(J8>j4NlUtPU1X3JQAv{=l5hvz4u0^fQ!von1{wr^aTg z9`-L(f4M}i9G9<@_nLJrR$$U{FhU9}QhCl482J84!=Gr`iLtZ+ajH9_eaUcW|6Z?6kqi3a3!;K{@Af&UFpMBQm}-Oh#>3Na|*p>mw?pJZDR%)S8x zlJb?|B&zHy!f3X)3AT%0+Tga&LE66-2vD4i4^U50Aj(fZXw z8iV7230u0x_8O-~j;DOmvXC&1VlU-SlZ!bkJ+=v)E#;1iH2A#@VrCWF*!cNMbWvL} zP)J9~VD#W$za$l$I4a6tl{~sH@B>w;k=3`AAn=_Q_dV#Im)7^VO3QhbB*Smncx!fr zY>|e78={2zOlJ!LXvM>mk@m!;_QO??T8ht)-L%lK=Rg_U9cD zpxK|RxRyb|!NJD1;K0*C&oImHBuc(TkP~|l zQKT-HC$GyRvt%Hw^(^qHkiK1yy9T-(T3TBny69+VEWzs{bY92w-!Wg_%kTD{+D{-< z2O5pISjEkoT9}w31Er-x=qLV=h5{lJuS0rYDd`_HVIw3USahiP^T&Wf)W|3eZyIEm z-M7B*_NZ?Jo%jfJ5DY|DB7FexhPHyj^j(nEb<#mc(e>PgH4o*&vhlm%U@#5a@+5*Y zevfpb+&&#o53GAhQkbJ>=r}kyHM{+q{&?hofMNGmdQ}KfV@vmrI|Ot(Z;bQPQp>s@ z^L3$5_|W=`n^;%2_H|K5D7g6T^-nxQ8DiBy;sDX;F0YgH&||Pr48D_IU3HeYK~_~R zcP+GY^oWEue`5dKE$Kr7b_-l=(9x^5+s#c)T_J>z@-zQ7+6G0K$`8uh*WHE|P@O}O z#~v#UzW3U(Qq)!}6V^2z^En$jz!Pk@uRnP3AW0nP1dUQFU@1Ud+)FqCxw4GXG`?*Q zM4tFIBj8{`if~;145lPJktzsPlK2u7qEJo&X7|JENtYAs%b}KugGxdku+v)Qy3?}5 zE9od+HRtu@zKFEp`&Y}9=+Y%EL*+V8cId{)PpKLkk1H#ZSU#^Lv*qy7JaS23^ibW>Q_tfDq(`3jJSfJ?zE;Bc1rkeb>E92UTT5AR5i@#C5?R*FbUAD!0F zP@BTb{U*UIg&rQkzfDoEeEVI5&K#is>FV4Zj!6&3ER z!0cS3_!&Y*V)Xc+@++$XB>d@c%|dj}kn$G*!NE1m31#*4^mJStu_3+)g{X^*i;7Bg z_t1$u1;tW^YNzFSwC_wWxCn*~5T}rZ&aD6QL)#o+O*iNT02Uh@28c&xd%^U5Z0zAm zQHPPCA+1zYu-J@c>DXRG@`*Q1w|RPUwbwNE=Qd_qztjHSfuVS z1zR0r<_B#M9Aq5U<3J!nqC-}pdrtV;wGs^!_&jx;h75r=N(MELNY~?KiAG-eT%6#^ z6mH_gV@k>=>h}?ojr;C(;YoQOUo492o_rGfPEww2Wy*i5}61^(U} zQ$F0=Ak2e#(a?SGQIlOTB3N5l$x0n}9Z%#{EM)?Yrqhn%Ezc4Aq+Vu|3ITGhn9*&c z4xJ1~JYPI(0_cm{lUXJp1t9^$Vu6Varz)v^X=k?u?LZ{N#L!q2mXH9XbM48N)Mf*r z_gGY1+!6Q}Ml8Mo!wGgTI0cRzVAe{LuKPia3eTSd8vi`7%*#FJhlvlooorOw;_eZ6 z`;Ot_z?=wzC%`Ux9;5@H%;x2>;;6@V8;nN*UcFh#)%`lcW|R)p&}vReHj&`knRz)ML5**y>c z3=lRuI~x|dm(BY5)o~GAG?^TYdjI}K>4@{{@b2z_F1;nEA07}wv#c^v%$8(s>O#fWLKu^Eey zVL=!lIAsV&HCNzY;EIHD>(bTh(4u2%Vgl1t2K56}Q0*evLWswU3~@Gr0~vkw*Cnti zK7w1x%(UCTgNu#b2F(v|-fUXGJpZNyQ1pa+!l^Qr&ELCsvTB6I#Owok#;99U1*&{O z0W&WzFF!vjGb>ktm~r0?8|X4XWc`M5&RJ|wT)oYaUDA0evs*z15%1oS z(R&T@VG0c(Ee5q8=@|RhMrr-!8awK00B^cqSuCGeDASt!9 zv;dml)>l5i&jT`q6!t+BJIy`789Q&12v(hmuVZ31g8&s2wEgdZU%rTF7AasmRFt>t0F^Qz>njonE+bP zk54CTroq97ei%47&>*?z%uNOi_9I3{;DOe{U^T%Cf5uUd@UUo;w~+pr3#o|AMg=IS z)9vBN>K=UK+qc*DWqsYM1z1@l60e?1N(-bg2H($CoQ@$d`QZTqrJ$$-E;zJxHZwb$ z4;b=>DS0tAVj0jQp!rMk%a*kz@*QZG1z=Eo&L9SWh)f~s;ZY@ef>&o8TU z{;$+kHF+_p&{x+gx7K_yv!a5>Fv=FBKG1``g8&%;=?`MjS{QWB!R=qTboHQs4!O3| z=?}~eERLCYB27UTQ_frdya?~LCPRpYo#5(P-hEvZ34HbkodszFODLD%fPVL4dgfLOLFrKbcU4j>CVd(^+<$93Md?49}b>w81>bJ`=O zxHLC+4V)O@6>fQIm|h2};`bpe=-h&}QG>`NlbjIUD=IWj@Q#yai$Jsn#jbB)pi=xl zgGz@T(zFCruqr}m)B>`#J5=V2&1r&b(3X>OgDfz1*5|q zAakzzQapMjlcREc^p}xNh({LGsVzs0Ab58HG=Kr@A+i-iLpxep^wNdTzt-a|a%yyt zHVs%XDSFLz0myhde>~rQYO1R9ci;Ty1{{+(pXKwdGKmgxsO72;xI>78R>B`^_kf)wbwsmS$u)|r*Kgy2{2e47ZCDPb856b&XJW{%;6?!5NI=6FiCHP$P`|!; z5&(_GEw{V}F78NF5uJy>D@yum-QQ`pL=n4zRos-6F$?wjREf37yj`oshTQAtT&vr(#IqM{DY!?9c- z`hk@*SWMr>#zsu+C%D@tUXBcb&u(R;G!JYkG&D%hFwDc>M*bFSukHA?q=MAcDM)C* zB?3+aB@UdA+lcRRPs$k_dq9fte(Jgnst0g+FY7N?gTHtWtY3Cqq@ZlbgWYaxwiVhr zjsSgXXZ6tytPPhe11SYML?J`dj~__SHNfH|^z}w23&2Ov!i)5^BdsU^b$S1GFM$xQ zqcaZCVx(hzny4zMXY{W3iEU8h^xB279NxhoT&sKGhE3klT-(krxbbEy>(!~ToA$en z5vb?=`w6_?cvL*@c@1L9rxpxdKi=nxzP7p&@87f@jMMz6ZBI2hMuuJRZj^Qa?x4=& zel6y#K{Mxdwyy)r0EPb}A_xh=v~c}DR0tT|{d5t+7`j}z9ha2$0~uPafs@Tt&Wa3z zat3KPo^Oj+m6xAfqZ)Api<-+^tTCiX_dn3HKn}KS{ZRfim8>!UYAF4HuimMef`LCf znHQ{#zey_KF=$i-o>IiA2RiVYDHLtctkmAov0t?c{R~}uFE!SVuIjINN5lBwp z)P}mh5F+p$#NZA4Fp_qAt__s(wd1}hCKsY~S8kh0AwZ9n=nh2dFV+RhUNx7SyC3fy zx)-j1LZd7eZSah{@+r#3OEGvR!cb?iD1Nz;zkNq~_h|%=VBpM*;W+%N@n*SGltc3@ zWX(P^MFjfp^_dK(y*YV;N{(hy@S|*p^}tg0fG}kOz<88=j(X7>B3H3{zM4 zP=49J@Ti!y{{CCZI{l|s>W!BORfG<6azE1FC&aZ;Hxw4qeySpRrKBvM^dFzun!Im9 z>)qYQRuR;uai3#R=jC*ffi=bhha5bWqsd1e`Q^7FJ=$Qgg~t_`fV2V5B{mswGg?5C zAP6(_?TM8zcy1Z`Kz24i{tcOs-1X0mH=-cZG!<&~988pXfTaOMZIHMN{NL(m6I zyJ&iH_}d=8ju5{hhbjpDc=+h3ck_&(ZK1fX+2OS8IJ-B~D316YXJK}>@L<|zR?f}Q z-O69Tp7oUcO=@7yL&G$g{1YaA*Y#*IeW(faTtUaBNG(RWXU|-KP8sw^$PN2BX%LD4 zwYXl4OIrlevUI{Lo~#gz&y@zv;j^tV6IHG~9w^fIA;}ySuxSV3ZGmv~fQF_B3mnya z{QS`Hz6{+#(J~IuwvFtYTTiQ@HEU%-KSK#Qi%Hi<=qu3VjEjj$?=yoZd&uzbRZgIC zGcu6JWpuQ)8RB=^m3y3>qC5n=!BU&{2>6N>5N2r~J*tK%4jNlqjS7Uy;GZCaey{Qn zS5YCPFo2dn5QJ84W+>6cw3b?qsAy?fB>8-^ZeqE<4JsCcc10x08Ylt>haFgt$a4b4$;`!iw>MM=+ux)msOgnD;mh9rV?cW{V{WdPY zJpW@{6RMNPrk38qJ;g;ROaBm)IHTKF#aj;V1dHvxl1*S*;%YJ?{Zv@v%gI7Q)}yWmvCOfyo`bo;aVll_?nU^TifsMHCYwM9mcQcsC|D_wz5C`GxoDh zF^gT+NvjNMAK(wVo};O@@8O_n-+BBrdhN2Z#EWJyqwLfPesJ56jT_q0bc+ad^nnTY_dq zQXkXNy~R^dRW&vvBq1^H%AUE&2t|2yRTVTwLFYyxKnBRS4kw@6t(c*N(L1~ZW_JjB zakww3{lFHi&K49B!tvk&9kkdu}5T1X9m zEYk zshii$xQdBAU%RW#-yteWEazNAAiQvo)0mfW6rMFuR$g9kMdy1g>{}zDItPGC&g*>P~->cq}b$r@O8pTXxHjM2#h4Z#mDRXz~+cDo$3 zCmnh?J3EV`UmWW^b7RU#Pp9c9uXV=o7x7TLLgROtZHFgh-Lra=aB*r%7{k=ueEHy( z=Eqkk>7Rrx3tY10P5ZauNdZ7)2Y_!phEW@uZrY)2fe;A3Jr=_ji1{*_7S^S4%qLN; z)~(c?Y)Cj79o2Gn-iLiOIH(pYO?daN7s!9IH2p>TO&W_5;1aGcZ{X&b`E`V|w=yaD zMS-JWoS8!)d3`!igs{J??D3n1cLOZh?02~1)>HBqZ#dt{rcQCSeRBCG6Fs&Pg|Ca1 z2$A-Hqy9t5$9inD7ES)FcSGkLXb2o$xMhJEu+I3%tM1N3?o_EX0Uhnz`Z_zWl}>z8 z+Ii!OkS)`z^rF~{xk*6JQMa2nCp$f{%#4{?EP9C&Trpw z-a(@rGYa}GUUa3++>H10^OJ^d5qQD|n@oCox;JDDLDWtUa5Sislfg$$3~bNU)m_0` z0i0+sF?(iKpH77XE!-}Uh(M0&6$}J&2NQaCKu!BA&+}ISj73q94o_M8 z&)G2Aqk50-w8b5Dy2pWnYo`J@&3=MLq@ITt6_vXVd#tP&BhodIr$Jdlm|LGrEf~mk zD1TGgq*&*~Yy5Xd{O;0kT6&F3;}@9Ve!%L}Q;$pe_f=gjX;U383Y4i zjI?J1b)~NEZD{y_!T_EHFgIk}{ZwF}rdT8AY4XDE5%APEML;XToQDnan{mb6CXsM< znMQ5n;%r}Y0EHySv;4^`)8B74d(^a^G!oa!(&fr0>D8SyMzZD4C<6V|r6w=`J~`Q% zjhN&~eWv(3io`LQ4szCZ!xm}Yrt5!i6wT?M%WT&TpxsPXQqs)Q5(PU@dYAE!9im5x zV7Njj^`M?6kfPxC2V4q$V2O(*Pz7>YjV{!QUAlM?GA}ME7tm}nP-|d;%kaQMfLDJU zu!kWF)J?I6giuF;V@mpYFOc0idC7PRN=gPvRRU1dB=NgrP+av*MQ66(?^D%{-I{T$ z_5!;J82YIr3h-&{R%}ArR_bfe2Vytt>$PKQ7Fk?*FhdP~=v7yW7kN*%eXi`aKYixP zn|4$r;h8mewBQ~TE>7?JgjQEj5@_Z?yF=8ifdQ3``}#bdZ)DSfLtsH=-;a`PFzT@N z+?6@4#+9#Qkz`1{c=7PlPYfNGA;bQN%g(Xq&=&0C^pTT37p!0|A*7|tB{M?%p0srD zq66H#e1;NelHiso%NTzkb%oC2KXPllUYgn#XD|I5M}aaNZJm?eS0y>juDlzox4 z@izb7o!2jcYP|i{?E0NsCmWw~N$2(~Oif`ibS>}t;l2F`ixZxnhC%Uz3C%b0-$6Sf zdPnSgfsu~R3?>p9SHfFo!E-CfsM;K0Uvy&T+kFvj@Q z()Vg*^O_O^0-Atq<~90#8{7>rZqUvG&+mbLdU$kN5I7t-*h!$8f~WNWqYG3BJVD9s z^u#snS+UFJGys1M-C8vUP=kZtyrBi6L`rI;$QU>NmQIOTlr**C!&^|R&2Uwx<`}RH z!IKK$`k__aY3}!(a;>`r1iK(k#o-RkjZbOCA>(5uI6Xh#bV}C=H3s@_T@J9`FU@lT z8}$TnhUQ_sz|FyN%z%dhtmJ9D=Q(W&^q5zmorT`T1YHO(I(hJfjr>FDJn#hr=|)G_ zQl%sUPG#t)FfsYMpQn7t``>u`?r<#o|8GqrMUtc_k~EAY$;y3|l2r(0uZ(QjGbJfS zk|bo4WN$(xNfNTNitLr0{k%@<{(hh1cz(z8&vQK2Klfc3*L9ws^YeMXU+eA4Rl*Ak zH=qtYh1m%^&p}cT=!!V~lSQ7zK_S%E+PZe#x`X!y2D+g~1n7mO$L7x|D(VVO4*VEE z`83(1sX+QbZfK^^P*D6DA0K!bteKJmKu?l=X=#bU1KPFBoE(~+d@;2@K3nI#PD_*L z2ngPD8?b^{cvZv2Y`wCc?rzxej+{K{2(i^rGC)x@9o}c6k~QC@l_Z_W5PSx;CciA~ zgO6Ps8kk>K-hz@c4=Mz3FnIgEUDCTD5P&;}^bxxoAQVbgEEX{_--LHM{7gKWuV2*m zQ?O$5M6uMc3;bUCHhDVIdtEDC-KdM4Lq6in@i1Vp(>13JQBTs{CcSMC9G{rT7=OO; z%NuQ8PI@W}3yTBcF)WN;U@+J`Db}w&9RB#tdHlj8Ws}7VCk59*0;rHc!8$TB^83b* z1$wFt;Z-?L+sH#7BFgYumM3lR@{KgxK|)B?1F^L4mi6CHgwCa}F|0ODPO4?vii?g{ zThWs!PQ|q~aR$3>167XKYmK~!i|aw(?_aJ@H16Z$e6Abv9md#L*#!i$$#rN+o+Q>F z{}4Q0LoqJQaG0q%M&d~=dlfY+_AN)RlJfSuWL<%@uRmHtuI#R4J?-^$IjCaG^w+n; zJJzmU+qB;1^yypCbk(R1OsAU>QZ=Undvs7(Sf!g5o9FtC8$XnlOc7atFa+!Z!|!d2>#_SP&vfF{{krZD=YCE6JP=F?XC`Z0k| zsBU*+(VI>OM}cE`fp@ZZ{?9?cOQyY^Ur_j29OQlFX;#3=Y0R7AIR~;DHKk=`7ZKqG zcl3|pVGR|P#;;%5{3SJAr<72jh9=GFfG% zPhm9?n;*)+!wHY<;lNKojlmB<8<=(_qv&p=AATtd3k!U79v&WrObR?{ZkFWPH*S#K zCZPs{G{0*x*7zI$LSEvb!~C*4aJd>AGnS{u$E9RsY;0|ZqiO$aSP?T*elj~-%N<18 zA0Jel(>@KKKHx1&Q)<`phOjxfTb+5Hwi|SR-(tFE%^LI=`iyark*rV>@fgY_zIs(h z?1frd9dg$@P*DK>Hx0Z-f_Dr@1@$NMKc9`7R}z>!HRo%e9TL4&+ej~rYt^@H+fN5A zlMf>04h6=YWONRBtNr5DEB!t8LWaiJAI!~@-)ism{N?Z(n-2ik}cBE0bpN86o}A#xZn$}THCmeAU!iSjbA)V>&fIPz8RJJ{VUiQyoFQ?zB;5Gfm-2FO zFgh7k?d|Q@)=ATUmZLUV2?ZDwkI$8rve=UXrSJ$|IyAqxMNL!lOG5(%_M*v2el8T$ z*yDO6!l7NBPd~CsuYws6Wd|g`F&FnOR^dE_v8n0M@Om4Zk4j40Sv{j(K=5BX4$2cG zvSLjyWbv?ZYCc#YlyGH3*GO5Ckx^U59omgev9K@+zL;lml&lH~b-3~P9%12-O0iA1 z-$C&8??Sh{^M-iCYvefT@0IEX5}+2&mdi&*VmE_MMs|g+z(TV&@6X=e69NM1n_XO8 zw?PY|{wsb(E&^Ig&I9U*H*a>((nb(_4S>4Bpc~wW59{4IMZR%kenA0z$Vieze!+C} ziabm5&xIx<%Z-j$Pmb?xBv%rIhelR*^Q`Uf>#nY@1_m4MjD;y1R zdG-*i!rl`XH%nEPxVV|lYawcCXlS4#-9M{`wRbjJCun|lcJofY^%CKK9*B|4l1S~P z=KS#Sqg}K=Z6{mcGaei%xweB0tT4v2va!7@+J$on3ihF2*H}i2OT#WsuC9iSWL*zg zhjov#zOS#Ss7WK6lwMu-GM6tG6rW-}^Sh;dV_RFBsE7y-2fXkj3JMAevmr`UR#7<+ zRDre@r#dC;>R~uNGn3QEvHA$@E@fnH;||zLl1!=W+rpcwLq{UJA$;5T&^yFF3MWrD zB&lKl1Za)3Wv$1Xwl-IYr~kc>WdGnge*aRli_+*e$qLXJNH`Y11A$VDi;ECw8kha3 z|MR3)r?4Vo&oUlloVH>KkS6{M--l7{K|TY7!}B4!Vk|@CndpA{=g+tBuKe&Vx=ica znwlI!5?>}Ki@7g}2tWN+S;+;O?Z}ZKyRJW2*$s~klvsVj-Wn@AGvG%g55S+`rE`W& z9;}YGc0`!&pY?siW8Dd;>nZ_djzRk|1J@1*wi-4LOb?zr9tasoVl zvbBXNXBTn{r6q|@(eD&{T`PbZ3;d_&L#)QZ4gB;}5@<8k>~58HAvXgFvXK9zRq6g- zaoLhlI}khg&YhmL>H3s+y)7;I$rfp)EN%4Q<@Udp7r9T?(VJf@!e-`?)eZ^XjBxNmyQE+ouYq+9q8pIcTEn{Fl~KB`I>>UVWmB}mmM^dSG~Il7safBTbMr+$wC{Ky zC4R+KZy)RX;yAu^-ZF{WPuVv6^y$O3b}{S^6?srrB9r}RC>F$YH z9LK0fr^mkw{y6tBqJL2@Q=l)ZPd!g1h3hhj^;n z;vBXnNdJf03T75&CtSo%p^X6K2XJe@C-yj^M?wq*J{I;#V6@t0?2=emAfMn(zQ_r$ z+-^~)%u-2FyKU&$cHpkTvUN5Z?vlIS-cLmw)6cLSJ+6jEKsiyP&?##+q5}Ig7Gcoi z$2b_vK71fDz1aL;7`8Q;?YPp%{~VMar0sC$;@=>Z5@2W7$EPB~6VSk$eQRzRorcm7 z9uQ#H8Q7aS0%Bug5bgKz^*=fOC^#%ybJLQOeQC#u7!@S6f?`_)uhEh*a=d@q<$F^= zD=6p%hzYTDpjp_vcQ3rJkmoVq8BskIl%Fr&WLO~TD9B>ZjXsQMe_+%htRPU$eXgjO z`1K1GW;sd8+`;4mq3m9wsSCawx-2jpfzHLfjPvJkxS6HO#{$#HdM3Z!zo>Rz3bzS* zmC1SJR7qN4tnzI4Cn>Iq&H_1)dLe_%fn~ zf$He);tc#mL|mhe>`4(q3=t%f%C9c#NR^<&0BnJG3NRmG%}-T9QUQoFZA$%tVteQj zE;cY>WQB;Xc8C1386jd;o$s?u8a3S)ZEH=8pS7pym0?YMHaye3Nk8|6lGld`BcM)+7eQFX-ON+} z5}V9TJJPr#sO!t^v(b50U0%pj1g$F@Py^>R9SLZ_lOtC;QE7bR;RZ<%yuoK8ylcz1 zneSW3yfjYnDyLT!kEbM8J4)$G|SB<&z4N3!GMH`G%aY{R> zy>R_El9Lxh$%!p9t|pA>K@YHfC^g@MvgDwpC<4nuxCxpw?v!<5fs0JlDWV*~LyySv zL1g6-*gTVQ0aE|mm*+e1c!Gx9g#Zq@cwqp2h+{-h@XyiFQ>~^_+>m1sf<_$9SXL)G z{D~Z6B+B&Jl-Un8rP+^a0=EZ=WsU@3saR`JTDWBn;0B4@bj4pI9@MY@Y5~+ON7)YJ zc4t|1h@t+L@L?zg_5xof43Jp8eNB(LzD(z~q^ReOgLlZw&qsoa1w`~BEIhEuK$+ZD z=$vzgE%@YhjxtGj8=o+;1?CstL%aIx%{puk2#pE{ z>W&oztDVueeZIX4@jEDH0^-RJh^3R`%>_05)*0#1CT&ntt zcdi%9C3U6WvJ=T&N2aJ^a^!PY(`98%na^&lecocuk52tyy-8YeN8^dCzRkiB(Ol5j z{Es)hP|Gv!%QMI!2PpiMN&o8At)wU18cfX0mE8WgR(URfHlQWyq2z;tfS-w=l702u zKmq#4hr2y$;kzXN0DNqHu0W8wic8lIJj)%b6CRZF05&PLLp@Cx-iVb(pdrnB@lpTBj;RAByy_tngedoVSN3=j53v~nqE3qiB ztu(>&dIYcI$EP5vbO6JMi)i-X=d_i`9@Bwx^X~3$xGt#N~O`R4YihUE6h z!U4L;u&Gq{=&skZMm7iD*A6!8OsghkC!Kz=5IQKZaM3{EUa_K#N#cM(O}zy-`sS|IGU>#6KIL;;*9e}W*;WnK^u)5v9u^ikikg0#xcxY1 z3>iN>X4Xq{-z3~VA%Pvs?e6;fg1BH`hso0pN&|ibbr& zeMb@9G(mW%!JIO;TyJe=u@Mfr@B84v@vGrifbW1Ufae=WMo-{Pvw`WBw_`4;SqGBz z0_dUdMmcloY3B;>S94C*%v2~yNOP}17))tq|1+nTo79+K+d$lp9Aq7U*=E>;Kp6rm zep`O()292N-(+j~+gLA28TVe0&Lrvln#vg!8s$G5UDHs!_q({<(iSp~jkmhjCw|ZT zCD7#eT{Lyi+rWA)Ug}$*?>M9VWyN9dkG@{KtbNCLyD%5+UC3K3n)9b@C;L3byJmQJ*sRb)GiA9yJ{)U6NmcdJ z(gI2Tl1d6{-$5J4_>w3`WuFHP_4Qi`7(qy^l$*aY{KF$jcI5_x$hbH!+Rjmn@H968 zTlx4VS5#MNm)n)xtydOC%dtiRPaRf`-+Y9FE;FIUg1l_vZNu|Sx&JF~nAWd8!007= zz>nFvp!PUr3qhnv%j?&-Mbd;lS4hyeupFD7KC451;9PB{ zLc(K7_QIJ>Cz>{t@KW8rNsEj8t|4!&A!^Y$Bt6b$uiX;aIM__OPKu=6!y6i|knnS2 z7SI#W!~Dn{6iqJ~bkm;pwsw%;Cb5dWy18vYXYDs9p+|B3kicMc004pbm>BGIpY+qQ z`+y>_6B-Nhk)|o>^LFjZmb6cAQX9Wbiku~da7$;0-qaWo&dBn4^_TCPxib?#s9i6Ti*j;$ zF9^M^TPsY9#F3b?!-iTFTiW?m{+kB8*Y$GftAwct04d@L>NhjZTF>G~OfwK`&$QiL zbw_ic93s$uY?v5>RhC`*dye>kobpLF0X~(f_vP}i&b_>5& zNm1PN<$CbsPCbQ3a)EISM$gv}E(R^GAI&JcOdZTVRUXPRIAv?!>uUL17?IWclL z%?B0Sy(q*I8y(F;Ono?d00-3#FwLgBu_^8b~ zoc5x69xDB+!90$nf3tVV&%uGw;M9H^YSx(3)6G>{zgm~&i$o79u3{S1e!OeeEH6H~ zTG5tIrv3FWrBQY}32qHX>tPt+LG!T|iX5Cpf(HT(0QI1e4o zzI>+YK6H~D0npH*1+$O4jXDNqqd3lMC=^>&t_E?#YlNInQzIi@zXPxZKxxprFU<#V zAdJVbqNDb%daS_D^Z*43z~_Bu@AqA)RQIv5qp=j(Gt?3Q=oWn{rlDK}){5imHqIs& zvGA~?qjq2RlWS^uK7`rw)8jpjgHzC7Jg(^G3m$@ph&A-zH6plOb6m2g`^2X%y6sB} zw?Arh9WwoQ33}@#e(P= zjLkm@2L=MpL|x|y`~s|rTzmog3#nKt0JRHaWfq2&&^93r4?;y~GrnQ64WJ=}E2AUZ zys-o%J4_+DDW5boGQry5FGnFcLn8;>Iyz7s;UE)Exz6#c&1DmrjmLmy<3$MQ@Vm~i z<>8n!GfN5zYD8#)sm3m~i@%XRa+Hx8RRBOG(Jl9}4$Gd)oIg)^o8k1wiVhDA9Rh0r z@G3ain4gKzjo6vxT<1RdYi@jI2D1@x-LH{IH{;_C(KtZ#o0vFA2qJ7x3fei0YQ!_2 zth(BH>G8(=Lqbap5g!4s1k0c&87Z1K_nA9Hbvr5{;V$i0CdMPrg*5ik3UXfl+sCFj zU0t8~e_#SLRsV|#%&55J76t_38{~_KxdiQvMzAVO_+0HI3^1?pd8=BUA29(ggVN3GG|vA*rQ$Rfv*py4FJFef_73$gK7@|+4Yzu5+RbPyzR8PW*2F?u#4nDhLyhabJlmG@`liAHo@y4RbrP=10Vkbbk&wRJeW-f-C%Xx%^USrJgcj~}$*ABmNVVWt9Q~-Dj+)2G(J@#o7g(J? zEgdv&*UGI7hrbTfOO)%U%Y}EcP_n+q%?Ev!ICZ&3Nwfab?3=;^W^D9SpyKn*47$|^ z2G;7>I*&KmWPLlsNpI)8q*hD@agX)VHgn-7Y4V(62!DVzCn+DJ?&|5`@RvN2UWMpB zI)}&#cmMdY6(uSoV^bpQ)Fa^^d})EG zb1S;BDoaXE9|}~Z-^s(pb=9N|!ZG^pzkEYW%_+XCi);PLLinyqby$FwEJ6s2YjK-c zts%2X>;Yz5OXt8OVPY8Dj3#fP*&8j6Y=uLCP?pe3+&#nQGq;QB0mwQp+IL-J(1tM6 z@BQ;ZsA*Supi8IMg>yx0-E%?ca0jfP*cD)dIK$?TJZ(w#o~riOjsZH1BuOYEB-sZ7 z9cz|l_@C)JRioekGJq#Bzv=AmF?s?(K!6uibMf8DvTo{Q0l6p;x^XzB_$=g zzcWaYM&{-mo}g*~$G*7GxtgD5d2Qi7b*y$X!$|{`np0prE zCi2^um*;>zssj0jwDk`<($chM*lA!3n%(%)m zfT1^hh)lZNTfEee{G6qL1`M!x9IpN98UuK0fG-4QhQ?I0Eu=b5l0~GAjpjz|PNqiIZN4PwdZcXS7_o0{d3r zU?~zscoog2O?o>%b&vP-7?8Zh_wsW6t$|W^5Df4L1$4yMvYruJP}lIVEQJnZTyE}D zR{@ZX{2YHT>>JI8Ak-Nz_RWtr-e-}e*y>Z-QvpBXai-0z@2je+Dk^+5-o?d$VT2)d z_tqmT@8(9vF_Gz~c3;_{Qq0MT)?3pTv`VT%3&cch(?IDiT(|&%Cj@h7g~fJl-!38l z`1HEJPlX31@xeV(5abbGTf;xx+k3!YlJg!$=olVgpTzx|nX$LI@%Os(64#yg4PaHI z5CJoWs%re4XvONyHK(QA^6ik`zW0YRBTL=4%9X>CN|Hq6283U%o>je2lKs6|w{HFWgEz0d=T|FY^BVq2-usXv zOR^L1nbC`u4pX=2shFAf{&~~X`YVs8J_uAqjh^iq9F&%)FX6>=?MaJcpMjyLukX*7 zYTr0bFw$@Mm>z(Y``Nf2>R=fP&?}aJfsAbaUWH#PS3&zY^f5UQBVZ=q`n4Vh=fMWD zAVIVm#N^8ByYG$llNN-j7y@BrY}a^9`V+o&z?7hdeW|puBjEq*jris0E_A@DdJ@|< z__XMC{h`~u%6LDc4i#Ja-IYwBp-Gq1FtqZy19GrI2CKm2wuCVa!x)GO&F!~&e<5bO zxqnFBlI(czP++n7d(+ab-s;nTFPrdM>@2Rs;GO^=fBf{x+GcOq^>@FLwTd;bTq*bc zIc$WmbhxX&0Q+V1lINr+LJAQ>f`p2BLCM~?hmY8Zx-?Dw($TmQOp4s=$@t)`zvLw? zEgo75&U@p3mdexB&%U^Bbk25Kn!&$Lc@UplA)TWo<(#W>~w9GGyF+){^%^Nm!1aa!~UD5IyV1Mz>rO`xX=e-x*o8>*N zOiXN#eidTrFxvY%Rgxr$br-F-x}^I~ICOV&Hzsc-eH5K}c!TWRopsgAd)KttqkWQO zPyd*VF?*1N^-efj6!6P#BMG&=2S$ZIh+58%U2eW|-FMiox2}J-xdjZdFh@?8A~@O* z0W-9p%4iP)k1s$pHO<(Lg{T($J0|*V@X85?WLE;vgxl z+=Zx66olXyVG)h4=>yS#(E0?$cqeD)kZ-8O#ZwwV87-n*2SA>foQ&{qpDa*n6m z7Es(_YfE!GP4tA z{fV0c*rUjC!T=Se3`dBdRVRittzWxl#vI28ku&#MpdBG22!llGleWpXMYMc~6Gm7% zgbfM_{2(2nvm_i0W@ymeAyfy-`0v~d^m5CCTFXMXsL1K)?(ID%Bzy|Y{W2n}S`8c; zF2JGv8b4h&OlS&GVvAUs$cJCQ_8+iOAgG||KS)m{I|$7v6hOp@7S%WiGkuv^R(7^# zv0E4|A6$tO-OztZUbs+PwUEl{HfI5gi%uV$t@sD0^L;Mlxc%d?KfbR&xO;O_ZlA*Brp3=BKvPFz~hTRkR(jSbd2&U=rfV&l{qWowDR zH*C$M)2BDAeT)BrxD>Cee9e%9K*@m@6dZ_{yn<{dw2rmn-{8zXZqfiXFdY${iHnO7 zWB94DKmnI9^y%mxiwfS)K&fzTFwstwfCX`)?%xA9vE24v7gHlc&xp!Y%Spr!S1nx z!`;UM>th+(0gAVnXo8Zd@nuKYHl06D~L^IVr2Z74V0s~$W?g$H+cUeTAjytP$imocNb=UPe_{ll9&dt(M3$d=W z+y5Lftv?>#7Dso6sW1W#NrT{SF$S?b(G@22H4CLs`{s-;zo z+(&`8bBMHgrVNTi!0a18T5w`I0=zs=OtQX#9RT9QbMj;`tu<_Wp!G*h*lmtFt@WQgu!Ppb5lLVg*jL z2+Id+5tKC4yfSS5NR9>=$}cKfs3+|!+|p%!3aKTa#L-O?17_`hU@2oBGqV0BK+O(f z6m=)#T?`5)ogZ`Q0X-)YBi~5dd!h z$>0(WhnToH>kf5W+vBY45vMl*%O70W{8*MMe}RI)eyYgc5?8-{@h|d|&8O zFDE^DvBrk8`&>sdE9$Nk?))o)BV&~mx4#Bz8KPHOknVGzk5ysJYEe|wySfO3$#c?k ztLIkpKSHxZ%+|&V6tFl;@NJMOgT@}r4a7NivyzpL1jz9~KslaK4!O{easN=VV&!6C z7duT|!+Q;xd<6l>g$unCvnUX@geXU^V3$1Lt+9s?<{z}&`MC?DPa!0krg8s$%lI5p zC!NPSC44Akv2T2NE;{(*#~7acj6hn>d(XsNogi$1aR>vl5At&7+RLjTV*+U#&H9Ih z-(SZ2*2vEU@KPJ~X19Gdd0Tr!19qF9&dzk*QYytFBGDL;nl59dI|y?(Hp2`FF4KE4 zRf#Qa0*d z7+Mn|7YS#`IF3-!c|lB+#0G$%qkK@okY}JH6*QNRuAwCfF<<0i@Ih^k2Jgs|jObGQ zz;?`HJ$!x46+a$oy3ETTS*`;8y2vSbw#o4K$rCWCO`RH^%^R5Km{#W3-sGhNCyZu_ zz0Kb0&DrUvSYse3PV*PfX)3;ZD3Gd_Y4_dxg@3gGGdsWyQZ8rT+_0iQw~7f3?{{7! zraifdA57r`@DzEB)TUtFX^#tUuwlXM%Y{hGTG2Nh2-6M<~$W|DW1PaC)CNsb8c& z4h;;RGaJ(?D?{h%VSv+M^N~2l5PcQ7hQQ#gaUwxHgB_}>LDK0>Ny)7T@fZhpX}2DX z`8~q6-JiCTK0>_QG?Y<}a#xrm;(TMTA&up!+rM3wpy4oR|IpC^5eHEiVbCbY%zFNL ze^qumTbFyHX2?~aeTUo;s6f8*i$Ws+V)OEq>wxPtHJ_J~1bCmtZQi)9jAEYpLQ;By zXnma3yTV+(G_8JS<_(s&P2!7FE3`EGc2Spn|1`j388f%l-mElDTns$d3r858r9m=%i1NRX-6{$RbVyi=G_WB2(-`qh+{ zmKLrjEop1{W)vk41*VtVTn8==Yx=N_gR{zes9IxcXDNlZ_?jCTIbaPpvk*`)lrohj zaWnRxE!-{op48OGt*QNU@{oyouD3amAI6rFpHHNMfsY^Jk%sz&p9qVNBpg&txwXL^ z`MRQ4np<0Cg!HAa;PmyR^RzHdY&sAn2fB!sB(rTW*rq-lDCdU3 zwl8NDut6YW)p(uH`o7^ax?w&(6}Zhad)ybs5XFFMYslq3tPrTI9HyHMn~xToW-|Nj zlaR-6VaT)A?V2@ao>BA`)!<6qz}@uwq;5{Ef-T=~l_X`g^$?W;k|W&ukQG6C2Nz4V zl>Z=PBjv9%PaHmW>^gvKNWq5~9pm3opRbEw{#;mSVspP>Vz@L6fC5|`!5sr@zMPM| zm>L~`+WcbE+w!!j-LDG^E8QbD9C*)5J+D?IzBjdtoUU&_`f!X*9;{e9$r2`4<|bli zZ(j!Ia{NV3!cCUCF*LZLK9E`Eh`*$I=zPdy0^S7C`mWbH#F+mXXpD;Paik%jOvD&g z%+YBoEhPrxh8d`EdDXUWnO>(AKNZ>Z=y6!jl%Vd!H>ue4G85jkEFYm#H=2IN3-Qe# zG)pZ!r(^}Uf737Aa`hZ$HX@E^ES=mjBl6G!5A$I|qPdr|Ck=WT6KHH-?< z<<%Z5^X)w_qxhm$@~8F5{?3`eVA%nVm&V5Jyy7bgsS3%i?(5(biJ2^)J#$OY&~BzF_YQvlAc_;IGAt!xmV<5FYB5HBSDM z@b>$6ZIWb}RW5Z2A^m2Wq*fD*0;;a-h~)sBddqIXH?B%{{cHlyp)L05W)7MZ$W{@a z27!kX532a2>1uMkS>ZGm33)7X^bZ$gWzmd?O2}i`n2v4Ejya3Qr_>}dG11wXj&3CW zA|U26xx)t!vNBUWwJDHb@l#P!x_WfF7*8mYhEO6d90&%#jh|2Yh@z&evy-2RIDkrA z)xP*fsT+u$U%!^y%yD1Y_`7}1!fY!K$qf&Wv2nhQ8+P8hDd^U_mTZ%xc*}ielikeB zW>$*p)Mz4AkY@!01jy{m^a{Byv)1S@;YChT-0%z?IwR{(@K0%yiU=j_Lzsp@+xu%l z+)P3?oD%X#Hir}U7?(aHQB$?5=fY1E+VWyQNG9Z?}mgE-7)Uc$$YutK%R(-r5&xgY;IfP40nkaRk@?-HM zGEi)%kYbgKYKo6K13mBbjV;u#UobW{Dn6(+*yY7hyH%e3&=V?ZHaeb9ZMVk9x@q=w z&#Atwcy-{s)}sC&M1kxEQ?)<=WU~-^AW99wJbGs4q}u+b@g1mhn~U9-(I7pR>PEio zkAZ=2yVTsVfx?+JF);y`J|J0(XS%u|_#i_5ZkFtVNIYEQFP}a=!+18BZddg1A}kj8 zdx6Rmx+&bm-vc1S9XeJevW=epl8VYX`8vf5qFJutYfjub)S03N z45$W`+@{+k0X43G^5^bLjw3C)m_}xeNJ~5ju1Ji2S5kt=NGSYS^>GS#D7!y<4qp_! zm$;NDHieA8va96`zE)rV`Io$z)#())EXyK={_nxK@|aik5-k^k%n0wS<3!vlZ~>Kg zj6#BD3JjyIcpJ_ZgXk?y8eTOQ*jub(F5nDsenYk9jx)Gp=@UvMv~xKSwBj1T%m{7= z$17@7Brs-Mr%eyAK~akS$`T!jN{VJu3gO87_1bZwN&o7e3w=X9J=waY_fW`dA;&T% z=1h_@#NKjGd4E7!14jFdnL!wH6laSCf667Hfl!R5= zQ?G*J0s`r12rweCJ0i*u2QDxc{39VmPVqV?7*4{+!fl=hm6DN>0goq2XH+A-pMz^N zO6FjRKPV(L2Vek*ZSR;viPz>3bf3f^#$phXg8--qKnEL`C<4M|wh|-;@v7eS!oz0% zBsDuEUSM|i(x$Gnmc>xZV!$(amWc@=@@Z3w7=4!T>eT|42M*@l+fnEdKMlD)DoLu{ zFBqMUtRC-JU|2h$(Ryh>$y2Y3+$-%AD^ekIidZ?`e;lAC4rqjr5!ibQPlfnDG)U06 zO@c|#td^RgdgebiO~6n98QZXb!PF7g!uFO`X67S=QC;i2CIG{`h-n>R5Ei2C^HNPR zxMO&7nfyNPs2hTTkaBKmgG_A?sfh0}@U!()g%_h`_34mo%Bg|yv&?gq*dacAC@^Qa zPaPenFuJ0sXbF;g+-vALpll#gNAPlR5G0tRKd&!U;Pc3OYuekzkl5n;0Js-Ocbiv;OIA`~%P*krc#oNz+blOaJAOP?43eHR7jOS_3#g0J9Qq{m-V$aIRjB zhN6@x!0W$~h)>iXG!o&9OYm$m!CH}r5RQCSwZvZsJzpuXS z1kLdGC7vPxSl!PJ)1#3&3%I1u73ADig*BUPdtVT6b;i z9vJAeB!ssU4kGuWz-rr+81eJrv;y%4yHEaG-un>U2G;Xb^UG3zOf4%(Wi%|~r7qzW zRQTfOO2h8?-7Hm$m-z@0@S$CIEeeV_Nx3cEopr(dfl}Rlv}8g*kZD17V}gR3I`o9a z^XTY4O*46UR|23o!u>^9u&L&U_=#^VEwe6SMYe*)u;V;9dzu10HluIja)y1cmAuOQ zp0cXXZ>UQIhS)emsLXF9>8lpq69wJY^>Z2-61AVd z${en2m6bh8zoT`-8Cs8>8@+U%cF&Z}2-uI@7~)-5^xXZo_LtDnnfaaCKR-Kv3GM1Z zIB>J+^sirg9j&f2`0U5vF$z44Z3&78RDy0XVNw?Y*?KHWFi{P&IW zrcv4P=kFtw@_OvJ_X4jbF;;wBS#`t$PFM`VpnOsIh|NK7PC z4I%W$JNe$%E0HBtqL2UyejzWPDq;gM4{haD{S>>qkp0LJPdIRGY`M6U5@%WH{yu?5 zGC1;1S#^3QCK@%XR?b`;8TeZpktJ0#*+;Em~b1r^uf&;>R@W7Tq+&fx`AaKQ|{Jv41l$CC+>pP6B)*_Hi_D zcYV!1?!WQm=uU`+JSgcb` zM7;;p3w_GB;v1m5u$3l^CnJaF&6_t~T5FB^2ZZE=9miuZgCg!U3WivUE^s9f#wktT z_46~l?5&)F6CD5q^eb}b>a7`@AJ!O{jKd&-aPu&dH{28`KHj#Qr(IUYm>ht4!ra{c z4>Z(MtTu%akI zaJ{BaZ^X=R5cQ~K)2t2~W0o12r6LL%pqtQ;!onDN(Ppq=7&QfQkImle?XsqfxIKrJ zib@w*E$p6v1X=ivssS)vo8W6P;!lQo54Fq2TmM`mibeDf&{g&|nW5JKANtD0L0kJf zN|$epeq@%y8*k|VX@{R9k(l|U4$W^@{!ZS25K!)3^={t4Sj8{f$ zdZzTijuJBl6!j3Fd5cEO0kzwSc86$>UJUC+_YLRQUA-rl)8Xw;S=} z0Rdfk1oa0Gj|FH5)X7#u!66}s1ImSzp}lJHq@lJ9LmV$cW?*Y8#7+kT9wf&glAQ4V z;kc6TO8YugfpmdS7j~9a6$<>M6W^pXYNd)G{_X>h&rq-iRSfUh`>~>88z~c?${I?E zsyMeBH%v5MI~?WWN|O8c(s#ih_-Jh_K-1y&!h+P)UwB;Mt@QKp*+zPe7)HWo{e3@n zx?KHoI(PSFq>00=$qy?QHvPN4z6>N&TwLsy)6Weu95L?`YZnqGTx!HvqjN-?(4K3h zhmUcamcXTOkX%C6D39imt3@Eyh-Zx;*RY^4S1<-EhUAS4r6T7!_!C7ov&h882gTKC*v*H}O$aG4mWIF-*)FM?&NAPpfz5 zs-YnPrvncDhRn@61$A{1%=f|zZzpq34^6_;M~|$Dm?UVr+p%U&#l^(j$9I4?Q;^0} zCnfcX;8i%BC~oX}cEUmve+JeB=7T#6MoCuKY@h{&BymnJh)eCsqetm5BE}W0uckX* zDrkm<02l~9pqv~+o{Rs-RWbLx=keq;I(KY&xNNpe8GrVr#7Q(r(#>7KC59e=!O2+( zW(V<+HUUB@_6u`}x(fPXLLyuYG#wa!Laj&E?y~~=nU0GOm$pIu zJN@f-_*D;}Mg<^5fSah~Iap~_hMvp-r|HXu*dE-$Z6pjpbzhnYSJ0D?C`E+uBtQ*O z449KMM1lGE2?dl)DOzK|yx{RgcZBA}zn1hBm&g&`&e#BH9{vHtH#gP*JPFtHk>S_{ zYJrx>bHTyZd?;Zns+im`geFn2B4n*A_m+oLZ2k|gT|+Yhzg911jFzeK*&7-tLLjmM z!O|)blP(>^P05O~sk;tNReMB;h{w}*;uA#0@ozSgB`gJLmv!~@!rX@EXGVEEe`DtP z%Twx95sB_qaUMZrs!?b6#Dnk4L~tt(lI)ZuL?FLxBu1;j3cpWjb0!(4NHYR|0!-u# zvO9jA$;tbsAB`sFSd%C+vB?9T03G+PedNcFqdGuVwsG)FkuAb>dC8pEJz}r=51;a3 z5p^O4{bAsSWA`v88fWGi_E+b|zGn9Q{P`2UBW3O$JeiM*nt(yIv4q^hbNT53$Qe1V z%8Tb8GJ8LB_2(6$L8Zk-&uA^ zXv>AdzK8|Ivf#kE6k<+XDWy;zz{xN7jV z0*l!<+|LYsj=|@rNZXZHeU!_JlQP*hC{yh_3Iq^W{CB}!7lKSD)|;H1uU-|FKu9IQ zxfFMDF;t)cxlIe*6M!+rR%}MOb%O!2YA7#$o0s<2+$F;J& zYVpGgxQNky0%~a}Ag!=)p*3p9XdS4DrbpVq--QMQWI(lprL}9FHyv=%HBID#sW)CX@^4Yp$l+4WF}Y(Xl#;_d zVLB&_sw0*b=PSCc%S5YjNG{_F6(Pm~qkJmBKX0d!IL*B#^WuZ!axJeNufu5qc|&&< ztqeXtoQWvvd=t$eT7%7hQ#S(j4P`CCjlEmkvHX*n)PE#;H5Drjo)O&Jopkr zLw@p+*!fG2yE2`2w>*?4-TLb3$rA9)d2dny7fRjjo#v?KYg_3Y#%*^EE zE0GI|6s%@%s-PQMvmA_5++;l&maHeXOjWXup=?A3zG%dh+;jrEj&d?WG+~gGnTB zVt0eJJ8*uvV|lRyPc|{zDCRIv!G+aT6*Kc8wp5?_eE3L;+joVEmHu!)WdC1D7GLl9 zGM_J=R`{s&fIzmfRbdD^Y!JjyDmao&+-;4$IKm6g3}VMho@$Kw5U&&nAf-z&?jZM? z+`O|p)?P=#mOSExlBiA3+Q%5$8_)hFNZ7 zw)n*WC-Fqq#p5Skey1ubD}@wupztE1?+>~2MIf*Y7**s&NwuaEL?2eqxw2h*o5;8e zFj5mg2jm?xW@SqG-?*8H z-LnZp(Yh{qqcc`L@&pZp(y0Si0VI&T!D&Eerjd97g$y<%RJ4bO&2;0T*t2Lq;@OA( zZ{41?-_0TPK)wf}sL|FBJpdwslm`;JgtM2fzbe(m(CrSKu>EyJX@Kct;Bt-s30Y}7 zn68F$RDLNy^?azKdw4i3Ffa?i0pR_bv^Ikb0+Jwk<2X6qw}vcY8^uk?d&lbrTuc0Ku7n8H78O5W7@I-hy8m zJq|(tAbu9HkGLYvRlxjE8`OdwFOoRT<(IU$|Az8DOcTVLfwBeXBoZN%IOzc?sbj=D zEg{8Ebypri9f(e*HOoW=l>(5`m97Gl>Jk(ZUUxrH8-IzxZkTXYcMYQMF_5m<)q4jA zF&BZLo?xm%p@L290pn3!Xq}Q>7R0wbEY}C0hBgGGvKdl+jEs!z?53gFCqT%@kNFWC z>s*D?o?w@u{7-k!KZfHHO#|kolE5Kik`+Aur9Qxk>d_S=tq}jGWc*<28vwxLOnmf@ z0a6m=6XY!jIx-5p9t?vPaZ#T7S zcqDMWI&mn>BgthuKv|H+u#9$h2R?l&hz@nR9>SZ3eiAAF^)56}c0+;)vJwW|5oEOpX^atlF7E!_yIA+kvG0cs?v&nsA?jn8Bd(S@&j@6-cgP07i+x3qD?>9xnk?!3X|J0kZ-}6XClQ5f2|iG=?OpWO1U@gP z#%#BWotk^GueD4NF@JD#3sQ9k!4e>(loc{4BntkI3?79#2zp!TT1`x5E1c$lGo z1B(uSA`aif?w#p?2n;e#A{Bt8tRa0@^)o0Q+>nb5_vuN;aeY|A&zyOml+!*iX>}}1 zl}nZzcSpge{#w$K-kRwL8Q3IXGDZgv9bYmI6-NgLWKK+@4;;U=DdISUA_{m`fcRNK zEeJ*&J2dKxLxECdVyo`&ud4U#Kzv}QLiCn1rg0)jn1;p=>6ZO*A%}|96#sKIfG?wH z$L6hk{Vw)u(8}(BJb^StZ+Fn~7-9Dt$16%e+C*rqUgweHMRgaDJrtnUby#fPrq?e#=Xb z@-O8r9`LZS^)Cs0BmZCagY17~iv9-z>HpJT|FX1QLsrcP2{Iw#0KK#F`P59+RN~yZf_%hv;uT6D z$FK{m{CfimnAO++`|Ggs5mtZrcPFGYEP=^*IXZ#5H?jWOd?j#098wo3ehdyGzqd-+ z@gp+739$wy+qK%Ym}MO#I28y#aFlW#Iz&Z7LriVNS-tp@8KF#g8ceR&#GMA^v-)B~ z6N|Gdv8W1F4fDD_!$%V7cD&)l4q9gBMM%Td2Q?9=UKXO|GIpDm z#2p0065in=>={!<$<3xk=vjyy!D7K51>%9+4kNDnmBpMY0uwxrDC!}u^?gj);5C|81 zDol*xJb3eg(-$U{SRxA`( z^@zBDC~Gm8G1;od@C3oXt-Rg@9w{Eq7;4v?ay=V8o3Wg+Qz|4Y;+ua%2a(Qo6Kdtn zyEr^AV;Xf~l^s&qSv|Yf*7{=0KaLI#FpyJththwpif^UC?ChO^krR(qr-zKPt*`Y% zO$%TE8+Sf5lAg2#0Ypo9Objye+zvbz;B`T%3`FSCZ2Kt~w~&VMPU-<8#DN(4fJ+k30sha=b8+PeFgzM&z25=3v1Z`fcv z)N~Xd_tPfisseTsLYR(-2m)dVepq(?4H8w4rFUy)8~eN*)f2UnX%%?sH~>SD1}4FE z+W`bSs2JB|AypfWH*Wsm>^@IML{S2H#Uu}$J?HK-%KHr#f%ztkg`Yno;S%nqLgK_% zGtZ0FDt1E;$Wku_Ye8S8iSESeHg@p>fDKI6(u2?gb(FO9^88!8^|B?xvj+YKC&=&v z(7=w+WTCCHsWsxqsa(*KV*$2nT8!xMF zREXe~?M7N>(Y_I@4eCpz<$}G@mywad#AECdP=TTD{1RfGe_m(+U)GUrriNos!@g}h zLhE2yfx>hMmO|9Q#I&(kG1Q8LWdRT!9CIiSZr!{ILslM|T7p&ujA2EEym6zq`o~k~ zjszTj7D;;^tN@GaWX-2x-%v4^Wo2+=SZ3gu8X0oB0EEMsYs@TWS$bG$UKF}n0Zb_QM~Iz_cMHd z$M^Twd%XLP=Xj1gw!N>nE5a|vOlcp|B6_vd{ zXp&%q{%nd~Qkz-}yHo%-va&4- zbI|h|TQ-Me2fn&co0JfAEd5Mov+BO!iY(I_2G&zoR$c?!AR7GK zigZC!(+%MUQyiFCE1!c%GYS}_&sVXT(dHu52A`}opFgt_$DW=5W;Ny3R8=KMww;-I z;`8wA!B1#`A_)c-iwGzV(YD4n*C8%*CTRIh4i_h9=@9uN4$v4;<@Q11e0)4urY(HL~uoa9^I|Q)i_YUM{&z`Sxb)#0=Nb6 ztf5SBvOSy!90UXrz%>x22^B>w+ro;uIw)Iw=B#_#I>x9j0*2bI?>}o<<^%ZC^!IVm zCKejMjfo0QAg8S3q;%Shr;lhtY;r|c=z1cLE6&Pt!j=j;KBqZR7k=;F9v{f1ia8Dj zG_jdi0BiE0LYIvDkSFcxX4ZX%!%4{B;tD|~4f8V(U~@R%X^?P6&CsYgsh)hE3{n*0 z7?t3x!zg)6%FcBp?}WHvF-aC?qpqb)?mnLyB~G7v*_$l_ z^vI?mqNnzz^~)DAv_`RAi07?JV&1WXAK(UJQ6Q$Na2G;t-|6){g5;6c->y?Vi}Y7*$fZ}Fxrcg0L17w(j^o|IHTH^i{E0$Sv^rF>m3@J2Zid>4_=|e ztC?sGqJj;Y8OWyR1_ayTFo`xHP+?Qt(imFjhWX5d4PgS%J^zgu^!n{Rd6VPx=WKDB($Q$8Vl#NeLDnJ;~ zLC+l9?n{|%gbkRLGpJ`^jGAV)H4Wijc(2aAn2rrdzz1#CE~84Cnm;j53GHT$mpNC^ z<+A;p8@GM6sE~A&L!p27?i-9xBC8-qcfoIlHzoEAB!gfjfk$(;a}u3g9MnG;j=~@4 zR}k)`*}2mhBm(>x>^RRw2pr_$sm~4~QY*o|D@k>I2ht|wu74mij7{%?ao)K()I&%O zYs<^e6pX;r5f~`oIvwOP8Fk&$-@h+l2pe}HW#Lo6%A`+3j~0RB3ImUUB;X7D7!xDZ z(Qx~RJ$%^znGU-fGhwKk+jW3eNNSFs?(X1ZOQ42JR;nr?vDK9JQARE2XT>_o4kGO`*zpgc^uH-~|4%EV2dhm!P}=a3P~4!WRR{_kIlhD1_1>!NvNitjc1=>3UNKRh9T*;8Vt-hmq>Ad-G6-5x^ zg@js~&#RSZ8QmOc)%T@haqS#v%T6YEk=V&#QYSaRJiJ~I{BK~?Qn$RlN#XA2+^>Bw zJb3spl)#rQ6O`3`J{DQ}$$j(i{hrvg-lg6#(jL=C-Pmgc( zLY0NiTsavZGA2fL{_xJPLA!1OaUwq7x9Mr#NbtWf9kkY5n7D9@ieS-o>=@l}fgmie z2L_;UI6M0Z6t2UY1bOJTVz_{V<72b$(~XeQy;7t{&_!!VrGKDD2e+LrA4f0gz_J(W zTUa03w7cca-{JD|^3n(;sh#*Qvfyf*#Y1{}_FKg%lKFXSn=cW;!T5yNu3=q>$ESVF zO6_5r;dUtwtyhfCcVQ=)pHG>dPH>rkS$oR^;=y@u%w*pm{k@fQeB@YUD<{r~KiSAaC%NCgF6kxe9(l+e}J!wsdNt>PtFH8}lez6+vHWV|*J zw{F|p3rcQzztxYD7dfJj&INX6Z{}ki3(Qs7e@KqMCDYW?lW4=Q|(uzz1Ys zq@vYv=f(u?2bbMGzWY)swx5jhoW?Ha;~67&QWF(1%Fn9tvE8Q%|2@94`Scp^J8cd} zbn2eugbbVLS!$^T91~UlX#Dap4o8Usk%D!|U(WxvXv?)}eoJz57yHZ))qnz>hUB0v zV+E6HYGGbKYcp(fAxZmXcw&P0_qNl;Cr)vbUY^T&CN7!2*( zuXlN1XTGkuSuIn!DiSEDb|2y6Bc@O0q|xL?S%>tfNN+a-Lj=HHgAx)LxjuO)vxr7q zXec~x5X2QSO0qgrt2+%|r0OO?TdqNO5h0s#+&Y ztZj|HC##e8hx*>bNn-^@LK+?!@2|{dITpS((911T*X*$`H`jQ$f%MY&(8si&t($VF ziyYncE`2z&<=6qm`{l0oXD?#qCId(e=5M!dEQQ4|F`R)t5B|l~jdwm^blV^JUAwiAdmnlw)8LPR;8&ggg-BS^^rzf@;9r*} zY`yNFkGDjPd>?c8Ne;mkPN3A3PppR%-!}lOFn@Kg!pO_Lu&|#H?>@2CrVNmeUp~Sl7i|Jt+UNIh%J9`J3U!O)n zo+q}}{ON};=|(qeOc{}YL8}FBD;V(ljTwK!^Z-!{!&d38{E;=d$0U&Ah=IX~TiRk_ z21(i*WCJ0xf*=DiUdn^UhzC+OLb?tKCTK;-{#cm{v0gvMh~b-KAiJV}1S;ebsJ5ue z=J5x-U^qJ${^`8lGojflMKn}FjKY)ny6VZzNT8O8IS(|uar-0pJ$337!8?KN2%#$> zd`TB&j}uYP5`_wsiLgA_)@KffCMF~hdjI$!fyKqk==@qbelJ@%@pOAw&7(i=FUAZq zwdTfW^K3mFcRCeiv~jt+sC9qHa0jU8J?77S;LDXIYiDNJ*+*Gwr@PnGsG}H_ydz1X z$Z`{koWN&W0zY7HCuqNu^)aP@K~X4NFw*=jB&~6uyS;?(&0)q{GTpIEZz_qz+T(@o6;*O+_XjLGALq7bxeh*;8!yvq}o3;e2K@{Mtro9DaX9;zLF?10Vyk8+XhAV2+n?g{(z zrKXwKs3>$srIv1{^t=dtleZxLC^Jz`r-AaB^dkw{cP_tuHgyEghO-pu)rrb+`y%J7 zD^5j4{QD+sMW(V&UK36^}lV~0G zTenpWts5@dEp=S;Ll=dre5o3(^yZf?;{%k}39q|!9Y=N=8k!L7?)m&6EDQv)I&=q+ zW1|b!IXCA%nO{EL{1cD{8X(x4rstyfyLQ;$%(^v$?gt9=LKH%{>ak(IXndi9s)Rpi z+dz3*M_(!E4#=T@JPuWmFnRVxFu9~*Y-*_{c;*S3E8w4@IGCH&f;|S)?%iGx0Yc>p zXr++l0EmDDLthnq4CZ*18dJv$yA2#4Xr-!OV@0C>1+0AF<6KAv*^!H{+aoDn{pnI zV3r$tLclznE#)E3tDum5WGnY=e}9M9;laWCe1;o8T+U@WawHABE@ZF$0|VJG&mL{b zPM&d!0$2ejb6XcnCu~8a+6u7^s9(=T3QauHxpWq1FDOVONv(Oe^GM@JX%q_UtL??k=87o+L@CzS2&e zlhjg))Xbl&_K(mPZTq0pmbN&U$-t$EK|&=C;240v#SKaXcd0xiUl9Y6OU}5FrF`+Y zB69YibCBjwB1ZAS?+3~5hkHOxSy-6nBnXt>ChJd(bj7Qio=@V0d{=h9_cPC|$Yrw8 zxL2=~@PImPSklY7zSBt8$t>yMjqY4vd?@bTodRF&t}pU}7(LdVDm6mx2@zaI?xJrH{aivp4Yc!BUt zKocPc?)!Zr7-T}$njy^NZC5Uwd#RGddG7wBH;jDY)iukHzwF&x4S^gve*E#gBgES(Z+)+)q?#h)oXBkXNc2XR^pe-NzPnE;<*l<+XU-WnuA;63&^_f8FwDSbNiA-L2(~I#gFx zChHh>nry>X{OIA?`Fm(!aGF{)8)qu4>Wd|jo6Lo zTNGr)V(^u)4h4*led(O~3w_Y@IcH9t0`DBxWCMyHgm1kp^9CXlL;si4?6vHcppOIJ z5oKBjpE-*jjRZ-3 zywZJuxv$i=m|vHjKU^Nly1?~cy{i9NzVU%jqyjN`v>SB6oyT7R$-uiT zgJH^W)FJK@+(|LOnT|dNMwrmbR}bHgAz@HgK*JDw(qYU;%-W#&kE;?#@`u>bh(Lj9 zmD(K-NO$Jjt`REHQAtoj;yrBjEDqz0x6z@Sl#{3e8Pae1? z@q&jjUi2D$X9P!}kKfT+cz730{Ba=IY{3;*;Ee5Q1L8YK{=_v2Dp33*+-D%oRwUIS zkYZv&1WEiN=RNc~IXO6#IA|LCGBWEjwo+0SgCIh{>6ml=6QZGapU)||nP~dKZ3P5x zi|}`%Pnc>d4AXTn#CUUlp)1C)J5M^s$!oF6fIP))ntNe;#14@^^$jXQB%Vf>iep? zFDA<85%N}afWD`fvm{S{_=0&A)I}kH=S+-^F|L*R`t@(LRA91-|AcmH+NU!yO2=^N z5Em;hC=@`U_wE4&Mu$c?v+Wp8Jzk?nc+sW7d=t!j`tj>ntvVPK<=RGTr@A~oAc*P& zmobVyd!!*qE77FDbHg2%qn%1flx2}+|1>%~1`csp4Z<@)xfjRu0`3jG2(Ao(M6`*5 zn2tV(jD({!w29RdgJZ)-oC<*OwTJG3YS{=R=e^86aMFBdDe6By`nq5X11h+wD6Xj} z-PXQ19#KfXnr4PPd?<4^OEEE1Vj7IM;{k|)*Sri`MV_)gHZ)13udLZ|NUTQ5Mcb~; z-q;nIKeo0xi-8^6y0_S6(e(&*aO zblCu!!+=y>O%0*whs+iA4GJ4^;Dq>`+zRe^3=0c{KztExr zv+0H5SAK0~TvR}thK8UpVv??jne%wI0vJq0c|mX!FLX1rCOfxWyAo_@f`pGR{Dqcr z{$)cG%&DqzT6LvSqGJ7dBgMWvi4@TUgdg;0Up!fK8BUTBrp*@$XtrR)}mfp0TB#bcB2Y%5sc#-MdGXc&S+*H3hLj`uiaQIXm8_8rwij z2Sed_Oi9e8&|$#8!WcuesSbQ<#097RNcS+tms0ymv)ZfyN}}D1ys|aolWt!{vMajx z1ep*s`A}SQKAv-we(Oey<0cXk62iJ|PcdnRPRyLM+T#P_`d-)wn3T-wsB!+lB8?@1 zrE4WKQ|T};xX^MPz;oJiY!C<<`0(@!uA$^+v}f{u`CPH^`<^{FW|gDaX5r`HSXRmH zwsF=O!$B}le!9m+%gGhjR^x7U@MgyByc@nk+!HpU$0_}E{K0Y`*&nb-$wTnrp*sae zT4%E_X}L5tT8K1?W^2{Kx@1~(WAy!)5dg*y+_3xil`(bJ zISXf_j{MBDv^1;a>ye0Y2-bMLyEu;wPe)mq)%lesmrAVN=OP|ph4%4#o{oItQ-74e zFFY9^Pfy(H?<=u~%nuTMeTCwa)f>2WCBbt&Co z)-z}s5Q0Pf@Sxaa#235f@z$YP!Qy7(J($U!UZcfm2Usn+7ENW%z9oSl{rze3^QFkP zGak}q)>9!S}hqsU&%03@+dX*9gR-g*q1++2uHvmtG+Su-TgvQIKfm&EU zj8dV}G8+y{uzec2aM(S?`dskDuM9au6AMwteR`N27g(ot_06+aN@T3BU$2kuTR(bt zJ5T*}G;FvvUzIeUFDd#M#H36_hI@g#kLc#A-fMbqmaogQV@DH^W_Iyd7M!e=YeFBr zL40%6=W{h};}^5={}}@PPM7K2tjr|7b%pT@hhp(N9}w=pj*Ypedb(Prm1(xU>ESE3 z0B&#S|7Y?1D#*yNMnH*@cv{lxm%kF3OxsQ7z<2iZ?@;@qLTDO+a+_XSsHm>hycCk} zu4OynyXg)C^p%R~4*T?SBQ&)j=N%&JzAv2*Yg=!(@>WKC;Sd^(1Wwjz{FOiAA$l;L z^HRYTY}|C9q8WJ2l-kcrmGiWt^bB#%b!ps7#~z%7trQ`k{;0xOHl7dNkS||!WP{sQ zB8MF-pj^qmk0?|zI)u`~2`3$MOgz){H7AqQ+DnH4fj^V@?0Up$Yb+9D>DbL%w{9gT zN2%;!(#d@)aPh~{mjMlVp)QY8aW>)wA0A5c>!mvsS!0^2&PYX7Vp)aH?k#4ZyNzVd z0h|i~`1fy*5$tsIc1*wW>z>9^G>=-k91#eYhX84#ZHnjoJDhgh#G$lZ9l9)szO)wR zwd}2`R!66aq`ijtz&F&QexiGCh8h3%7o_Rp!d-|AMS4fLn_`+R)3V)T;LpZ>Cby#^ zGy8T({D#np+a9sc?Tf_?P4E^y3!hZfy@5UBra<#Vi4S?bwFAQbnic@=9V0@SP2O=^t{Q^Zd%*BS|Ko0L1^CGVyjD zYB&TruEaaECqM-6&A_okgC51BEc5t9CmiuC_n4L8s~^+T79FxL0zxpeR+Fi(FPNTz z%umc`3ZIaqEkJzBP7wtUs6pIjphQ)J`LDDNF%XNkJ;ph8<~`E2)#{24w3*P@utjJI&e-)&P+9k;<2tAu*^0t zf`W;Z2>>hFaBxYvS2X}&s4>etU$^UbdWOKsw)Fd&gh{kFBhf|s;4DH2&zvKs;Yv-86bc7d=$iq5~z(*PX;|N=HY>#Kc5H(=Tu`ocQ2R_h)QnwjA%& zA&c2c=?d|OI|L~z^^W$RXeIe1>xVPE)AW~~J;GD))bG>NJ{;hrCuyD?7I7f{K2s5+ulz66Oj4|)W2|ia#>s8G~$#BB{0tG1bzrPEE#{C~$`FjeAs zv?JC-)oXm_4XkSL0_p4h?u%LPZwzc8-ad==_3JexGGaCc^ST**?}q^X{QT*L*wISs zw{ZWzGbSS2j|FS}rO-g;%?4b*@Jj#&KpqQ7aJ<{goepwYaEp&;!&%R&|IJLS3>$(j ziCsLGu92a#wLLAZZ6$eawMo&HAorCb5A~%2_=XjUpuM22&FPFogf3iKtJs9z@Yq;Y zE+J_md554vwxWOc2;+IZ?=~z(kmNlsnIi02Pz+dk+VF)l%r~kKS6W*3N4|82CYSU~iy&KUrGTF^8E|#yJV$B!+m{#|tiJq<; zBtx2-njqcT+Dcq`5w#t+T2F8Jn&*y4-i+_X$Bbe6h|Vp73<@Zi;R0wgjkiy|@q1jW zZrqyH2jk@HHx?8yJss%Xz+EpZAI1~&{JA-p9SC1=ak<}Sor40In2!dC_dKH%q?9+n z0U-48^Vcr|p2SOByz8p%rAw1g$PjfIC;Csq8Wk_uB)qyD%eS%m+zLoxlGMPBgH#!W z)a*)+Oox$vrEhX`>XgNuggv2E7|fQA1(ureTf*IyrS<)j zWy;_D`7+mg@13NoWDV``cNx3#FnhRQ(I$SuCfo45_TqURFP$$1-KjZ0o}ZU<9f^PW zrzOMLCgkOrG0O|qa&NV5Pu<;n^XX=vFMFuh`mnKuckj-uN$8a|+_Of%PQs(4KRA5e z!<2p7l*izoh7lU_4g7rV-%K7{oRgBe#Y$h(AB%kdf)SznME6#6=gy9IQc`5M?@F&_ zC?SvK=_8S1Vfu9zq(QJ5qSFvjz7r#9dTCHYcF6w?+3nY0*aJ6*#M#ZGish|sZMZ2c zKAxZc^&9=3`TH)*#xjQ(DbGz_pfLK}Sb2=4F0O}<&y#2CXKSHgU3TfrT_Ebf_KKHc zYsE0sV5{|3JFPB0urLwH!qEm?Ge2*MRxDZ|vdW)8ImQ4Kw7UtM8BoM53cRfp6hCHW z7;BZ7uf555tb4>@hhPlH!uOyBdp$0FiN@BFA=Gl-?jhxiN(L@$683N=2+F0&X= z0Q2AHLs%^r|EjSiW3?6{wBkZ#<53=g**2ACTt3WZtLpVSGAo-}itIF5%-O)ap;8Ie z6K^9Pn%Q;~G`F=?2r%Vbt6NvsYNkS8niDCccxav_CLoJ1z|gO4OGRCUxu1Hn2?Ihi zo#~POX*c|7NJC;g(^FCZtnjBFOh-?3U*h9x*e;b{U>~LV&i@v}Bj!~G#dgZg=H72o zwKAn+lswR7LveeMi!07Tl!pfrtaEq>v}Efcor(b!$QVNNmYtnlP_Wr;t?*zG#y5^Z zOV!0CM>d}N_9ZqqZ1-h;dR(=$`vxAwm0bx)fC!>F-j{)q z5k^tiM|iwHFn20_>Qr1}qE3M_jHR13^nn}P`e29x3t1DYPei4`X55LG-(Wj%06du) zFh$`tq6M6Omc&2@0^8pEiEf!8gi*;eozBzfOns!HZ?dEbS zd_7chpd(Qu!|KWWWYtxTpF68Udw2b{Z4(x`%TM3JM_2pO8>iS2u-b9}+{n z?#r-&He_UDGXbKS?c-0e7FWPG2AH$=K8J`%UQA3cP6(nus9y^3s?7@D3H>t61>>^V z%!*xr1ptN*eHmRG{e z!~}p;+Wa64ow+4F> zJx!zO6+54-35n+pW(h#H>*zR^8FWyS?`-#(nV^ z{33+lf`Av)>6fF5>YJPA-)~os*0HK z9TFFhq9WYXq2+M%EEO^S4A*Zm*0_ue2Qc^WgkYQuD}`@*@hXtzDFmtvmvUvA_y%(f zQO$h+PI!_%))LAL-${_#_4JT<=0e{ALShjK5Gm+F?-6ON^qDhtk-|Y1E+ANA=`iDu z?F?8LwwEAa0A|t5Z3qYmfHFM>yI|4L+}Q~WQ~++1-Nnm@d*~($#fc+J`D{H0!6+Uu zD-MSqqP>HA69Vn<;|zwL&@2*y!X1oe8Oa3WgBch-ZtGDpE2YBu`bZpJ8Zri-M4GdaTfDnd-Hq=fA z2FpR(5ZV2##+cSG(#Jl7i4ry!dlhWxGhnWM;6NhU6(I4=<_=

^FFU0$LvfGEJqo zMbP(tGidE_V6fH`&(VGo8tR(HXqcI=dw3@8tW>y`x#l~=^o@EUNDs{srTmWGn&*#5 zKy7p}L&Q!~KK`oIEd&Ewwww>JBlZAo%{1XKCg&Lf%aZf$>*rr2UwU4;?B`QP?PD~(4MhFKh4P}b6XeZ`=!pHeHC(KV722BTdtA_1oL+xt{% zeeQwmt{Beu0hPfkWa$!fpN-0FK)=LfGGyLxM5@n8^6&TC;CY->JAS|7xGO|L@+D#391mW~c4z#dy2K`RutXmPJ7vKBxDF}BXj&CP}2&Dl(p zC~y+FBdrG`ODUP?nH_U1Or3Dy074WM#UdqxSy(W(t{mAheU#=(z{ddfhv(|oyXb3~ z22iah?U0J8uPAk_Hx(dX<p?7`cm%0S&Rmt)_!qxzK?I~zam7rsy2f-P-r7S!_Y?qbn(-N^L7 z#?sW$aR*+5yI?+d5Pbz~{90o)ml~kO0orpI@6@*UH3~CzrR%&J6gK}1_B#4_H`6IN zo53IMD)}0wVBN#5meBY_4ZYZF#rWs_$EYn5TRWtwXnD&T#3?tk`T&PTwLUml*4rc9 z8q*-q+1@To*6Wbp(&=7e2Z;g@cOHu!e|02VyQYH$lExhdBI5bFwWgWj*93b$s@2wt zibrD>c(Ba= zRYmVhSobGjJq*#XOnuO~`r6C>WsW3cMSXjufy0+~7dnFoFt-h0!mrgQ9B;eGhem1wZ=3?zf zVq!M9QGn19K=cfZz5#7z@Unz1HMO+N;o9P14KMxx7R{+MXLd+IgTUbRZ!p$;?xCeJ z`>TKZX=Si55;!;y92mk>`=D(tT8xg%-_DHg3dvqBntGTU)U^CYaiMDO8WNoa{vao( z;gag`?5z5+H@6%nei-;mmRzRy>PPPKdGb`Ox0IZFzVB2*Fh&F(5DBFBonDC1yn zKaDmQ87`xPx`#uF(+ZCECtR_Q9+lB3fQJE~!7yP*R@Nmoz${6v0Bos1X?{p$-=VHwF-yBn*0jb^d&O>)2*iRK3t$=}V~TDJt-dd{7xB6ueXH_MaYs+8uI_g)SMLi?W<@ z`+F6aba0Vko4_|jY5*%IMHY;5f7*B##ks`Kpopd^C6AFGd@NXl7N&0wW7 ztLm~TC?HFQ`$tD3Fs}j4IW*T{x!K=OLr;GcWJFAO_L(9Pf@z381Tlc7fFuUd6-?fR z2fW&uX$pbTH)y{?I01nYNEZl*7^g;60UKGN+cw}AAWOx77o?~uDJTN(-i1L_`lYBz z3=14)W9x^qi3W7k?|kqN4E!^-yfk%Cbz?;cT&Bb^IR(xN?6TXyJLEkrLh8sJ9UI$E zy_x@8_y;9mWJYG@^6|A-mOM71wNuvKUH$@~lL>3OMzvUT4*h_B&NyShJ)7?FHI24*s_hpQyBn_yLi*ttfQ+@p@Ob2ZT z!r%}SZdOy0q^oaZjxdtwRG_#7_5}<#Yd?PEsZHcr*7#6f{)HRO4LZYhA)Tv>@rqeq zip*~L`wFD}pu!l#EFl%fu%$(93ni>P7zP*^>>*UZW6uMwP*&a^>+j@|H7rq(?l$-wfT{hJ z57F}qcdDHd(>61l1XCeY!Ywu*N=uFKebBX~zViX!1f?4aU*zbfoDQFlZA1wKrue7s z?(PEfHKZOrCnqN;d-3kvy19}V7y%yk3Rzdl31u3{IUL){h+qo@3Q`1vJnwt@`Y@lv z!1$8tiqToN$ItenXDk|GKrKc3spaVj|2tFDXiU(6*@c=U(^ZLhyM3g^%AJXoux9-= zwaotJ&D^$42K|J!tbtmLsHmomjStl`wRSODk{#LcH?}m!Fag^$y^5quYSj7+) zx2%e>#|EBl3tWx!I>(K;iH}F}ZeG>@raRy7a2ai-g&_uCj>44?MOwjp@Rx?K{j=Um ze}9+)Gl3wIK|r|bxBox?NcQ(1r>}%|0&X;bEM(y7ObKcAqusB`8{c@dbTcL8nbmJb zvRkn$CXoWv*iD2_?FAe`^Y<}$_u(-Ri`3aMTm7+|*Jzx13u!BNa0Vl;4IJ>(tMBlo zadBakU6IX)%A(KiMZcfN1~NnU8GI#e@%+LJtqPEr{&+2o5^0eijekEt{isq_*PYIZdRXR zC#){1+C;_WM%4h`5O&z5UG^&*i}m>NhcI$xy;4+kat+;mtef|CBNX?dPwqbBaNqPb zFH7U_`wEk3`(6Qsyq%{Po|CDcdWg;J9sR@HY5xbOz%cbBjsxO5>8kHW%eyJ!=2{td zePS{2^(%~KegAGq9#Kf-(*n?f*d4IIX}u?kXUOPhi%f+ICD5+DCTVbCp*e>c?3p~> z$!`a)W0tVKTTaArR)M1Z+>-2}LR~V}V~3&8S>-0S!&q7mDNRzW6> zPnegr9(rlHFZh^t{T{ngnKv#CJ!` zG6H+t;+o{WCr_X{csgCPx^Egn&G3mt3{dED2mFAN6+GQ!HO~7oXvKpt^9&jkTJ6=A zFe}Bih${uDXWbCmR&^1Ahj@7#;=-k4A$Ii*5bUSVpXo5P4G9`ttb{)YcIxrtosEIC zG&F>s3u<_f&;Yw5w*Vr@05*g@0oxA}!4xI-8N1$d9O0?F)=)=}wxJmR=ECS=4c%3c<3Zm=3?-Bm<=ChXhl*j+000V#`>?G| z{F$XS)~3(`j>i<3(z^EF%vlp}oln`=9QX1JHD7$5e?YIL*!WIQC0$W>tbvj60Ba$iGK2S7JyP9tytAcea*oKCfJ8=zJLW;Tz` z?6+^`mX`QnAm&sec|#r>CS+q{gW=Yc7cchuz*8nSD@!K65tF?H$c0V_kKPyj4WKG3 zzQ(3y@6X0Jf)AFwt}q@hAxDv3Ock5=- zQf%TG_uo7XW1AIZrta4WFzc$vZP|AFRjBUY# z?%)#==z^4tiOB#0B0}&7Lwh&3yxt5EI$PYNNKOS5z%VKTR~e@WsU1vkSenK$G=jAT2L^-mFvSM1hj@S$73qd| z@EDOO4$S<5O5nyOBcuVCLxQ{D0)}*;_FN`V_3`8L7cczp-X-({02^6E5PWk8fe}Iq z!^7sKlBU_(!2Bj5T?k$2Z*Ip-d!Yr;f`SPSy$j%-88_BI{|L`pe%lpHiuf5=e@O?D zjqZW(F1;{*vu&YOJMVk|TQ5618#ay=D4n=R?f4@pDjzdv^}cn6`A4UcnB9(rL@UDX zUaUbwy?(+$YM~N199gfqKm%9A4V19ND?nOC@`hH_mC=Ljp|WJ*;+_R?1ArT!iV8IW z<6)vkK0+V~+*Rm#imXu}3x_ln3b^p-%g4Gs_UTC5BoOFqASj@U;>%}YnPgFEM7Q$V~Ln~=i5#cH2i*@nv#+v z)>DT=kpV|^8OAIvbK)4l5gpn1_98*Q63Wr+q-XTuFg+hkn66>S1vmc zH`QAwq1u_Ea*k|j-sIKeiz;XAJpMuH!h%jPC2e0XOt){XLKFTY&(8HCTzcmyXC7WU zbLM7Oy2NJGAH8KeOYE`#O<}I#Kb?l09U%h)CW3k&)x28fIqi-D{q55a_u0p9@_1(r zmkMN>n?}G6y~siJ7Kc6p37R~+m9;fGu^e1npGKTe_t7#ae3DjH17!&Wlz2v@f$TnN z=g;HF1F1IcMcXrA@f%LDfPw@pRN2q1xE!P6L|s?5#hr2!687PuPQc3!lz(iu2}S<- zY!pO)1Z`^Qd{o!e5F$Vn6oM-e*HmrY)Klk8@F2w0TCguFf?scYm&B%cD6)?TG{a9(?k?E`yRc-n7n9Mv5;t3Ackfwe61|!ARC8ccNltzAKC*2sWY=#9X7soP z(uk-&2f zg!C?>U5u5ubZHwInSu1Xg~i24fw7|ZOt_t(T6Rx{4Z~R~@FGC41c49s>eKhSmexYq zUzlD(kqj-E5yz9LX$;fzFu0JPuf;_Rak|;IYCmR(+ttLi2KxH>4Gs?Sd77KvjBKl} ze!j8m#Mri%AnR1LlKO{sM13#}*MD}!P06UU#*&@-sCtJa87l;dF~$_>@%8OJK95wB z-udx$4+QhU23v`@cqILb+YZQ@MK9`l+*_3PT zTwM?NfH8HG3hGXnfR(2-?-?Dn{2+alsg_V=h1LV=5u#)X?%eGA4sA0c@YCOf69E}3 zumC|b-=b2$KC7G`zK%4^?3cfPuM17Y`7bt~)8!Kh@4!70^D*JfmrVKjDPG@bVeTMA zuEWiuc(pXKz`%uZZ;3CyjnYkg9%M}2?)0sYx3aCR_uWP*GFDWpaFM$8c4XTWq}8zQ zXgAdj9r7~$F0dK)kPi9mwoouv>sha&&;M;FTRX!y7NrwDeb87#PU1s#P0|~{_<_(W&;exFGF7sPe*8}d6}`>fWzgf(6GjP)yD;9gwIVq*ixMuxfA^cLMe}1hn@BgMuiZ* z^f0p*5^`Qubp{ZaD$R#}J$KvmbL`zW4~0~=>bc`9al>F2Q5@1E5FujP25lXFPxhxa z&+DSBt*rJ*m<6wdpEjQw%9r~L)1UWHX*)Q2vyb02g&!noAG1Oem@)A5bav+5AON8=46({)NQ~*FRXKr+WFxo2JHY+JhRx>T2Uk69hAu)WPVf z`?##?4%Iuy@o|3dQj{gq{9qeO-BRNe)KYA6g4JKl8r4!TM&HJ10h2VhvcjSPD_(nT zj{xyz5f>d^ui)|jV4cF@Q$Nw3t*NWq{qz1s(*QfjQ&dnLOK2&- z*SHFN>NTzcpHx!Fx^J}+2O;tmvQAwoR@*;}T&_WSN0ij;IHp2c$Em2j``{2l3~g|8 z<0=P>2znc!-W6kr2h=3sW60mI%``MLrpEd2e4r-x#a#f@0WbhihbSNi*i4D?;nV<$ zsHl1%4M2T+BS+6i@`#cPfA$Ux*kn9r7<>CCIy>dn?VN5MVlR+L<=*QB0!;rKAci1~neJJF5@RH02XWR_;A; z;3mBd061Zr_P7Rfvr zUgvynw;W?pfD0KX4(AMR&)|>%!zjdB0;JK>3P&2@x#)7{%$X^(GaUZ#bH&$NE%*b2 zg1n%l1x8TFy?Z~UID8nP z)5?HBk3*aoI?JbZh>Ntg_Au&#EQ?@VAuvfq_vWGO{yQHQ`gSjy)9ucM*g6R2$oU#b!+3FtD)boj;$A@&P1%v`ikRhC!te(wm5=pq@iT`y=D^ z#C%9oUtwVNID?n8Ss)D{e`UKM9GUJB@TTWq)H zI_kZT@h%^O&kI%qy!YppZQL|>cG>cLP4=ny#ONSL7-NbsCv8=$%} zg3cA~25WH*jaD??nNXKvYrlCTj^HLLDhiYDygSuI8rbGTL3x<;`3XohrW?j5Cr#Nm z13ej}d25!gDgUzV9%O1=at!yZX3r!|A3ie=s*>TBrlwdW4gjTiY}4P7s$sx~3yRB_ zCCJvW-i8ol2oxFl1R?9@%JnF{_boysX0Mx}5I`s^t-s$ckEUPo8Ogfn-Me;~P1zYx zVJFM0f9kp>vUjCLEE{h~UE}w9=BaHQhd-kJFOSk;)4^8vq=J2+RTw2^eGhn2?L347 z&^HD*#?i5K*Vb)!6GmEE)TPKmt8jy4yNd9Tfyk+nwDGDECDXOu3iw9d`X86#Egbj| zU?}##83QmH2d|VNlt$UEaB%FY52YyiW^QY%pYIT#o9hZ!dwcC`zdqg#3d+?nwyU$V zTO;kiT~&G&HY9Of14^_?Oojz!H@XEV9YD20x8V=;)sXP>^YBm|t=vwrX;;abPC0st z0|yS2J~h22*)u+VjAsATDppvc4jAW5>whD;=5zl)NUmS_X|ZgGOt4az%20UNP0-Pm z{3#jbWALG32{JqOT#JLI%ZPpj00r)R6%|lypPdL=Kzy0$aI?O22vHV%2?)wQ(B!&g zYdb$bPf1Q5aqKc*@CaQL9r5lThGqyqO3cczQdr#D>S4F0#BVY^!0+NEN%80-H00Ae zx37eS5)MRL5mEnVgskGllU%e`M94O)%`)0rTG9AK8=!Cf2L4YQMe<@P<5iT&H?aCV zR|C2w&Hu20cmwan_03(imR9?GsABGk%(3~9?rw&SVafA%FH-vT3Z=q!z?fq6?j$xR5f7cYb7E5IMm zLQKFaN=W*=>f{(pQTf-Y&-(@envx9GpZsUx?2K2C#DJ~;df&fzpLP0_LA1*)zqw+sHO?uN`c=psOXkq$LeH}8J(wsbUr6`L2leLV1tBXs7s`QO# z97yr_R@FoAo)xxwBnl|^-OOALpxk}bs}N(&P}AYVK2-09zKA_iu8PAGjgoZivQm+T zZ3E)hQ86(}=jzo(V{UIgUw16?*2TRfbpm!L8SJ_7s9`whsEL5Kj*jwsf+>%eH1En+ zUaG+c&=Mege%zI$Ex-$!%>Ohf!?F)Q~!4~_I^W{@f7bCYoLutppYMxy!3tX3WB0FTVc@j#bjJVvd8B#o z?5GE@XJKhsSzS$Y`_#$p_hMn!mM$AFYX5aTv`yqlD1?xefXGKn?t7Q2?i-3I07A%a zDUW^gj-L`7;iYSsteaV8QBF-wA;?I|9OKM#yZ`yC(zNO}+=*aJNF^xlfBAByXYrQI z-0l=c;t72@&PMN+!luVP4Zt)A|p|{~nuk3d$xCFoCiN0jL zhe6QZ*~nMH_*{fkUWa=J-#^=&BpJLdt#)G5cBzGq_t!UU z@w-&0o;UBd@w1lE+=R=pYV@vK;nrWifXq`hu+7Wg&UvWp8b z$#SmW^E>xXL0;UPvvSO(p3V)+~@w7;VKLSBe02p+6S( z_%RHTP0cf*;}$6Gh<-FMY=FK3LPEqKF0KYhxsv;S1;Yf^E$1#?WL0Pa5C*gh053?e zA8TtNxQ3*8aM}(1F+6Y#WtK&@jQxO31u^X*!{#9_AtK@i;W-#zl-YzS5ZZhgk3>ri z#R?lW(pYXdu|xUW&lD|p@DR%ci8&qc+RkINi}nsgBq19LCnr< zVc@kel2}$&2AUQm(x9Jp9Mm>+_JDTLPK+OS5o7IN5hsi;b{)mPZoHa6}Jk%e>BGNxne!gx!c ziDx~(vw&O)LI-+UT4$9XLQx8=bJQQG6{a4UPNp<>zHAyHthd3^kD~gDMd+Ib@dYoW zu<%DR?fXUAbeG`f5u&vq?CrlVgOX(J%4`hxsOApSD9XY4xT9$GV}|*iHM&Ew_{!)V zLs9{kJjfsy+rzvL|9B%=Epa(Tq+*j2s(1K(BsE(e4y_uR!C(Yl5VOsJ`3h2bUK?bF zGigoN!ZV2nefMnS<)^5}SbeCp<41pu@9=C6yxN_nY;}&Q@7VH#W)lGsrn|@kOr0{7 zIQ|`>U-7N+1Plr`#WCz?AF4oU!V&fYY@FcCC&cy?wh@_N;bH@{617n9HjJ1dTQqYp zMnWu@BAcoX?yoyYI5<#cj`IHtf0euFf8nn_zT5|?cx)z251|Oa@Q+Lrp)Z6%W~4f8 z=$YWl(hJ3?af00gfFsAeVXyx)sK=rmtEbn6a%i!t23aj2LkxDp2{0MulYl}re6Jz) zO$ZLaIft|_)SbWh&PKccjpRCe$zrfpyK{n&Zo*~*Vpn|i&lS^)&dxu;B*GBa9F&M{ z05h0M6B4Krzk4a+OF@0RAA+(`5?*h9U1&eRU$<$--9hQUarnTA{|^q&KchQ3vzB+c zKm9A)ZppJ}F=Klb>PcbOCP3&te!K{g0;ZL1uP*i`*_(X)dCB<|xl>Gf>KbpJ7J* zw|2LFlPlXqw#h24_b@59`0JOTfB&D-ftVQ{XNHHUsJIQ^7~;y|tl>1L_p`CR26G9D?YO!XWo5tGEoHV6W~#W|FP?!; z=>5=8GwPUY{{^;3tot8eJ5|`x>ZxZ-D4oC(;<>G8dW$!@4#L--fQ9fUr{-{ecXoDl zc6#=I+`jGR41_(8L5Ej96IB`^0D?pw7J8J&)+sor?!_;{L{_Ra^k z2>R-}4*goq=Mdsx2pmu@^#aiakku4Ni7psU*{);;kLKoF+;5TwM@L5NvzB#YBt%s9 z$>91456ZaaiCN4pBmI(wZvqU1lxpaByN;iG2d-TI)3F8!C)SyFf;Pici=yH5#+Hs6hoqkqgNaezA_mF((5w007zC7#+>1 zu>475@94g2U6ZHb@rzU)W&~II7G$7(7yv8%h54Ylk4!ogCm*Y&=rcV zQW)LCufu!L@nYDWpoUMMPLYjV5Dfb_%hvkwX9kI@uaAoHeX9fqfFgv0%fxiokK6Rc zBU4l0RL)cTg^O98&gD<4L-D!Pz5FM-qCHHR8u~*1GLuijhi=3lKME)4JSa2Busy)- zc94&c56vWyA4Eml-DJ{E52<(|!8;H{;f|T1$g^pTTPP?HdQ298W9H6^GJjra8o{6( zra%H(!@a%surd&ev{@P+?|u7yHmBmsb)&wu@7UXo&l`xRs#h7N%ZsbMU4yw!Q?A>w zo@*s-1r)}dM1>*))9RUa^CxqkRvtXJOqzQ}G`)|DuVdalLoY-2mZI!d2@{vcMp5z7 zX0YF9@BTS^Ys+q?;E>Ror>gyeLPGG3hUhJCA1`LSpj8ei37UHD!Y^ zqx)jUZR+akD)kYlQot_q^=oH@YmAjaZL?a^*=V4=>qip!U{r=-pK6~LFJ+mVDQxYS zkY`(AkIc1sAB?gJmf8EZjbi_K93k z>8s)X^ozFqPfV{LeEDvcXbOxti@v(%T3>J@5_|H&1H+70&z}!N2V?L3kIyWpO5g6+ zt1+bD*^Knw?sA-2i4;9oOW3fctog!*bz8FmW0=0qFIso){}lJ-(Nwm5+nb85kW`Wo zl1e4RHjfF(9EHe`DTE}XWZo$dhp%$2d(Z73?U!gpNM{XXyd)_T|X zegAxWt*3SCv3J+r`@GKc_d9;aup5FcA)~&*!SLeDkXGjskJ7_}?EFT3?;U9l8p7@} z+(T|<)-s@B#A|(VKefB*qFLT>e?K4fJ%g|vvE?xA0L_P1~|!GHz)Xc zbo#-eWsDTT$uC+NKZ#hSW5d`-F5ykNNPho|q5OVANvOTMb`dZ1`aS7+z5Mw9bNJW4m;WHHs1>VyJ548PD~ za+$S>syHNFc2b$mRlkabomXvQvDLh`@~}ulTjP@{8&VGQKKdE_>uA-0B#^j?gHi(mLa7AL{!3*7vt*`; z?<6xnYA`KH&2G?jF;D1^3nyviR(m~gH{G;;;5i7fE*hgEl)W4 zJnBt?j*yXXywG{WzcYZ1EkkY8Pltu}gqM%4Z;&@1@DJZ@G)H&!bo72(#S*L8kTJ=P z%f2W^&GrZLM=fG0)@m)Ul5c702vTKaC%zvgotG4}>Jhc&gx`uz=&VEuITSNIDAf?x z5WcVZ1CT&;L0ItGx0X{lHf+!)Pow0Jq**CEpM!-v3|Gv@5^w32Z50v~tj*&l4eMPXU!hYTRisCEUY0P@{S59gTh;yd2B@m+84L)$u5Atsc6 z#>TH9&7i7&K-1_rm+jJ`jy4d*I6+!)a(Zcwh72=-#Pz+QMg7s5f@Tlg;jr$9oC<7@ ziB2@Y{)i$%?iXSR8jhwOEHFsh%(bvXbfrSR#YeTXw_oqD4C80?jgD?^9ARoNyU$E| zh;;mz5ul!hpa(L@myXB&lSeH?=jw2N<$#y-qZ1-={B`SoTK={go_WJHOFtyjZ6-D1 z=6SU>lLv}AF5cC)>a{;DdsG#R%DioK{2c9C?>c<9?0KZSPyNsRKq7@ci@E$h5n1){ z$2)ki9h4d@mNsX}QPWv;k@|3c819xmr&@60lLl|J^cr*ApQjlTVTh-=L#qvHq8UDZ0qj_7ZKfgiKBfh{2(tG)^fSNv23Bth$O64`Co)NV7%jm=bRIA);zebhvqWO_Z2l2_=r3VKk#OY- zavX?c^@fITXo<8kF4fx1B3qQg%y56^aN(!%wQJU3enN!BAAhHW3E|CGDEbk4eR|IZ z3Hh^{ca2s@olIh0@>=Fy@@m^r`2*CL={Wx{@=rY{)E_w*U0=m=e}ql-u!Tj)(=*Hg zF@IK|`#k5RrIh4`UeKT5u!IOffF?6440A;2Y}nz(7-#;sAWgp?TF zhZ4Mu7tAx^7lB(f068WR+qF&oh_&hA<<)}(O=!a4UV+^abQqA(jSLMV%AdXb_a

DF7Z zo|Drjz~OanNGf-&ySwiKgt_p{(8xW?GNOeLH7g+c*Fg$tDEYdn=VuUUQVduh_Z@7j z9zTK^8j;Z${-Qq0$w$uneQI%i4UaR&w>E$zp_D{^FXh8Rf35|vIg?}ZhyOtNpKGA< zChl#>{|ijJdwYjRMmDZlLjhN-`9P}2f4<(Dv_`^(7>vL3l$-n>cvTKSmVaKEQ%X~% zi&=d?h<%T%EI2MoE(~Rlvd(9ougmxPN>X&X8zO&mM^6?U+n6mx_qeuM32`hyxr=X= zWf6DQ7XEVi3cKpFyZ-hg$Ri;h8Vbg{gW^ne$o$btS_I+vzlgA#6N>D_Rms?a?z5;j zQzHL<<{>tclHg2|`4QvU@L!)aehghoT)`wyMBpFn zKkx%Fx2g7HT%cCtH%Oek|2~;`ehh&dzke9=_g^K6Di8$a2O|q`Ht__~V^CBV{$t?} zPIlt?-KRteP*-+@vdE*h zKG3Y6cr8UCiB0vaAIRzpFH?QtR^;}`0ALY!_;8rXKrG0bVOyQFYp6&H(U=hM0dA(Vp@yc+>xdSCPP@5 z)A8et&z?=e_V&E2W?ju9Nqe$e*3hWcYQw@91F!mWfuTLz6Wmk2&-O;8EqK;46Rf1K zM20g(x5zZ=ry})6?Go=(_!-1Ac&0z7V!%-kW!b7QHNsUM z+Z|zkO}ACvSI=Men_kb@lqfB;|JXWD7sX$b&!6*xl z48!TYUEd^lL2kzG53wlBR$DzmPC)Y{dW3#78|`YG+ZHOglqg(rO8%Ta8<)Tksg`H_ zuHv*2q1yy}0GI&CEjCuh;VZqkRQwF`AaD*$-vb&zf=H&(Q2k;(komzJL=S*T`)7BV zn|DVgj}CBAq}17MJsmuKuXFgE7k z|BN}dePJUt2K+88J$t0p*xR$j1VZcwSqUE%E-Izv<>m#^Q5y3``x`$rUa)N3Of}e{ zx+>k+sWGMbi^a!^@9Yq~v87k-e|_>Z-+tk9%2gLd9&{SH$FEJRwiu&)@c6b)>Cc*t z(9=|z+?~@tH#-v0_~Hc|t%$+xBv$1vBy!3$njbn;ipr+I)Ws)IQ9$4$=w)G$#}`jS zh@IQ)$cj*QgjeA`H&JF`CsEMx73wa1SReG1NkYYymlw&Vx}T7u!#)8^frw@SaYG_` z;^)sZ0~ra>yhS`d=IV;7|2Sa7R*kJ|UeP3n;IA*C_dzU~uoXwyRcj+w6axqkX=|u@ zLU~Rj)edtx$Ptb_yAM}cJUbyF;aG{8>N65c+c#JL`(V%f z!VPrv3IvLHAkWPMEUmstXXub{OY5a6<=H&++bqb(XN@Tc5{RBQAj@JLj`<~JWe|%? zCG)Ve7XsLa+o%xY6$rBzXrl*)e*ebtJS?Ul{&t&SycSTqae#fOM!&tQN}GO&z!Z3H z=bGOafdN-OFJ5Ki#>U^>ZEbM-ICFdmpA6Ik>9^1Go_Xdo4X6{)6SyG7W;^3trLVsk zf?okxh1>{ggmt)CaY)oYEAtz(b#y;RjztQ6R^Dh_Y}Q^h>32(4e!L-4F>O9_Y@8>0 z?$#q-TA$MFSNGb*eIMu53c+i?@|`}NXaa=HreF%I78N>>$YkXJrPlWV{85Bb zbkCnVw~UeTSV<@JbC{^WmeLPNr6B)ETXrGffrtmt(9oEfo?ibmn=8@*x8@Fxp{lu| zDhz#KnQCBRQ3>i0N>YqopoxB4Tl)p51=4pE75xZ*&kt3MJB&?Cum`|F7Oe&nYO~r0 z(WnudU2g8z#(=q^=RUoFO|j1t2tP3^1=0#S556fV>qoJ+Crdwp@dd%^P=7xhAfqo_ z0MTNXurM*Bv#{vFd)Ya-=VIAyWmr-`oD5Ab88&?I-^|G&K$x^JO^H*l2i=fk17>>Y z0bwtE3Szbl$qjINk`I{sje3oUDuW5R#3qn3K--+f2?PTR|Iy@k@KeJ`HSt{zMHho5 zt8#Bw%vT7vSQS3V?K0H_I*{>!S-}Qw?w1t!VQRx&3$A0SskqB3&>q2jLfYkR(?6z8 zpGINi^vm+mCWkatlOiyI2Eqw?*pn4a${AI;CaGq&>m7QrAkj*d0476#o0 zhgDW^><>>bVa@Q$QQ~kpnB5-bnq@2h>MoBLZ?+A)ndg=F^Ok?*JT|PlUF(!ZIfPrM+VcE$(OOKy@-6yr+IDo5U z!|daJK~WJp6`t8iL~Y%}A=N&Jrk2bPXeVcxu}gdSD43b8=!@ zvBDeypeb*!Ls(PPs8k*>KL@~~_w1-{=JHt0F~}Uy>PB%%hHGFJiwD4{GHn$EDDLHB z1FDP4zBTAEAaDyT7I4ULuQ}#WZccAAw8hYeJ_wOZQc$65pxeSy&M3`rvq$_UMJD2~9Wl zMq_iChYUCR)eV7IO(O0Fn>&$l3EICwieaX_=e-z_)kE;#$E5vLcY5cuu`y4W!Zs@X zikObvy-!>Fq6(kE@|K`c=Kb!S`VZBT!r0B6$Bem>t=>w~olb zK*HrbP7HhrIf(`UHyv88QDb2Ij_n<*O(a?lmO#i3#M!Ac7{ewGBWaxX*t8%n%VTe% zrKJyp98P3-J$L}`X(|)esl=vDA$mPIYL7#{)jAz_bd*8NU`k34sCso*Ew^vaAu`lq z?hEoAYOU}sdvjXv#J_-K^#W`lZC((Oz}T#99aQwRwS9kn4g<>%@Mz+s80-yTL4}YX zE%^f{{U(Bq6ATU?=9aj4KO^G{HhJKt8x?$Fpj2jOPd^gr$$LrUAC6nQ9n#sK!qbVk zF2|;8Rr9kW&O)2v!8~6R!WbV6;w%-3ejKzgDW2|@E(L*+BR5isswOI1>v|YZ=SfvT z_AJXB>q2bO5j{(rkeYb0AelrP{Q8wr(rDbz+Ph66m^HK%0H>$0rl|cD9v+IH5F6Pp zb2h<1kbRp1&kAedn84vj@aMTUWz~Le3KYe70q}dV$jgO%ZJF2qJRciMS*Kw_HCis- zU;GFpUSqC%m{p8?`UElNehUlTlXjO41@s%7y{K%ocHmRlE!Bj3Jg{dO8JXMLa>Th5*yG^5&qy?DA1pq0&LIh8 z^;ArPTstXprw9q}$F}EVpI_1>wFmsJLigiLPxB1GC*7C87F`Dggh}_)7cZQV*hFAB zJ9oA#(CG$re)EVl5J}J`5A^pNFcZN9usEm2V0iqn;5*;ub<4*KH(pW8O_=>t$dN?x zaRj3(jqbXjmJY@Tga^FMcFEu8@M7QNIrRAb=5Y|@AzGcm0fw0|W&b-~PUg4O>P^9VWy@7nrx1%=)Js#PN%V>2zOv~cw9EFJi{LWOzqew=r9Uc4X zF1?o$^YbXZDx!MvxuMR+*(^?CdjQY$x!u$T$C<<=X>23sz{BLYc<$7TF|j3|Kq=@h zo@W=1{IEoH6>+H4sXr&2`eCHiv{#;on;T3G@LlZ14Fg{k zH@loF^~Jr|CtyIL790aP$sYm5a_Sb&Zf-&rmFs$RJ9FA=oX@>l_VS<6gLN-kF2u#P z7CABT+d)VLVi!s%P`0IUK7k^DdIgjbprhLB_-~HbCYYq4o_3};i+*#fFTo(;jp$Cz zcc)mINA*WmY^L->S)m3RaC!MJXw(RX18Sm;0l&rg`LCBNB7Ff5;briAfG!Ll?6^#} zh4Rd@@D3Nr70m3arV|r4^A65RU3siwn%sF^kr8uJ9|g-(0Ugzz6iG=anAlane)|^t znsNg22M7i6));2h7XF zo_(K?@!03*tJAyQn#-|Y@n>m@wbWbX8a+rk+35eV6k8BGaeG1vns%by0z-K`;9~%h zE0DfLZ{4@}FafSVIP6Abr6VYLXb3&s8f{RY+ttbA+N0VREbnObe|~Rf@t8uVyVr6% zP}DKG{l6QkdwcyMUBoeoa}BfiSqOEGzD(R*G~pLm+h0)lTS0O|{tbsXn%2M)v63E zgVXzd4>!RjB17xPVvRQfmd2a2CTC>TixtsxXq}K-HBqFdUp|<`Ps|x$&v#7H(PSN* zEr@qX$WBd7P0=0JK6!%XE57a%EXI`2PR0D*WnpPxaP7don>U{{9UFJ{X9AUB>z%=U zAYR3~IV285-MY0Ux(?1mSbC*46<~wm76O<11U->Jj}4JcRU?XfJufe;)*sGJm^TcG zpXfe)F>KHMA~W-)%%7+_pI@Fm_vJ%-0|y8G{*d>J(Vj(Ca-hr9_b7OLX6o=~D*j$y zZO2cSCrPU{qCHR%j<>AvX>K|g%_D!M@gIy!S`YAi0&T(7$!WcUkEpKFgr1}0*5n3A z;jS&&zb#4U3XLYr9(MWlu#x_*TaF0Zf=QyZbb?-u4!VAe2kyH^Q~GPM6*Dk%x}#Kq zvPAgl2-r~=7@&#F7BR{Sv&_{_PD>+>*+~(j#8?qkVYH&fI;tUMT8a|PbJ4<1al zWDM_Jn!Iksikv4b9CVw=N4C|^7H*2Q(G~Q zF$ouug|!mc5d(WvovUxfP6*VT-A1Ym-X>Gb!N#US28ExQl@)5=t7-z^BXv{-DD#9M z%VHBG(bt$x0iLx)+lV|Hx9$gGfV#ogKz9cHF1oW#RFsuNhIh-$eGeRH!{U9DXFdrs z8bb1NOG?I(Foy*P=mvl)ZA5F=t!F5*v5OPxS)z5KQkQ~tjq?K%+7^?vFU2$P=zx(N zQ7wWAh`tPYeP9DYIt}UPUh--5)V`-W5l&<$zW!ouOT`r0aa7i+G@A4A(Ugxus&PDRE9?1kN)Xw+tElMa!Ik9X`tc1ml46&)>IFGFJx?0t+Txs<`#ZIw zG|YYH&Thp+Zj}KZ3vL|v(o_@$ua!tas}I-4tC(G5qo7Db@&=d#A2Vh(kl>XpTfsrN zFRE;WmUlg&-0}$y=3qW~mA613aXqWfP18S6@fx>xm;8>N9fc zcOT9;`}xhHC4?MUZ4d`p_>{A~Y?Xe1H?o$GY*OQ}o8HiCFdI0$yQ)LuJxPTku2+DFO?nC~oSVnWJ_lcOW_l^f+ex2Bv0e4>PM z1g>2jlM&i1aXT~%E#Ry49gf8u=qq*0YpKP3F<~*i-g-z zON$2>bZD&)a7lOCuN_lalfx-yL)8?t34QR>r$QNk1tegY0K$beAHizl<0(phXLc1< za-|B$XrP_=8|F!9@DxPw!2*7htdiswUb`n*txCpWuOO z2~SAEXbSfqAO*mY7)5^_8j?o=w*Q6((o43M+^ZilO6YU)VFDTl8#x?>Jz(BtXTMnB zip4^e-SZ9ZbFu)HU{_slf~xZ?hKn@X%=Zs_RM z3PmX5>Y-xU{OAvLp2Qr~elf|CE;R_sT;vQeGM>X^1YMU+_AecU6`xM(!RTsbV2SJKj|VE0KFYM^+0>UL2Ma+$4ICD`9me-zO- zfeXX}-ToBQlQD4M7YdTADJd+JW{!Ss%p{GB;zB}}+^WB&Qf-Hzu7#Q;q5j)>$rPPb zyiS-}c-4;(CwK{=5ryJlw7Evq4`qytR2OUO^5hB-USc)X$YQgsPXYB|Z%Ir_a{3xd z`W>|Ioz!=QPaZ@V6F8&2<)?PF!Y)qDbCqgd-_1W03oP9m7<5ns66$fENp|}@xv;8i z7yGk+My&vH!hws3%0?j5^%cX&b2=WqQJEFi%9E3CPRt8^>_{`sOY|B2v;0ZbPoNLs z!nCOy-&yE}?NNtb!?JTbWR>_daocNbLa;G>C9LwTgNEg$S82!=&?J8J{ry4$!!U57 zU;qh(Aqh6IRUi03dlpg+goh-P6K62i2M$EZwtF3g<+l~&%u(`_D{=MAq5yT{^>3;G3A{vd%|Kvru z>7n)Il8gU$U*zUBg=S{>>(^J@WA-g+;&R(S3RF6m5@(wdyPJHZ@&^Oca&hmIi@*JH zgS=wT%#4>R`B3SlsVUAlvvcg}>G-L$@9EyaPPQS47c&C@<3D^9Y^9f; z;bSBBS3=_3Xj2f;!)>rvI39d20Qz92eZ3bb=TTZgII=zZ;Eu98vWL4r2=q5iL0}?J z-N<7q0wW3jcL@!~JR7O75sCC73CF(n5m$a#IKHsMCE}p; z32Z<(7^pEs%b%wFj`$FuHBQob`oWqMm$iM{Hd)_^9sG8n4#WxR;w!zzXqa`m*5>+l z#~DNYlrjtWUM1bP4nmMhL2~Po#W|7fefsw+n}U!@aUmte0zX=x|feon8Av7K^0$nvXFEJ2cGJ2f;E2Dnyz2l)^<~6QKn8(u>1-Ys1pqP6LM?l z=)kSP8w{MT8F3-j!s5gFk4x3!Pl3DtjzAZFW+3LU|2{Yu=Cpr*v}WVBmBtTaPN=O2 z1fTy2rcrNQ>H6hC$3a7=L;m32bc()dCAQ<4APoE*zTP8^@!*Zr|1vNx%s?H8X?nw4ojxfl2%iN88Ex;CMPJSR+^!IpoT}tU3j^*CVn&a>Q6NG!V&m%CuoSk4Sr1qqH#e_ha_rB6xpNhwq;c3a%@>;HVzve#{cc?_dMnfv z77su7_uozo9QImE>dQGbTa1bTaNO+1iNU@*0Rk;3__sj?>x{@m|Mii;gn z17GKF$kIN(`r1nDyQJzhV2L^4Mga~lER2K>!V(3X;ZD@D@OVX5*s{vY?HS8S5+yj8 zL975;2&@;XSjav~&_Te)6AtO~vewlk$z{njnv~1i?_&c-04lM302!iuxP19CW(#>4 z8Joq#Vmk`&-9rK;fXTg1E5r9W2R^)+o9hB|X56nZY)46&bLS3vC>aQ!RrrX?u`MAl zD~q2RfkfSBG5lJ)re|gbi3@5U^6$jVyo{S}CwmxfBy3hmjq+!fz=W>pY_R}<7)2h& zz%UU0Uf`p)fb;vV^dr)RZz^>IAHwCEM2g$QM#jOZNj?O>7d5q|{C3bRA!3S=m3wNk zY90fy^uF~XEcJti%L0=STnOn?-{>@^EASo!sQ&ZCcKkO)6Y}ow18vw3wS#t#Hf?Zt z_#tR0fN7hpA#Wx!%j4tM!fX;Th^iM<_@LzrVPHZ0Z2-gGrPj+I{u-7qFomWFs%&R2 zVDep9uy`fVkn~1MhS7VltD@)-L*=#PAf0uT=fEw3-emCNAkO(Fw=KUVkW~tCCY?@0pAHVxTe& z&BO%86H6Hxu}b5aP$yyUZU>{$@9n#}iTeS+KWzj^Fw)=Olzr{*4Yz@yq6qy1ODAd! zB6n)}F4m^6n#qn@xY=}?Gp<|~@dfgY31?Ik;Y~Wc9Nr&tqb{&{Y0(Ciiy8xZ`;3(w z%>^L#8CRLWf%LR>sPn<@4gUc_e0t3-rvJFe`)NY)T7*ffQd*UByRZ=2i>(|NL6ZWB zT8YMN&5MZ42Z$$!Xy;{$yvf3+?Az0|w11R1)>vDLV}e zw^B>o4oi!C=Lb=sKEih*#RuR{)0d}@W>itul$O&-f3|N-O1Xu8C>x3hcN{`zaWKjS)+e)n&=earBzfE zw+(E%ZbB<1(|>BW)t=+r?e82M)pl?YuxjLC2T1H;g^AA=w{mD^*H$a5ci=)bjSR{e z(^+SZz17F8UD4w+xNY^=DA#@(TauHR=>QT8#a;uwb{?z_YHF^(iDc~e{rB>}tCO+H zq3wtpuJUR!@!CNgDSfxgkEJq&s|nmxoVRJ zvN}LNM+oE`97<8cznpO*`7u<=Pn#O$hhTI_CJeR zKlVFpl;0phX}Y%i?|Z;-DbZkSM0E5&9;M+YW`f26h5)C4fDz0WWFJ{RaOcN#1_DQb z_jS>kBxaWK*fS%zV>eG3&|xicvz}ispQSZ>z>n(+u^XRk`?SufVb?F*Hw0q3d9%7u zoNl7%2MlqJTEgXZ1(}?PsQ@PIj-T=MMny#YhR?9~-0x`|D3RQXucJ8@b2mKFZO9EDL6V|{Ozz|Loy8ewkkG*zI3Kt!1iebd_1c# z=d!fqWp8S#laZ6VbZ0pC-aRS!e%lnZDLhjKKm#KB)@7(GOrTwgi7`Oj2zwat6A>Kx z+x*5`vFO)@sfpV@Wp3Z-(FZh=W&_up)1M-xi8L9wucKt;B(Az0R~?s6<|l3ki`AXF z`_+6U#&RAk?m67Da&m}*gI`mu=4DH`eQx49Jb`lz40jNv>4U6so0NqQZ%e3@+FFv3-mY0n)&W{-D&7n+_`FN= zx5jYX`m{bN6gy#$FG>&2vSZi@>pD7;*bMjYzrN-cf{)krEF4s_FnStWAJAT(+IOzq zYQiv!0tjyV_WH+o*{H1xL(_ zu@-#*K4!MIpI~@XGWbU9)W*K$Bx}dFk!=O{nVRksb^z8MfLGMX&l?&JhQmQ`6@h4b zEtYQKB5QAZb|_=-!e9-v1`J(|jV1s~=v!K*!iW^(K9D8Ly9t>R+U?&!aSm7s@7%eW zib8MV;W$Ystk)R*9~^MS^o~=;(}Vz0SfMA>(~B~!fRX?}+>00CBC5BELQ!kiEUF%g zDj`BTUI58huG7)yG?toZ%P+}vD*WdOA3mstlA8z{#C^Zq!K zV`5+lRO?a-mlT|Vre{6fJ7gaidu6Q(b++Hwbu=xb*lN|fx-e4y8wVFuo4SWF)%Zu zUApuWl_-qmP(EOb*f747#7RZgwT8x1RJSPFfCpMy_TnRef`BmZ1a2pu?AlTI1se<& zax#oIBAM5V3!=7!mG}1T+5)jK0fo8X*WSusSX&^|5nEzHeS7xuK&3qiclf*(sfKd0 zvJiCNt`|mx8>(YK?M9}iTiL@PV*@Z)3=cC5oPaZmSXR8f#z?mI1YZIdfbgIIaEmWn z8h%^O9$6mC<;L)x`{P2#-Ev9pD&1t2I^6E*Ug{lA6*{JlJ(CGsJLIHp^sv82!x)uz{*GR7pgO#bP zK-f9GK6LFFTh(slkGm@9s(T%Zfae^q#1ULvS4aVa4@~!Btb&#-J+mVZruL9TH!-PmSyE^ywlDR|q^;EI8WR`ebz3gu>ICcA1T z%H4z@>;f6KO=X|{`3G^?kA9m7#1c{L2M@w=6f{J{DZ-&8aSxs7TJ(UJI^q<<{({yS zH<#81I|ql?ZEb*4fu3)zB+R`vPu2EXL{BWNoClSpa~!FZ0W&_ z1lNQP2wfwJM6`Z5o@SJpNv$_9!F4tQDL$kJzel5P^R#P&A|6{nQqVGyWvpb0Tzcryf<5n~@Tr?*X6|}q_47GXshIn>U z#%m7$^r7e8tZ-dLF*xSTv;Erk+lE8Ca?WHZ*}4B#3wc!7cHB^0b(k&ReyhQuEyOs& zy)dEf-+K=q?w$SEQ~p5aaO(}t!Re*JV!7|V3r2|o?(!V7({hPP-}>FCZ-s`Yvtoau z2i@wIuU-UyTK$4%=TXbQUu@tK{rd$if%Wef>w-6|esN`uBJJuIuV}c}t$txfb9c?^ z7YDBVe|eb?8YTGsPAjqVO<8Ge>t*Rk$Bt3__AL9AkIk~bmUn6iw$>bPrN}Y@AQ@x4nxv$lvZr4i^2Q>Lpx>*;ylQkzf)m~^c zyEZ%0Fu#8H4-L)O#KibGtDLV_QR%~7baX*$*3pv55TD@F?7Iv3q92vet z^Q!wDA2ws`-o(VjKu1LdDe0R?L%-Yc|f6_ zZy)03;E-}L|B$3kJ7qSRaLzJL`k*Ic-NemAO}aIm2lr6-OJooJTFG{=WAY*|FE7sg zdK;8$T9MOeA4tfzZ)DuG>B5hvNk&P=B_8SN=?n}E8c8Ky5)Q>NmjsDwa^gDtvJV75 z7GxW?v>Wjs6xY0C+au9?wCn5u7mchn^%QnmH6$IQJ9OyKyqK)4FJ)zUL2WrTHPx}L zINy8n4_;N9fAmYwP;>tIbLSqZr)?!o4Suir{#`d`sa``q(nm4Zr0l+!9rtbehVt@Hi_=ke8R2ho|Rr?4iSldu-k;&d*N|x9Ca>^YQ7GdOf>) z_sp#wLU@6578a9@x~7%lk*cW`6&0bOq4DwYWo2cAu93Eq`T2S5W3;ewovm2t%EveB zZ1aj=PQB6a__$&!DdOEz@i>3j{^$Kjc3U$m`-ZoLN^kEsHD;fR&pLHWH9vMePyhP& z%3Uvn+lw>nGiQ!7I}you+dBgfz8*QAYSkM)dSQb`wtc|D?Z}(L8z0^QS{Tf=v$JEiV+}p2tywUC zuGz-;Nb1=@0x_X+sv^9_N5`iyKQ6FxIbHa%lB$fW1v77Tll$&_&$a4j->aE+ss=tv zNVY1PiFf==CVYQ!lOstX)>J5txw+B(xoD)?E}_UfQTsNYH}IVDD84v3(q(a?M?&hU z#G!<`XXXAvaV~{(hg(W(e|W!7tTfkskvgI3!TxG8xQBTS%`3H!A3rKnZY`renbUoj z=El(|=Cu+s!eV0dG&Q-SeZ0K-I<&J5#0?D%=f(EzQz@$aRKqn~-6_ zP+Q6DG3om9%Rb9{N#`zHFgYml&Bo4d{!g5ffk71I$=hz1@us$xOK%x{?G36wnlxs& zpC8Pjc$=!1*++%TL{OX-2%+E1HlD9f-#s|kyS_)v^+jB3Qo>4w>YmhdA2LlAysyeJ zZB1qv9&n%W$nZtKNn35BCejHB2@}pHChgcOPR*i+4?71MGv(yu>g>ucO-HDv>SQ+B zij9wthp>s?t2!bpD{Nf6w?jMETOXx^+lGVrF`J zdS)hykzMk_E>2DnkRM^$kGg`>+iJAk+A+iDM}rh3VHDEHfie~ zD+`vmKkg&T)eKpVEJiiszw>?8(MkKrRIZ~_t#&IB?zmIK=awfnT>G>*E+CLVN+OIF zc-&H{tQO4DqOKWr;$gpQYtP7it*W}D^5v%c$y2AYv$H2BCdh=^+S-M?S4l}>QHKeX z-K?KFD7$4mB0jP_IP9vF5J1?zJ+)#?;p3g%78Vvgk1hA@+eZri{rmT)PoG*_jV!u- zD9Y;o;h)ygZF(T%sV+BMpB9SN;bxSnBg7rR%WLTF&v&!vM%`yU{jA#iaeRZiC9}V> zw{}rxK6HF4l05CNrr0dGyk!5QRgHB(M!(W@j(Oa6P>#qw>y(w1ZES1;)R+~IA0PhlEZOMSa7$r*eSKldaRgc6w_>Wv=4e)|0)Y6zDx)RNWJ0qAR_zTZ_Kc|;OF%H zh_}wQxVX4h5@z3-TK~&K+qZ4Aw6Upv%h}r6xF>ljXBpw{GR-O~WsG zdwW*~aE+z(lXF^&2wfh2BH-<_ZKb?Tqu*nHYM^xH$7G&K>vocTmG zV>YS$S-0Ee#uvwR+TTZ%6uLAPx>Qrt>P>GEXFL>&G7ox=MC;vWE0WPOs<93^)){#G zZ;$4Wtg0GlKRt}Uce!SvXW(~lM`F_M-Mh0nSyd9J76i6cz1XsA*RJ1h*u?G0*RRX{ zzTx5Fv7Jer?bLhYomLzi9PRDx(*upJZf;^POSu#Hl&dKQMh7b9{61TrOK~_`Lws70 zo%kYTX>hsYOe3wQ=?RK%=HiXW8+S!tHA=i{^oV&9udlb|?mf=|19dI?ga_VIgU6Y3 z7n-~p3qKrV^L#GBBT8dpwmPFz!{g(HlxY_R*<^$LEKeI4WN@+?m3W-W)={%yG@>3S zA#MK16%y^l%*`D;c0`UQy{@v)Ur>lN^I6EG`L$`sT7UWf0*FX;ogUEkzJCQnyQE|>gk6$D>nEDJ|GM`nDSNYZ z1QJzUx2|2gHsvg4Q%uaxqVb_22Rof>1QI2utE;QTb5t%Rh(9u8B6o5iGq=iH0fWoZ z(o$2iE?&md#l^+MyC-r6u%UdvEL$KUL4B6xlgcN)h)pL$A~-!K!g`r3cSP+v_338O z*|3Pzsrna6RuhDJ;%7pOIY{Nb>$pO{ax&FQQ^>yF>gg z?evx48#h(OnuCnikCpy~T=74C=q)q%qg&@16-_8a9wU)b)^Fn3UMPS2&Yh?|>ohbp zB2{@rJh+$-5-9O&Q&Lj0Oe;i!Xlp z(rvVTx$|{^m6cVGy~Oz@!2<_Q%YFI!_0G+k{JOosazLeZTGtk)2U&LPsEVH26VTA0 zHUFb?EhBo=MEMQ-e7wd>(NWE^OYZYh^75wgA_UUQ;$of0o0oR)*r&8EuJ}`$H1UpU zS>eW@xVOu@hzlO&E4^jeMb2L`MEvLIH%dG)*KS!c5e{?F6?)J5{kazYi4%F8pEGzM zwR<^LGQ%(~l_g@Y^)?CT{76;%mdM04MDmjhm;J*?cQUk+t*Za|Pn{|$-`O=jHg-U~ zY__Mq)O!-|o>SKAz~RICY4x$Ov4H=DwzY4lfFcKzK&#bUA+r%wL&C9Hi{9MMZvSZH z6L|pzdgsYMUrI}5fhdUN_;_P8Gczx*m%jNXWtW`FrlT~|Bh>xpe$g%aaFWzn$%Lk+ zCKUTbM9`~OM*(Q7#I7k@q-Zs`=k1(THTDcztY}njtx_xVacX2ZyC>CXyYidKZ3 zdtJy|g9Zs59s5YtLuE=FNw16VL?t&j+`V&lBtyP+v$25(rDLoyi*kQ$_*pG}^4y{Q zX{JYo>^6RBn%?DC@gO1Maz*o%efhNO+i$5n*G{!4G}`gzsO7^BN17TDDped4t@`@4 z{h7tbIH&Uc(G3~e=li}TVVCXm4NP{OG1b%4BayDXZh8a+`ROP!IDLLJ^ zq#!LV&F@sfnYl$;vGm65*Jb9r_bJ_Mp82znvTSao`PkT7T3_ze*P@EZqmDk4BBtjr z&M4&+Jv(z&vgOj9s0GQW#oV)68Tk?=l4Yskg=hVo_04bm(aBlZ%yzq}WovEYS6>CE zj&9u-qD-o{BN%mMCO4VP(v`MWX{(uk40-Tqoil5jZw3<)h3;x{{^?>>Jb6Q8S)j88 zON+$x$msGjA<^u-p%nvK=7E`YD-UQA)~)tSp6x6wvn@^)(<5z$CYO%Vo1>W=5Rq~p zXu#5OG&BIZpx@Tj%_(-F}uL~V-+E#2yO_Ju{Ig?(;9X`Hz zPSnKFOQipaqtlAVp>_4zs?~Yb>U%5G*o9LHG6v0TE(nqj9MYL#dZIlZ72q7}|IWHE zTJ}lH_EB^#vr+=Mu(xrMixo!F{{8!z)5GK9N&qbODBqDyxg#d2P_b7=rlj@x+It(> zC6=bYn{Ky}R6)n^o9~H!^oW1RCAYrcW4PIBENZg1hEv|pNG|(K@^#s5YeS6_3|@@& z+6_j_@9WKiF-PD#%|VgaH#cF`knTn>%-9-a;4^MH|EI3`yLQ} z(Pg|4Z|btN(D-@r1@pu$an6z8axZ}zv*mlD!%@L`J!uyr7K4@^URdYlcd&}u#2Pfu zp7mdu@4YlV;O*n{J^k!Q6!M!lkI;*rK0P2VE`IRfK?#Y1uIItqk1Hz=ibUu+2VYIe z${GTZaCUb7a$nrl)%8U|f$;wQm0!QAYiP*(%w(+y22`nOXi!EEE8p7Mmb>2H3LT@@N~veiw$oR5wr_tXAHUG3-nppahF2Z|AGf~9 zvB_+A(9FCbUTq(>Pd}q>EY8!@xIFyK>2o$0>gw7|GOX=WrZO(@)W}H1J&>6-JD2$3 zYmUdC$4XT)>)%VRxb|;mYv_MBaaOWGDaC!-c_QONkFDK+t=%InK2avMI;NW1F@tW&WoG!2jLK=Ks)6jnvwT85THrH^*W2}*g5nW?CJ!Y@tiA9g!tIlpGn7ppz2KZ)gDVaJ-4_>!GKbvy>^3v`fjeU z)GUrx$m6m0i(JGX7Z*+bNK!E7l;_Zo@wxGC&3yeA*2{CCGe+@2h4P*8X)3I*+@I zv>qlZwiUatTeohynwe9sHRVM3c=yLax8jEnAD&H9cklbAP(i{EXv9l57C0Ifkw5+G zG>mnH?cjR);zhVxj&Vs-p;HG)k7>E@&!0cN8#AyddOgp^1yj0wwuHgZM z$SSI;8FSkpsg_?}L|s^0S+y0pDpzrtsNeqE5M`}cL`*g?F#IfX?L9VND|R102n|(> z%6lHjdb`Yx{hF|Y)D3-)a1JF;eU^qNwOEP*rw-xX^`zjuJhA?Ve#;BAiH#2x*7b8q zOE26t>0AGwGPvJh?~cONcpx?YD~J<{iEb9xsUv#jzVldhGNG-+Q{H1p&%mHL;fPNP zms$PeQ*_K65&R{^#pTd6>&C-~$GEwuZgP>eJ9EB)?-0}rg`hyo*UqTYuIi!gV zb~AHW($wgcynoQ*L;c9Anbn!PW9QB}pl>Tn%Lj6oFJbL88M(;8H*ZoI$kbU19x*E` zYZHg8t-U>ehaPm^K$0_l>h|s1c-=hPS`vR=etuhpzrVBd_u?5zx&MlaOe#6$1-w0w1n!gE}`fIOsFnEUv8lkxc#v z0$jSRn3$G9_tQtvavfSR7^^G3-V+lWE2Q`IDE(bXI;Sp%eZ~XgU;vai=4=V z*MfXipVkQ0+LLEj|EDHVgsfNLw`}NC#$(LCfB*R8q}s`o+ek*mZj~J!6GC(bJS*^z~a@vN(#VvW6~Q3P|=@()qlMSOjZu{ zJDrd9y3KJ!T3)`Rv+>)vZ?T7Ls?yNi!ml%*^jnc(7&5mZ9Ncfu^6u?t`zcuFE&be)en}PRhbmzlFK^ zW)2QH|CJ>OBt&vik+*|G7KXW&ytK457uV6`T3D|`n18g<)Ya*KcVp{z#`h>`6QALH z-i13N>ylXySpEoixULt8|rE4W6C870B|9EB*EvU~;#sD-0YXmwt-|)ozX%HCZHqyyNxw0;i zpU{;+mhUx*OZhE%m3R!NOkn**L~>E@IOqHJ?)~xOhiuaKt7NJV1l{oc87-};uQH>z zcOBNOVgfnCc903{*RSXF9eW+V_+_rJ!_P3!_6+7MiX?SQAZh%kP2vgft4hiHbt%6q z!|Pt7n)wT?&CQ=#Rqp`(TjQhQ{GTiU@g13AYo0;2)<|b3SN4UAIBI}%@^>Y#a{N8? zUe2YIl||Iwc9Pp*lMp)W;lqdF*`WziAF(!B8N0dl(g-})KmzOsB9Yi5JpA*;$C?@^ z_<9S0`uh5HSAh>-)55D;7;0MJ@N=sV?|(LTTLopMgz%=l z-EX|>J;l-_EG+Ee;-;9UQ)*L1*Up@T(u9RJa>SzvAei}q6QqD+|0D=RBF{}9|@o7D}_o;Yy=pvoY- zsisDRpTElN)#lBcYXJ?;o~^%kZ_C!L;ZU<_H?qm*$R&&Z8*LB?juJ9tBx-7E$}Nq( zgW6+nqKC^4 zM5|qDf9@RjM!J_ivm#?-@-G^O%FU?QFkQWvf7Ba}Q?I+IkssOZSJ6TM%K}^-vrby-ZA0 z^!?}0pVjyN`!0o9S4&IG`g079-~0FPd4z@GnyFGUbTXmFDCy`lp|&M33?3?7ni(pB zd6KH>y|UzOrvqt&boA)aGiUDJy;}>Z+-G_q)szC@c;wR5r|RmK{PWSIq&Yk}a84^) zXqDT}P^Q1=QI=EULhDX8HiHXsR{$2B%IB5XOu5J?_m60Tm>Fd1(Z^HtwH8ed=2md* z+{vf?qs*sp%DLR7r)u*aC9R3t+|r4gc|zxGZJ(?>UrP!e8@p(C@invj(gBy>pXl-l zEG)gFqoxK13snNS52Tz){7Wl~qZn)gRQn52kQgcFM?pa!u@-nbS@5w(kGOI8VHKom zrq|kXbf09@+Vbz)rEB-6EIU;!>qvWCym&yAs)#n(=acBs8OwZTt4i2`u0Fndx09Xy zd#-6mUEOmd8K9u)si|!whaXSPzdcll^t}Xlh<=M!|6eWLx4(n3GONsadG9J*G%OrB z#uh8<{aoyrIeu~-?S?SMW54EFx!cI^SBOP7kHqjgP85(r&540R8$qx$IR=pt3Mfd#=sim@2X z9j)U*oPIlxT*@Py!?3}uaPdBKrWTD1G{;y=>-Vo;y=kfq=rU@BHVVf5<^W0+Cf)dDTB{)uM&Blb&d0*wsHiLgF0L;9jL8S8VR*@jil=X5@m*5U7X;fbT?4_1CXoK_3wU zafp*}%rUL->;1Z$NREqZ0iuSBncATqzV8%HTcLCJ2jyyT(Eng8(BVmpll`LDCpSgK zcR)F*CkU05l~^PP1F6o~|wSdZ_Jey+F_?I2aio+nOSXqGH z;R=}a`wjFI4ylW;0H&mkLBM=Q%XrB<4_SZ4nZ3|aE=2CDPpbiimz9-Gk)|4DEA(nn zg0Tn8-)tRrMGONiT%0Y$4P)Y7e0;XN3sGF$9qNdHP8J^@UzqCckPwXKCBR&9`vw*M z*u$>Y9v(9o7r>Z_woc=oe_6BF>&VDRq^zcAJN!d$+<>mV33vPG4K)U7@X1>oI25sd0g*l z6m_^?VfhJE4MS1N_S-}7|Dd3t#3Md4C8Oo%AzooE6I+ilbIAU{jOeM3_MiMBNh_dI z(I#wKZdAnt`^0bP$1^}r{ECX{3LZ+@YwV%c&&I}Sz=()DK^s`NY~FlE82no0)@E4o znBQ`-o(m!<=Ukoa`)*4Q=QrUGW!9hKi4}i;i$zUlPoG;c0EAld-Z3MyRRnT{r3tl- zR1^pwKiL+T>gdr|C;)8bzRPq}fbr_e8jY?Z`~2pQAA7NC02#>0fj&ix*ooc86q)(R z=`RcX193~l=p}Vr;b{aSXZ^mNQcEs=q%-sm~VMP<(v1?B)&vu0FmeHaA)!priY7g4*m_WjAWf>LKnO@H1gXE?cu#4!zKc-CKqSyW zv3&tH8JPfvh2NW!`6jr?>(7?Uyf(*c`luK9map zNkqQn7KaXb-+C7`h_Q*e6-X#5D#BpYz9gN%^mjs!ofDOm6cHBYki5_u(8Lw7ZTt3q zH)*?|HyFH`=$p9N+VM zJ(NYX`3l$GJDTZBLoaIZ?ws0+we|E~VuLV!8HpHohy9l3s>BeRD+7o;O`p_g&#V4G zuQ8#MRe4od?)08Uk$p$0V+aGBVS&8y!;pf0XQElLkRHhUuP9h#R##PBXXZ?J@SxP6 zvZ5+?kcY<#lN14&va)6DESZ1}2>ZjPf>0WZkip*F^0p<)|936I>sSc1<3)_GsOPJ4;wo3WapMM1LeOg^>|)9q3P^6|=VN#$4G$FpX4`#6Yvd_e zzPeD`_iy~3h@OK7{yl-vg8Oi@HvKd6x7YFbtfihnUTBu2U`#_@m&;cvCo&sVnS{$R z^y*aqSx{9~r4F-hIxE&5JwrngC0}m&V0LX?lMsqTva_4vR zlfB=w^v}+>L&+`2At43h{80S|m!jNUv>J;*0*kPb*XaOEA)mqQLI@|*&F6EqS8gy9 zQ}>eSWn^Ufk7hQmOx0_cn3w>~B8;{BfZ0(8hans=K|w*(;jA9V&E9P?tKN!NlU;r@ zVvcz8;~;8&etu9@?IG}hvgM!uF1&0HV|YTXin&xIGM#oMZlvcoZnLNfE4>bdPC0zu0mcxwfqg9g^E;lG9jM%{rSx5{pHR7 zOOeqke;rhUm4?U*bOc>Y+H*wqV&_pb!hTJA`wS3-{~&vmV9Wt?5@sRc5n3<6tK;{a zcmg+q;{bvN*%Gp{OV8XGYIt^2cS-iG-{P<&2+Yk6Q5+}tA`BPB?|nhiVV! z<*74~Y+@?eIvNSGKdA)S3H|vk!shAMbTZvNJa&-K8p|M@YX-t)qJZ`S0=mfH_wG_v zkd;wShQE0l8k(9M$Z4n2OC=MVKYUQ~FIe*IxUBnZ1v`_PdF=XzyG#(B_h-AERtLhw zh|Oq*F(-VY4QT}6gC|QN(5mVlF(89oRZ|mP4FL`Y6LLQ{0na(MZQENGj|vA91I>JY z#R7w3g7fvecb3o$FcW(2vx!;;6Sxl?7#p3UtC4A1MgL=;_X`TP=Glc4y23T&YVGrJ ztZ-UCS69P@^yvA@2>XnyS4(Ra8x_!%)}G)Il>Y zg6=w%ZNh`;P{-?e;zVV{{@(Hp z40$1tbkO__8OdARd}wHB03o^X^SKo|uFDsOt@~6IZpX&zdwI>h31sSV-?$eGcDKMC zz-W5UES(6=zYpb54OA3OaucH@lGOwfaP;tx6t&c{8{wK=c+2)wWzAB+)Uafb@3tgCDKiBnsvyOM-@E7F;4m>cnFe{ubHr$w5qn|Z zQjksP!Yu}jH>dYIfHnc=I~a%nk$ar-=eUq=#a8zB_k+~QKp*9iDRTJv94kmqTv}R6 zS$cF&5=5r}n+6#T*l!Mpo!?v6O3blYjGLQ^(j$NF8pnMI{GHHww~*X0<#7vR8uGll zdOK(ho@AB5z*j0-*+yGR{;SD%R$>@yXMj8xn7u}}7m$|l2+P>R{|_CwimV>}GxR~;^;jJNK0dFxZU;xl)XGgL{2$M))Ocj5{2|=}*B124eZ6@`w~(lAjln)M zpl9IAoTZQjF}5$zY~vstnqqhP$WIFkzl&&PHiB&5hmi*U3M){?t&hv`khzVGXWuuX z-9N8^RPn9TC>;nPuuR&^FSpdyxnQAxuQ!`NM(rjVs!$bR6arpN=`Lobz|B$V5QJh= zAz*S!UA-VLZyO2t1Snf8>nWVB{uvGV<-D#&$dc)e_VZ##ju(lCJ7sX+6VaY&&Logq_VaWkr>oeHqUMeHPpYZ0 zA3R5vz5eF?dnC2ya?jOItJkxrg6YnVj4x>^jF4En*r)I|IScfTAWRA)#~g5IiV;N}veE4Fc9O z9^h-Dys6)2=tU^lhEBo_%86z@qLkE}!gdUzeMhXY-`jJe7G2Y|GSIV5OaqyISMFmr zA^m;R9LDzc1DI&`-*bdQH^Z~$6tg{j^5ktlaqsjuU=&nkAhHStelr`i`c#Gv-V1!< z^waH1&+rat7Y|r`xZUAT!7V=M%Co7~AQfq*a)@W7`+_+~I;fgFS74JSQugDd+Z_1w zBn7lvbr|gFwb^>R8_Gm7E^n!--ZV;xh(PVrU|7N~1f0jb1XfTLkfJBHd-ujGk8ClK z`1dUz*Vq4vL<&dV-m8GRU^ra5QPKPn*V(BZUExcW=>%>@-ZBD@*GL~j3kuDoV8`y< zxl_k?dHD1!#=4gBXlQFLfb3 z@cRw=J1aZ8?o#MEG$|CFat!Cy=%~97W~c(hQJ$Wj9t>ee;A!~(^k+b(pzH&%?2(d^ zf?x!^pIl2-?V+H6gzO-pyHoSWp#Pf7x+dPK8i{hWhSa-=V(sNwSq`9ln&ziZf4jL| zAkSbJHKH8C#`8sSE)IwoO#nzBE-}%`(vo`nFdfEU>({Rr@J+xegKQmx=}v!`V(Lal ztx#j)A3WG0Y&7?lS-v6Hl;SjVdgKH|=^HmrVCo1oAsapiNfEXt+xG2^*@lvKFM|NM zpo+qnPN^s$kkYkg`f4q(mng)#j?}eYQi1AeKFyb@V`WShU~332HV`Tg8TsEz@Nbj` zzyF8m$Ve#Sc;brDwvt(Z8;qXc?54Pi08^WhXhOYO(Q@r3Rh=37aS&(*__OFDNDNN@ zHj*H~16D7z=QxyYTz$wfD#A524tDmzkrCm@^ZCsXTp-%V+`q4cC~L)=W38p$u9f$A zpcB*yGiyJpwQaD^$H^)h_5Lr}8fG**yCIOq&DjgU8mnaGUwm|dC?Ieir3*g-L=dLW zli!9k3x7M1))zpjl#mVAH?U3sF2vryuao84Q?(u9xl7=$U%%j$eYelYz9Kga8JD(& z6VC*q$4%xF6ht$NU<5lE93FPOaG@N41x!Ug$MZ1eHEv{*2_WA;O1)ngfv`LG>n_V( zUU29rU*2rc)bB=_oLxerr}C+#$r>~-q5b=zwBQ<3H+10D`)#}+&k>T54bgFN0*>)q zWcXs)dA8hRdVf{YC)eJ+Q{?1K#Q1?{c`RrX>w`lS!oLDiAOV`|PI3E)*V2&SzJkDP zXFQHhl$U$;H?|9UNS!@>WZG(O1bRRA6VaRih&fbYw&D%DQz<(9CgMB%8^ZVR3CRv$ zS6nhA1%puPxLmq61oHg=#1$@DhwtPUy2c^|xj>wE!#l_^O4ss3R2kOd@%ITZf=Fid-7;`+P0aWQJrvltU!8kQIw7ik$8!wTQdS63n7xi~xDRY#QqAsA|EZXpl| z92~+mPhUEBUq>r8$ex*;ymk9_1G5On%C)nPPwH3rm8qv?qNtEyczt^hsu0i!@-`{> z%9ShFQ=kig30zM$$~XYZ4-^(Pc*4xA6JJyb^*qIpg6x1Ig20iPlJXk|DgKa+`u3HR zuL6!-*IvbFrLuP*ijc=9PT>y`-gI8?-h#yF&6Mg|7xUykq zsbX-61S1g=5@mVRUpEV(_m&EO3fx1O#T$k0aJ~3X7JywsQBe^-Fbmi-MNNG&Zr!rwzNC||WMp)7G%#AE{FX8Ke@kesT?{qJqdjkk z;Nxg_w}q3FDXxq~BH5)zDQSia)X`yTBKYm+PpZe%!?vHFzZoR~TjV?R2uQ*1K)qOx zOW(HkboBC`i~&rY7QqdRLc4mU;)VXS#Ht>^`g@EZsadA}z!;kIXqu*#v+!IaOjL`V zF$+L7)~{zE;;x#in%c&K16Ebr`rD~-GGXdngq{jLV1G8r3xbsQ0OnxPJSB8m9DfBg z{;((-`c0d1mV~I1K)J*0a0@RBsyM%k%htl8n=(sTF@;sRGRziYjRsoBcf|#%sNI~L zrsFq2+Qa=)f%$ucdw&1kz$s@`+=i?K$5}i>s)#qT1i0u`RIZarz3-0xfT7GkUyeDk zO0M4Hp8KV5z);fyDt06Q9nJ=lj!qQDFX2r~3o4PSKdr2VEl1@`s3^;hEBxj{t0_2b zu2UzG1v#DB7+KmwHGVe%=ZLUS%e>h!2^9{|Y6A9za|ju199kQq9L70Ou)KHel6CHS z2iUL1qnfI@@L>x<-(#!lO4tK<`$kr#h+wB1iOebPCSBXQaxpM_q#m>Df-0W=fo<-;Jo-U6zf<9{8?)9ZIjyAep+ zcI+5I#_wJsBD`vJRD{k1O@Gzrr$^uzeIJvk9q}b^mUWep0=n3z%MhiAWLz4=onLr& zskOEUb3$N8$U>0(sir1T-XGi3SaFalI1)duAq4|{ucxKG!+K~RB&q`k4hRZ50Q*3i zKMWbfh)T$*H@XPtFtiq&zN6#2?Dmne%Ol~8kHVV?qiJY7#6Y=+H%2Z)ib<8F5Bywf{!CA2R;7T2Fn2N<%6;4shX|fq8h* z!vi78JR=2Dj8GgEyf?fK1U&5M9BQPvxR`%M-p7-JP`@C*X(`sOLpN0=_-57_t*D{J zEjc{=3tT=0&vS{!mSxXP4TGzBEJNa6Za-6?a_gQdq|~~G&r2&_l=7Nh->)ZQn;1sF zn;z~xY?E;4ko6z$#9r@;!@D*G9fDJbZlE43+5=?=Tb5NEaJpn15Q-cI${ETonFEw2 zVp4iQOiUI|7jO(xN(dqzaqd!tj``nDX-F95wP5uV`$%d*vYmlVul7B{TzS(lbAQ8n_dS_px`< z>IbJ*hh>@QS|XW1q9;xTTy0q-hf$r%dbtA!2+jB#1M@TQ9iPK|3QWfD=r(eYt}1zZ z&jUgufw-9jhor2u6!N5P9C9;K{tga;lX|A69mt&nxgM1K)6;{93jKwCjabpZKh9af zfr0CZmhkGi@6r$K*+N3v&m02CzO3v62M?w{Aq13^$n1)YiqdJU&fSg-(k&K|sPJ%$ zjYRm$V5nAgJX(|kmPqf@<1J_0e_>PertcX<&#a_;#ivnM32-loJ;yAqtX`YBL(UiP zC+<9W4w2`znO_haV<2KX^tGRH!7JjJRVr=7C1qq}gooGTmp5+MV5UYbri|$;exffA z;Kn5N?P-Z`-FNcJB{~R8C=I;*>=ddCij80(q8Sgimz6{?Lnq&U_2^Ohq{6;TOE|OW zCIJfeGD$^CE26CINa)Azi&7N`oN2KbwIHS9kCX!GQ}E_J z(AE&3bpwe;K>arEn8BOHCt}1L`=v`9j9I_B(xfK}d?F$pg-)h0TE#+e(cRtM-TY)N zkQ3g$6&#>?IqOdY!4- z+Whn9Ya$?VnCYuepZXA^*}@Jd`eV%eDk}m5k85dZX_tV={xiecMT~b`_KdP#3E}7CQ(wtyrAOb$7vU|(6BWF+x zbrjbW{(geVZV0#F=Bm3h3xM%KQ7^A7#TIS8rc4fl1XaaMxv*|3`EA&mzlWpZ$)iV) zz~aH_->AgB{dM&4QQ>IAT3;V$pfD)iN!!=NPEAiEF8$eH8&@iJ`|Sxiza_pNB06%w zvBxK4Ns^5Jtd*CSrw=@h+F)W}NLR`gJ#H<|H)2uLJaL-PXI8u<0g z8DVG0nHZCAxQa6R?)e!oj+fXPUq^!PTZobH_m&nJ6gw;zv<7G*y7u<=mX^QKax-c( zAxh>Nm=x&~;z0-&#l~{Q;<*6O(H9wrfZ@>JGnMG3u(tDn-dIfh>Y1y0@O%tF~*-?cX9;md0ViFeIu`P(3(= z$om37po{q9(u=Jt{9Ng~M}Rqu%V_t`=}o`;%;CQW+;~-VAL~gZ_k?+WF?Z{ky()k3&o7wYzrw8t^YI9UY<^O;Dp@aX~t2F7uJh8=slM9n-to%t+!Aea<0K z1(7rLa0*0dX+h^mMaMImn!wO>M8&;Zyf0n4smQIZ-3WgIix*0m=zP5np(iZPRub|% z|1@&rx_19BwE5GLXSfK0MlYX^EbLFk5X9%;5J(`Ii5O0Alz^n@>98yc>BpTx^T&do zJ##Lb+Ez(>eZ%TOBP`^j^j>v71<{6pYBp^A)a*5I00WUz&i6-wW88*?S^p>zQ}vQ} z55`RILLQDehHEgr(!oxu{)bM&40RR-*+U-_9UHFp^t?|{*G40W9Z89HLynEO}RIOh@b@_t5$bnJJA?K??kBk@R$u&oj%}Ge$2>r+o;ECr5P6@l3lthd+On#=P zuOC}c;g4#@-MW47xaS2ocqMTjxukeY_37t?BkZcYxcy;mY5D2{`sj3DZ2(~lJNtP@ z$KkH7=SU$UWCFu~3S@#z_~fb|o_SQBMnCHOzjfZ2`sU3Wq(GZKU|zn}>qO_-zB?v5 zS~+L~MtWA8$@si4Xg-NPx*9_&yL22p((Jk{@m*zQ|JxwuELwbwSC~~8p$-p>Fp<6I z?e=GhP>ej;d;k1F``to{h>V0o7*i7So<#w=N4^iwT>!PoB}{PSMz)LN)=&NS@4PU! zVz#lcs5u;~H^E0$e52a1_{^;>bobiZ+t*%w=pn)=epITOW_92tC4K=Iq7ql&Jm_ml zNws)iIN!(C94Aq8rk%_x6C!sD3MAq$ZaT4I-<}Ee29q_8-~#*z2yAa>hu4QNkTsEv z_U#4GzB2b&E_*f@1(?%w)J^ZCH!o!T=;0B=X;Vfa3v41pA+w zsxWcf8FJfBsf%31-^khLsa|{h_z!9goEbh|R(g8nuZwWgMXjnLK*r&LH(q@7&%(h^ z($-i1wulUIw0fMN@VROvqbJbQziqHztgNoqb<-#>Ej2oIDulxa@Q}?z5VxX!{D`UU z!S>*TeZmPYfN}lI6y!1tz$R-SDk&(KgKl?C?oU~xc0ehVW4WiAg`|RO5vlS8Sy>yl z?Ak)YY0>k+MFPB;q0v!PO=j)0s@imJ9NUNHEW3BdqYJ7zh!E(dJ%&{1(O=SStv^DE zUEd3t5O+=U+ij-yym?9>^`&udxq%NiTwYp|J#qy9?le5x6i)>O0}LnwaFGp5&=ZTG z3}Gdp1vkHXb%ij29tyF>+Qw!p$?O~EFP_CsCGZn2{Ad~(Ih;RFy>LkS1c@4s(fK+_ zTpz*mB%SW-@1OtmmJU=HL{!b+3ry0q)N3<=aMY=79V7au$$QT1vp)bH zq#yzbzIBScj)@6!x$$F#X=%KJv0B>NM-b-%cH6dXTgKm~=f2A)D*7B)jvR)F9rg3v zAi9+viLMTuh()?K>+t4nU)E6XwW3XYzm!DAhbQUull1fx^!-iQaVmTH1qFBElUkOSU)VSD-=G}xM;yeK zX8nGBu(&=}Sp7`U1eQck2LutJ}X>TQ}Y_@ z5T+NIaQygtq!-vco?{Wmf>8pQztZhDH!%GJ7LhUV3!zYC$Dn4f3xF42+?S+?jzcDZ zd<1VoPdb$-7jNAl9&q(l+`Hyx16l81Kz0m7e8APk!@ww6X#xrxcT%UZ{5qJ& za~=-gbJ{TSf0e5$DSbc!*bLV>FMaXv!6&lhGFUv9XS`p1y7ZL-K0s=N2m=vP9)yU% zVH}=tFj1vzwvKBNbA%{+^aZfj+ZjYRPt=E_rl@x~N<4F)JPFPpM8h6iscmS;qQw2c zeG|0dh@IOWy05`xIvsMz$Hxa3%I4?hv_uj~)71XI0DI`9q$rEdkDouOSlhh;7qiSp z;lRhXnD1u9h4-I7ku*_O=H=pgC?~v5+<`izm&ZWfaF%gFZ__5YW>&)>a{GNna0#Mw zmjTWu|Lw|^qcbCIOd?gFYJEtQp7iwe+&OmM&JJ>aK;%toz^O}|2|hYLz2}t!BJs2( zRNwLhXM3slA!x31Cf6UZ5$;8%^z$%UP0FM)uH06I)5Re5z_|%ja-X zMdj(UXIGCQ4g&M>et7sQHg|B{<+fglbMPSY(ZTM-=IkIJbSU zWPc1RV`|E6e9Zv5!msUDjWJCCQP|kn$OPeN8P`82U;g^sJBgcn&u}B^5?5jk_2Wly zs|6&Enjpg`WR3411jE3fBNHH_-`Fz%Bhtyq>4x^~9ZvbL4zZDu>u-%=at{7V@CBKM zgXcxPI`@C^_U6%4_x-y#88W49YA|mhLo#a;wNYj=Mo1-@5+OsPGDU-V2q|Mi3dvBW zC@Gn#6j5X<84ICved_t1v(E32v%c%M);ZR_p6Bka*n5B8@7Hj>uIqY%iE~p5gH+Zh z0dY?z3@u|wNY$b}g#g!msu;+~5S})SMB#RUA48D0n5ZqCXsWAAM|9zrUZd#w?+IF2 zTQibz98aCb#Cf}&yil;`_uhG=C%G|xbcOp+xRewZkE+UwerN;OW00bBy@w9NM-WYE z`%qu$y$gWZ5Y9OSEOn}A==gYebW--mDDT)&mZ!3WVXf`5xXlZ+|LZo%Q1QclvBC3CuIa~v)0{^7#GUq^L(24A`2Y$6k3YByxir0nh69g)# zc)i___(V?zxVy`$bBtDAggiklM=b|^MZ|i!>}k})NVb}($Mxa3?s3!amusWY zyDyhyWi>Q5hE-EzZk!~)Eh$@;c1;}=81FG3$Ir!^oPkk8_u(6t0_greO)5kyLZgBS zG3GG{??vpM^{t#}GjK`?;Pz-W@8ni7adDs!ECJ=U(o$f+^&E$?f^xBS1hNWtO1af` zKJ{a|k1vrJXfas}x!VU~8UiEkFaMr%+sMSzXzua`_z-H(mHioEHOO=lutaHh72;G~ zkbau-&C&_nn1^5=O(KUhTSukdCK1I)?>Hl~GLM9W1h7Wi?+7p(ly^&d3UM+W)2ey; zbQuNwcNHt>8pFfiSBk9mO$t!f0q8v-!0c1Lae`+`H+v%jR$d-jHv9Hbx!lHgml99}kcm-YZ)oF}x}PYSn?4JXD7lyL5Z8exTZ$c|LU96U9ktKgDJ-lf%At^NDHQ1L~G*eTDWje@M?NS#&Kly z(_WDQY`DqSDsHuo7$nJD3b^oV;%%hY0FY${0_LlDnz3OsH#a*4iu$JBRW!DF_wKdd z2nh=-`T6W80_$6i2TpI%U!|7;B)?;hCJoJi`iH4S|0G#jbQiLWIl8T7FGJL9^bXj z5u`gJW;X+kVx#mvKCSb%8$H)eUt24yRD9Gd$0vk>M3<2Y>=-=0AS94yz<886+<5Ht z={MTHG`Bi~R#eq<-t@`p>hCizthqYy03+D1V#Ihusidf=?j(bz_TNY7;O3oy&;bZR zC_1;iJju-dthe{wH;||>@L+5zoU<6h`_gKA;>{jrR5$L*{Oj{3(52L4O4Z&T-A#pO zM}6z303?XjztB-269V;8zV#E@LT!0A^w5|<`k{~@3UOZ&(Ey=~xSoc=L}*9|?6E}T z?X^rsPMaTIygGwmEQ1rMxcNO{RD6M(@B&Pk%Fk$UJ$npzrl6pqO8Tu%oESR z6nmC{jsnJ6Fe|KdyG7qQXtS4(rv$QzZyzgSy;gA!{L9v(W`GUcqd6*aLtEPvQq`mV zy1FbY=aH|0b_GX4b<##vhpuOU(ACJbE~CmvP{yn_JS;U!>ndS>0Bw1jjr-12ctD^_ zt?x&n1+B-zp1pgkpxz56ZP~J=l07{)R}=S-bllb!)a+X2lE}#QVfC=0IX#f+4E)(k z0YY!G-tTcPb)S``(wdq-9oahO_bnl15Glx;(y0Ron41o8SQXS9AZCL{ZoeJ|y30lZ zzp~zGGY1C^nF{j6N_Ql0w>xNzFu_4EMBxI;w#KYRDFXAVD>_;A`ETzzRJJx==h<4l z@kH_bJy0AJ-RRK9g}lwh3q~Dzk({CKHp{CFN%eTwJzyNp$V#|h(8Dw{O+fikUQr<~ zCf1hB%pMXP+!k_a8ydP(iv2)H{liO+bAVOhb|oX@Gu{G}PLxvxNT?<3Ga#gX`0xSn zq?Pi8Ev80Ba!y@h;Z;uWi$x~1k$Z9L8Oca5$Zsbm$AJa#c7qlV1~uB%j@hesBXaj( z_DD@ljlszZ%`xG_)ss>rE%;f>e{+B9HMAXlAD?Kt8O5YAa^jJlYMWtx+7t6 zo-w$(>((xhP**F+2Q*aE2@@AtW=^F3-H5Uc)Lf#5&Vy~<3GWdp+1%cgCb-d zv6amZ^I?`h7J&f+Djys=?0zlpzqkNEgiu@ps$-AKLgXw6L!dl6maRx(NI%+$qy~`% z^npZDXhc?PZH=rL&=F>DSCB=ak?6yIt*M7C-LU*vJ^i#B)!%`Efem2lq*1o8#tHXDBs(Q+T#BGH{3T4XlU&9Ux?}sKwS|` zvbMJFdFu)baeU&y?=g^QxC_Aax$ami1~$ixY9S^TQ7ZRelvCD0t7`Z4Byo}a=ENqx zuPn}IhI{qx6;@{EAZZDV*21+e zj7ePq*R1H=^A|7DAi@XchClP*DoZm)gK7h_i$Ir7eU;7vIrOID2rk9|e<|{^hH;&& z=#jngy7xDRs}p0Nt;R4%#yAOBHP|kh-7_Gkude($DlWjlzH(*cVSyFsnP@2_UWIPr z<6F0v?Y5#|J4U+g_&yl*d~^_V|7 zjATR`rkkR4f5Ul(djc#hEJ?Sy1b6P-q;lBP(^DqA$zVMJ9NrASem&$z3TA^Otr8(e z{u4dk%6G~HF?>nV$O3Q^uAYJOhqPONH#ao&1!m~bB7L5mEX>bm=0PR`{l)RTW6tHP zGNMPgaN}X+b4Dby9CK2imha!cLkXo@#!Pz|I6(b&P>LKoe(Ext%%={zLbKX=zo!iXVtQC=x+#5D6Zgm?#nDYzv_`>aYK^G%u-}2@3&_(_ZX( zwhvuL41ugE$;e!!FFo>*LvbHb9yE0XuFeYhl||evV#GZLU6F(0>E$KT;#YOUG$~K17x3V?KELSoImMW1N9cp{OIXrd}{Ev*mM9=x*OeBzf3JIV{)6EltdVZ zmUX_=7&hg?K;xLj`Lk!`^xnKfPboIYcxk;QO|dr8B<6)DPMvCex*d7!?dxr2+X_+T z0^Ws!H)QmWnJ$=cZ|jA~X^ z1~Eg&`20%zQ9;xi34O>aSJgAndZJ(KoHn(w;nywYsCj)WI+`#b*u>@R>@2^PwXJ~T z@#Dup-g(fzSu=h@eHp&Ukmdeo^}zotMBMLuzseFuw)^?Y}z8^NMNQ$1P{6Vm}m;27UQNWg}gL8wm0Dg9io-n|GZRc76-V+dv37Afcp1(1UF^-debTp;s@F zMsPz&lhLuUDQHwmTrX=1M?^;6)zJ~dqX%;*+&3_5EgiFk2h3-L2FMcP8NM?^T4vu= zAQVP{4(J~mmpq5ikhoEQH55|z8PqAs{c)@~h zO&!9FqhmNqxi&>TM07*X@KnCz{a&XBf-aa^`=Z8Rj8YCf+ zqY=nIZg)>&u+QEKK?8|{!+5LpK>#S&)bM+tvC2pP{HqTiczR{>yD7XIT#tY8KQX44 zBtxCnj1Nk_Y-D5vC#$nA{r>&aCMKNU0DJ^pM_Rlr1*+-gL)-rr7&4hQ!t8Ww*J3)> z=27i8{|Xd3aFh1$PsjWMj8IgW`HQ$N!nNz{Kx(PPOs#F%pX+*&qkmp#`=yAss0JXu zqX#Uv+=)q2&}7l7M5S=2_yNfv-vXwn2DU& z9S6S2q01-7QO7_Vd%XLG6GD|_g)1tGZ^^JgX_$q{HYEtOL(3;rfIX+-l=Z**mR;LG zx+VF$^q0G5W}HBzAPc^JMIIR&V{Fv6k`5Kt*3;|8nEA2GW)G_sT*^v6)=<-CF~Gjl;A} z5H&bCLD^Q_l<=XqH#9g{hNsPr8ko%eQDJ-nVM_50t4V<56+i}D2<*d5C}Ktsm|LW% zw285?b9WvW@3Y~v5SJWCu{eCV&HDjn(_n@8e51)Lqe!vLl@L{1?ki+*wCMmRkxy*2 z2zQ>^0&~h0^ny4=NKAvbF%qy>gynUOLH>#TY`R#7*AOvdOUpP^5NNU33(%U;s zd8)RpElS_-u1+WV4$~&B0eT-?tRxej8ODJd@=xKLw&X$|t zLN{UF@f9#+ly02X{{4f%{hRflDKj9XgdbWTtr-^+gW#>p0PlgT3MB!M9KZH&fMwd1 zvhc{ru;q=wEYU#)J)}USiptO;ga?cbsu4%n4#3O>^gs!@-n66j{(N4BW=#8YDbCr!eyo z2+2+;<%m!2O^Y@-#HL7&4z&3CpAn+-C>iUv#iI<=?$mm@o-SYM%3*$5ZsVvK`evdf z6(2rZfnw$A_3K!LP>mrTQEa{|%*%U~oN_Th<=+#)kFre`N_>r>ow&2K?sMTkDMrlT z`340fg8mp(F@a(jop3NoO|2e<#x!BE@!ZOaA^onwscmdryJ6eT-Mh7y zm(JsAgZA*mb1VZ`PhOrAWE&{x%|hq|S3VpNW=Gpds#Jjf^y^PRQk@Du7wZREc$Bt5 zeK+rh;W!Sy(vBS?Zaaa3!@CD0kl39chC!2c%R{7+C(jjND96>!bT1t3Qr+V7AEyDyv(x_s95QtfM>2oi$9{7|L)-p-Kpa6WWiTzu zU?fsEp5#e0rF#v)pBTx6odY7#T*BfKv@!KBfb+>m!sTmqwpA62<#3Zep{c`9+8ema zdk*;J^$jz2@IgEYWrIQ$MOA@-KJswFbf!XZLD%;b0u8L+Zgy`F9T6x~tYI|i)SyYC z=alDZ^9sj(^$MV7Aa_xrO84>eE2dZ8r>}p5%aThrU>eQD{FAMBb)rFLPm=uzKnaas zeIcf{Ju@OJR-i1?X{-{p++p9fD1*jg&PQv`jdx3K8tLo}=mPyx_(sp@Io7aFAo(cI zKtWIKmqL6h@!1cUEEr|PvzdSm=-`lR0*58ON@?UztQN#FQqy&4e8FBc|?DbQ)6?-!_mWD$S zpA&pSq~9?dDYtS3_B)%Z@cst^i1U~eMkr(=6^CF-2Otz|PC!Zj~7U%51l!>2} zt){h!Q$S$7_cpW+D6570Z=Cb+h%IMgxrF4fw)zX4d~wgD`v>p|qp;>UdJ9r7<}HKB zbj1hENJd+Ow^>)ld6~e94rX(NP86=jCr*?&vj$f^xpXl{_&*LgH^(9X_~LuqD<~j*M!s@j>NI$3Y;w}32{f12 ze_$wqp+T)G7CkYNk@MNjC%ddPf(8-GUv^qG{ot?^FUhD32V*QMv8$saW|0%rpSnx& z%3M(XBdV>q{$XZf8n9TwS@~3Tey#j*mUZjKn7H{P52}F}2fdPRn(`t}bJRT?dd3in zX4A3b$7lM&)N0lcacT9SjZObQItbAL8-3Q^$F{|2KUn}(TUr`4L;4JksC+%9ZO|0( z!e;RovIMTflJ)T9Wc8U(D$~6+s98H-zIt`JaulQF0$!ry<*+?tVCo5e4bhy?I4NpbH3H5Nkwjx^}1Lh=jO`_V1kRY|}&Im(bSN`vMhi z`lv7I8nLnE;wO93Ls+=H^3K<*KjQtDzz?1=Gk?p64=L|O(RM&Y{P`;AKm?mtBF*4Z zfFcoVR@RHL>Fz=T)J@ia)Tg#tH$23D!C%F{69f6^_^2^p_;ZH_j<`YfQX04y(OwYD z%x7x{2pc0$Tg#Q>a3Ts}t=d(7w?=GtQ>vLjK@2W3ru7VrMF%c|1$il@me2<= zg#a?LDtAkdy9?jDpGnRDf zgpa~ZO$FKEYm2s!VE4ep1oPeaP0ud=DZybZ2u|Gg=nU7J7GR+0%tV77fU^X0!wJR( zz%ct!3p2kfXqYswy*4ytJo_?=j3FA5KRONVLwShF`&;gzj@IeqUzsDy_9X%E4g`kiYCgUu}nx-pZn@%;SQUA^N`38D2(MWm+DL@cA@DBKcc^AE_J&KQOu2|RtJI`{j z2{TiH!n5GFLLc#2a!o}7j7}{UwTeqh5ZD>XaFH#DSlc@Pj(k|`54aTVV=^roO->v&`k z9ECceGh&Y}hUDzbi&s`3yeqzWbMr(LZURLUnF28dwcOyhZ`;!PAy*18150M=<}T952w@Bsj>PDmEJF2HoOm`>@VfDVt3$h;Pp?a*YR$1};jx~iodbI8e zIe{3RIeF$x_A8;Q3!!<=EaY!~=n$JS7hWSiybWhSc*@Pr)?k3dId{A%e-LD4Hz}M8 z`<6d{fc&|9!hqb{pcp8-1B(ug@Z~WsJX7{Si*Tq=gFL{^soll}$Y@%bTToeS(f*y*P9R}KDFZ#3nD^Ae<%${_8L2;&gJ;%24~iOU zCd^0p5hHNYy1TpKM+@aTUBx5@40lp_$ijkx7rk8k{gqjYlDtnIGTm>lJ^kJzp2QRa zHiG@vD%g()4}tS>uHX?Twp9^gL8xtPZSTa1Fgg^aNAyfX$^+EJ8;nHU76loZw)S?A zO&J*&(4cd44}Y+SMWb79#D+!u+*_)F{e68^C_2#y<3xdNboYe$Zmo1^#|knoV2P!X zD8`vic5%7%m-e+GqmQ;Cc#fV+E{oYURHzA#ExI_ex`jnW20M`#aCHfjhZ%hp>(&kg zcVa@TO+p(Gu9}T^I1t8r>lfGpm>Wz5kiyk*KSX2DTK1K@t0mi{AEhP0A$C~O*1n|| zR=gG-e(6;cD$thhZbfk6eP7}XACoSLYQ?8`Ec52yWf_=l5;i*T@s9K|yjty^LiM zIM0q}oEN@nNpIW6!VkA-JmF1nDmbTnzQdWN`2F=oIEZ8yeFm+2s|z< zOp&vAqGDtBxD8zNIiZ}(7~7d@ajveuKFC5}0|BbOK69p+dvs$k8RZKtgoL3- z*EeT{!c7{$YUU?NJT;K(vd&{H*}%NWK)Ys%Bcch5q8 zU*IxRs0gz{uKO;D{lEyuUVk4QeTM}HFAZxm?%pK~M(hW|dprxSmgxZ^C@PXoRX=j% zJVq%{26wczk$xoM7J>$R=apbNjCB5TD?K65dTavDKp^HarX~vyZxFPOskTQ$>6qb` zoX6XP`}CuvN9uF@O*rDt+2)udP8Cq}-vD}AYJzp$*&c-r|GUP83}^`(o5Nh8A^(I? z8H(dbFszb?FOh1A8L6D*;!Q=VStO9=A6_CN*+}vAl?MU0zk9iD$k@tC8TQTP^L4{# z!`AVbO)N%$x*N8I#4vixbOX4Ius1?af~KxhyaDd_o^A_}*w?-w+E9^K|Ju+3Zk<nWsPe6bs_qtl;NVex`;GAM-iX zxWG7{B@i7ljbv1Evi&~VM;pFk+~u6DI=b-^aRlvvfQ>J)7iEPCMUP_e2nuETr)2Op zFzAM#Q(kVA0X&Xyc_bddKaWxqhjRlB!=?kG!6eL0;jxZ9Vd|L~c|+NP>j(UWS5{Vb zxV&=%9lhDj<_K0NFtq+e4W6YeEFr@_pcI@2nKU97C!Ep^Kx@^2*4NxgyOSsP zD;!i)+XDG;vKk1ddOtKuexyIt|1U1UIlz`nK!-}-Rbgh2;jNB*Mh@LiTgH9XzcCAd zv5NvDA1wFQ@Z~T1U9a4C@NFK`){UvaFA|Q``NLk!=rsB6j)!FMi1N{=6oFHy+obLkW*l;38 zm5Gqn+dIy_wAe%l6`a~&|HUjhouswyd3=04-(J7Cc-|}94b>ra^?-P2YRvMG>^YMs zXu};@Yd|Y8@87v)iyB&LQbu868U}c(dzTAw+0mv##_#FrAxGwQoLjOk9m#=hTCK8Sh8 zEvEepn<2;t5(Xg4N4Xzh36KMg2K&t<3Od;AoAj640{?yfqtJ))5Xw+2=dkNZE@X&o zJdcrX-!Foa$B&=@4mfDcpP#_W0h;WAt<}J{@vO&d&es&9oqBf09?u@tIY?k%ZPE>W zpgpb}wM90YKPP{?b%Ze01h3k_&36!mvca zdjMBp;=XzJflpKgYmB(=-QFDn5|m6nJd}2U9aM4E!kUQozn9nl6}5YXe@i$iGf ziw-YViI{SaLFL5 zl%;VJ<;4xYBeyrNI&OX`Zkz)N+D%u`81ir57Pz*Rkqm;s>fD%d)jY)RXQC9N>IYpu zJiczxNce%nv+B`?Oc;#?SxO#?1{uCmMdRtkIF|2yPd54yEmuz=+=49q+n+sK!!9Ib za+)_<(0i969N5@~dVcpA>mFiYnl^i|){VnA_(nF-_{ zi2!m=!;p~Zg|hn$S5RX@)`S885{$Ta`^?R~@j*1dr0n&9j4mjMChL7?TN`9VcP?Gp zkA*S#8N{^VG&*Yd$WU3(j1wAmBqG*8?5S_Y@^&q)F|=G5?#fIWLfVJ{B#dh={LB*t zSC^0!5|gIa=$$1j(hYBeKMz{<9HJ*h6B}uX5!!i3e}Rbrla@n)ps5{VSbFRw6#3qQgppbgP z!!kZUe*)GKFRvc_He=UJWbiV$7gB)P%!xjHAh(eW^ns}JSCOL zb610V|JTY!nDP?*y765~CgCWES$KCspN3P6w|_izjmbfx6N>2pK>_yQ`xppE1SIv7 zeSCPG?`i4j>FMb3J##T_Gd3mtwmJzC1jfJ9DnP>@nL5+abII z1%j$6LndF^ZW>jA`Ug3IgMyMo0nULI-fLG<|Gs21h?cXniA`l-(4fQ}-K|Bazte9Y zfq5}DSum0@D~GG&)q*Fpe^3~}6AtP<;JyfAim6U~qo*!+nv*CUa*B#5RPLdM0LK9Y zD?8$?78C(UIc7hJjuGWYj7K+g#YB*3|bdU z(CPe>p_7HFiMHqe&o4fC57bh#hSd+cA#sRIRD%R-kPDv3lq_L?j$c29lD#gU14YL~ z^>QR$mita#jE*e4hXNZK8frU-(a29!cU?w%Q#TAD-9fy1kX5GFdb(gPSv<+S)I=Q5 z|9ai*w*U8j;fCI*;5T@{qkuR-HrdC3!dMnanU^;%{i#{pZiLalGDS=$@Zry?V%Zxu z_CN(AFW=eIli=mOI)UQ^NDsV2$dd40r1>L%5pm*5VWBI4d%M0{$*`3_KmD1Gtd6-A zOaglP7NwU@hmNoN-=4vC-Tw$Z$!M5}aQdH*;Q!q(WtYTobd| z*TVbn1F1>CLqp+RxGLZcC=a5k`9I}&oo$iK|1`LH<^Y~YkmtI)x`u`)#=Nf>LOQYy zsHoG5GH)IiuuwlDxI*YeTuPi#@X0cWD?=RWdBWdo*V!yg2SFU4pS_~YFOB780sPF} z3H#CG!*#f)3N{)?`EpcIko-F#|Lu<_X_fTvuP_pr%NK%v4nM|nt3uGyKykt&MCXgR zL88pl{!c|QL~I>^?r|_v;^Ks>&Gq#Q5oEw=ICu6ejza#QNpi5^1%eCl_w@vmVxpoR zqAv$_wM)d*I-h?-tZ1I1ta*=og0YEca2(f0|JQHCmPv_AN=_3$J}j@S08y8LFBOJR z7T&y&R5JbNP_#YJ7bJb_d$I$Y&h)L{x_o1=FAS<+(k@V9a6M|V+KC%Jrvo$Dq&?oQ>UJibDzUn2^*PpWxMJMZrpTw< z_cFlq^K(AbbvX7A7sbWK=7;qQ(V~E2Amb4fPcr<^%Qzp=_@$|md2}su5NNH8gkyu( zZqC-$R&RHA-^A2j*B45B%pWB9@^kNfI+y&TF|dVS)m>0p`V7*a)%Kn(YF?u#@IYI- ze&vd$rY4}FEx~Zefbm8z><7`~p_PpCTH9`r%S5Ju@c@eM@Fys|K;7!e&|t;_ZNRIq zU!M?O(*!Nu`Ex(|^%#87LQ2Q9m1`$1Gpnd#0&kDxk0ZN8?b7>b47nmW3Pm`)47%Dg zp(OP6@p*dwc2B@h{`s7*-Fvi*8?_?Fcbm5?Lb^eWw$7cuLIVGN9*s9d8FO)*OPiex zTsyOhiiOC+6Gtpu0)7s6ZUrO>{_PVCIDkxBcyCxSaXRG!eJw~q_`-P61GIJA62d0S ztYw)4{WLv@k?29tMW5{3IOkmV3!*Qw$u5`x-jyx-zW1Ej9vdo_O4hF%19qL;R@>X_ zhi0U-dv6zsklsIfWP&tJOers5`^ZHo%ymqkf9aDtu%q34;OyabuYxmPkL^l1S_6l7 z(8b!p<=M%5jwd+p?rw4rqzHMrxov?F>KP~(FwDMkh2he^DP(sf^^esT;7jr3=h7k# z_M7R!*s-KUp7n#_-uizpFdRy6FqV^8(+W@a-1~aUd3@(RL2=@J0S%m=l22;<^1QKe z)<+qW0RUfmdT%NN8jeP23PaM1K~2P!D}-?Dx~QlqK2&J@0m|KPg-=-xYPZhWHz_U- zXYx$yUTvh?KC&bBz?PKJ`++wraeF|^#su|v7gcw}crXi?-NFH^_$H{xcHp4y*zp!$ zt*sAU^r_{(0{sw6fM9qBb1<)?<680C7dQF`5GOwz-9f?ewx^B)BfY!$!aAM4#f*Ns z3yz*GPEDzYzE+yb1@6DreW>MGF@P%sIRS+drc)D>?%8X7#Xz(TY$H4W+r4wJHe3Ktq8&bqQu))%Wo*>&}#UdA4w=VvN^D43r>Ux{AQ1OK~&h#th~`1L^7RHZ6KM zkX6w{;#>E1`^bUb6y0^+r-r`m@H02hODj{FiUWL{Qd=dGSjz*x^x(*)cc|}uf?DMw zH4Q88?itym`Ymk-T5`|d(|#SedL8x`z3BZ=jhp?k&I?7NhKiQP=8(gagaRLG!{M{$ z3W_^VyRy%Z4x%res8mIfd%rubr_9-=o91n)e5s*1XvW5To=hueooZOF!6jlD{$A?> zQ{i~g+|0r*#6)}fpWh=2Lls%sXkU!J8n9*r`nh(%kXYZ0`7#wLnja zbplRZxrn>K6+7uM*>Q6LL5e(tV?Nc2CobCxdK0k>3i-7ZKIFi6VC4{hblzLQoZBcZ zi`zgSxet&704{9E1*O6a%WE*TJaYIjR%?R>ig0%~#zyOpiOFg*zp@W$_ls?x>8ehD zs@_W;!bAd;Zd9;XMg*VNt0g56tB5H`ii;Bjew?H9^%zT|@J}@qvs3ZYD=f-8b5{JQ zc#nwrYJcp)vPouG^IZijUBvMObrDbszV8b9bpT*VSk%~LOjs*48De1U;NXzivch;b zA&^CKY?IUDQ6DsDy?dp|L%0>F>y2gj1kX6=3F|E+b~Ea|8zdu?(Dh1x}6| zWK2h2n{pd-48|A~G{CsN6mZ_SVLY-k%#71q1}}@OsFghTTHi z1CysO2CWzNi{i=D*4CmJi7=qgA8pUCYi<33aYgg`YCKth*W+ltFzz_#|KE|y?y?r$81EhZe$iRKtx5RKbTQWjeo@5#%)2zwBe}6 z(eAsK(<<^FjG^a%v6#Uhcwh=Z-NVHNZC52IIfNAm^{&Xqt3Yn?6%o-32cUHiin!0r z`1xczy;GaQrw7A*FjVVReU)usM`>mC5rz|9gU{%gul{N5zwkma`oeme8u2-sED3Ek zjs1q4CN5&fslQ1^L~u>?p8xS=yw|LVjL9e!m$gPPmj)Zpl{wLl^UPT)(vzzrP0;Z( zlK1cZ6e;cYwWaxH1;@+OitMYNUufYqY`I-lwy3bMB`@E@`I{BISeHw#ve2tx9Af#q zAU}T(|PkTuIg9k^SoRiYoALb+%rAnvPBX^K9 z1osZl*r1AvU4fs{Fu#I)^-E$D7RWV}GypulVeE;M1*ISg zYp~V>f&Ky8bX4fYSRJ$ge2nGZih--k*jXPO1r7_ghQPe|*s(z*BVZIj^qzoGimp7| z$J5=qCSb3iAQia_zzk+PH&asNZ#JVb17MF;Q9xjDt?>o#fWeU_Xq!#V%=AGLZ|wxZ zX#ckn2rKq6z@-$&7NBDL*c@^IoT-wU?1+Uj`e@u$tUrFy7V&8+WEli{gn1(a8Pyz6 zM^KdsVjt``V3CFZfbr6h@0yLjj7SalJ5hw=-5HC;p)IH=FMs&{8;Agq0FuaaaLyza z3IXT=Ljz23*m(sYgM)!|cpI_y9RA~}xBiSA2`L{Do7C%%Np||#&!Kral8lu+cH1qA zv+&rPnxG*-iC_k-5OEP^f6(t7DLNiv661-}anTbM44K`c>O%fdsp9T;b~}_{(*udV zspW4tNgpybjW_z`{w5)dd$$s2;8WC;p!jEbc@Bb=4WKFaYi5d5@qV*2t?i?3AdupU z#H4JRs*wWxaY>HSxlaAo_v6*+g-0nAOrl}!=G-ug*vNLEl2a=zN;tMn^F-ej^9NZx zTnltsTTddM!HvKR$X&*l9=toBS7`kqMsXp4o&>2ct0$*Ha$COBfe!(AX45Z;J26&tS3FHE0+fdzYb@JM{^F%-HU2J(lDjR6_%4>m;2gTmv)$&(rd z5N_2YgF)2?=#JQe3v{FChGhJJNcGOcsQ5xC?;$Pp-oA@WK87hU<^#AT(BOhp6uV$V z;)ytN*F`BsI~!2W{8;B&X?Jh$t+7q`Gc5F9@WfFf356!`Df@tZJdL3s8wDaoWpSFK zmP40Qo{A`nEEA8KL3a18wAk1$BZL9K?u-p^50-%?Hb}1c;NVA`7D3p;4#7gu;E-`- zL7T*hx8f5wg@UniX#4ZH{jjst(zu+bQ5{ihTZ6l#?% z?uZEc`T$~jw0yMutvB?`t{vYdCtE%_B{$hw9zINvuSUiuF@^)&d#?)_&RsQEFc>_J zp)X&82=1TX)Bv^3%i~1^M=UrhEe*>5wc-BM{Z05NrdQ#ZfAr9e?c4F@V`46o#y{1l z{tDuyH~_$f#A!3kxu?7<-l1#4<7={;NmmOi7l>h_;EsO;rHhgaN`#o0W<+DW*;EEt zKTHN6YCm6BNScz$cH69~14` zNFnrKbvpP_Qw>rW>zPDH^5)V*p)C4klg%c8IYSMGWnRv~=Wou5k6;SQ!l} zDDF;5AABZ&48e1F%~qA^nHg{}_gkHkivOZ5xnLjNWS4g8Of!%T)KLuSNWpB@bI+?d zKMLm>uQ~?k>aM0dvIpX>{GOmKqZJA!U48!fz4n|(+<<7Jh2AlC^_P0KQecR({h3m` z=0!^tRRp@HbwBx{KrM=xzYc&4?rA%ijOqs6Xm1FL@d zEy`HBc!myF>f>_w@_M8ETbzaa&%T~CVt-pm6_q5+$AP7Vtt+kq0Iwh)gE-+yRO0Pr z8X>5QzX&8d9Y2ji%FZtC_HC?Y2}&eoc1~}ldx#T&)X}7yx|2wY$S`09qc#>mD}`YQ z>U&M0z$`R??vGpXZF6!C*lvcU7B<2R4cUqFn_KR2OaQkTl#p6eWB>>*vEUGq00}O} zn&~pFmy@k+ZAq{u2ZO(*tAq%7X_vNJ>=uh|PEm_sT2S6sL+-O}r5d^)g4j2Hc2}MWnSSlQ~*V;luZs@e*qH@=Ykvsg*c7KU= zCf#R&@^~LJwi%u*T#b*d66#k)-NP=$_dF~D)o5$9ChEVO&M{=GTD?-FJ0w!X!6}@l z9Um-m%NJ^n9)Oe!m#!cx29`#;qOsc9+Qa1?y;>!~U82)7yS8p^#De=`o&8ahhPB7N z+(;|DBIQld1x`${+bC-SHxTCM4qwnhjhbfjZU|D3?#-eP4xL%k(0%QY1hUQSRQndz z%o|yEdUOvd#hS*xy=&mS?n~<#EL%R)&;MD*QCQ-~p67oAOuQ1((rhM%FJ?Wfs~hb7 zQ0A$0ut96xiM)xOdNuBjWR!9+ZV%3@O!y|8Ufcq$pQ)*-<)KSQV|5$14c}JVA9qg9 zCm@?P}xM|Ay$wkZFam~$o!BD4|OBfM3IOUdHOk6bAK(US}KF!M$4LpejkiS8tSXwGR zTmS(!=H;-k$N%Eq6PrVC2f`D|<*Y8>=Yn5o??KQjqBu3VXFuT}Z#<}ZnqB&l``*Vx zMlTa-N5q!&^rN)(6!S&c*5Ey>3L)i!*zu}s=Q{Z@)gPu=4G@Fscwh4aCV`N60@5|b zJ<*COE?h5j;$>z}1g7jPQl!7=5CQ94REOA-R43uDCj0HW=AYP&16>>Cm~DQ28CJCz_p{?%$t=G4rF$} zeamq$`jvZjF~^!e&&hBP*}%ocXcgkEpKj>Jh>kc+r%+uxHbguRL46+a$JZ@OdU~F*7#@ z#hY{~`S#8EDAR`u=Q0IbkG7TuUS!r~SZ{fE(-|hm7vIwri2d#I4Kxjm2ly1U4tjD^EAK6II(?p+cf_zPxruAHNW1y2iJZ8 zT}b&;>^1n^goT7$pLR3PIVoFnP`+mGDX!}8PC3MGK|m84#qMo(4-iW6E>;^TnK8vG z-HKBiZMS$KgI<_fLI2n71aa zrE1!H@*>F+?F{8goh|sYshe2V1KkNg=S+F|;)OlDz^x#>VH#Pc6rb zfb#^e-Q(uv#Wg(I@SE_4b(g-|?m+y0;FagCWen*sMVh)c)q1=LZiW$olim3nTSJvg z<_7D&W0V7z`xmoOuo?tw;OvI08ltgPRaHlPJ|@sgPd*u|tyc=f& zehdVziCZHe@MoZo2EKwyD#6t2(Sv~n+Hw8EdV}UU{iZ4_D>ifEKlcjBd3loqzhYKz zcbK2zD);q+BaSshyrn^#Wo$2E3FqVB)bqaC8E3d=s$3>UT;Q$eaGWZO(sBYs4BUXB z@o|`x5RQ!o2CbNf61xPUTLbwGG$^vVXe=u7SGUNs)6+03_rts!M;rrD$X=XIo-~Ed zn43G|$m{hyO3n54YH^v+3}aj&v*{XeGEn@GM4cTRurL>F%`J#0V6!T_0{2?CS#Aiu z3_OuBU%e%B>@rvhF)>Ey4-X&SG})kH-@IvZ?3bHl0jjX)4GpMA5&KYefh2`;_!CU; z#Kj#^8p04BgJ*_DSlex=pMu=}%6qp1|&zNy;}9v?>JOrL@`&+~1G zzhED_BpJO6U0m#|{^ZZ#fsTzq2@;FVd8Ve>g@yg-Y5{YXjqleum~9j|mv#+}5?0n8 zb90-5h~urWC_9Zter_%c_qR&#pDq?HoKW9_J8@nw4{S)kYGqcc7r`SQqr4@tf;?N;?Nbl21 z@gz_Ojo2JNa|UebPGrW#r6r+Y;2=}9@&J^Sumk~n4^EyO21bFnXy5|bQMp|cJF;8? zueo5+Iib4eb%ZxUaFb96P-e%6*%^0FhJV)B4f0FV6M z==71gw^%7!j-X~B6XJ0bD@8zULJ~wO2Q>h)@}wUdiLx%f)&g>IVVOT-Y6+6Q$^K)F(t6t*d}!bWbPkuCK?E?1P(-Kr||hRGB#oM z*4ogb*ZzH6PSQk20g?%iG4lbW)N&fEKdSwH7~ChMGmeS zyx3PazWmH@F8J@fBR_wFTQCw5c-7F2K!k|of>*@x?wI;VB3*dh_}q_lF_|;D_mc3! zroTl?C5k-4XaDv4(!Bwy=>?LD<~JhZ${fr7aLe0cwO}M?JuqK_`6c3_kG0+pZ ztzfkIR%*s$hg3IUke|DqmLixK(v5b1G1lmdeKwPow}eM?Yvs!Tc*?^(G7C*nkO2(w zsqs8JUIeqmo4Sy&h&-zMN`Ccxv$BtyjZHuDJss=11&;-;7B=90ra*=O)8njd4@#>& z4BIU5`asPF?y@yy%jE=s<&Ci%K<%_n8V%-vJBZyC*;xs$d%SCs?+L&8t)p@v%jVOE z*7q2*7=jEn&bH=a(cuEDm4WFI_&??*CZyM(*#DlN0IPlY>(?Ib>&yXjEuwcsdetR^ z9tTm<0a3r+6QRVkofU!p9x2)y*@TH3;`9zC$i$>A6@bGchQfe_p&d!}a5W>{e9gOH#8Q*!f(DiM#=IFAoi)fYDcI97VBI^0Q4+B$8)pH_7@1zA7ba7H%e3kTj|CB?%<)Kf`AR^F$PTlqnTJ)Z(Mu&N9r>Z z%mE4xn09`C`2Jd@wo(-}87?mh-^-@4Cd+Y;G0z&ReUg{rsG;T3(?ijVtLeM8BO)Ng zg8*>@z}{>E@eyLa=f^LsEGdEzkAQ(bf(($))!x1bDuhjaj8sz-7x6t{PbK3U);9j1 zzm4sHa5lzQytg2#P6L@p{NI1RXz}O&>))yVPrn#kPy9K-wV%Lrf(!kdq9olM7gplM zE!$}ViJQ6c_xsdDp)tTRS$Gt1Mr@13DYU_Tv*2fYM0@=}X6G0K8SiZKCkZMleq`zy zR_d4Q{__a&ofH_2=xgNI!|J2G|26O>GOGjj!$XU%5ez@U;_X&US|>$5?<}uKt;9SwO{hz_lP4v4o$$C+FWDj z8B*R9!meTX&c2jPJo?N`_>xbn;C-}l517mR{mywvqzq($q+s$KdR20Go95=_%c$V- zHN;1UH$Q&o-)kWLG}Zs<1N?7(&i}nXb|>O+A)+hbh=o&dv?i35Pqc7ZVHC9bzM&xt z7b_-2%o`62K70omAbJAJ`;gBd0vl-f!K4rhQRPfMLl(xL!5{eX=_p15+S9ASLuXBa zor4#|Mggx{I99}<5QZQ*Sy_U?7&&A7qVawR85zD);Gvqq2I?z7!G?g46G&b@m;Ucf-4Mt0NeCS z7+65Cpre!7?tnM}t2hkbCop|``ehB(^0l2AAWeuAg=0DyROfIm77Y?91$z$+8)!=R zqws!yDmpRot;--96t+_r;6^bRg5uNLs`%kUL+!fNRD3Q-`j>tgV-6Uz0utWm6_`F; zE!RRWB~gf5+~od&TQrt1U%-m5r6s?o>IDV~R>bl(h^u$j?Z?`vO<=0O7ZNNWdsHs%BwEAx)#K!}&(e>G^lyU*)>x}36iDe+j6mocb`VF zLpTM2D~O*}*Mcq)mf@$IozG*=Hz)#gWQjyp{QKlcJe+AK9GqF3)=(~ z)%w7?A-n)P0%UMtKb`D95PL2@!iopDh!B2>KZ^4>&$pWX+*zRfwS4B~g;!~TlIK_M zzw*nj3UI6=7LdZibU%Zm zlM{wV$M8$=FQM;4fW-X+i~x-X?iq3zaGKLjsz?$zxM1_G1o2Cn4pZ3N^Lc21`F>zk*sMaUp;@WVPFs$xCm7j+*Ntm*(rBBK}%f(6FLoc z<-1Tx6TX7{dTI=!u@x|h4UU3i9<1z8x=)>j{Rno^;5mZ&j?wXN%-`SKt`NjSz%&u# z&F=uN#(w;eeh+;H)g!~RXJ?>W&Wn27(KakBa!hSJ%1bKFs!ZWUTIx>s&zRpQ^HDP? zE8wh2Q5k&C@)hFWN&)XQ;{O zcjh6C&3X0k@V1YV$50hZF;0dlq3%`w*^h}5R%;IPwHT}^9FNS!%1eB7c>D-2(catM z?14=eWY<9?ZqR?xP|~fA@hLd8;oqjCuA>3o@QZ{$jQMP%_YBfsg?ZmoUt=`2sQ1CU zbx=Bbkpym1t(s;b)~UVRXp8?i3b2tf`WK2(`zXkWnm5x>42WhyUS5pPR(v4BFKlJ7 zuSaq-Qv{6}Q|rd%u)M5yKfFVe3EIu@O#YrNTfX2$qVU4zPo%}T3sfB-1Il1pjoOo- zFTybzx~AG%TdY>knA_|kf_qR_T3W^JW}lOH+~HIS?UC2`r>(`lq${rQ*1CtKrJ%}1 znY#>Z6+QRr4#2s;NDsPrB~wue3HV=cA}hd;b+r_C2GQt2FGLmXM6d$`0A1WR_7#MZ+jtvUkXgj}fwWMHv;zNF^gG zDxoqWp7ZMa`yIz~JpVubxbItcea3aYuh%$V=Xt(B3_OZyG%(X~85oneXIPyi7ZzfI zy9(mN)*mubsaSzTs8ul?(3k1HD18?#5JY{r99ol{Oa_@*uv6ED?E-d%K^S1VGt7%u zE>qK3WP6$}vb2Iyh*BmDzf{^&0|2Yv*&#{C^9v6atUIG$28WlF?DKj203+Y7F1`d4 zW8<*js{Am0Tt5IEce)WEB}~R>0SnP>ZC1?5%NutN{JhkHeF%hR4qSb7udk@6NPZc_ z{yRh=C^4&`rWhTy?ocZ`7mEDttE;Z}?c;F+LzYWB!)|81UTwHJ4yS-BhqDdPQDA`T zfUXwwQVg3gajUv8vIbRWtDdYv6dp6|ZU213Yw4<0+RE;C$CFa6&R-bsIJ16Z)Gs@m z@8w|E8??LNyU83LIDYwi#Rt=oV@77-O0Tl?b{WPG?)=*P#p(R8`p;j#pi_;$frBA5 zejv2<_9lYd@z8OLk~*5$?aOzc;rU@w!kioho7bKh4i>P#{Ft%;pjQh7C3e4j`N>Th zd{@dRK8rJHHTC20G9rxFf!NM?cY%BdO8_%t3^oJex!s5OHUSh=_nbu&lg% zeBXsLXWXGGw+Uk$V|$xdGK~(yjbe?11(H@cIVwL-QqMI22_`i<8slpEHRX%lNl_UY zgzxaoCTf&caGdg5IPpBG#o-}3;kWSH&uW0*F!=E}cNYnT)}OGRg0AJ?#0IYZ8_kare)s&n`J?dx73CLob-B5o4_Tg=sJD&jeM?5m~e03*hnqDeQaZdVzyIQ@>F9>@_X*6Z)QlMa-W+Lk1(Z zI+9?LFZO_}h%AR>jL?4J?AaErBy9(E^$>_^7Alo|h0wnj-< zLjzb7jJ?oF9LU|Z2&VKyo5I^3zo3Bk*1l5GD$Oiy7(ZW3L*nq73o@lAod`KGc((<8WJ8GeqTBSPK8}4FB+Xlnm!C$K0#B zrF(zBY3aS{m~jeQW&0*aj*F^+7xSJ{4-K`tyf4Sfov6gri#4~R#cMmB?Xef1Doxg3 zgVXlQyV3S8qxXe0I-$MneD!K8kLSmyjBAzPsMfgqx18EE-a9a$3yQb8I*b60(PAF( zab4;+Tdf(2csyx&#G2VluxtaP6`2bWqSpIo9+<~UU(m=gd+MI=gwrSH)R9b@>gojL zImDv)bn{Vu{tY~KuC6yg7keM_7g^V98$e16pN&?GP^5!mn(3-w$j`-?`@Hf5$Lf>&3U4=G{s?GKRR zK=%O51KjNbc+Egj16;m-{D?DC4u~7GQIIZDQo_0^c`D$#v^I}Z4ZE#!D0KkAK^}~A zgd-8ewIBY4;?SH4#&+y}1KzFsvnAMyLAv^1FO7+hiH;53l5^H0Q>xbvq2ez*d%0-dYdZ)~-2EhSx8VVqS zt%{N0SFL%I&>RQF-UGbm?<&biciULmqkp?>O|Cw{rST#l$Em`xn{(#JURQ!~37dqV z%U9zB={yY#lS+!c@g;Ud2tLfm)I`Qmd05j--|L{3KwD;t6RvfyHH@!L<)NzJK$gc05_%6a&P0r9N8cBT3~{u~(q z1V$9y@`%(}YD&95BO&4Pbe~8-57>{WY_*;zjeq&_<@a+Bfa}LJx&_dB-cRCg zf~kkWOxVs0J_10%w>%0r4?OuOLVzFu7i4-s$H35fYqdN15O&@js$ppYA01BuvAqai z1U(o64Y3*KbK9kpi+&ZNtO|BS>=}w-&QRA94cvI1DHEXROesAjI>%Yt3X4fqF6Uku}aL_Qc%zoLvHNYv&I5iy&|?by;{A z;J$zIHjd*4|Byz`fw0sMeWr!g8Ry~j4=ZhGmYICUZK59;v7JR}|L+E4dUE$#ZF%`T zcy$8TE#MW%ds=b!*RNIuw@ClvZ!lbn-p`$d+ugbamfFigC#+F*uijc1di!=?a>3sC)Slr{7G~y2Ky!EJsPb)PwHM}XF3Fl__2 zCu*qVV?RTi4aebaYz{?=_GP9zj9fH+8~C+32%b{&?aAcBV-Q%^*!vq^6{rY8*!(#2 zZ1rdE;J|>?yN@Q~pm(Q&DyBq8f4rX8<{e1ZmQxCVp(v-so#U9L&DNitad6msL}V?0 zHmLo7xd7YRm|O1^72STfcjLy5r@?#vO!1++`vUxBn13y@msCCwZHAr7>R)TfCsrdm z2#kN#stz6WIu6N`yv{;0nOS~Y4T`IAmhKlX+D>kVjtC}D+yN*tv$3hKxdS|=xs#K4 z3YP}wN2q$qJ(vNmmV$`X<8r1$HzzRuafD{>{KRqdP4SwbAG*OF=arz+_}@{kl$1Uy zEKOIS{kFoy#x^-N<}b9z3Jc9=RW_A8LfRSH!nWB*bCv1~(0FccQe=aq+o<9Y*WczU z7nfsCNM*^iJn0Tq#VT+Hev;52uMpfe!RcB18|+vf-Pi8r$?NgUTcOGrc$n(FEs#E+ zOxGz~Nh$YfTk({_;`_6^HkWN88StEWwjrPg)KD8++YCAH$HV*R7d1-rTJ zpejSm1U4ZnyMkd5=ts_t>8vPhd{KI%JwcZOXS0;)YiR3kbCm%haC>&m2Q?nbt960q z<&bLztKXvp!q4D&K*^T*>MXPlXiYeeUiZQi4dChH=xEQ?$TQ;g{hT9fC zOzK<9QDdbbTBv7TrUx+>m*>>rhf4wCKina_jvRcydkY&|JZ3v^%cH%^qHb;BSzsu9 z(1*uYT#I|te-~bbck1icZES2Bdk%6jGd~`&SHx@?;!4reqai;|Dg7}RH(GrnoOk~| z%Q9s^4|>N+HC1= z{vQXr3if{=?J)|QlV}?nT6TV`!#vK-KN`+>Kl3)4hnZmun&^tVjg5cPz7BqU<5HQo zap%?J*<&%FdgH^iv~WicNT0gD{5bSCBTy*J37!Aev^-Fnk9%~X`EM1V7nw94WGFcS z>DZ2jD?d1$p6D=?VonR#;wOUk`;TRO0ez4)R90FPEGI-qzrxt^^NUmyE2~D-6X~0G zZx}b5#H#}0WGR1w=mgqT*0J(dp}^CE)p9|ouD?RFhN=iI9rx~yV*2J~BL!bvd^q&$ zutT96inlXgAn`u#A-Jj^cCAxUU!uQo7mv#AGO+tz1E3vWPIi>oklrX6FV5s8 zV^E;yev*dyt~&jmz~I_iZ(s^gMrmk-3i~>PH2t=xM={r6PMt#zaa~2uy@N?sDDUCq zBbzeVs4yvcRp_6uae^1+kBJFS49C$7qf*+mi<{-i$@gQ7JeV#mv0)^3B2DJ^bmdNd z@(zPh&4%7EecR=y5!cL*?oSfNxq33E@F>$L%2372bmEas>7Mdyi|HTe+~-n&A7C~v zEjgmT!{Pdwo+JH3AD`Y~TMUnC&_>G!yvAft;peY(oP{?ddU89-DD~-<$%2;<-+kO9 z&)zC)y4kZc^3hm;(SX;IcB~mlP2Ap{E!esRzjPoc(0#Xx+`@DZtw>*a+_8sgy%sfv zC^Cpu6bec(7Lk^f1uq`D7tg8QwpXv*0cL~q7iCP6`}8vH8octbP#mK{q>fDunkrhh zU>xx13>>r}*CyO$J#oc7)K-x;0xcV82Utr6LmRJvb=1%3TCial^yl4}yKgT%5oZ2_ zJkO{0$Cd0TJ@?X>+r8Ps(8T0$W!RV2^DZv8nRru{M}6oh$C9Y4Hcd2dEKM*Gp5tJh zdoLgz&2CJ$1=;95KaZH03kcee=^Y&${;aI<1puQ}Q3zlK91C;l3k_Q)xYmW1qJou{Ax9P%94Dg>{fPK3u@(xBTB^5j#` zo~?~np=vCii&wh`uO)*5)`9B7hRz?N@29kNuKOiLfQ5W6qQ&di*TDPxy;Nbu6U4{N9};9(b6?q29yIL-@rr@O}^arbh@E<2UGb zZO}EZf6OwBq6Mdvz*KR_bmYYT0KeYu@m}&0yzTdGi z+%)F_xZwwsmXB`IxM_G*po@fZ0Ms9VqX7-_38v1LM*bn2mvtgWz~kVXd%0}y;2>2g zDj-0;?x*0~Dx}*kSktZg@TAtPvqm8E4;Gbg^L&p&)vFB{0StRW zn0PyEXhth+F~#90=XeSs6sD9f(e~`KTmt=O z?Jsx9j#e{F6qneDZB=0V19w8B47XoEnV_lwIFiC)Li=%|lII6HMn1pT% zexdE%2T!bY_c)7gbflPC!hW@0Ffk`5yxnedHm9U9nkPaKB-un!l@2-xh`i@6yL{8U zFuW3f*Uf2e&4KLcPI*BM;ZI44 za@|8JHRAY1gzL7nZsgUuC1NA@)|x`JfpSInLPV&}8RgfAP;Jsxsf!<Cj~vG;D{URsnfP8MxSu8}boqaC?dn`$_!<9R}MG3AORvWmPtL@?(ad-A|PCy%lEd z#9bqlKGq+K(ybzjb^4Dpe!n)|*8x~GId^XkJlUYHxc~ao^C1)S5BvG9T|Tu|elu%R zt@o31k5ebvzzzX1L2c;xC0NCju?UL26nV4OSQEFb(0w#ldtl*5Tlhoz26)Ii*9ntU zUS-9>=9XgZDh7lvFv1zrec1eWA$7KRw&par=f3F#m$_dqM>n%O!yKyxKaT(DV5q9{XH!fXM ztC0H4bxX%Q$H$0z8dj(2Czg;RLG6=s2H3TI0 zXkmb~;e=DWEuQA3V4!%R9%pgeuy{Is;z7a71v!4{tpz5Xn1O`;Ja19uvb8tc5kVc* zrz$PhtuySFF)UJ6-}>?$Zve4pFWm?LD)52hv?!S*WnCu_0B6-Bh?8|ualS<|AFBlU z4p5PNbP#qZcb@4`L+1R- zf1*P57OY;=M`1`!Lj%Q=egCHyGRnaR(^lTz+37pp2bW#3HH{bcw={D<)LnA^!R7nD z=)SCgz3$7Usx^yOe=ewg;P8H_k$m{;RQ=*iBC^O?%l(u?3e^0VV4?cBdh(up-fjW0n+P>`9ieXA14GVkjjUkF8rt=$$@ zzegtRj>Qgq?8dsflxrrK;W;?0M-1AWIrFG-8nMUA8;{S!ZDC$VN3hT9q&>`Gx(|IK z4Cc$E_V3@K5Q=n!1sm=cmcWvhyPb^v92l8u>4qi6ACQ+%Q&4iZg=yAft27;LZNh@x z>L?!{)gX1C66UMi3G_uhdF^Xt59zxQjF0`rsL5m7Q^&Ws)!4UT;|(6*qMa7%!B$Zl z^&|d?f6$tE?OnX|ky8f%))4^Gt)jlZzIR0+gurZ%FTwZkA2x+Am^#7UBhFMe`xUZ! z_!Yt~Xhxa7b|Zg&Pu^(JI2^B+O*XCCX}Iw+8-}@|qJEYd)nNw`3jE?c9^i3coa;S> ztq@RdOb%Vgblb}nrh*~IT{?eFtw)&kgZgEdj4EJM9O;msz4xrdmMvQvR#E+}jv?jx zIV#}KTPKf4>}d04ea~=gRFuT;=eXYg>eWG3#$o0_%ej0FZ!=^7>xQHD+aR+RVD8zV(gesPP#bEyYbG-mR?6E&`h^l+XAit zX>%2bMu@j1C^G5Lgj=1u8Oxk59>tBWM2hJ?>e2^@iO*9c=}+kK&%WGhn&~=Crqkwd ztNF0{kNPNa7C9VwnV(q$2=w6Kwg=<7^6GcRxjC;uRs*UVAfwuyF?WyNNO3eFmcbXQ z^BVy5!OZy6%#1EWJh0=W=;%jQrE_MHcyyMlxS~>R7kj3TO_k zOX@IU2U6eIzVX(?3k@31ISe?;q2BXMBTx3i1s66FE$%h|pkMx-HX4t0lCtMM&%i)q zi&~HJ_^H0ZK?UsWKCzr9TJhboKhJ;$u8Zh_40u!+U1EJP7Mq$D* zi3WnHLhXiYEz4MJB;Y0}ss;#G$fuPPX&>zfc!#6Yc zQX^SMiIWt?(UUA0F=?2k+6bH`-`T2uG-NDwv8};*#ur+L#HTcos{e<4QN5Z&B*ZHo8i`{OiT#)hfgq+}XDM))qk1n{Vb-?`HS9|KUI&EFFI zB;X-0+1dZ*-R3}K!lznpt|lJADCzaX?lPo%WNm1zy&}_FpM~eqoGDAc4`i_P#_>jJ>d; z0us?Up`Wxq#5m?ru_YRPXM1~rA8ybDZ0pbBU?DB_LvZ)1yq*zT!KQe?m!B|qGCDIPA+5F~*FJO;Y zRb3640^XPD!`{=BdLqTa9d-~*Na8RaJP1ENr4M6%8OQ&#&jX>5={(|-m-r7fcQ6>} zl--5qMm1Dsm4`&)$;r+A4SB=q(<8u^2pSj&1+D11auJNbeQ30}GgfH^+SD*QD~Lle zbX8&?3+=^Q3ZeZ$EsEzFq*~%df}Jg3O#?HB@(0j8s>Eannq*6=!gcTC zSUtpO8x+Xa#jx?4n$i1qMBq5uTU*1ZLCCF6hr~})%-|<{FCNUv3hLRtOeBrl+Yj2b zhHRa`GkzTWb}S;tRzuU1Cqele+!5@j8C?QW60REdfhafq`z)i0)TcL4ki=~ejS!E1 z2>az@3lZe4dwvk2I9zwRN4Gcq`(MO=Fp&%hb>M$L|IaT^#fd-m)5dzNLl_5@mm7TW z_mg@7%k3907;m|NXs34a>pWWjAmiK9GQK4GJo`q+|w&bXMFha4|zwoF;Le@gn^Hy@+BS z?gcv*NNaY$=|rz!)@pJa!2&|&;t9YavA5M1_W$<;sCKruNBu{CT?#~yj@*9m9~)A} zPxt;|2?{WKRxWnh`;|o zKa~Ie@&BJ+^i;)6^z<+dcU=V>I|n#$;F}=81Ei28N05Lq=fu3~3S4~{-DO1dT!<@7O2Ccs3{?N- z%xXa?4pEBIS-{(9)3aPWP?bSe1XUW^63y%HrD#tRwu<1JZVp}Upxhha8(a^!2pJiI z7Tkkp>tCoKTo!QeXfyHhSEhPZKa3HKa?QStnXry(#@e;@qWpYm7<_`eJeCSqn*~h7 zmK+fkxMb}8x`%;|#l=y$XM$vjgcHE(gHM6DvQ-J-ag;cd(ms>lc;%&g`25Zqo!kcx zZvEkiIT~*s1Dd?3#6)b0b+fL{PzLByQL5N3hUVO9qRKg0Isvl1x{R2xXEF8sT?K(^cX zm|jIqO>^2fP0f4|0B zoO<+cAOga=XkvHQgD&_0fA|3TKJDUrJb1HDpf)4qj5Y4BKi}P<>9}9@S)k?(6On~e zMb?LSisLFAe{i9-uCWkUngWd(=qNEcM6oe{hkA9QRrlXGhvyeiw$4sXo;CnuDy(ZT zz!%{Oz~^9MW##{2>ca_|7g`9^V{r43I4XR06FMktB>+Fed*giJ&)>i89}b~?i6i*+ zq*itomXF`Rr=_3H{Zga&>Woxd&K9$W@*1j%MQ4-_PUl)5*6BER?#%8>m~TQIKEIj0 zf}K755^5}j22^I+kmi8T3bqI81j48yye;O6CvN8DPHT91{DX{+de7$+h~u(*jVc5A zW>;P0OT{9h8N-Fa)4r3RKX=LA&Ddbs&!vC$>K`Ccm`eD(r_k})flGl?28?2T;d^R> zEv_JU5CT8$nUpYx$dP^g&Q;q?gL2;LmVLI&8N30m0!E|p?g86_pM-{o(=#v-Sryb> zp;b29cZ)dVV2Odd30Yd}vuB?X(B|c{fki@l=L#P*n`& zRbWLV#>KTCl$(9Xd`*uqi#sAK+YHklJX4sEQ+DC3pooZ4&%q;{F71BP@jDtLFq)1F z#u9|=HP_IwNkA{fnVB$?hK(6)EYF@*q{Z@0Vs9-0in1b$VbAl!gclAFV#tHu#m5J* z0P|L`b>Tk|a_v}3R$(F7&jXMuKz9qu+A?4V=5J8&&SQ)KWCi0&%oS0Gp~_O~4EHer zw;b0EyE5i}{#2&L&PxKdL9PPUiyHEr1_S;PxG7*FxJbQw{=5iC3}Iwr4z|}f>gY+M zh{(ui+50x#J?Q_!p&8p~Q9XCqrKF~A*(I%qz8R=AM$pHUgq*F*O#dT@c*>xqdk@nE zm@0a;0EB=z3E&I90<>|^NONQH+GiZ;EYmN!cm;u(nkFlX4Hu=C!a2R%;&3?ir7QZ47to_Y{rX6dL0`#t5 zUqzfX6-&S~j+qDw&ka8JKwdq5d%HV|Nt`NNHM7=2oS|l{==cfX16GU}j-xx+xG@+= zIuh{7g)?xyOMj}bU!E2u9QHUYMwo6`uy@1+lja#H62;GpB*QooMNC{As|qh4Ge=o6 z(lG)hkf@<8gD+y+Z1BAPz|HyIZs*9xwp;wj{i1J6@2WABmc`c7C>~`yE(4Iq(OGLA z5-+*zGGd|=XnH=oNJ35PqmKOd#ei;%9%?Rr-hBHaLylIA`>4E}9899va2Sk@F-v=P znsg#=6otBt&EsY%@QP4?v(l-KUOalRVHIj(_fEgJm|EAHOr(oHv7yh$bG>hLn05UJmGoKzN6jHh!?Xl&E6)`9eupCm`Wy3 zgJ6t54RYiW5Ec=6fscUDvbq?5|Ndu!L^#sac;-nE*9<)h-FD_%k4ymzqdbhdDPy2_ ze3#m(rH91$;Z);}`_A^CFm?x&Vpx$8n$Md4YmUR_g$mZ#OprD$d5$B)SMDmve>R|6<$g}%&B+RMm1l#nb~-{yTVyt1;w6P+@c zi>Txi+|`NMP(nKU`+{2$<$mv4%#0?(lBZg4|041SitMLl{u!zKPq?6ch*8QcH7x>1 zc&(C%$IMn`=}ajodV$R2C;VX!dfKP!@W+Vv%D^6NQy9gpohr%6v4d4I+D_ka?4y5~ zB8iE0{p7Eo9q;~^3lMD!#l)i%A^@36^7EP83kkL8UFn11^ePOY!ucq&_nd?|$p47C z%fUyYmvZ1ukGN=c>QqTyo`b8aj+|8ZPG!)e!59N|=#7h1pwdAJ37iVPXmuAUS$_W3 z?(Rt4`+Oosgsedk2m|ETt&w@iy=Wk4GhyWon3j&~3=c1_4X{Hz*if0^emⅈVJVM zR}t9O>Lk7;9Qyo!63mp=kLMv`0HD64a07K8l4h%I6>x2M5fNf_OPH^9#);_Y7zoUA zGLpNx98}oba(OFOr}F%Smhz)?K8oB=p3p>quX zb&2{DeQ8Ojrq-`ZAPQ4c@lsC3@Y~-jF8)3~elsQMFfMEsmR35y7-Vs3uDF{djvH%he=#jLU2$dL=SIT)j@9XmfX#bDj{ffnh zKz?4lP^FFFQaB12{^-#)d{+$Mx18S)pORANF{xQ_mo>J-veGFY1{%21;H~fCWk)w5gH6C^q@fMIoNW=X+IE0WH8Q5pL|j8%~4FgDpi~UP}nfW4_rw z#`b_#kyOm}HWxX8>K8w;m7qskNuRb{R~Sla$Z%D|tc zKqE6G>T3}S3ZfD@n(?5xxDmQ13}7+HMYE2xgd&`N!%grP!aPNark(bzZr!(cuRM6S zfq`w9cP6b&qax);L_!C;fQK(EI)Cr0GV4OnXWO=I(7=!v4rq}5n5pgcV|Hy;!=4KBJk(L- zP!$p2ai=q0q}$l>H)-C=GEPbn6cUO(o+An$0woh;UGZq9n9@yxYAv5(W@;j-pUD+) zGgb@*_%0F>q`>>u6z)c5)S~>hvKQG1A8!phI38A_3KkpiHEDVOeiZ{x?uf?tm04Fj z)f3CpAn#ABY5kdQ8<$f~{3TD{|cc5d|f6o!i(= zx36cZZSI{*kz<*8eZL!dto`*hQa#+JqDRJkQe&Wq6VK_#j$^6s|Hmy+qTlif;>o`|k;W9*YFsQ0~VI*H>y4Soy8lZ21 zGCM0Pn)SG$dFYJ!OF>Y%2*P8|CaT~kQ$3kaPzIs~*+e2ziH;82k$~1~ilNk+1~lbY zRt!f~oD75?Pso2IoCa$xNia9*L)25hMuGVRm<~XHC>61b2Jtgsi+;b5?pWjjo!4UQ zUC*A~#}oTp33LUI)36wX`~I<2tBHroVKlHmS3$dnsc+n!z$H=5cjAz}**`e|l{Ii% zy{qECH`?j_F{boatZ6<~d=>Y@>4x*uGyf?1%@CBW0#*0L?A-^%s*<`gFCrZofcT zRVk_*78-gkEBBm%RiqM&RDEcknzr_Brz|A}TI!QD*m~R3*T=8?n07x@y~NHq;Q|vS zS7NUvF_dt5(}PvEbCnZjhYM(g>BHNg^xx_I1pt4Y(-qx$2rUMfxvoCI4UkI|(Zq*G1`^UEkw1C5L+}*y+azwRZ!=G zVtz^bGHT5U>RLZupXJGIb3QAG1L$ihNuNTi6}hFMcq2iiU$63;Q1M&}UV(-*Z(46ocUPT}UNy0zAe14DTXWz8CR zL7C5dFEn-==m&h*wvnKKtiu`v5etu6OJ-%<41VKF==uRwGLR6iZP*7Ua_*+3Idk3u zzH;s~kxUVf1o`=CFO1+BgsO*7|8G*JsJvm+mJ2Vk2iBo!=Ye}CT}@GWd~57r(($2j zD1%hfRawB$%}FM-!@9bgNic4Rf37(9`hEq5KMggHv+kFYkIz~Ert)g}GwaddNqjFKy~1k!RnGj4^Bq%L!- zYm)r!KEDz#yO`;{8K$1Y)^Obe9Ezwv40V@!$jVE!FF__Zb5j#lJZ)rqxQm5FdNq&k?JVe1Fp#=4?Q8DkP%_Mbg zZHYsNY$*$;nOIeE<3Mp%-=%b7m6IgECj2_~^iu_^|BiaVH^&#dOvd}E{K$fQ`A*lA z{3G44A};3w_nl-T2}r9yyko|C8YYYyz}#ySI2U*3Mrhl9%?z*LLG|?hu+FujNBctO3aaEAWZTNGSf#APmC-Lz z%r|v-%ywV3DK|G?eZlSQ?2pJ{O@^~O&T6KX%LG1KF?bYvRuh!; z3nL5I+vNvNw9JP+{$4no*LfL*hA%DX(>V#0yj6Tp;G_mB{Jt3 zXK+vZQer{2=KYq?zEVAvcAo*jMA~xYKTd1jrSlB{qj8ds6d*2bW@eT|Y_SLhu!1!; zYj}!##Wv?U(zDW&6kD?C6`b`NkX#DIlaTXIZEm1nP+r3tiJL`NtYohwG+sHy2ub{t zSFh&2*PC`W$d|5na|zZR@F$CzVf^95s~-l^A53Lt+F$&A@?4wWG3wdpAMGzLd3eOU zaVcGh3Q7?2n3oz--??|j>vDPtI3*5bS`qEJIDa@rAlrhAi9LxL1~lL;fE+COEuQ=6 z`38?G1OUah$GAu0g~LNsxFfKO*|>2da9cR#vxgHdXXct5Bt?+3g>+IpargmMq5F+G zHN=~gF;A>d?fnps6&igKCXGO%s zl9z>nYo;`F1@yRhE1Uj)ce{`}fRLw_ev@Hv0XuAOXD~up{5amud39E^XYQKI609z;Ha)r1ab1>12_+TMg*i zPSTJ@KYc<;!n%9+U$nUBA7W2!LLXMFvCbiI=bLEL9?r;WjPFwN&|#y3$}fLvej7I| zASehZ2FflXOO5Y@*M3r#a7rGe{_4DW=9&s3)HX~{ZOV4--rbS!L+L+?*G2_`)qiLM zI*|^@#btZ$NtKXQv|s35|3ER-#l~00LUF8a=p?IZSnAY$?#YkJ(|5%;D+?D1Qg_OY ztzTd&m=<>9tuz$wPnTqIf>%UqI0)frED$YTJvgl}^~~V!B6F;ozPo}p78ZSi$$}e{ z@EG*1%#I-^IiQCg9v(It5b%EmKo$V;$Rm6G%H2L z79h4bLi%P)BVLM-DVo)KyaO6+$PsD)2CN3Ot03De8yKWGIi-}EhRU51%orhPK{r;C zGAPJci&uoFe}99oZ)Nw2n+x%G{+O;m+c-Z6#s383rDL zA2ut&Y4GV&8glyjZ7dx_&x$XbQzp^(SVM&A2`0V?kZsv@#9#_SccH_k%+ktgu*gh# zZI63XbxL7Gp{PExp8Ce$QBh2Dl|U4GugSOXQBcUxW)R8B$q8VP1@?1ee)|Y8y9zl> z$lzB#e!&}Vxg3KgPviW{>EgUu_kS$yrMnx#Os`t<(1DVcjxMIA8c;1UAx$`!Z?xX1 z5_T&q|H4^*Y;ZN$25T^!B8PvMrgfFh`w>8dDg>ctm^i+rVQr8j&_lAv#8J2jj}~} za!PmHh{rnW*b;FVXV8K@Jm&?52AF!iBdUTC+2{TsqVN{NU8{8)UG}*;!Frpw2TU|~ zxc>r}47?Rt%clLQl`qTo1F|%wKb#@$L6Cbz=||1;-Cn zXl#L;Z16)w1nL{=avBQ*pcW(H<+jUSE)siYRHLYh_ApAH0sfHoM@lG#vRaj*zZOB( zi!(<>LJh~9QH(Yu7y>a8_t78O(*3^sqsL1tDq2soVV6xY{MhksMa9Xw%Y}Ta5BB@B zfA!INz4|;q%;c2Xt{Kd@$bWZQT>}_`6GepYBC|tm)gMe`pbLTbJY2RuqJ~8t#1gQ? zX_kYJQTU)-xeoiP&C0>2G7I8QA2RlNQKNKnchKhFi;E-W_VhoqDR~6QQ#vQ6&jE7) z^?=$Bg};&POEa90_Hg89)U?rq5;`j@EBh^~wEtY|u)aX|1IzU8FdX~*@gw;D77pIW zjy=Lu3(TYrs%dCWLz{==Cl^P2g-kTYvjV1GBHx`QzV6sGkUFmuWyCSBW6V=_@V_u>*FuWpxaUwT&i>qF9XQcmR z_vN{xmY?r8jXE}cyx%x}qt1x4DRJ?`gLDD-0Hj0a$PrZ)75E;YmaYIMk9r=%g^7)I zfTm|79fE-DgRH2o-j427de=*29W30K9A;D12U*n|yK!&@f$ZZCk#ztwfa*<6>EGB$ z%whpmrNzb`A_o0@YR_Tb7c_XB7fu^MkBcm84aUu3>}jPe%`z>#janB zVnJy$ff%8swFiIu*+&XLNI`&u2e4!? z(^t$`_LC*Bg%+qUhMH(y&1&2-QSxDby7$_9n1%qrz(zdEI0>>Pz6b<5_(9G=k#=O) z=98g-V~5LZgWW1%&jPd_c(XT9wZwlv|5gZ`IXC4S%MTwsqlM5OHsKyJn1tU^tDqMX z=QQ_c(KpU)zczquwEQtr0U&wnqzCQ{JT`$eB*w?L{fY(mNXzJ?0^P~Kiw4+TLCW{K zER2mVl8!bu9x#&!<^=L2hKARdmtw@KVM#4h(GA`c7E$!= z$_tK}br!+>L4@dtvc7&)K}u=?Aw6eb9}ot_K&|V8K*u_0FB5g{+Nw8<^lj?rXJ-i& zuy-B=!32o|221~jmPII`u|3c4dG8`#5ACOjr-mJ|Y98msxUvd@TVg+Nofo{}z;@fT zffD};ic=^DqtcfvuNurhJpfrIkdH&R9@*L3L(mizW{miM4j}Z6#`i&d*yWUV6t~ACaC8?W_t1p*A6G-kduRh-hehc`Z2nHC$eZshf zSYQDH^K}ZPd?~%0C$5zcrvMD3AbNA&pPj8wpCT%4BNY`D1z2cODS~^289K#1K!4*p zO(0Lf7&#RTv#2O+Fc<QviL9{Ek66(V78WOM zAL-xyIo5w1cZm!SG`u@j_=s=4CoMuPs*Nn9Wes(p8AUKn7ZecVPId@A>Pb zU?!d(;#MKf9_BfpUa82&&O76j&@QnOb5TYxm|3fj_AD zpz8ymg{5mUur+*2e>18V$&1*mhX-->T|5vDeb`AtPa^UT$Hahf-fzR-Bs(3Ly^=-& zE@SQrXVB+|+|HoW0RipHK=IBFBc8{A1yKJZ$}P;z6-4~C&1|=a&Iw-==V=E>fpxaW z6iUD|fjPfzCrP6ky&I};iWXFtWf@XR(}5zlxH#50)YLaafQn`th#IaZ8^?wvTzZga zNIgKnB~?wpi8|U{PLGcUXdu#IkcuIOzIvC4iG)NdaZoK*LA2kUO;6MRw@DuFY z)Vso}Al$I@-xa2iPWYc~iuj!$%xB^~{{0*NmOGn>|7}2ZG-JPgu_Cvg89(tmnJ(|& zGSeIbVw;rLtTn=Z9o4Z*Zy}!FA#s*O!m~7ehuBGMX?Xp5Lmh;8*jc$H;MqsS?Dled z{3~bK?RbxVr{0{*+h_r2l;sCZ$W8tN!iijM4;j z2p5951`xYX0;%oaxrO+|*5>f`;OGMf6(SYOBY?=stoUD`-2%KY~cGV5qh(){;7 z@p}yE|KBfu*)y>6MDcBlwLb<|Xus@ine^!DLOIvc;`iiHMiDJf4yQ+jII8@6!er!v z{_{AWn)ERcj{D6n%^jneWzW42;Awlm^4jA=#iG03T9~ZHV98>;s zl-u}Xm!8dk#JmPfuG`_^vT!~@-28Ww^ItxS^Pe`w+HH)UWn{`tE3XX!N;z=~d;efm zvWWz0J;qsfBey!0{+@|XOau_ZO3KqsBhQ%nctRaZMFIyD+^$Q{{(JuPhUdRCDrg0D z!-Zw*>G;%Cthj=uV!@w$l3&+kxv3a>vdUd)(bz+@IAcZnf>mNzR|ab;US3{O3&K6% zf$0cF);i4*`?6CFUI&43A)=#O=^Xx9t=qZLvaHd_;yPm4MNiM`X#U~4k`#1=_%b|9 zw77_zEG*a|3?;F7l}o`)$l6aVx`OIRR#LLRvy+VS4cTw{4lSwWS{K{@aslq)(xCXn zbO^~5jvGg@4E`t#Hd3H$MiGxvoY*FvC4A`_H7P13?R6EKou8xAyv0T?u6&;r^I^GG))rPL%C zYD(z*^lCsihNm|qD<4t6R|1^@BMEpiy1?aS$#cB{yWkvNfpCp^wVe=-Kc+}$F*O7U zAUW9#?}(D->F1vUYJ^H0M*;vNItxTu!0bRp0kwx*5V`&fwrI$@v0uR${yZ@OtC?|> zd(qX{vyrA^YH2Cfq>Snr7&%s)o^^5pn7nzX)HkfE!E|goK7lY#o)AX4l6US8(&R)l zXlija*k-ZPLG$>&hgXL22 zt?7IqN3qSufQgJv7+~e%@+vEE_VhBo0T356t2riML-^d(qUT=tSQm6dIA5;0PoX4F22n^IX`ZbMPg+xM*~*Y(LKk|L^h`}Ji*EetO( z?|W*uPdE%F(^}fvh8GOK$maFNJ;Fs$ZMqz&lPs=kr=T5yzPWdJIBm#)-+pv*QoubJ zb+opwZh{w36rr$Pg$)hYhsy1l9|j?=AziW!r60L!XP1M+uv8DYY7Lv4#;OgCwsd!O zeSLjDvi0N_d{K0umD?;$LFuAUJb4nkm@K?DkyKPvLN4q#>AZ)R1XFU{9jz>2@0da6 zlQiY01t!_|=)6Xk82v`+7%&b{aP4esDUGG=TRfS!u6sNXwdelvw?OfXMj6-+p;m_2 z1p%N@RVv{xaR=^CuK&9O*n!hQXpRPK0J;TBoKa?jt-!?Z^xPV3Ge_rswc4AX4e60l9{@`AsonN z7bd=-X3sg}4b2Bg#n@^v{r&rkyLs)EWHRRm)I{YC*qepw;OD{uRJ~NMPcJBpjNaenY zid36+>{YRWpnDqk6G!?xP#cGb9Ui^x%4L4kl*dr>T)r&D(M)*4PBhDLp167tApb5B zG*b3ro6Xl&PWnG4(_LglJjB`<>;(wL6yy2bBk;+jC!xj0H3+Ou{C7%BMt3UvuS8(z z1ek#2h8DtUKdn;!CzO45nIGbg0Wzqplsvmb-?(Jm$ge|z-qHHpW)5+c-#lCK(39@1 z`BvUtcAVM3dJRI}X}afaZPd~-aYl}>S{Qavw1B11&z3}q){1~y&N&^eu0$U^F`~N1 znKA4SV1+(53v_2+bw`oN8ZfRi*R zqeQ0ST zzlz*wmkD2Xx5wpKSvHv6*%>J+`gysl6RQiOs==ZPG)sz!QSwQERw?Z8aH70@ONGNC z#(4XN0KH+JeaJ*v&YpYF$iSuyK69}lH0g?4m}0+yw|k_i{O&dWTHQ#fQJP)KHFW z4`bsxHoqU`K;gno?f|FF#IruF!vSxk%_QNALa*pb@hG=*lKOtD5G~q0tO*wVT5``3 z#ldOt>Hg%mx0}Ve(|7onRSHN^7e7S4r6V1Gu~(J2VJEIaf-`8MWsTOBRQjr2Tl%!( znya`Jl^;6pLV*LnZp6ykb8j1~3j@9celRdMOUt`UFX5nnoGkyCKKWTHaw_|e4Q+;7 zfM4z_7lm(fk%y-zYeDeDw^G#EWKc^NWC2N$C4=Jnm1w6S?yq zvfH>u{c1<`R67+H3UgX*tiXLE{l{pWb8gd?t3_x_*`VE(A< zlo@FyI#sV>-(DtK0?vUc!`+aE|LcmEIlS0G!cxA>xI%v>19oi=ZnONu_Czsj?W|Qq zI?4W$H2UMmgFpFjHKXtoFHhW+bk3O+5*B7~!y5dm!zA^yHi`x`bpu9>WTV5dlrAHk z3Xwd>y9?&VBb>A^Djta;8BP*t%DdIohEnn zm-_YZsAt=3ZC~nhGZ}gtlMuGB+hL<0!rBzPH;O8&sy?8;#~J{FDrGNp``=D1X%ha6 zX{xciV!9dhCb4?v#Egs+%q6&$AYihX$)4@u9&;QVXRXnc6f-7C0|+Hy$ezma;@BHJ zoa#oWr`klV&>1*52$Hb3=tRPIwi*RzQ`5&lU>Fz~DYyaPA3Uh<1*8_c6#$(H#k=M| z`YSd^bt?7(19ey=gCv|C#IDox1UP9}B|X~Me*+~Ll2@JQR1Dab(~X>tAnBgL6o4jGo_a(H~({EID-Cz|!e9-HpYG^U^(#6T{D8 z>K*obKzpY(64d<}Wg9T&r>(8 z_}W+0c?fL4ucPQ{X+wRV^50<-nn5`1-O`9nC}s=F2d4Z8>Wf``efiNe7yMZRX`Xa!Nybq9=c8#CZPlaDU69LTz_oE?9ax#QC|`F$NAchJ-uIA zUQU+c4A9g01v2|E&%F!8035+Rza$%hf@W!wJD-&B=0RSV5{amCr?bW3<&IRnUo6X0 zjKAm;8*D7d_G_QkOK2X@N<Q0U3=yoAy(uU*}%}S_2V4QSim;X!`5f$?lYjg#qs04{O!##{kX(L z6L0TYxR@rpKI-WyMc0BC-F5ifKvXk7XeNV0%Vj8JP!dpB9;x zW#tQpiaRa%8uwhUldeVKGEGkX3)}bl%ReZrc8;!&XlDBzPt3sq^C@v$&kYm} z-Ot${cW&paDjNGEEqk{5-ZM%>mD@Eiu<+pSd*x5;yeHIqmsbk~1YUpcPLz%`a{Wpu zX&1gnHeDOhfMLI|pFzq|1H&F18nAf!j|VngBY#bbg5nl!EI!xLpFcjJCc()Yl?eC( zC}F!W9SmmR;GJwlb5Xf%yUy$W^U9oE!1hBQkUJKs6T1ZVw~zjmbVt>SFk@C__h+_C z0PKZ7G`+!fdVi~(aItlyN*etjQfsH5%UfX%1mZSNR!A=$PkJKVZS?xo}IHeN@qvuZ31!qTe2~-d4J22D1o4xNf^E#tB? zr_A5}@X3?Wz;);A&Y9HvT{h3WBO7?V$su}TNeQLEHOKbJS~Y~m)EK4neg^YqMM{ap zM7{&_D=XZOczMAcA6MTS!ZTg$HWbM0k<$WdFKSL-0K#u$blr}HmhujR^U8$*nhVm^ ziMnpv1G9(D&8O`Hq_&Tk8)p)H==$vQ?qPQh)gVftST3(`g7U=&BYec-40%@-Gm!;t z#$_`*yYM-qPWV`z4bKxV?!9xU{*Qk4PkYBF-fVH`y~)kPKJ#@#Dpo285gx5AFPBAG+P|Y})ZNQnNorW5&;e19}5Y>Ww_BI_&6h zT~@}Pv3Ox*EcSRlJ1Rb~&&lz6VD0U6`&m9dath7<>R%@~I3#OWf02-IoNZClD@m4N zAF6JlxPAF-w)OV*>pRCRd6<-)@%X4)pVlt@R;6mWu?<&6US3Q@OWAM3qiuUK^3Fce zn{iHHpJInk4cZ{&yllJG`3dv|6%!Co)8l0iKfs71Xim-3(^F45&j2*Y-MJFznC4t& z(ZUCKP?5rLs;lNc9I=Y-KkJy)>Bh8)=K|YWGla)YZADv5H*aVBIl{K(x%Y~b!xna5 zz>tBntA|Iyxs3KdkIG678a{k~y7QuEXZl^4>^+RZJz>_4h3mC|MyNfb`Q!=;zz9c9 zG%G%-4a<_B^tr(joacLCw6G@IHW}CzbpQqs$QRK;I2L!)-N*jwhnyvpUP|)vkp~ZY zySWu-2Dd@L9IyMib36*Sn&;;$(Xd0efPk}j%Y^#S{q-qd#0>sy(qfC)Tk7Jw?<6%y{i_EF zyL`LY^#LoZ8+KE8wrdTB4Ld}sPlw504N+EHsFadKrc5N~ZZifVoO6oC>1%GRA8KGA z3|^6K@fCtH{{H!clq3ofUS$Sa0b(K?1!G||`L*-UKj&Z1<(Kn8sDQULwYNWRAF1zI zazat@+!t;{|KmtijUl@AFK47Td3!wu!)lj6tAM!EYVIaMJOZODx(U>T-$MQv^6(e( zekT!^mKVBf>F(8I<4y3dw=2WHBH~8hBeQN1g${cU17xpTIT#~>IZL10%3r|ASH$|4 zXe_tC8N7xPgK5maFuYi`{0Ch*OCTquoMUy3(SlnWfdWypF}ZoTkImg{`?Fp4K3o#r zB%Y=YT5#u<%jPuPdNOnG$Wh#)cu+qLkJCB-TD+NZS+A54aPX&3C=-SSSXg-)r-T&; zsH|ZmfDKsY=S|>)60@|lqzf6#8;4n7U(1puUwF~*qW~7#<&h2f`jmgehhM@YpoJ+y zZpli1n2gQOUtL$xX_0cF%URZnpN<^)V|?K1)heS_=&+9f&sX6V4D&v)VZ&`u)wMxE zxyf&!a9R6ildl42$p~&op$~fW!t6jy42w~|0Rjh&EyF{Us93jowJ=zBX$f={zj7rN z7rSz^5ITU$3r-rKVnoah`kVdi74Qb<6|7#fX41roE2@eXP+bWM`lnAdXT^3=KLhp@ zGj*1ktR9U=H)-kjOGoe}hC|7mTg&tX*zfVUt&s%f)DyS7O4HHPk3%icfafz9z6&NMhwa!QXR`g$rQr_EEMM3=gQ)Mi z4-UkkZNy9v0oVzVx484QRvl22Q^Onk^I=O1#|avOrJ(N{weaOQ9wIeA}o0qtCK_%p66^g!nsyw#nFCW%8x5 ztb#Quqit7?5MbxD@EQ$=barZlg}uU8jTQi05*0E*>>7#@zye_&k7q*Rqeswr_;9a& zThQl+B_(622skHHyo)_NkOH{DL<7cBtI(e9uRcGwHwhJrCkS|huRGh(5!o)DVLO$K zYS6tPFir8lEN^w1hDO7F%?a*$_>~p_`@Eseehl|WckPNH{dfu}7QgbDI!Y@A@Q@)s z8EV}lU1Nhg1x8BTszBtXOtI0Koq(iUu;3wSiZ(bHD?V_~$s_amtzW)+1r0Ky54J4U z7tU(}?~Y^hNm7!qz(-%jJfwNL!zQ;SOH_LFz`y4T*wLsFBdS(%J<}V7YW8wj!JcMQd3qi zQpyCL{Di&cW|FC*$74G8(Q&xn4#sFoj*2bXVYDx{K=7WEk$ba~i{U7OTL=}&o&IP%CB(|?SU zzrA5z|6g4(HoW!p>C@Y{2dJxOGoVd|36~n$kb8& z?R%o8r*dv*nz6cp%U-P9)m*YA0i^QZA12wali=j}znR==%EhQ}mTy`AL?Y)8q5J0- z?h*T=UEjE8GTwXGv;5>5y$VS%7B)e#JU~b11e>z{F_3?$S39GaBg+ic`5woRR@lfMICO*bs%#%wJK= z_z^WlZ{EE#x3v#z|9e*oFES-XWD0LH`$VaA2|X^UFZ?M35oj;cyoCKQ_+I|u(g5OW z#@91E5Ni=NwK=ea^pw=X}2VP-G-p zF^Vkhg8f4OHF+{S%rC~p#ZiAoeuE<6$*HgPh2Fsk!xLH_#_v2C1sfM+aK592}Y*uC~Qdl*@u!n`rc!ANMGj8WLM` z!C)xRSqms|$6)7QmPV?g=%;k#INs7085o?WUPRn{2T=0eH2}dzvc2{-T-rW8I$S^p zkfN3ewznPsRt6VmHNU6uY5TeCsq1g;Y52q zJM$K$+t8<*_sB!QvcvSK7M!%rtFsAL7juaPLDZX%A78Mi4EC(3+exG=HC@}KO1Bp=ypOd?n-z8YdiKI)4@W7YxSj+STdUW&l0=$Bm=$u+q`#B4SKC{z1p}XbhIkEdYtCA8R@*e>F*|YT8GiHPM5goR z1!MR{k%uuE{~e|=1&|tTO_hCPj?gaf=izebSiBA(%YFWQ)$3|ll=bVE_29t}b@fd- z|-*b#HO8rfB{I` zNt2EuyPc7(;rAWu0Ip1DvG#1joEWBL>0Nvq#%A_G9DQ=r4c&4k8{hTV!Vz zJ`1lAa>9Kiq+9_SK7}? zjoRKrgzp6n5KsNi&*{Yv4|lv*SwAmhwYVr{HKdT`-EWIY=NA@EM6q!eLK)Ckwr`r; zoWaAPyHsV^p_fo<*^wuNX)fw5N@d6=2V|X2)27W|yciyC?YZ&!9{HHjhdgEofw$E` zG%Pj)Iia?$1|gdw^E4#_tQKTFg^l?y#ce}}4O>!j!hsqYeIpi2aD@)evuA4+yb<3? z-l)6v>eXD=9y_uH4it4Oin(uSTm=~8+9i-O(nL9X=^gR9nQg=+>LXvDQ$ag@!6mzM-OzeS zL)fxll8{gn9zWj9EhW|f&>dQp3;;m0F!R)$ythkfpD!=CEulY7jp1=46vt)jnys(L z>cv0!Syy+JWdSp1Dmbteo5?1!ZO1OXRL|-8e+Ts=b|Sp*G*zJ(V1hOcbYx1SE)oRV zUNOtZGR%pw{-%vT7a^+aHbhOdrzkyot}?T$ z8tfd(WC-X1qdsO6CWt#c0^}5zQotd`hp2_^J?14=m*lnMwLuxgZ8W(EW5y_pxO>Hy zmo3!xkQRadX-G?lM??^9B}q05op<*cX?J-^CVVG?PVKMiVe-^>YBo2DuK0V@^=Hvy zX`~sdUz$UXjkk}_gu9~{z^LFiqVd@xsQi8%X=-frhWCJz8ewcI_2$!aj^XFB%rc0c zFk5Uonf>}k;e|%!qJm@~F=6cL?^Vc&Kx&WPy(zq6@LC+%<}Yra-ELhqNu(!K4D6ln zveE{C9KItWoo8o8-qo3D;dU4T6^k)2qSR*_#<*P}RuQ)grd$pi*=Kub@2BlCA!}yd z5?*I=m~Gw{5DpOMbmWw>W4_fI$O)2L(#0)L)V`6VZ$6CL2hun^+{TG zAr*dCkmcViEL^^FB|D8YM)r4@^7uO;T~<~W(VE7b=3it~j`4)j(ad{b{(zow7riMZ zm+Z$Ef?eDD{qu4cYph4|gX1xM5N0D;#m-TT0P(PSDw0EVw6rj1a>2d@sLFM!$0I}K zUcDqm=nOEvYbQd{LL7is96|)t(Xoo^c+`#%eLvbj6O6^-R%JaIPRQuh3;Sv<-4|(h zf?g0@PGiQr1ymW>2VLcx5*yUlRTsz(-I3kD{2SF_eNzz7O!%2&7c<6jFHzS^&X}e+ zTE&-)U9l+h4Bb7@n6R-J(D+sXe+F;kj+)d~f|?G~UTSLNC*KaNeGut#v>sErIswSD z8#aJfN(;E#Yxwbj^Y!P1Uc9)1+T_4{BtW<3DLsl^NX7*q69EiH-;q9Ws&mC`J1R?P z@|)@9s?003Qw7?jnoDW!Kv?q?yz|H}-AWZAuXH#wLL}OESn*0c9y4#hew|l*FY7{B zPC1Z;LCsnnw|62(jW#Do`R3|!qD~O|*fvf2b}|vAGZo-_g(y3XlcwYPiq?Oa6%~Y; zNZ$#z>^Yb=#lCaFg9pS@eOFhN;THlPJrR)h8~1d_R(xbp&=s7FmWwMWDg8qB@T2%i zYw@GIJF_L$&uJGtYe6d(#NQhlvith?xA#x$U67EFFdyFt+Ar^6T+ruqioktsDV?4?SB zJhL2qkz7pV49j+5x1g$)mKLKUe6oAV{GLzsb+yy%oSnyd-0u@Uq!-hy^!03Ge#071 zAlB_kF!LUTF%u3|7Q7=yBe)a|Wq85eZZ-ZDb)l|u@<{opIc@61nNTochb?6mp%dnr zL^?k0T>na~uQTi>pKX7WV$t(((wat3uB>(UF5qyD=Hew$J5f< z8c?{@-+w^c$u#49)p`deT4!IMb?fhMK-t8z#Dc0-(A^ ze&^{5Yj&J<=xKeDv4$tELp3(0^egmGjuDc4(=`V0wxCNr@S4hsp%mgzo3(g#Ih>uNtc{vpw;nFR9Exi7tp=1M})?f zRPVle#FPNUMs>=+UifChTJCsm2@Z%d0Jjs&Cs(}B@9|cl+-^@ux3|F=vIowWJbE;6 z$PlQKB@PbJRPH+0!Y}*ROCz%| znSU^l7etO0Hb^Tge`dcXgJ0q4X$x+i8=&1`?*`cv+qKO7@m=y>ZUL{L7VeitoZ7i4 z@+{<7xQ!#Xlw**03*`X4T?(nr?t}h`9|r299?iK$wVI#MA$i~HyCv!>}xe(-@b${KO6Xse&ZhlE94VqAiApOq%~2(3P61#M(Obh9d@L{`lepR!^&)dd>_xGNuye)a}B2UjQiu$hLUht3I9S$ca zKSswO%DsO-DzJzKXG-ded+{dmsk3FR$}6e+wp~9*E9^yFBL$CBlSV{a;tITJcIs)F8cx{-@ADlh8!- zJv?%{Ea@nMk4%X$x3tV7@VP$UH@;$^FpMui`vsIEX>ogPLXO+lyMSN&GNX3r&};tV z39{;BSl1(C^dE)TwU1KPp62&JYS54&nhRU`L~|;}reC@Q<5ocDc0g{bm6h6F++v3| zwZ99L`{IIJ^S%M&e|b!vNHLr8b;G&UHhcz~-~jWzs(2V%ayn4 z-Dg?F%mv|6{ZW07WhRpT1iYP!$ztR(yBiQhGem zo6h6L%{@GI@It9w8*e%;H9R0KV>kGUlOisUe;}`6fE7I-D4`Aq)(?@r1hPWsV7EeR z?RZhGT6}7%_Nsm_H-!{2g2ZZpFXdoT)^|tuWq=v?=dIB!&Y}9j$ALMGJ5RTFOYA6; zt&EoIiGy4JXnnWsjmL(9_knNF49+MlReeXRz-4Ee`NuZ)-YqIp>D4QD^v3;DP&B0$ zUcGj0$8afMDgq`-BZoQu5%LU*e?VW*PQw-$;3gMtf_HHQtP( z?F5zMG9YH*P6~|Jx4C+-ZEwQ>Tc@m(y9X2gE=1VA&d#c|TYKCh!@OA0JuJ4cvVQT6 z<6hk=(9KdjDy`__cK76-M+>(K1EzOwbffJOO1?$whr$q6ZDP19bS{jtev!W0+fsVRQsmG3%w^XR~t*gr&=Ijf#MntjE2&~?2X_EV zqO6`d`DARg!jvGHnGz1_tpyr*ETj)nZEL8jQ)1jZa<}aC8`$2!sv33b&Cpd1P)~=V zX@@$NCTXv`I{GO-@xRKRJ4)-kZd2-Ik?U6Ue&qdA_r12|^F?tA{xW6QVV}WQPbJBS zjc&NF*KbJIEsw#O_T*kFZ|E}qQ1#)uL1py=CyqK^Y0RhEi%n6apJvszR_D&z7bc;JgxiFFp{RT<(b+FGIQyM~62*t&kL{Dkq7&JNlou6%aRwb_$AK0TXs zHer{vt!>uk?QUXHGi4%Qn}1dBWO=A(Wm9Ech}uSl$mZr7ll^`?_fT)=cKZC!Qqf7x zVr4nXESfP{JPLXl?PX;np4W7l6XKqonvyNHO{DkY?siK>_TgM;d=zRM9};4=V^lwu zElVmWZ*|Z>F2Ua2-Y-u_ZB(_lU0!~t+jhr#?F&|T(Ko#zRlNH+mM=0a_SO6&tDD7j z8sGLbF8$7k+^kt`K(Jgt=pL*M`H>6-!l3kuh3R?IO}?)8-u?SEr&#;T+U+NSai!)$d`6o$Ytw<`mv0W7OJCHFLxt1DO}- zya0W_zPwV8Wt?Z7BaOFUjQ)EmFa-E%Qhdvxansd2JGYIwIk2BsXZHv5N^K2t-R~b% zR8n%Bb*O)``$U;GQ01HztOrSCPx*INdAIQYIA-+s&5^rYYW zA7-Ynvuam#;Jcj!Kg}~R_V$pDr&#I+nI#oZI*93v}w*0C11Vw?~KQ{l!k=u7a6K< zN^J<+R+{kaw(v7RPp=kzk^n{1m|~3$wg(-Jlat+z&+O1`RBwd;NAVS9wJ->yS4kb&Kmd%}mYAY=d+(9(aZdpM*@-_7^*D zKhZg2Ph#E38RM-(4xM5@!k(g&`I#QzN#M~eUa{yCYh-R_8*6Tzws2nRsS}a=yLIg> zdo{|znC^%o+*q^Uf`g%UCL>pcsyd&Y{nTqeK$P+f6V6+PCi& zIgau9%=q9DV$+K@*-Sefe5_FOxYTBK@IUh{T1@=A4ol2)IvuLCPUyz24pP~2TMIuUvDMh>R(S;w57eDEjS6xN){=v1=$HyxmKjnN7nGwVrNqa_ms224kX1BPm! zJjjd+HQVy#4O}zWYliWA`fZYofAHYmJ^8CgCK_*A1uLEBqk|ITO?9>QkrS$BGC-Re zeFH4^F0i90?To32oE)T-Ql&K>oK|B!e@}7s_GZtu)d%TI-p}9YM+%?JxT?E9b98BH_CBPRXa^nvn^&v9V8F{szpyiIgW`9yiuH zS(#XApjwJjmE_^z=xAHWW8|(tYvBXxUwN`q+Q92{%(aCTq%k@b!{fJquw{lHTRXe{ zjb-U)sVJ9?pv;dGYEs>yX8ktbZ5TyWmD|3Vq03NrOyB-`OdP#&qm>6Zvd?|4pO|)s zP*s{VO+z=;Uh$U(0Z(Je3~I&FXKepxp0V{@$a$et8PHei=?AP zc+~q`I~@ujVwPQAHURR=#AF@_%)J^%?RxNx@84NMzbptK@4r9-O_&n&Ee=0*puAgJ4j& z8uyk*p4%~LMUdxNs=M&v2KZGgmHFFPTJ|fiF3ZepTqo!?o<9+){l}k#!Jxid=e-yb zNlCcMqmg4zeI&ih8TaSf9m>1z-P*WiOGN4y3Fae=0kUwKrd4;2v1Lm;)U-(o*x{-Ob(l+Dv@WSdRr8roWn!w@OMb=J^c3LSfQOuP&WC_y4)xd;RZq z#ZR7W9Ne2do_!@ZnLm8|xOrWjczezaA*xmQTgurk63LHuE+jmJ?c8}IA}BYO6YNq^iHVtE ztweJis|qjM8P9&`ZL@MPKw*^cv|CY%dqx~Ds=FAQZh6y!RX_jZFKHBVCjK9+&TVfO zHt_}M@|I6+f=hFMgT(+Ja7sO|8m9(?K@`GdpEZGj zl5vO2*6~k&?HHNi7a}DbG%Y8EMsVY|MAm4lN>ac2d6#{l#K44o4O$^|8`wTVYUkg( zr^5{+%IJ6?L=38x$|Tt2U8|y@kiFnAz;fnHWN(S|#_E#q`1aUK7j|BbvIcwyaZDo+ zFr13Fw>OkMz%Zc&Z~UCF-jhmxw5|J+TEmiJ=AVE)xYmLKXlO4gSdb&sQvJ60W1mgw zeR9bBWa=L~J3o*U!qK!;dn{~gd(_c-@#~t1L=vZQKjGbf<(G5SXs`HuHN=dA2j}oM zVG#1RsnJ_@w;C>pcv1Pob>aj+B=fh6yL$--jB#Av>)`z#-@N(8(5m)z!$ zR6}L=O7uh=A^l!mKrB>t@5ghE(qUNgLr=yWWM&MR<^_i(hrvl)oX{T-B|(x1%9F;A z-(%EW)9S!b=GrZPH8nh)cN&)#A$m?dMEXCP~sAG3j$cT)3JYsvK%vgf*UK} z7H6NVaQ9GE9-yFy(}=Kqt)H>D zt|=pyc=F_05bLd5CF`_Derz43hj}j_8+DZas8O+buHl1jSO6Pv7Bf(reLVf{?b~JO zI$>AO$s}C6cCAtG2el4tI4*UN-)1&Tdcx3a>FQ?qz3;rg0F~qU1f{-aGI%JU_&PB! zpKDPIcSO2YpLIkqQ9aSi+_}G&)-ks=^}~LC-UAWt>En~(UumNsQvBwzh7iqG;ObF! zd+tg^KI|Pp(GGlX65M*d^-dBzHnDoAq+wO5T z$Us*tU*6-_q|5`y<-`)x(k82%%)EAuy{SzuaQ62u$Q?Rv0@nj$5*D zA^t>aBxZf-lvAgiGgk*mjXgZ9%@NJV0LJzQ3_$xP)wwgxGb%&0D{O#Jn!4N9?53F6 z3C9chckWbB_JA(XnOlFU=2e_bO?AM#ws-H3U%oiE$?TgfRF}||E6DtCaxB{9E|QxW zi^W~WXcvl%MIaB{PSpFk`T4jhFcM!eS~+ph_&aSj7e0NKq;3BVpbX*y#xT{=Qd?Jd z-`>5#$jO;AOYu-Vd$#-XB=#>fB{Vph2otZq(|(tB1eE29oBSnMtTd+E*qrM2^XiZP zv;dqaAu(TV9-w%rXdFs5Kw0#1oy=q)f8{LVvN4)e@I&XasJ|18fc(r{HeVJsnY z7#Fpi@`u#{S$7(LUC+oUfA?SD58klHjX%gs2|PLwcC)lC%s3%}$}q1ffBUD_i<#(ai_ zotcdNzzZi=^sN%=C!B7`WMW3D36CYabvu3hIJ+BnVqh~VN7w@i%_N+Wg@p%j9>+Vi1q@Q>|$Xv=3;Eu^{roG7Ktd5_vh%~0OWo=S=j$W+t#T5ZMyhc2pt}YLd zELy;4`unqYkACAkjoX#=qm2LC8rS$GLD;7E=SMOm^3Q|dAO64mw`L!!{(hg7u!8q2 z=+ld~0_=bUoR<1z7ltBl)@*e6^Yy=F_8&B;pscKnxa#cungRiak{ZF;`37Akn3%S6n$H}_icTXGEx_QOq*F(;zU&ft%J?>0e7xq*G`>m@dR|7PeW?(_kAYR!?HMH z4WOn=_ys+goV=hNbk*N|eMQd^U0%6%P3jO87&B3KOG;)=n4k{H4J!!@((ppCOy&67 zb(0glxwhf&iwW>U@XH*CPSb2Br)GW`l|e&qVHFi442ah>1q|a)g14%vxAZj0~)it;jOX1+ap`t*|7jrFRMIF>*{c0PpR-W4zxvsesA&_LRQ zmIC#&IX7$mvE#=X&iFzMPde2M`^K01SY3_PGJB05-oJkek`{bh05P<*JYTY_Bfp`I zeOlec_Qu2rgEQoI{A7RWf)McW%1Ge6QcC#JXV2K9C-Iv9D#)c7c++s>VR^j)_m+@^uh!G-mku*^E*(A= zA;!8H+6GFd&~c~&Lvzrf%U4KAhGL8N0-3PbRPj+puP-kf_brLF?kz!7;V2)j!7{!{ zou55^A!3=AC>x>vQ|;4OzilQXa8nghxE&xjn336ngj4*uE-`}r=h|r=B2lM)Zhy~J ze0caHiZS|4wwsv1(l8l@hvMDBz|qbCjNmQ~>Sq~GXS?#!04D#WOXHae``*^r)YM(f zmfm~M9#zkww8s)hr)kM!HkUPBuS4@M14#=M(xExzF94lDyQ$vZCxDaG#&i%_NRHAF z{_LA=o!K)+^daR(t8V4P0jE&(jA4w8w?~8tb?30XB9TKO0_lH@e?7mjy_pO*K-zt< z;fM*-z9x)icTCnojRcPowa z>!d@eZv@vM&w^hc#dFt6OLytwX{hAzbiL}qy-wIP>VW%}~&$A+1Y1Vct z8eofi*vw#9?A+k`P}(^<4{AVSi%=*IVTA5Pt3s!svBBBfdsGC8g{q=a0SI~M2|#La zyvoj~FJvK*Hl^OrO{2`IgIEgj^zy8G=! zO5=3@ysoNyRRO=KX~m{eOiOg_gJsnOGucFB?OY%V>XVu4vT|~^1bq{tg!6UUAA#c@ z`=P!GDzQ6fzoJ4cTQcvYoCP%#nJaWbke^@OnFY5|gg4oP6msFhGke0P=pW<-9UTLa#)ON6m2kq#a4`@-egmLw$$eItynlV{wn~rva;?7BpANFzVl|x zXdrkD-ajaCt-!ZN>2x0rFdVLWEo=Tztb%%H3i2(~r13fBd#CE^=m;v1#A$~{ud)mF z&mp@|pk2tDD#2&`uVxTTtH7MqpN~&~JfMj}JF9TLjw}V>CY3K|_~G3<*&aP69JUib zjeHXXPeWaO+m$_AwoGEex#hf?rJ{`3S;8(kc1QIpL{#KrjRI=Kbct4Fp}Jdt|3OyP zTl84O9{Ka*LSkY{nce2&JDJQgmD!k)(L(+3`O_z*c=Mx=X-wB41^G1~I8+s_7g)?l zv`vZSE7-9Jnn%7f95Kb5R07}@zMEbN{nCnDP|y6<>(JC;S{e zTELJhT_!)#(O0fs9d=A(c3((uG8$(XDvpYtAvTDtEz-T+mn<3l!H?LF?HhU>I&V%Z ztLwp_0F_zhZBu@i8o%`VDuX6=D;pRZ+FLzRyCnIV22Kc4pvDl5&i!8#nZezOXG*JA5!k`kMW-vDu|R~zq}tjv0Et`ByUmF^7@nEGNKhQyPK7~F;(7KX#a6&{zA ztm7PFxxnRu0{O2UtLh@q-6+Vw)WDy}n?xq2@OvpLVyGiHKKSraK@sS1fn@oi`h^oP z;Oo(wTfTK~<#JzSp&M0U;xytoz!#_t_06VXA8G@+(>plx@EbdJps7E=h2^6nxPCEE z1-KJV2F2XcD{}z6AuBNuoTlY-ew_(@1Xz)pni|81jDR2s?;K zv|p$l_`oFAVJ1hA=Kpt>OfIs(hOfxCh%9&y+gDUr`1xr3x}=9Ov$r@;pU!uvC%geB zlTS}Ur{#sK6B#~tUw{NJxV1i>O0=e`is8^zL~;7-@OTyk$tfv;_0AZ){~9Cqd<%Mn zbo~_mJEW}>CQkg-*eKYnPoI8%pK|By@>=#|UPZWS9LIC1`Sj^!g*ISB3{@c=KQ;(2 zRQg%+aLHX#+J6!jq{uU72>=dfLaU&K;|T~<=8ruQ5&k3_P{7!j7?xc?zGe6fpq!2y z?yIsRJBE5{_wH_SHogufsE#@NI6Sxzby!YC+|N|lV(;i!^Y*PET%>FjlyccE7O=Fw zOz{#*uPiH{7_lEeIGd?ck^7jfnN+NX`Q)d+jf1F2f*Bx4aBfe^itM^NKi(56YKDs? zI(B>jY(d=}stW233dL--j;7{tY;3*E?Ef0EJyB7^xF6g? zfc9%wu3WH;AFc8g0XhG@GIJJ$ltBg35H&295L{bf7|=_A4lasN?G%crOpweX>fsb4 zQF-|C5Kp-chc)y_6T{uw`+&XiR(GgwMMrP8eA``#F1_Zvx(57iT&BHQ;~=lTvb^ip7k!*ykf`3v-t}l@V(9T>gMkCR0H8Q@OI}D?as-XRmie+c&<|)5 z=88~nB3fpPi`$y<>dvU4c@@G22)GMdpZ|hHV=YyRup}5#h&{+Cj|rKq=CTTD;d=0~ zGO4eqRe@QV4?kew(RJTsAGcm`{ouyv0(iqmVVef6kz;4eBB~1@10p8`OyQWg2bImg ze$iP7riGtZYnt%(2pT-)$e`aQI06EUgCCakoaPSLHJ2n zM6QRMtmNRq`dk8uMRzOIgugcj@u%=9W`mVLv~W_vE}PMEm{Nmyww99V(G zidF8@=cR2sL#hAX^gH{RGx2T!YA!1+UCryDYxn#jyMcg?JX5HF!9-YY4W7Mt*-#D~ z?}Y&w-oNgb@v;JtynhG_PhKFNJZ}?(z%<5aF_V4Y3x;H-Pxo$Uq17e-kne_oTbx_w z>8WKVg96;Q?eh3X#l;};)Pr^%@2ZZ}q4VWueP4;)IXkv(BT42_cmj{)+V&5dNQD3R zFnJ4|2!)(b%Y568GKkMA?E2*Y2tj4}ui1`7_m&5R+mGyT_;x8S)?}C+G3w^LcZGLYXoZzNdfO&GA98D3q>8i8EW(VY#oUW3z+!*OAx52uvOTX4jA?Z zNjjIPWM2B^%gHAd7|T(p369y@yf53 z+(#5&b6H`r@iQxaYw0$qk|*m5g(<>S>U@lzDMzGgb?Gh0LWkw{_2L=@jm^w^rI6Y2 zf2tbOgN(+3BFD2f-8>Ni9J7FrK=wSlXK~=TuQ%A;%(O2w0Z-5=-LrdG$t~M%TW;6Z z)lIgsaX#y^iO}`&^_^g}BRl(=95PQ7laOF<`UCtiZ^@ywKx}k{GJDv|lV>c+L_qws zaWLmXanpF0Su&cLWh+1IXIwn6cW-wy859c2$`+4@ z{<}~&=27NyNl7OG%a#!xgo#5uEyw#9tCBicCGzoOGp!R-o*l)+zJDz<(bGHYw%-|S zgJalJXLgawhI=^e;1bK6gXt}&LOzUO!jI!0%l~3K4tNcnQCL{Wa+wlWVIa2h11hmm zvpU-vnI5*QycaFkqX#;k-nKqu9vY9Q?|cTlAYj^oJmac3spBKVe~9&)Q9XL~!a4#` zgRriI4u!JbGLUBc1DYCsr--Pi4{zQq<=TC4hWlPkaG{~&Ruzt!B@1!^1NOM6h~3&s zTW&8p*8-$}!0H2YZ>0lz)zZ#rSRD`s0g2|dy3|P%myFhqyQ$O*kcVcHV?V*@O>H1V zc4WolC~(Y10^+V+NsIoK@TTYK0pb{K;XQ+(ahbl=);gS-p9K@R(8Yxr$B)gG^-bJ) zfLdJQi@0aVBpiznb>^{nwO5zB=&1PZb(*%3OqnY~G7ZS*owZzj{L#nYGYt$gF!iF* z8}RWip4G&@&pmhd0_bAz@~E}H(hvMGz`?2pekDEBYb$mveMSm9Skb*EyHo}J7vwF0 z^xEK!dF>cBkQ7E@2NDqQor{KR)G}%iCK6ca+@Zq;=70f8FMcrRLkkH7kZJVj(T?W> zu;+Y@i3DBw)2Baq6|#qV$ja6cY5{7dYkeN&!Z2+t*4Pw4**zCyqq7=D2}h4>5(6Z{ zCi<^f(G>~S`EuvLz{{X580V)b&hGH#i_o%BaDTqp%UJQ>Yfo6j6e?$k&w=d2<@|HN z)Odg|o*w|?*Xrt#49}AUK$^moN6nl%^(*iQ--}_afrADu&blBhsaRF-4tGwSTr$b# zupJIQ0!nFf@Bs2G*ELMW;&|Ke<&~4M+$T~nRZwedv#*{Hx0?5cA}&Q{w-#xk0C3vE zLZ(=idX5-EVvDggnvcvqEsa^-;t;(6{r^sb9Ox4xR{M1vl8zo74%3Z6I9@-(&UeBzZp8AA>bJ>W4H~??EGSy}^t}6`e zqJAi?sd=|(@nR++D>?WsrPP7uNTvYQjQe$RW0^Gk*oPO?MmYC6-Y>wj3!hW}H*)H@{bD1kfM@0Z*co&aD zpKJRP00If*Dilx*i`4#EiHB$ecd*T~dOmpmd?XTit7Tbakd^k1-qd58qM&rV|^4;`|R8BZcg z>AhD5lxBf`G{G&*Bg<3@;#2tfc6-OpotPMXqeP%%fjq=7$V`Tqi+FwV;jT)CRm`;a zHdzSvfhj4ZvrT{=h6!j~+dWTHuHT>PiZ0dLs$|?g-7FKSYN6z9J_! zQkk2FUb?ihpiW$^^Co>h1qG~ioUxIH#^%X~w|nKi_aoi+Q)=l9IGz$AY>I|3T%?Q#ziO zW^BC2ru~J#eC0ZO&>=9-uME^s7_bUyC}aRnI?KL2MHSx!ToRxh-iGd$!92?8PNUk0 z5=1BJG+J^;MNOEel`DfOtk7EBF>GdW2`X$F@K^Z-+dO=Hl)HDw)OwSOwT;b@X<8?P z3kpexIj;K0A+7sDGmDv7_$O}E`s|s~`kxknyn@+vRX=?b_4k1r-_-^Z!OCvu6pwu4 z|A!fA?1z6>4e@2htXX}H13V-A&mg=bhyoKZIfsq7g5|HWi$1ZCW5%h@E3A&l{)Hp&a8f#mAopGH7Xc6o3@vD^_DcI5pZJU(0r} z9GP-=wocTsRbu>dF|(j zfh*~un&T3y#A$T*U}A!O-(3n7lMP<@qcSq#RMn5r6ROIeEu2=) zp8dcX_6LjR{P!uxtyR!BfF=vZXeE4Fg%b;z*6R7~@FJ25^ zsd)?77e3 zD?(KPUvg7F`0U7<&&o24W2DC9zg=3oW=8vMM1+BQHdFgceMFP9=zxbSkE7dG(JAP|wGhV8tMC|C%w>sO>x;&ZGJpmX7E zcT{N0PbMaFCNP>X;xy=qx9=3nUP4C_F&^KGlhEl6+GI{^)qQnjPB}Mt$JO#zO0JAr zrJC;+TK3?tKAUd8-8!&*(y46x3-_A$YG5!P--NmLckcjaoH}LmWbn>MwZLw`5ugyZ z(p*^jN;|CS3XdrTf(L?L(mZv%Ta@=N7Ze$4L_~*&_fJpA=tW|kM^R6>M{5)j1}I4H zS&JbFkm8O-3W|!UtZdEi^xII$de*Gr`ucWN4|%zI8)`~aBQ%l*Hc_O5d|qst(3|xb zMa9KER#yRQ0Ur{WqWHA1oSgO@#$J33s^;HK{);w0xk)+zGa`CUFzPjCwV|1ei^3rb zqF*<9VWjQGLpR$<{KKI361GsUy3C&s73%BmKCx>n;dSuaxmC!2(L?5~l!honJxM7q zJAL98>T(~u0n?BHaJ5~2ELN2Q4dY?^N9gNL!%&25o3UnP;w1iXaz`6|nQWR!%AnfxN;uvfD{xkQ} zoo}{(WG<3D`}FZf(84r7q$MFA&t?!ajvpu+2GRAPNZo^1QykIDV+r`^M7y1+-dK6M z;?semWY+c(IIX?JfFwvV0^mCqiqGokV+Wm?A5T9LISepGZ&ziDM`CDbdnR^1JUuzY zz#!s!)G-NOA=ug8`#~KaamzgoO9k#NJ(~53v4y*~Zr$T(-+<52T;%GEkOZBF2Eve= zMcEULZH<;9x~2bPICh(tY2q}Z<&7;H+s8lf7}`tuT%3ph^50v;|5Opmz{DViI1r&F zLfHX)3K+;=d9;s%&4CM0MdAx2C+>KQUiwcsMCoXyibB1 zyzFZ#Wc%Lg>W9IMY-8OlrgML6Y<3-(p;FfEUp=pc8>Th;3Mj6xUR_e# znX#63py8!i z_4x05ITDV{WJo72rQj4CW$I>UdyIikL|KoHiijw$4K!cbKV_O0%Ru>Gb*nChU?Zj! zc6a{JjVhR4fa%D@T=We4{yl&qMNiuO!J!slt{(^9m~wBXyScpwdAu{+>Dme(lBoQ^ zrrN;kj3g3xYx_#2QVP;r5*YJyb3MP_6ca3tU}4!MF_*j={)4mF8_OZc&l$9Cug`V@ zaEE$!2eqx(+|%XRix-(|n`au^ZWE%OQkeVV#V;O)7)t+*7gZI+3*a3V#8H)ZKrn-Qp!HlSraC_r`7cdoZD{j{$_SE@;~z?w)?^uap6B< zPe|MOW9xWTef{t}*DtR#oNoO9|G-33Sl^Lyesm(*PF~))&0V5d&sgEXA`%>-6`nCK#-tZ@qA#f&L%V3zaZrH6+O+yi`Ex&SO~Y`0Cu`brnH@wu4>>e#*GdlyvvjQ`yZG z63orhLO>Ll_csY@EYjGaYC*Tb6z2<8zT8PinkooKs1dS1DL!O?1 zpAD!yV=SDs!VdWNT<|&R=Qs3_sz+IWI(_1V>ri3J)qJUZ@uX+@S~`@$=I8-FOUF-zI!U0YGeeyJ-* zp*H~ET6xl_e!O>ovp|OcdyEE*+aff_74pI)m4Q1$Mi;G*)TH0TM1w}*PTP;QCV$bA z^)}%37EjVb&`LAppSUG=`}**gx71&3O;a5~4?>YXP_N&S2QZnyhdaCy==K@(qP$et zeeBAdC2VPg(#qa_j`_8y!E;#)7F+Vvj6uP$?c1reW5%vs9T8R?@C%v?{0lh#R(Y-C zszr9(b{b?z{8pAI=RE*71r+{@_K%}qs@5e05Rki2RQ;W|IK|KRdI&Xe3z!a|C@0SA z`0isjNIh6G2~`cEF4$|`kFFqKXsljdSiHY6Y=Ywn`gXzsq97K3=h989n=FGy5i=8r zhA~NAk0>Yv@B|FDugf*D_WxqRojdza%PoJxs*}dd;&$)$`sDry6U3%$NdRD}HvP!n zK9{qz-+ip-L0FoZC1%ffiQ`bN%ULzW(Z1lY06YSxxa3{(So{qX6)g|jQ6~>tuxtxA zYsP2cjgu zr(kulbm^J+_<85=7vlrYe?aXsWbV}`oBLVzr@v%*^z-LUf>T}HVW%$t9!dImYdx9Z zbo9F5j#SdtKjuxJ4(jIu81r2V9sBiF{`i|!G~e_eagaIyEvNCQQ9Bml1Gp3ffD{2` zFy+~KR{UVCJdX*){(0k<(EOm9V9(m-O`8~JG5YB!X0~($BAGAL+f*gbNwRfv&>$-b zulxq_jLL<7HJ8_jSjC_>>MR;tK!XbjTlkJVpPyiJ93q7_IrVLT4ojY&n}7$6v9aT= zE8mj@pCLoT?+_3?g`U%6$N9m4^FC0GLxlmVD6%nN(5T40dz% z!zI8ZK>S;G?W8uqe$M^r_r* zFIxm@Wr9FOK|w-XoaANnGeBjxu)CQHM|okw0Ekg}J?cy(Ed~o=2u?aw)-`;b6&&2ghczBC0#|#R@Hp)WBQfn_D<9ZCXxdAa9ix-+}ZS(rq?5Gj=xh=;Q2r+fUOLSy`pnrE+#SO_!cs zYG|WPobBw~%EM$zf**TJaXYF{Z|_h1qFg<@^DBZ8LL5Y;0c^uW24-NWnrIF*j0i2z7@%7cRB2H3mA{OR;255l#|^zx#7o;Q%K?nrra#Ss_Iu?*+j^8;e27S zN^in{)&PYW{;aFYXvSc4Ez5uSzrD2yge}j)}HueyBc2%vuXtDWBQ;TY*3;68g_ zTpS6p2n7gTI`J#3S8Pr>kDs3gz+qkz;mCS3+$X&A%D1<~`DGFt5}DNrJ_dk@5!<}k zvpttASqGBqcg}BmAX)X^zQZ)R*Jfp+M4lO zunIQEdoNf}3r8lfb>Y&})VX*WOpdK>`Bj{kw+IB4(7?EB!z&Om!Gnt!Le7OP=k!sl zoZPsVrvg797{HMTU%!4`IQiQJp`L-EY^gM9Q)Cgrzd9@$D~SKq-I>Q@fv;`cOr>Kh zPn;H%6Rl$@EePqPr?gNiMUp`{&Xlnvlu*i+3eyoKm2et`=td51lCg%!mIzIW zXnDVn%;&uSzyH1c(T6^IJj?HQFV}V5*Np;YL3|5>_DSQ$J#F6hh}TJ+5!)x+YXr=Z z4xxtNlY_~pIK0kavP<2OJf__2?Hjl@RAT6T|G7XZ?y+;WHL7Y%a-n<}wA7k)W}W=V z$Oi^+@kr1aujFU3#VDFI8V2dgm7hx5I1{Wpap!Pm+}2qmFQj}z%MyI>AgRw>{ty?A z;LV&?*gbC4jE)|>N6f%KVY7(x8Mkc%;BqK|m^;6L+S=S)F2sa_*UG};Tv@y3cx>Z? zZ9gR?C(nn{VTc()>ea?_<&~>f8?cQ6D36U(OXV3TK&b-O$y63>EWZu`Cp}QiPG6p zRGUefpvPySI_<@E+f0VvQ5R5@);2pHl4KZfUqApPdy)Oz zP~6;uSK9nqP|(0LKw>8D{5@Zc9RB!=VwTq-m!fin-YGmX30_+5Lpw|jpfI;ULtC3; zd`%=`a{XUUUI#4br^yzORfpZ>)*(%s3|93&{5?F zegc8W<^)D_nlL#+8c1zB_2AAf`&$cnc%Bx0 z|GyFw86-Rz8(UsGh0}}w(1%Nvv`7Hf6(1Mi6o69icwtk{-+mg8coShhmN&03TU+|S z)tIbZAIh5&5{HI`&wTs;@5^($y}!Q9o+ZshBf=o?HE-+W$ubfPz7Y*kFXxVn<+gA{ zC{>7woM#GYKF^wu->HmgBN>sF=ms_bSmxv)H`he@v(yUYg`dZthA_sV01}6}yUu#Z zghI{L%K34UBywcpsvJM8La*lQ^?!!8LVA=*_$2xgs?k{v5LW4;WNsdgit2m&Ry?rC z<6#|(Fa)y_?))<&m!H?+0`Kenl_z=~TtR004Y6-X9Ni=ZqXaThI zzjVk-HX$_tEsVQ;i7{VLJA^yB#?7s*ILbN~MRKq3`P?&zmn5IX?GpmxkzdcEgM z&8@+6SmcdqC}}Z2GNIUIRB(Zo`fOXZK|99+PC53iSD+` z^V=5H?>gpRB0Jc5qwSAjvbVN^lp~fj2oZbG5pXwwP@$exm6eMdLt5*BvLD~(d-J4I{Ai2iWt+BRWptkzw&OKBO^Y% z`wf{sa|wF$pTfcpZ-X6-)90N%t14J76cQt)x@R%ly!+jY$T_EqU3LU8DSe6iFVd*g zNhs#T|Kp1jDR<*Cz#v|W=DXGecWG6Tx?}MJJ7?$Vg`O3cr~Z&*lQQ+53g&Q*U$)>* zhiELeJgC1hEYtQva`Hjv@wDWDj9R5l?T|cAIE*yf+pqjJUN@XklpZCZIVRp!A0h)( z=^Z6QNOwMPnwP!(Beis?>(z)yEnXF5!ML(D0krVxVlCq9WH1&K3~Fi-!g-4p&Gx9e zEj%50ln;)nOEsHpM>Mc+S-?DE@Z8zGuVyDa!9Y`}=8$_*BW|Gi_`0f8fRP_nogbfW zx}NsWrSgJfONPiuJ+@oj%|9RpMe+x`L=hBeU?d7LJ@ZG=KbB0H#1OSjmi+;9Z<^|K z3-qM#HqiAf*0|ksSLS``lP4DMMQlg0H#H55)-hCjVY?IkrPwy*tJ>ynq13Lky?tLO z01_-1aKq?U09?O~R<%79Inx%0@{60^mx_%!aQtFD)A)Jz9>YA3$1-U6G60KOC8WN?@YV0#-)X0&@i>pjC^x`%8o-*RJB`G(z(zxdNbN4dY`@_>S z*OSQkSH4{%SGB!&Z=T(ya_R2yCjI*E57b4Se&1Tzbxfo$;qJu7KA7_Ntje;od(s=y zyj#_M%UH!2o-LAJMij)p_Uhm9_=THA5ayT$H`v0xVtPze@6 zK}(+fCagpotW5@dLbY$MLZp86LP`@4^A}Xq6!DJYQ#JtfC zN;L1;edcol|x>_ok+#7!7ucgw!(GgeGd<_TB-MricSEyZih(kZH{*QMu!Co~|fTfBJxJ zy^3+dg6M4$Cig@FGFY)r?e*6OY>))9F;`?uIv@L^Y_2Z z`ca&K3*-=#UkOg0c{-yPBLY9uG9y#1+v*vc2VW*fF})ZXPODGJ>Ebi_!u`a@2z~L~ zbyFK?I5<}2Q;qD8MeVOdp|;;Ef_gcbo~U78tMAp>lEMSqh9+_wLTzgotrMb1*mX_) zc+orLl_&;9s@su<;HjYF|9v#KZtVpy!-VRcfiEIvR=w$;8x($P7TqpTjn(%7%e^|n z9!N+?W~~w~wG^T!_`;MN)W*BL0jw@xol4iehPCc#mZG@T32j+lRd=<~yPp~Py9eiG zXX_-KFfs7@Ju>=O{)g5C6tFzzrpschm07Wmp!Q8?svkDFf1?u%H#7u!Z+|nyu}D>!0o}FHXS{j-*@Al_xb7Sx%uve8r z&WU&&bFO#R@MQ7zB>%1(VSf!Ae63e6S>dZuAJA8K*YKuiQ)seQMS3!mc2M|Z9nQ69^X?|}^Ho;1T& z10&>|2qoE25|YAtFRyupo+buZH0)_h^bLPW#KQb7-mV`-7lN}jCsW;osZ0> z$#gz&U2sKR3abEAPZ2=w3W7pUdf~NmHf`HhydsC*?m+UXNrJW)Ox4X!NVdrsuWKD& z#$~7;orh;@0-Q@f)0*M4jErhO>xz=f+N%UpHl$fzu6(AKm8leBGPUNVgS64t*@Ujy zxaozH-sxTbo!@JIsT>_pR5)sf>Pq{mSP>QY=gY_C7Fr}ZR7!?>6YY1r4>$8xi6GJL2;E9QSYnXXH*|9n{WnMiet z_nkgJW(A%^dO@rlo3pLDCkchn+R7rYB7<2$LN<2FYeU04!z*HTj}|> zM=f^wqL;-+KoG>`ohy?|g)3In_)d#i$;|BQXMlNEBv+^YR6gjBe=XKZ<7VsSG}Q_Bm39nKNd5&RA4z zsQ+1B{F`el+;Z#QWBY%`VE09mUQW(#KR@j9r-5lA1*_fI2j<93LK`oGV0W7<<1~-L zFonjowH0KDZ7p40KR&Ll`l7KLPdf5 zFajQClabGm5N|1VB($B*^R{5Gh`O=U{F@&8A=Ls;uczS6kb?nV7K|8HUSKNnU*C(K z|Mr%C!!iOcbJs>KRj&EH_0z&9^AAqwulFjShakuFw?|7 G_ Date: Tue, 5 Mar 2024 16:18:10 +0100 Subject: [PATCH 2/7] Update changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5f662ee..828c0dbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # openproblems.bio unreleased +## MAJOR CHANGES + +* Update Matching Modalities task to OpenProblems v2 results (PR #328). + +## MINOR CHANGES + +* Update Mathing Modalities task name to Match Modalities (PR #328). + +# openproblems.bio v2.3.6 + ## NEW CONTENT * Add an event page for the Weekly wednesday work meeting (PR #299). From d6f410e46a19e0e61c79b7bc9bd671a423c85e5a Mon Sep 17 00:00:00 2001 From: Kai Waldrant Date: Wed, 13 Mar 2024 10:29:43 +0100 Subject: [PATCH 3/7] Update with latest results --- .../match_modalities/data/method_info.json | 24 +- .../match_modalities/data/metric_info.json | 18 +- .../data/quality_control.json | 72 +++--- results/match_modalities/data/results.json | 230 ++++++++++-------- 4 files changed, 181 insertions(+), 163 deletions(-) diff --git a/results/match_modalities/data/method_info.json b/results/match_modalities/data/method_info.json index c6635657..860e76d7 100644 --- a/results/match_modalities/data/method_info.json +++ b/results/match_modalities/data/method_info.json @@ -7,9 +7,9 @@ "is_baseline": true, "paper_reference": null, "code_url": null, - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/control_methods/random_features/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/control_methods/random_features/config.vsh.yaml", "code_version": null, - "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" }, { "task_id": "match_modalities", @@ -19,9 +19,9 @@ "is_baseline": true, "paper_reference": null, "code_url": null, - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/control_methods/true_features/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/control_methods/true_features/config.vsh.yaml", "code_version": null, - "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" }, { "task_id": "match_modalities", @@ -31,9 +31,9 @@ "is_baseline": false, "paper_reference": "Demetci2020scot", "code_url": "https://github.com/rsinghlab/SCOT", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/methods/scot/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/methods/scot/config.vsh.yaml", "code_version": null, - "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" }, { "task_id": "match_modalities", @@ -43,9 +43,9 @@ "is_baseline": false, "paper_reference": "stanley2020harmonic", "code_url": "https://github.com/KrishnaswamyLab/harmonic-alignment", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/methods/harmonic_alignment/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/methods/harmonic_alignment/config.vsh.yaml", "code_version": null, - "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" }, { "task_id": "match_modalities", @@ -55,9 +55,9 @@ "is_baseline": false, "paper_reference": "haghverdi2018batch", "code_url": "https://github.com/LTLA/batchelor", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/methods/fastmnn/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/methods/fastmnn/config.vsh.yaml", "code_version": null, - "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" }, { "task_id": "match_modalities", @@ -67,8 +67,8 @@ "is_baseline": false, "paper_reference": "gower1975generalized", "code_url": "https://github.com/scipy/scipy", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/methods/procrustes/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/methods/procrustes/config.vsh.yaml", "code_version": null, - "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32" + "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" } ] diff --git a/results/match_modalities/data/metric_info.json b/results/match_modalities/data/metric_info.json index 648dbcc2..e3684c02 100644 --- a/results/match_modalities/data/metric_info.json +++ b/results/match_modalities/data/metric_info.json @@ -2,23 +2,23 @@ { "task_id": "match_modalities", "metric_id": "knn_auc", - "metric_name": "KNN AUC", - "metric_summary": "\"Compute the kNN Area Under the Curve\"\n", - "paper_reference": "", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/metrics/knn_auc/config.vsh.yaml", + "metric_name": "kNN Area Under the Curve", + "metric_summary": "Let $f(i) \\u2208 F$ be the scRNA-seq measurement of cell $i$, and $g(i) \\u2208 G$ be the scATAC- seq measurement of cell $i$. kNN-AUC calculates the average percentage overlap of neighborhoods of $f(i)$ in $F$ with neighborhoods of $g(i)$ in $G$. Higher is better.\n", + "paper_reference": "lance2022multimodal", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/metrics/knn_auc/config.vsh.yaml", "code_version": null, - "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32", + "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366", "maximize": true }, { "task_id": "match_modalities", "metric_id": "mse", "metric_name": "Mean Squared Error", - "metric_summary": "The mean squared error (MSE) is a measure of the quality of an estimator. It is always non-negative, and values closer to zero are better.\n", - "paper_reference": "", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/90f8643e4c7ea1402487955080537c0ce00d1b32/src/tasks/match_modalities/metrics/mse/config.vsh.yaml", + "metric_summary": "Mean squared error (MSE) is the average distance between each pair of matched observations of the same cell in the learned latent space. Lower is better.\n", + "paper_reference": "lance2022multimodal", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/metrics/mse/config.vsh.yaml", "code_version": null, - "commit_sha": "90f8643e4c7ea1402487955080537c0ce00d1b32", + "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366", "maximize": true } ] diff --git a/results/match_modalities/data/quality_control.json b/results/match_modalities/data/quality_control.json index a1d48c37..4d8729ab 100644 --- a/results/match_modalities/data/quality_control.json +++ b/results/match_modalities/data/quality_control.json @@ -363,21 +363,21 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Worst score random_features knn_auc", - "value": 0.0069, + "value": 0.0061, "severity": 0, - "severity_value": -0.0069, + "severity_value": -0.0061, "code": "worst_score >= -1", - "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Worst score: 0.0069%\n" + "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Worst score: 0.0061%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score random_features knn_auc", - "value": 0.0457, + "value": 0.0458, "severity": 0, - "severity_value": 0.02285, + "severity_value": 0.0229, "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 0.0457%\n" + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 0.0458%\n" }, { "task_id": "match_modalities", @@ -433,11 +433,11 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Best score harmonic_alignment knn_auc", - "value": 0.0467, + "value": 0.0501, "severity": 0, - "severity_value": 0.02335, + "severity_value": 0.02505, "code": "best_score <= 2", - "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Best score: 0.0467%\n" + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Best score: 0.0501%\n" }, { "task_id": "match_modalities", @@ -463,21 +463,21 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Worst score procrustes knn_auc", - "value": 0.0663, + "value": 0.065, "severity": 0, - "severity_value": -0.0663, + "severity_value": -0.065, "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: 0.0663%\n" + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: 0.065%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score procrustes knn_auc", - "value": 0.2744, + "value": 0.2751, "severity": 0, - "severity_value": 0.1372, + "severity_value": 0.13755, "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 0.2744%\n" + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 0.2751%\n" }, { "task_id": "match_modalities", @@ -493,31 +493,31 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Best score random_features mse", - "value": 1.0, + "value": 0.7867, "severity": 0, - "severity_value": 0.5, + "severity_value": 0.39335, "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Best score: 1.0%\n" + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Best score: 0.7867%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score true_features mse", - "value": 0.0941, + "value": 0.047, "severity": 0, - "severity_value": -0.0941, + "severity_value": -0.047, "code": "worst_score >= -1", - "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Worst score: 0.0941%\n" + "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Worst score: 0.047%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score true_features mse", - "value": 0.5746, + "value": 0.5717, "severity": 0, - "severity_value": 0.2873, + "severity_value": 0.28585, "code": "best_score <= 2", - "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Best score: 0.5746%\n" + "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Best score: 0.5717%\n" }, { "task_id": "match_modalities", @@ -543,21 +543,21 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Worst score harmonic_alignment mse", - "value": 0.0267, + "value": 0.0871, "severity": 0, - "severity_value": -0.0267, + "severity_value": -0.0871, "code": "worst_score >= -1", - "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Worst score: 0.0267%\n" + "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Worst score: 0.0871%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score harmonic_alignment mse", - "value": 0.9548, + "value": 0.9771, "severity": 0, - "severity_value": 0.4774, + "severity_value": 0.48855, "code": "best_score <= 2", - "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Best score: 0.9548%\n" + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Best score: 0.9771%\n" }, { "task_id": "match_modalities", @@ -583,20 +583,20 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Worst score procrustes mse", - "value": 0.0486, + "value": 0.1067, "severity": 0, - "severity_value": -0.0486, + "severity_value": -0.1067, "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: 0.0486%\n" + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: 0.1067%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score procrustes mse", - "value": 0.9763, + "value": 1.0, "severity": 0, - "severity_value": 0.48815, + "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 0.9763%\n" + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 1.0%\n" } ] \ No newline at end of file diff --git a/results/match_modalities/data/results.json b/results/match_modalities/data/results.json index 9ed8c2e9..65ae6029 100644 --- a/results/match_modalities/data/results.json +++ b/results/match_modalities/data/results.json @@ -4,19 +4,20 @@ "method_id": "fastmnn", "metric_values": { "knn_auc": 0.0383, - "mse": 1.1745 + "mse": 1.1707 }, "scaled_scores": { "knn_auc": 0.0343, - "mse": 0.6988 + "mse": 0.684 }, - "mean_score": 0.3666, + "mean_score": 0.3592, + "name": "auto:run_benchmark:run_wf:runEachWf:fastmnn:processWf:fastmnn_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.fastmnn)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 43.2, - "cpu_pct": 69, - "peak_memory_mb": 6554, + "duration_sec": 16.6, + "cpu_pct": 142.5, + "peak_memory_mb": 4096, "disk_read_mb": 238, "disk_write_mb": 58 }, @@ -31,15 +32,16 @@ }, "scaled_scores": { "knn_auc": 0, - "mse": 0.0267 + "mse": 0.0871 }, - "mean_score": 0.0133, + "mean_score": 0.0435, + "name": "auto:run_benchmark:run_wf:runEachWf:harmonic_alignment:processWf:harmonic_alignment_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.harmonic_alignment)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 98, - "cpu_pct": 958.3, - "peak_memory_mb": 5940, + "duration_sec": 57.9, + "cpu_pct": 616.4, + "peak_memory_mb": 3687, "disk_read_mb": 208, "disk_write_mb": 69 }, @@ -54,15 +56,16 @@ }, "scaled_scores": { "knn_auc": 0.1089, - "mse": 0.0486 + "mse": 0.1067 }, - "mean_score": 0.0787, + "mean_score": 0.1078, + "name": "auto:run_benchmark:run_wf:runEachWf:procrustes:processWf:procrustes_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.procrustes)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 13.3, - "cpu_pct": 175.7, - "peak_memory_mb": 5735, + "duration_sec": 6.6, + "cpu_pct": 245.8, + "peak_memory_mb": 3380, "disk_read_mb": 199, "disk_write_mb": 58 }, @@ -72,20 +75,21 @@ "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", "method_id": "random_features", "metric_values": { - "knn_auc": 0.0496, - "mse": 0.993 + "knn_auc": 0.0498, + "mse": 0.9751 }, "scaled_scores": { - "knn_auc": 0.0457, + "knn_auc": 0.0458, "mse": 0 }, - "mean_score": 0.0228, + "mean_score": 0.0229, + "name": "auto:run_benchmark:run_wf:runEachWf:random_features:processWf:random_features_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.random_features)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 23.6, - "cpu_pct": 57.7, - "peak_memory_mb": 5632, + "duration_sec": 11.9, + "cpu_pct": 83.1, + "peak_memory_mb": 3380, "disk_read_mb": 198, "disk_write_mb": 57 }, @@ -96,18 +100,19 @@ "method_id": "scot", "metric_values": { "knn_auc": 0.058, - "mse": 1.2526 + "mse": 1.261 }, "scaled_scores": { "knn_auc": 0.0541, "mse": 1 }, "mean_score": 0.5271, + "name": "auto:run_benchmark:run_wf:runEachWf:scot:processWf:scot_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.scot)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 3133, - "cpu_pct": 2889.1, + "duration_sec": 10270, + "cpu_pct": 1855.7, "peak_memory_mb": 12493, "disk_read_mb": 219, "disk_write_mb": 58 @@ -119,19 +124,20 @@ "method_id": "true_features", "metric_values": { "knn_auc": 1, - "mse": 1.0453 + "mse": 1.0473 }, "scaled_scores": { "knn_auc": 1, - "mse": 0.2013 + "mse": 0.2525 }, - "mean_score": 0.6007, + "mean_score": 0.6263, + "name": "auto:run_benchmark:run_wf:runEachWf:true_features:processWf:true_features_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.true_features)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 18.2, - "cpu_pct": 55.4, - "peak_memory_mb": 2765, + "duration_sec": 2.4, + "cpu_pct": 161.6, + "peak_memory_mb": 3277, "disk_read_mb": 198, "disk_write_mb": 3 }, @@ -142,19 +148,20 @@ "method_id": "fastmnn", "metric_values": { "knn_auc": 0.0138, - "mse": 1.0755 + "mse": 1.0812 }, "scaled_scores": { "knn_auc": 0.0075, "mse": 1 }, "mean_score": 0.5037, + "name": "auto:run_benchmark:run_wf:runEachWf:fastmnn:processWf:fastmnn_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.fastmnn)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 24.5, - "cpu_pct": 147.7, - "peak_memory_mb": 6656, + "duration_sec": 21.3, + "cpu_pct": 133.2, + "peak_memory_mb": 4199, "disk_read_mb": 217, "disk_write_mb": 65 }, @@ -165,19 +172,20 @@ "method_id": "harmonic_alignment", "metric_values": { "knn_auc": 0.0064, - "mse": 1 + "mse": 0.9999 }, "scaled_scores": { "knn_auc": 0, - "mse": 0.6653 + "mse": 0.6451 }, - "mean_score": 0.3327, + "mean_score": 0.3226, + "name": "auto:run_benchmark:run_wf:runEachWf:harmonic_alignment:processWf:harmonic_alignment_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.harmonic_alignment)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 49.1, - "cpu_pct": 812.4, - "peak_memory_mb": 5223, + "duration_sec": 52.6, + "cpu_pct": 711.8, + "peak_memory_mb": 5837, "disk_read_mb": 187, "disk_write_mb": 64 }, @@ -187,19 +195,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", "method_id": "procrustes", "metric_values": { - "knn_auc": 0.0723, - "mse": 1.0115 + "knn_auc": 0.071, + "mse": 1.0102 }, "scaled_scores": { - "knn_auc": 0.0663, - "mse": 0.7163 + "knn_auc": 0.065, + "mse": 0.6901 }, - "mean_score": 0.3913, + "mean_score": 0.3776, + "name": "auto:run_benchmark:run_wf:runEachWf:procrustes:processWf:procrustes_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.procrustes)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 6.2, - "cpu_pct": 268.5, + "duration_sec": 6.4, + "cpu_pct": 258.7, "peak_memory_mb": 3380, "disk_read_mb": 178, "disk_write_mb": 64 @@ -210,21 +219,22 @@ "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", "method_id": "random_features", "metric_values": { - "knn_auc": 0.049, - "mse": 1.0008 + "knn_auc": 0.0514, + "mse": 0.9882 }, "scaled_scores": { - "knn_auc": 0.0429, - "mse": 0.6689 + "knn_auc": 0.0453, + "mse": 0.5941 }, - "mean_score": 0.3559, + "mean_score": 0.3197, + "name": "auto:run_benchmark:run_wf:runEachWf:random_features:processWf:random_features_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.random_features)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 15.9, - "cpu_pct": 74.7, - "peak_memory_mb": 5632, - "disk_read_mb": 177, + "duration_sec": 6.2, + "cpu_pct": 162, + "peak_memory_mb": 3277, + "disk_read_mb": 178, "disk_write_mb": 61 }, "task_id": "match_modalities" @@ -234,19 +244,20 @@ "method_id": "scot", "metric_values": { "knn_auc": 0.0517, - "mse": 0.8498 + "mse": 0.8522 }, "scaled_scores": { - "knn_auc": 0.0457, + "knn_auc": 0.0456, "mse": 0 }, "mean_score": 0.0228, + "name": "auto:run_benchmark:run_wf:runEachWf:scot:processWf:scot_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.scot)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 630, - "cpu_pct": 2797.2, - "peak_memory_mb": 7783, + "duration_sec": 448, + "cpu_pct": 3046.2, + "peak_memory_mb": 8090, "disk_read_mb": 198, "disk_write_mb": 64 }, @@ -257,20 +268,21 @@ "method_id": "true_features", "metric_values": { "knn_auc": 1, - "mse": 0.9795 + "mse": 0.9831 }, "scaled_scores": { "knn_auc": 1, - "mse": 0.5746 + "mse": 0.5717 }, - "mean_score": 0.7873, + "mean_score": 0.7858, + "name": "auto:run_benchmark:run_wf:runEachWf:true_features:processWf:true_features_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.true_features)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 13.9, - "cpu_pct": 57.7, - "peak_memory_mb": 5428, - "disk_read_mb": 177, + "duration_sec": 2.8, + "cpu_pct": 164, + "peak_memory_mb": 3277, + "disk_read_mb": 178, "disk_write_mb": 14 }, "task_id": "match_modalities" @@ -280,19 +292,20 @@ "method_id": "fastmnn", "metric_values": { "knn_auc": 0.0432, - "mse": 0.9461 + "mse": 0.9495 }, "scaled_scores": { "knn_auc": 0, "mse": 0 }, "mean_score": 0, + "name": "auto:run_benchmark:run_wf:runEachWf:fastmnn:processWf:fastmnn_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.fastmnn)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 78, - "cpu_pct": 82.2, - "peak_memory_mb": 6964, + "duration_sec": 46.4, + "cpu_pct": 111, + "peak_memory_mb": 4506, "disk_read_mb": 356, "disk_write_mb": 119 }, @@ -302,20 +315,21 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "harmonic_alignment", "metric_values": { - "knn_auc": 0.0879, + "knn_auc": 0.0912, "mse": 0.9999 }, "scaled_scores": { - "knn_auc": 0.0467, - "mse": 0.9548 + "knn_auc": 0.0501, + "mse": 0.9771 }, - "mean_score": 0.5008, + "mean_score": 0.5136, + "name": "auto:run_benchmark:run_wf:runEachWf:harmonic_alignment:processWf:harmonic_alignment_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.harmonic_alignment)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 251, - "cpu_pct": 778.1, - "peak_memory_mb": 6759, + "duration_sec": 218, + "cpu_pct": 413.8, + "peak_memory_mb": 4404, "disk_read_mb": 326, "disk_write_mb": 118 }, @@ -325,19 +339,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "procrustes", "metric_values": { - "knn_auc": 0.3057, + "knn_auc": 0.3064, "mse": 1.0011 }, "scaled_scores": { - "knn_auc": 0.2744, - "mse": 0.9763 + "knn_auc": 0.2751, + "mse": 1 }, - "mean_score": 0.6253, + "mean_score": 0.6375, + "name": "auto:run_benchmark:run_wf:runEachWf:procrustes:processWf:procrustes_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.procrustes)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 9.8, - "cpu_pct": 206.7, + "duration_sec": 9.9, + "cpu_pct": 201.8, "peak_memory_mb": 3584, "disk_read_mb": 317, "disk_write_mb": 118 @@ -348,19 +363,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "random_features", "metric_values": { - "knn_auc": 0.0498, - "mse": 1.0025 + "knn_auc": 0.0491, + "mse": 0.9901 }, "scaled_scores": { - "knn_auc": 0.0069, - "mse": 1 + "knn_auc": 0.0061, + "mse": 0.7867 }, - "mean_score": 0.5035, + "mean_score": 0.3964, + "name": "auto:run_benchmark:run_wf:runEachWf:random_features:processWf:random_features_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.random_features)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 9.1, - "cpu_pct": 158, + "duration_sec": 8.9, + "cpu_pct": 130.8, "peak_memory_mb": 3482, "disk_read_mb": 316, "disk_write_mb": 110 @@ -372,20 +388,21 @@ "method_id": "scot", "metric_values": { "knn_auc": 0.0758, - "mse": 0.9948 + "mse": 0.994 }, "scaled_scores": { "knn_auc": 0.034, - "mse": 0.8637 + "mse": 0.8626 }, - "mean_score": 0.4488, + "mean_score": 0.4483, + "name": "auto:run_benchmark:run_wf:runEachWf:scot:processWf:scot_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.scot)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 5053, - "cpu_pct": 2626.5, + "duration_sec": 14136, + "cpu_pct": 2033.2, "peak_memory_mb": 17101, - "disk_read_mb": 336, + "disk_read_mb": 337, "disk_write_mb": 118 }, "task_id": "match_modalities" @@ -395,19 +412,20 @@ "method_id": "true_features", "metric_values": { "knn_auc": 1, - "mse": 0.9514 + "mse": 0.9519 }, "scaled_scores": { "knn_auc": 1, - "mse": 0.0941 + "mse": 0.047 }, - "mean_score": 0.5471, + "mean_score": 0.5235, + "name": "auto:run_benchmark:run_wf:runEachWf:true_features:processWf:true_features_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.true_features)", "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 5.2, - "cpu_pct": 215.3, - "peak_memory_mb": 5837, + "duration_sec": 4.7, + "cpu_pct": 138.7, + "peak_memory_mb": 3482, "disk_read_mb": 316, "disk_write_mb": 25 }, From 86012350c399ef5ac846bddeb76da15b8427edff Mon Sep 17 00:00:00 2001 From: Kai Waldrant Date: Wed, 13 Mar 2024 11:05:47 +0100 Subject: [PATCH 4/7] Fix special char --- results/match_modalities/data/metric_info.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/match_modalities/data/metric_info.json b/results/match_modalities/data/metric_info.json index e3684c02..f500402c 100644 --- a/results/match_modalities/data/metric_info.json +++ b/results/match_modalities/data/metric_info.json @@ -3,7 +3,7 @@ "task_id": "match_modalities", "metric_id": "knn_auc", "metric_name": "kNN Area Under the Curve", - "metric_summary": "Let $f(i) \\u2208 F$ be the scRNA-seq measurement of cell $i$, and $g(i) \\u2208 G$ be the scATAC- seq measurement of cell $i$. kNN-AUC calculates the average percentage overlap of neighborhoods of $f(i)$ in $F$ with neighborhoods of $g(i)$ in $G$. Higher is better.\n", + "metric_summary": "Let $f(i) \\in F$ be the scRNA-seq measurement of cell $i$, and $g(i) \\in G$ be the scATAC- seq measurement of cell $i$. kNN-AUC calculates the average percentage overlap of neighborhoods of $f(i)$ in $F$ with neighborhoods of $g(i)$ in $G$. Higher is better.\n", "paper_reference": "lance2022multimodal", "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/metrics/knn_auc/config.vsh.yaml", "code_version": null, From 7a408c2e152186c6c4d8c6798a3e4c5f3649f45b Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Mon, 18 Mar 2024 11:22:01 +0100 Subject: [PATCH 5/7] update results --- .../match_modalities/data/method_info.json | 24 +- .../data/metric_execution_info.json | 506 ++++++++++++++++++ .../match_modalities/data/metric_info.json | 10 +- .../data/quality_control.json | 114 ++-- results/match_modalities/data/results.json | 286 +++++----- results/match_modalities/data/state.yaml | 1 + 6 files changed, 715 insertions(+), 226 deletions(-) create mode 100644 results/match_modalities/data/metric_execution_info.json diff --git a/results/match_modalities/data/method_info.json b/results/match_modalities/data/method_info.json index 860e76d7..dfe65b0f 100644 --- a/results/match_modalities/data/method_info.json +++ b/results/match_modalities/data/method_info.json @@ -7,9 +7,9 @@ "is_baseline": true, "paper_reference": null, "code_url": null, - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/control_methods/random_features/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/control_methods/random_features/config.vsh.yaml", "code_version": null, - "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" + "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" }, { "task_id": "match_modalities", @@ -19,9 +19,9 @@ "is_baseline": true, "paper_reference": null, "code_url": null, - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/control_methods/true_features/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/control_methods/true_features/config.vsh.yaml", "code_version": null, - "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" + "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" }, { "task_id": "match_modalities", @@ -31,9 +31,9 @@ "is_baseline": false, "paper_reference": "Demetci2020scot", "code_url": "https://github.com/rsinghlab/SCOT", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/methods/scot/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/methods/scot/config.vsh.yaml", "code_version": null, - "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" + "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" }, { "task_id": "match_modalities", @@ -43,9 +43,9 @@ "is_baseline": false, "paper_reference": "stanley2020harmonic", "code_url": "https://github.com/KrishnaswamyLab/harmonic-alignment", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/methods/harmonic_alignment/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/methods/harmonic_alignment/config.vsh.yaml", "code_version": null, - "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" + "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" }, { "task_id": "match_modalities", @@ -55,9 +55,9 @@ "is_baseline": false, "paper_reference": "haghverdi2018batch", "code_url": "https://github.com/LTLA/batchelor", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/methods/fastmnn/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/methods/fastmnn/config.vsh.yaml", "code_version": null, - "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" + "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" }, { "task_id": "match_modalities", @@ -67,8 +67,8 @@ "is_baseline": false, "paper_reference": "gower1975generalized", "code_url": "https://github.com/scipy/scipy", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/methods/procrustes/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/methods/procrustes/config.vsh.yaml", "code_version": null, - "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366" + "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" } ] diff --git a/results/match_modalities/data/metric_execution_info.json b/results/match_modalities/data/metric_execution_info.json new file mode 100644 index 00000000..182c5227 --- /dev/null +++ b/results/match_modalities/data/metric_execution_info.json @@ -0,0 +1,506 @@ +[ + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 23.3, + "cpu_pct": 98.8, + "peak_memory_mb": 6247, + "disk_read_mb": 265, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 17.8, + "cpu_pct": 203.1, + "peak_memory_mb": 9626, + "disk_read_mb": 276, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 20.8, + "cpu_pct": 109.6, + "peak_memory_mb": 6144, + "disk_read_mb": 264, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 14.8, + "cpu_pct": 151.8, + "peak_memory_mb": 6144, + "disk_read_mb": 264, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 13.8, + "cpu_pct": 131.2, + "peak_memory_mb": 6247, + "disk_read_mb": 264, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 13.1, + "cpu_pct": 155.8, + "peak_memory_mb": 5940, + "disk_read_mb": 210, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 12.6, + "cpu_pct": 189.7, + "peak_memory_mb": 8602, + "disk_read_mb": 251, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 9.8, + "cpu_pct": 216.4, + "peak_memory_mb": 8090, + "disk_read_mb": 250, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 6.8, + "cpu_pct": 362.5, + "peak_memory_mb": 8909, + "disk_read_mb": 250, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 6.1, + "cpu_pct": 398.5, + "peak_memory_mb": 9114, + "disk_read_mb": 247, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 8.7, + "cpu_pct": 315.5, + "peak_memory_mb": 8909, + "disk_read_mb": 250, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 88, + "cpu_pct": 29.3, + "peak_memory_mb": 6144, + "disk_read_mb": 199, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 22.3, + "cpu_pct": 303.1, + "peak_memory_mb": 10036, + "disk_read_mb": 444, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 20.2, + "cpu_pct": 453.7, + "peak_memory_mb": 10036, + "disk_read_mb": 443, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 34.2, + "cpu_pct": 261.4, + "peak_memory_mb": 10036, + "disk_read_mb": 443, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 148, + "cpu_pct": 77.1, + "peak_memory_mb": 10036, + "disk_read_mb": 435, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 173, + "cpu_pct": 46.7, + "peak_memory_mb": 10036, + "disk_read_mb": 443, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "knn_auc", + "resources": { + "exit_code": 0, + "duration_sec": 60, + "cpu_pct": 148.2, + "peak_memory_mb": 9728, + "disk_read_mb": 346, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 389.5, + "peak_memory_mb": 5735, + "disk_read_mb": 256, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 9.2, + "cpu_pct": 133.5, + "peak_memory_mb": 5735, + "disk_read_mb": 267, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 5.9, + "cpu_pct": 140.5, + "peak_memory_mb": 5837, + "disk_read_mb": 256, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 9.6, + "cpu_pct": 119.2, + "peak_memory_mb": 5735, + "disk_read_mb": 255, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 3.1, + "cpu_pct": 280.9, + "peak_memory_mb": 5735, + "disk_read_mb": 256, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 1.9, + "cpu_pct": 476.5, + "peak_memory_mb": 2765, + "disk_read_mb": 201, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 6.8, + "cpu_pct": 131, + "peak_memory_mb": 5837, + "disk_read_mb": 242, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 6.9, + "cpu_pct": 149.3, + "peak_memory_mb": 5837, + "disk_read_mb": 241, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 3.3, + "cpu_pct": 211.6, + "peak_memory_mb": 5837, + "disk_read_mb": 241, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 6.3, + "cpu_pct": 147.3, + "peak_memory_mb": 5735, + "disk_read_mb": 238, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 3.5, + "cpu_pct": 224.7, + "peak_memory_mb": 5837, + "disk_read_mb": 241, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 5.9, + "cpu_pct": 110.6, + "peak_memory_mb": 5632, + "disk_read_mb": 190, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "fastmnn", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 10, + "cpu_pct": 89.4, + "peak_memory_mb": 6144, + "disk_read_mb": 435, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "harmonic_alignment", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 4, + "cpu_pct": 265.1, + "peak_memory_mb": 6042, + "disk_read_mb": 434, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "procrustes", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 4.1, + "cpu_pct": 279.5, + "peak_memory_mb": 6042, + "disk_read_mb": 434, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "random_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 56.9, + "cpu_pct": 17.1, + "peak_memory_mb": 2765, + "disk_read_mb": 426, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "scot", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 116, + "cpu_pct": 11.3, + "peak_memory_mb": 5940, + "disk_read_mb": 434, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", + "normalization_id": "log_cp10k", + "method_id": "true_features", + "metric_id": "mse", + "resources": { + "exit_code": 0, + "duration_sec": 7.3, + "cpu_pct": 95.8, + "peak_memory_mb": 5530, + "disk_read_mb": 337, + "disk_write_mb": 1 + } + } +] diff --git a/results/match_modalities/data/metric_info.json b/results/match_modalities/data/metric_info.json index f500402c..a9b30699 100644 --- a/results/match_modalities/data/metric_info.json +++ b/results/match_modalities/data/metric_info.json @@ -5,9 +5,9 @@ "metric_name": "kNN Area Under the Curve", "metric_summary": "Let $f(i) \\in F$ be the scRNA-seq measurement of cell $i$, and $g(i) \\in G$ be the scATAC- seq measurement of cell $i$. kNN-AUC calculates the average percentage overlap of neighborhoods of $f(i)$ in $F$ with neighborhoods of $g(i)$ in $G$. Higher is better.\n", "paper_reference": "lance2022multimodal", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/metrics/knn_auc/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/metrics/knn_auc/config.vsh.yaml", "code_version": null, - "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366", + "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1", "maximize": true }, { @@ -16,9 +16,9 @@ "metric_name": "Mean Squared Error", "metric_summary": "Mean squared error (MSE) is the average distance between each pair of matched observations of the same cell in the learned latent space. Lower is better.\n", "paper_reference": "lance2022multimodal", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366/src/tasks/match_modalities/metrics/mse/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/metrics/mse/config.vsh.yaml", "code_version": null, - "commit_sha": "fbfebe255f1649f70f4cb1bcfbf3c5823cc9f366", - "maximize": true + "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1", + "maximize": false } ] diff --git a/results/match_modalities/data/quality_control.json b/results/match_modalities/data/quality_control.json index 4d8729ab..84d65e9b 100644 --- a/results/match_modalities/data/quality_control.json +++ b/results/match_modalities/data/quality_control.json @@ -363,61 +363,61 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Worst score random_features knn_auc", - "value": 0.0061, + "value": 0.0206, "severity": 0, - "severity_value": -0.0061, + "severity_value": -0.0206, "code": "worst_score >= -1", - "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Worst score: 0.0061%\n" + "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Worst score: 0.0206%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score random_features knn_auc", - "value": 0.0458, + "value": 0.7137, "severity": 0, - "severity_value": 0.0229, + "severity_value": 0.35685, "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 0.0458%\n" + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 0.7137%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score true_features knn_auc", - "value": 1, + "value": 0.1295, "severity": 0, - "severity_value": -1.0, + "severity_value": -0.1295, "code": "worst_score >= -1", - "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Worst score: 1%\n" + "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Worst score: 0.1295%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score true_features knn_auc", - "value": 1, + "value": 0.8047, "severity": 0, - "severity_value": 0.5, + "severity_value": 0.40235, "code": "best_score <= 2", - "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Best score: 1%\n" + "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Best score: 0.8047%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score scot knn_auc", - "value": 0.034, + "value": 0.1236, "severity": 0, - "severity_value": -0.034, + "severity_value": -0.1236, "code": "worst_score >= -1", - "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Worst score: 0.034%\n" + "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Worst score: 0.1236%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score scot knn_auc", - "value": 0.0541, + "value": 0.7019, "severity": 0, - "severity_value": 0.02705, + "severity_value": 0.35095, "code": "best_score <= 2", - "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Best score: 0.0541%\n" + "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Best score: 0.7019%\n" }, { "task_id": "match_modalities", @@ -433,11 +433,11 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Best score harmonic_alignment knn_auc", - "value": 0.0501, + "value": 0.148, "severity": 0, - "severity_value": 0.02505, + "severity_value": 0.074, "code": "best_score <= 2", - "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Best score: 0.0501%\n" + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Best score: 0.148%\n" }, { "task_id": "match_modalities", @@ -453,31 +453,31 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Best score fastmnn knn_auc", - "value": 0.0343, + "value": 0.314, "severity": 0, - "severity_value": 0.01715, + "severity_value": 0.157, "code": "best_score <= 2", - "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Best score: 0.0343%\n" + "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Best score: 0.314%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score procrustes knn_auc", - "value": 0.065, + "value": 1, "severity": 0, - "severity_value": -0.065, + "severity_value": -1.0, "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: 0.065%\n" + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: 1%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score procrustes knn_auc", - "value": 0.2751, + "value": 1, "severity": 0, - "severity_value": 0.13755, + "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 0.2751%\n" + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 1%\n" }, { "task_id": "match_modalities", @@ -493,31 +493,31 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Best score random_features mse", - "value": 0.7867, + "value": 0.2137, "severity": 0, - "severity_value": 0.39335, + "severity_value": 0.10685, "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Best score: 0.7867%\n" + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Best score: 0.2137%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score true_features mse", - "value": 0.047, + "value": 1, "severity": 0, - "severity_value": -0.047, + "severity_value": -1.0, "code": "worst_score >= -1", - "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Worst score: 0.047%\n" + "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Worst score: 1%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score true_features mse", - "value": 0.5717, + "value": 1, "severity": 0, - "severity_value": 0.28585, + "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Best score: 0.5717%\n" + "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: mse\n Best score: 1%\n" }, { "task_id": "match_modalities", @@ -533,31 +533,31 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Best score scot mse", - "value": 1.0, + "value": 0.2091, "severity": 0, - "severity_value": 0.5, + "severity_value": 0.10455, "code": "best_score <= 2", - "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Best score: 1.0%\n" + "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Best score: 0.2091%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score harmonic_alignment mse", - "value": 0.0871, + "value": 0.0037, "severity": 0, - "severity_value": -0.0871, + "severity_value": -0.0037, "code": "worst_score >= -1", - "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Worst score: 0.0871%\n" + "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Worst score: 0.0037%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score harmonic_alignment mse", - "value": 0.9771, + "value": 0.2025, "severity": 0, - "severity_value": 0.48855, + "severity_value": 0.10125, "code": "best_score <= 2", - "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Best score: 0.9771%\n" + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Best score: 0.2025%\n" }, { "task_id": "match_modalities", @@ -573,30 +573,30 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Best score fastmnn mse", - "value": 1.0, + "value": 0.0646, "severity": 0, - "severity_value": 0.5, + "severity_value": 0.0323, "code": "best_score <= 2", - "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Best score: 1.0%\n" + "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Best score: 0.0646%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score procrustes mse", - "value": 0.1067, + "value": 0.0026, "severity": 0, - "severity_value": -0.1067, + "severity_value": -0.0026, "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: 0.1067%\n" + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: 0.0026%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score procrustes mse", - "value": 1.0, + "value": 0.1981, "severity": 0, - "severity_value": 0.5, + "severity_value": 0.09905, "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 1.0%\n" + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 0.1981%\n" } ] \ No newline at end of file diff --git a/results/match_modalities/data/results.json b/results/match_modalities/data/results.json index 65ae6029..bf810417 100644 --- a/results/match_modalities/data/results.json +++ b/results/match_modalities/data/results.json @@ -4,20 +4,19 @@ "method_id": "fastmnn", "metric_values": { "knn_auc": 0.0383, - "mse": 1.1707 + "mse": 1.173 }, "scaled_scores": { - "knn_auc": 0.0343, - "mse": 0.684 + "knn_auc": 0.314, + "mse": 0.0646 }, - "mean_score": 0.3592, - "name": "auto:run_benchmark:run_wf:runEachWf:fastmnn:processWf:fastmnn_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.fastmnn)", + "mean_score": 0.1893, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 16.6, - "cpu_pct": 142.5, - "peak_memory_mb": 4096, + "duration_sec": 175, + "cpu_pct": 18.1, + "peak_memory_mb": 6452, "disk_read_mb": 238, "disk_write_mb": 58 }, @@ -27,21 +26,20 @@ "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", "method_id": "harmonic_alignment", "metric_values": { - "knn_auc": 0.0041, + "knn_auc": 0.0043, "mse": 1 }, "scaled_scores": { "knn_auc": 0, - "mse": 0.0871 + "mse": 0.2025 }, - "mean_score": 0.0435, - "name": "auto:run_benchmark:run_wf:runEachWf:harmonic_alignment:processWf:harmonic_alignment_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.harmonic_alignment)", + "mean_score": 0.1012, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 57.9, - "cpu_pct": 616.4, - "peak_memory_mb": 3687, + "duration_sec": 239, + "cpu_pct": 401.7, + "peak_memory_mb": 6042, "disk_read_mb": 208, "disk_write_mb": 69 }, @@ -52,20 +50,19 @@ "method_id": "procrustes", "metric_values": { "knn_auc": 0.1126, - "mse": 1.0056 + "mse": 1.0055 }, "scaled_scores": { - "knn_auc": 0.1089, - "mse": 0.1067 + "knn_auc": 1, + "mse": 0.1981 }, - "mean_score": 0.1078, - "name": "auto:run_benchmark:run_wf:runEachWf:procrustes:processWf:procrustes_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.procrustes)", + "mean_score": 0.5991, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 6.6, - "cpu_pct": 245.8, - "peak_memory_mb": 3380, + "duration_sec": 14, + "cpu_pct": 117.3, + "peak_memory_mb": 5735, "disk_read_mb": 199, "disk_write_mb": 58 }, @@ -75,21 +72,20 @@ "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", "method_id": "random_features", "metric_values": { - "knn_auc": 0.0498, - "mse": 0.9751 + "knn_auc": 0.0502, + "mse": 0.9859 }, "scaled_scores": { - "knn_auc": 0.0458, - "mse": 0 + "knn_auc": 0.424, + "mse": 0.2137 }, - "mean_score": 0.0229, - "name": "auto:run_benchmark:run_wf:runEachWf:random_features:processWf:random_features_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.random_features)", + "mean_score": 0.3188, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 11.9, - "cpu_pct": 83.1, - "peak_memory_mb": 3380, + "duration_sec": 129, + "cpu_pct": 12.1, + "peak_memory_mb": 5632, "disk_read_mb": 198, "disk_write_mb": 57 }, @@ -100,20 +96,19 @@ "method_id": "scot", "metric_values": { "knn_auc": 0.058, - "mse": 1.261 + "mse": 1.2539 }, "scaled_scores": { - "knn_auc": 0.0541, - "mse": 1 + "knn_auc": 0.4963, + "mse": 0 }, - "mean_score": 0.5271, - "name": "auto:run_benchmark:run_wf:runEachWf:scot:processWf:scot_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.scot)", + "mean_score": 0.2482, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 10270, - "cpu_pct": 1855.7, - "peak_memory_mb": 12493, + "duration_sec": 6711, + "cpu_pct": 4835, + "peak_memory_mb": 14848, "disk_read_mb": 219, "disk_write_mb": 58 }, @@ -123,22 +118,21 @@ "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", "method_id": "true_features", "metric_values": { - "knn_auc": 1, - "mse": 1.0473 + "knn_auc": 0.0183, + "mse": 0 }, "scaled_scores": { - "knn_auc": 1, - "mse": 0.2525 + "knn_auc": 0.1295, + "mse": 1 }, - "mean_score": 0.6263, - "name": "auto:run_benchmark:run_wf:runEachWf:true_features:processWf:true_features_process (openproblems_v1_multimodal/citeseq_cbmc/log_cp10k.true_features)", + "mean_score": 0.5648, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 2.4, - "cpu_pct": 161.6, - "peak_memory_mb": 3277, - "disk_read_mb": 198, + "duration_sec": 8.3, + "cpu_pct": 88.4, + "peak_memory_mb": 5735, + "disk_read_mb": 385, "disk_write_mb": 3 }, "task_id": "match_modalities" @@ -148,20 +142,19 @@ "method_id": "fastmnn", "metric_values": { "knn_auc": 0.0138, - "mse": 1.0812 + "mse": 1.0807 }, "scaled_scores": { - "knn_auc": 0.0075, - "mse": 1 + "knn_auc": 0.1146, + "mse": 0 }, - "mean_score": 0.5037, - "name": "auto:run_benchmark:run_wf:runEachWf:fastmnn:processWf:fastmnn_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.fastmnn)", + "mean_score": 0.0573, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 21.3, - "cpu_pct": 133.2, - "peak_memory_mb": 4199, + "duration_sec": 37.2, + "cpu_pct": 112.4, + "peak_memory_mb": 6554, "disk_read_mb": 217, "disk_write_mb": 65 }, @@ -172,20 +165,19 @@ "method_id": "harmonic_alignment", "metric_values": { "knn_auc": 0.0064, - "mse": 0.9999 + "mse": 1 }, "scaled_scores": { "knn_auc": 0, - "mse": 0.6451 + "mse": 0.0747 }, - "mean_score": 0.3226, - "name": "auto:run_benchmark:run_wf:runEachWf:harmonic_alignment:processWf:harmonic_alignment_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.harmonic_alignment)", + "mean_score": 0.0373, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 52.6, - "cpu_pct": 711.8, - "peak_memory_mb": 5837, + "duration_sec": 80, + "cpu_pct": 1538.7, + "peak_memory_mb": 9216, "disk_read_mb": 187, "disk_write_mb": 64 }, @@ -196,20 +188,19 @@ "method_id": "procrustes", "metric_values": { "knn_auc": 0.071, - "mse": 1.0102 + "mse": 1.0101 }, "scaled_scores": { - "knn_auc": 0.065, - "mse": 0.6901 + "knn_auc": 1, + "mse": 0.0653 }, - "mean_score": 0.3776, - "name": "auto:run_benchmark:run_wf:runEachWf:procrustes:processWf:procrustes_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.procrustes)", + "mean_score": 0.5327, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 6.4, - "cpu_pct": 258.7, - "peak_memory_mb": 3380, + "duration_sec": 14.9, + "cpu_pct": 143.9, + "peak_memory_mb": 5735, "disk_read_mb": 178, "disk_write_mb": 64 }, @@ -219,21 +210,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", "method_id": "random_features", "metric_values": { - "knn_auc": 0.0514, - "mse": 0.9882 + "knn_auc": 0.0525, + "mse": 0.999 }, "scaled_scores": { - "knn_auc": 0.0453, - "mse": 0.5941 + "knn_auc": 0.7137, + "mse": 0.0756 }, - "mean_score": 0.3197, - "name": "auto:run_benchmark:run_wf:runEachWf:random_features:processWf:random_features_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.random_features)", + "mean_score": 0.3947, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 6.2, - "cpu_pct": 162, - "peak_memory_mb": 3277, + "duration_sec": 14.1, + "cpu_pct": 62.4, + "peak_memory_mb": 5632, "disk_read_mb": 178, "disk_write_mb": 61 }, @@ -244,20 +234,19 @@ "method_id": "scot", "metric_values": { "knn_auc": 0.0517, - "mse": 0.8522 + "mse": 0.8547 }, "scaled_scores": { - "knn_auc": 0.0456, - "mse": 0 + "knn_auc": 0.7019, + "mse": 0.2091 }, - "mean_score": 0.0228, - "name": "auto:run_benchmark:run_wf:runEachWf:scot:processWf:scot_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.scot)", + "mean_score": 0.4555, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 448, - "cpu_pct": 3046.2, - "peak_memory_mb": 8090, + "duration_sec": 1157, + "cpu_pct": 4300.4, + "peak_memory_mb": 10445, "disk_read_mb": 198, "disk_write_mb": 64 }, @@ -267,23 +256,22 @@ "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", "method_id": "true_features", "metric_values": { - "knn_auc": 1, - "mse": 0.9831 + "knn_auc": 0.0583, + "mse": 0 }, "scaled_scores": { - "knn_auc": 1, - "mse": 0.5717 + "knn_auc": 0.8047, + "mse": 1 }, - "mean_score": 0.7858, - "name": "auto:run_benchmark:run_wf:runEachWf:true_features:processWf:true_features_process (openproblems_v1_multimodal/scicar_cell_lines/log_cp10k.true_features)", + "mean_score": 0.9024, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 2.8, - "cpu_pct": 164, - "peak_memory_mb": 3277, - "disk_read_mb": 178, - "disk_write_mb": 14 + "duration_sec": 117, + "cpu_pct": 10.1, + "peak_memory_mb": 5837, + "disk_read_mb": 343, + "disk_write_mb": 13 }, "task_id": "match_modalities" }, @@ -292,20 +280,19 @@ "method_id": "fastmnn", "metric_values": { "knn_auc": 0.0432, - "mse": 0.9495 + "mse": 0.9472 }, "scaled_scores": { "knn_auc": 0, - "mse": 0 + "mse": 0.0563 }, - "mean_score": 0, - "name": "auto:run_benchmark:run_wf:runEachWf:fastmnn:processWf:fastmnn_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.fastmnn)", + "mean_score": 0.0282, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 46.4, - "cpu_pct": 111, - "peak_memory_mb": 4506, + "duration_sec": 49, + "cpu_pct": 108.4, + "peak_memory_mb": 6861, "disk_read_mb": 356, "disk_write_mb": 119 }, @@ -315,21 +302,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "harmonic_alignment", "metric_values": { - "knn_auc": 0.0912, - "mse": 0.9999 + "knn_auc": 0.0822, + "mse": 1.0001 }, "scaled_scores": { - "knn_auc": 0.0501, - "mse": 0.9771 + "knn_auc": 0.148, + "mse": 0.0037 }, - "mean_score": 0.5136, - "name": "auto:run_benchmark:run_wf:runEachWf:harmonic_alignment:processWf:harmonic_alignment_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.harmonic_alignment)", + "mean_score": 0.0759, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 218, - "cpu_pct": 413.8, - "peak_memory_mb": 4404, + "duration_sec": 203, + "cpu_pct": 706.8, + "peak_memory_mb": 6759, "disk_read_mb": 326, "disk_write_mb": 118 }, @@ -343,17 +329,16 @@ "mse": 1.0011 }, "scaled_scores": { - "knn_auc": 0.2751, - "mse": 1 + "knn_auc": 1, + "mse": 0.0026 }, - "mean_score": 0.6375, - "name": "auto:run_benchmark:run_wf:runEachWf:procrustes:processWf:procrustes_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.procrustes)", + "mean_score": 0.5013, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 9.9, - "cpu_pct": 201.8, - "peak_memory_mb": 3584, + "duration_sec": 10.2, + "cpu_pct": 360.5, + "peak_memory_mb": 5940, "disk_read_mb": 317, "disk_write_mb": 118 }, @@ -363,21 +348,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "random_features", "metric_values": { - "knn_auc": 0.0491, - "mse": 0.9901 + "knn_auc": 0.0486, + "mse": 1.0038 }, "scaled_scores": { - "knn_auc": 0.0061, - "mse": 0.7867 + "knn_auc": 0.0206, + "mse": 0 }, - "mean_score": 0.3964, - "name": "auto:run_benchmark:run_wf:runEachWf:random_features:processWf:random_features_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.random_features)", + "mean_score": 0.0103, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 8.9, - "cpu_pct": 130.8, - "peak_memory_mb": 3482, + "duration_sec": 16.6, + "cpu_pct": 88.9, + "peak_memory_mb": 5837, "disk_read_mb": 316, "disk_write_mb": 110 }, @@ -388,20 +372,19 @@ "method_id": "scot", "metric_values": { "knn_auc": 0.0758, - "mse": 0.994 + "mse": 0.9929 }, "scaled_scores": { - "knn_auc": 0.034, - "mse": 0.8626 + "knn_auc": 0.1236, + "mse": 0.0108 }, - "mean_score": 0.4483, - "name": "auto:run_benchmark:run_wf:runEachWf:scot:processWf:scot_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.scot)", + "mean_score": 0.0672, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 14136, - "cpu_pct": 2033.2, - "peak_memory_mb": 17101, + "duration_sec": 9586, + "cpu_pct": 4558.5, + "peak_memory_mb": 19456, "disk_read_mb": 337, "disk_write_mb": 118 }, @@ -411,23 +394,22 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "true_features", "metric_values": { - "knn_auc": 1, - "mse": 0.9519 + "knn_auc": 0.0936, + "mse": 0 }, "scaled_scores": { - "knn_auc": 1, - "mse": 0.047 + "knn_auc": 0.1915, + "mse": 1 }, - "mean_score": 0.5235, - "name": "auto:run_benchmark:run_wf:runEachWf:true_features:processWf:true_features_process (openproblems_v1_multimodal/scicar_mouse_kidney/log_cp10k.true_features)", + "mean_score": 0.5958, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 4.7, - "cpu_pct": 138.7, - "peak_memory_mb": 3482, - "disk_read_mb": 316, - "disk_write_mb": 25 + "duration_sec": 113, + "cpu_pct": 10.1, + "peak_memory_mb": 5530, + "disk_read_mb": 621, + "disk_write_mb": 21 }, "task_id": "match_modalities" } diff --git a/results/match_modalities/data/state.yaml b/results/match_modalities/data/state.yaml index 9cc4fc6d..abbb0fc1 100644 --- a/results/match_modalities/data/state.yaml +++ b/results/match_modalities/data/state.yaml @@ -5,4 +5,5 @@ output_metric_info: !file metric_info.json output_dataset_info: !file dataset_info.json output_task_info: !file task_info.json output_qc: !file quality_control.json +output_metric_execution_info: !file metric_execution_info.json From c76d95a54d193f5666903ca2401b0b1afa0c773a Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Mon, 18 Mar 2024 18:53:08 +0100 Subject: [PATCH 6/7] update results --- .../data/quality_control.json | 136 +++++++++--------- results/match_modalities/data/results.json | 88 ++++++------ 2 files changed, 112 insertions(+), 112 deletions(-) diff --git a/results/match_modalities/data/quality_control.json b/results/match_modalities/data/quality_control.json index 84d65e9b..1af95968 100644 --- a/results/match_modalities/data/quality_control.json +++ b/results/match_modalities/data/quality_control.json @@ -363,141 +363,141 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Worst score random_features knn_auc", - "value": 0.0206, + "value": 0, "severity": 0, - "severity_value": -0.0206, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Worst score: 0.0206%\n" + "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Worst score: 0%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score random_features knn_auc", - "value": 0.7137, + "value": 1, "severity": 0, - "severity_value": 0.35685, + "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 0.7137%\n" + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 1%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score true_features knn_auc", - "value": 0.1295, + "value": 0, "severity": 0, - "severity_value": -0.1295, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Worst score: 0.1295%\n" + "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Worst score: 0%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score true_features knn_auc", - "value": 0.8047, + "value": 1, "severity": 0, - "severity_value": 0.40235, + "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Best score: 0.8047%\n" + "message": "Method true_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Best score: 1%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score scot knn_auc", - "value": 0.1236, + "value": -0.1298, "severity": 0, - "severity_value": -0.1236, + "severity_value": 0.1298, "code": "worst_score >= -1", - "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Worst score: 0.1236%\n" + "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Worst score: -0.1298%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score scot knn_auc", - "value": 0.7019, + "value": 1.2458, "severity": 0, - "severity_value": 0.35095, + "severity_value": 0.6229, "code": "best_score <= 2", - "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Best score: 0.7019%\n" + "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Best score: 1.2458%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score harmonic_alignment knn_auc", - "value": 0.0, - "severity": 0, - "severity_value": -0.0, + "value": -7.8462, + "severity": 3, + "severity_value": 7.8462, "code": "worst_score >= -1", - "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Worst score: 0.0%\n" + "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Worst score: -7.8462%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score harmonic_alignment knn_auc", - "value": 0.148, + "value": 0.7456, "severity": 0, - "severity_value": 0.074, + "severity_value": 0.3728, "code": "best_score <= 2", - "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Best score: 0.148%\n" + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Best score: 0.7456%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score fastmnn knn_auc", - "value": 0.0, - "severity": 0, - "severity_value": -0.0, + "value": -6.5863, + "severity": 3, + "severity_value": 6.5863, "code": "worst_score >= -1", - "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Worst score: 0.0%\n" + "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Worst score: -6.5863%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score fastmnn knn_auc", - "value": 0.314, + "value": 0.6265, "severity": 0, - "severity_value": 0.157, + "severity_value": 0.31325, "code": "best_score <= 2", - "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Best score: 0.314%\n" + "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Best score: 0.6265%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score procrustes knn_auc", - "value": 1, + "value": 2.9564, "severity": 0, - "severity_value": -1.0, + "severity_value": -2.9564, "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: 1%\n" + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: 2.9564%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score procrustes knn_auc", - "value": 1, - "severity": 0, - "severity_value": 0.5, + "value": 5.7299, + "severity": 2, + "severity_value": 2.86495, "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 1%\n" + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 5.7299%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score random_features mse", - "value": 0.0, + "value": 0, "severity": 0, "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Worst score: 0.0%\n" + "message": "Method random_features performs much worse than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Worst score: 0%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score random_features mse", - "value": 0.2137, + "value": 0, "severity": 0, - "severity_value": 0.10685, + "severity_value": 0.0, "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Best score: 0.2137%\n" + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: mse\n Best score: 0%\n" }, { "task_id": "match_modalities", @@ -523,80 +523,80 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Worst score scot mse", - "value": 0.0, + "value": -0.2718, "severity": 0, - "severity_value": -0.0, + "severity_value": 0.2718, "code": "worst_score >= -1", - "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Worst score: 0.0%\n" + "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Worst score: -0.2718%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score scot mse", - "value": 0.2091, + "value": 0.1444, "severity": 0, - "severity_value": 0.10455, + "severity_value": 0.0722, "code": "best_score <= 2", - "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Best score: 0.2091%\n" + "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Best score: 0.1444%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score harmonic_alignment mse", - "value": 0.0037, + "value": -0.0143, "severity": 0, - "severity_value": -0.0037, + "severity_value": 0.0143, "code": "worst_score >= -1", - "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Worst score: 0.0037%\n" + "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Worst score: -0.0143%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score harmonic_alignment mse", - "value": 0.2025, + "value": 0.0037, "severity": 0, - "severity_value": 0.10125, + "severity_value": 0.00185, "code": "best_score <= 2", - "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Best score: 0.2025%\n" + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Best score: 0.0037%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score fastmnn mse", - "value": 0.0, + "value": -0.1897, "severity": 0, - "severity_value": -0.0, + "severity_value": 0.1897, "code": "worst_score >= -1", - "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Worst score: 0.0%\n" + "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Worst score: -0.1897%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score fastmnn mse", - "value": 0.0646, + "value": 0.0563, "severity": 0, - "severity_value": 0.0323, + "severity_value": 0.02815, "code": "best_score <= 2", - "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Best score: 0.0646%\n" + "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Best score: 0.0563%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score procrustes mse", - "value": 0.0026, + "value": -0.0198, "severity": 0, - "severity_value": -0.0026, + "severity_value": 0.0198, "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: 0.0026%\n" + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: -0.0198%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score procrustes mse", - "value": 0.1981, + "value": 0.0026, "severity": 0, - "severity_value": 0.09905, + "severity_value": 0.0013, "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 0.1981%\n" + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 0.0026%\n" } ] \ No newline at end of file diff --git a/results/match_modalities/data/results.json b/results/match_modalities/data/results.json index bf810417..21168826 100644 --- a/results/match_modalities/data/results.json +++ b/results/match_modalities/data/results.json @@ -7,10 +7,10 @@ "mse": 1.173 }, "scaled_scores": { - "knn_auc": 0.314, - "mse": 0.0646 + "knn_auc": 0.6265, + "mse": -0.1897 }, - "mean_score": 0.1893, + "mean_score": 0.3133, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -30,10 +30,10 @@ "mse": 1 }, "scaled_scores": { - "knn_auc": 0, - "mse": 0.2025 + "knn_auc": -0.4399, + "mse": -0.0143 }, - "mean_score": 0.1012, + "mean_score": 0, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -53,10 +53,10 @@ "mse": 1.0055 }, "scaled_scores": { - "knn_auc": 1, - "mse": 0.1981 + "knn_auc": 2.9564, + "mse": -0.0198 }, - "mean_score": 0.5991, + "mean_score": 0.5, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -76,10 +76,10 @@ "mse": 0.9859 }, "scaled_scores": { - "knn_auc": 0.424, - "mse": 0.2137 + "knn_auc": 1, + "mse": 0 }, - "mean_score": 0.3188, + "mean_score": 0.5, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -99,10 +99,10 @@ "mse": 1.2539 }, "scaled_scores": { - "knn_auc": 0.4963, - "mse": 0 + "knn_auc": 1.2458, + "mse": -0.2718 }, - "mean_score": 0.2482, + "mean_score": 0.5, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -122,10 +122,10 @@ "mse": 0 }, "scaled_scores": { - "knn_auc": 0.1295, + "knn_auc": 0, "mse": 1 }, - "mean_score": 0.5648, + "mean_score": 0.5, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -145,10 +145,10 @@ "mse": 1.0807 }, "scaled_scores": { - "knn_auc": 0.1146, - "mse": 0 + "knn_auc": -6.5863, + "mse": -0.0818 }, - "mean_score": 0.0573, + "mean_score": 0, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -168,10 +168,10 @@ "mse": 1 }, "scaled_scores": { - "knn_auc": 0, - "mse": 0.0747 + "knn_auc": -7.8462, + "mse": -0.001 }, - "mean_score": 0.0373, + "mean_score": 0, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -191,10 +191,10 @@ "mse": 1.0101 }, "scaled_scores": { - "knn_auc": 1, - "mse": 0.0653 + "knn_auc": 3.1469, + "mse": -0.0111 }, - "mean_score": 0.5327, + "mean_score": 0.5, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -214,10 +214,10 @@ "mse": 0.999 }, "scaled_scores": { - "knn_auc": 0.7137, - "mse": 0.0756 + "knn_auc": 0, + "mse": 0 }, - "mean_score": 0.3947, + "mean_score": 0, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -237,10 +237,10 @@ "mse": 0.8547 }, "scaled_scores": { - "knn_auc": 0.7019, - "mse": 0.2091 + "knn_auc": -0.1298, + "mse": 0.1444 }, - "mean_score": 0.4555, + "mean_score": 0.0722, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -260,10 +260,10 @@ "mse": 0 }, "scaled_scores": { - "knn_auc": 0.8047, + "knn_auc": 1, "mse": 1 }, - "mean_score": 0.9024, + "mean_score": 1, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -283,7 +283,7 @@ "mse": 0.9472 }, "scaled_scores": { - "knn_auc": 0, + "knn_auc": -0.1204, "mse": 0.0563 }, "mean_score": 0.0282, @@ -306,10 +306,10 @@ "mse": 1.0001 }, "scaled_scores": { - "knn_auc": 0.148, + "knn_auc": 0.7456, "mse": 0.0037 }, - "mean_score": 0.0759, + "mean_score": 0.3746, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -329,7 +329,7 @@ "mse": 1.0011 }, "scaled_scores": { - "knn_auc": 1, + "knn_auc": 5.7299, "mse": 0.0026 }, "mean_score": 0.5013, @@ -352,10 +352,10 @@ "mse": 1.0038 }, "scaled_scores": { - "knn_auc": 0.0206, + "knn_auc": 0, "mse": 0 }, - "mean_score": 0.0103, + "mean_score": 0, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -375,10 +375,10 @@ "mse": 0.9929 }, "scaled_scores": { - "knn_auc": 0.1236, + "knn_auc": 0.6029, "mse": 0.0108 }, - "mean_score": 0.0672, + "mean_score": 0.3069, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, @@ -398,10 +398,10 @@ "mse": 0 }, "scaled_scores": { - "knn_auc": 0.1915, + "knn_auc": 1, "mse": 1 }, - "mean_score": 0.5958, + "mean_score": 1, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, From 08fe5e194edb154f170e1384dfc5b7ae9ce6d7d9 Mon Sep 17 00:00:00 2001 From: Kai Waldrant Date: Thu, 21 Mar 2024 08:15:59 +0100 Subject: [PATCH 7/7] Update results --- .../match_modalities/data/dataset_info.json | 16 +- .../match_modalities/data/method_info.json | 24 +- .../data/metric_execution_info.json | 214 +++++++-------- .../match_modalities/data/metric_info.json | 8 +- .../data/quality_control.json | 126 ++++----- results/match_modalities/data/results.json | 252 +++++++++--------- 6 files changed, 320 insertions(+), 320 deletions(-) diff --git a/results/match_modalities/data/dataset_info.json b/results/match_modalities/data/dataset_info.json index 7b505e5f..856b1994 100644 --- a/results/match_modalities/data/dataset_info.json +++ b/results/match_modalities/data/dataset_info.json @@ -1,12 +1,4 @@ [ - { - "task_id": "match_modalities", - "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", - "dataset_name": "sci-CAR Cell Lines", - "dataset_summary": "sci-CAR profiles of 5k cell line cells (HEK293T, NIH/3T3, A549) across three treatment conditions (DEX 0h, 1h and 3h)", - "data_reference": "cao2018joint", - "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089" - }, { "task_id": "match_modalities", "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", @@ -22,5 +14,13 @@ "dataset_summary": "CITE-seq profiles of 8k Cord Blood Mononuclear Cells", "data_reference": "stoeckius2017simultaneous", "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE100866" + }, + { + "task_id": "match_modalities", + "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", + "dataset_name": "sci-CAR Cell Lines", + "dataset_summary": "sci-CAR profiles of 5k cell line cells (HEK293T, NIH/3T3, A549) across three treatment conditions (DEX 0h, 1h and 3h)", + "data_reference": "cao2018joint", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089" } ] diff --git a/results/match_modalities/data/method_info.json b/results/match_modalities/data/method_info.json index dfe65b0f..fb4867df 100644 --- a/results/match_modalities/data/method_info.json +++ b/results/match_modalities/data/method_info.json @@ -7,9 +7,9 @@ "is_baseline": true, "paper_reference": null, "code_url": null, - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/control_methods/random_features/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/control_methods/random_features/config.vsh.yaml", "code_version": null, - "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" }, { "task_id": "match_modalities", @@ -19,9 +19,9 @@ "is_baseline": true, "paper_reference": null, "code_url": null, - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/control_methods/true_features/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/control_methods/true_features/config.vsh.yaml", "code_version": null, - "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" }, { "task_id": "match_modalities", @@ -31,9 +31,9 @@ "is_baseline": false, "paper_reference": "Demetci2020scot", "code_url": "https://github.com/rsinghlab/SCOT", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/methods/scot/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/methods/scot/config.vsh.yaml", "code_version": null, - "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" }, { "task_id": "match_modalities", @@ -43,9 +43,9 @@ "is_baseline": false, "paper_reference": "stanley2020harmonic", "code_url": "https://github.com/KrishnaswamyLab/harmonic-alignment", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/methods/harmonic_alignment/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/methods/harmonic_alignment/config.vsh.yaml", "code_version": null, - "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" }, { "task_id": "match_modalities", @@ -55,9 +55,9 @@ "is_baseline": false, "paper_reference": "haghverdi2018batch", "code_url": "https://github.com/LTLA/batchelor", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/methods/fastmnn/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/methods/fastmnn/config.vsh.yaml", "code_version": null, - "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" }, { "task_id": "match_modalities", @@ -67,8 +67,8 @@ "is_baseline": false, "paper_reference": "gower1975generalized", "code_url": "https://github.com/scipy/scipy", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/methods/procrustes/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/methods/procrustes/config.vsh.yaml", "code_version": null, - "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1" + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb" } ] diff --git a/results/match_modalities/data/metric_execution_info.json b/results/match_modalities/data/metric_execution_info.json index 182c5227..c7d6912e 100644 --- a/results/match_modalities/data/metric_execution_info.json +++ b/results/match_modalities/data/metric_execution_info.json @@ -6,9 +6,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 23.3, - "cpu_pct": 98.8, - "peak_memory_mb": 6247, + "duration_sec": 12.7, + "cpu_pct": 138.4, + "peak_memory_mb": 3892, "disk_read_mb": 265, "disk_write_mb": 1 } @@ -20,9 +20,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 17.8, - "cpu_pct": 203.1, - "peak_memory_mb": 9626, + "duration_sec": 10.6, + "cpu_pct": 216, + "peak_memory_mb": 5735, "disk_read_mb": 276, "disk_write_mb": 1 } @@ -34,9 +34,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 20.8, - "cpu_pct": 109.6, - "peak_memory_mb": 6144, + "duration_sec": 14.1, + "cpu_pct": 118.8, + "peak_memory_mb": 3789, "disk_read_mb": 264, "disk_write_mb": 1 } @@ -48,9 +48,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 14.8, - "cpu_pct": 151.8, - "peak_memory_mb": 6144, + "duration_sec": 12.4, + "cpu_pct": 135.8, + "peak_memory_mb": 3892, "disk_read_mb": 264, "disk_write_mb": 1 } @@ -62,9 +62,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 13.8, - "cpu_pct": 131.2, - "peak_memory_mb": 6247, + "duration_sec": 16.1, + "cpu_pct": 100.1, + "peak_memory_mb": 3892, "disk_read_mb": 264, "disk_write_mb": 1 } @@ -76,9 +76,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 13.1, - "cpu_pct": 155.8, - "peak_memory_mb": 5940, + "duration_sec": 24, + "cpu_pct": 113.3, + "peak_memory_mb": 3789, "disk_read_mb": 210, "disk_write_mb": 1 } @@ -90,9 +90,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 12.6, - "cpu_pct": 189.7, - "peak_memory_mb": 8602, + "duration_sec": 5.3, + "cpu_pct": 333.8, + "peak_memory_mb": 5530, "disk_read_mb": 251, "disk_write_mb": 1 } @@ -104,9 +104,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 9.8, - "cpu_pct": 216.4, - "peak_memory_mb": 8090, + "duration_sec": 5.1, + "cpu_pct": 311, + "peak_memory_mb": 5530, "disk_read_mb": 250, "disk_write_mb": 1 } @@ -118,9 +118,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 6.8, - "cpu_pct": 362.5, - "peak_memory_mb": 8909, + "duration_sec": 4.9, + "cpu_pct": 329.1, + "peak_memory_mb": 5530, "disk_read_mb": 250, "disk_write_mb": 1 } @@ -132,9 +132,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 6.1, - "cpu_pct": 398.5, - "peak_memory_mb": 9114, + "duration_sec": 5.6, + "cpu_pct": 241.6, + "peak_memory_mb": 5530, "disk_read_mb": 247, "disk_write_mb": 1 } @@ -146,9 +146,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 8.7, - "cpu_pct": 315.5, - "peak_memory_mb": 8909, + "duration_sec": 9.1, + "cpu_pct": 269.3, + "peak_memory_mb": 5530, "disk_read_mb": 250, "disk_write_mb": 1 } @@ -160,9 +160,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 88, - "cpu_pct": 29.3, - "peak_memory_mb": 6144, + "duration_sec": 6.8, + "cpu_pct": 196, + "peak_memory_mb": 5325, "disk_read_mb": 199, "disk_write_mb": 1 } @@ -174,9 +174,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 22.3, - "cpu_pct": 303.1, - "peak_memory_mb": 10036, + "duration_sec": 13.3, + "cpu_pct": 351.9, + "peak_memory_mb": 6144, "disk_read_mb": 444, "disk_write_mb": 1 } @@ -188,9 +188,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 20.2, - "cpu_pct": 453.7, - "peak_memory_mb": 10036, + "duration_sec": 13.9, + "cpu_pct": 282.9, + "peak_memory_mb": 6144, "disk_read_mb": 443, "disk_write_mb": 1 } @@ -202,9 +202,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 34.2, - "cpu_pct": 261.4, - "peak_memory_mb": 10036, + "duration_sec": 13.7, + "cpu_pct": 323.7, + "peak_memory_mb": 6144, "disk_read_mb": 443, "disk_write_mb": 1 } @@ -216,9 +216,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 148, - "cpu_pct": 77.1, - "peak_memory_mb": 10036, + "duration_sec": 12.5, + "cpu_pct": 367.7, + "peak_memory_mb": 6144, "disk_read_mb": 435, "disk_write_mb": 1 } @@ -230,9 +230,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 173, - "cpu_pct": 46.7, - "peak_memory_mb": 10036, + "duration_sec": 16.1, + "cpu_pct": 302.7, + "peak_memory_mb": 6144, "disk_read_mb": 443, "disk_write_mb": 1 } @@ -244,9 +244,9 @@ "metric_id": "knn_auc", "resources": { "exit_code": 0, - "duration_sec": 60, - "cpu_pct": 148.2, - "peak_memory_mb": 9728, + "duration_sec": 35.6, + "cpu_pct": 207.9, + "peak_memory_mb": 5837, "disk_read_mb": 346, "disk_write_mb": 1 } @@ -258,9 +258,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 3, - "cpu_pct": 389.5, - "peak_memory_mb": 5735, + "duration_sec": 2.9, + "cpu_pct": 200.4, + "peak_memory_mb": 3380, "disk_read_mb": 256, "disk_write_mb": 1 } @@ -272,9 +272,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 9.2, - "cpu_pct": 133.5, - "peak_memory_mb": 5735, + "duration_sec": 3, + "cpu_pct": 207.2, + "peak_memory_mb": 3380, "disk_read_mb": 267, "disk_write_mb": 1 } @@ -286,9 +286,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 5.9, - "cpu_pct": 140.5, - "peak_memory_mb": 5837, + "duration_sec": 2.8, + "cpu_pct": 219, + "peak_memory_mb": 3380, "disk_read_mb": 256, "disk_write_mb": 1 } @@ -300,9 +300,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 9.6, - "cpu_pct": 119.2, - "peak_memory_mb": 5735, + "duration_sec": 3.8, + "cpu_pct": 164.2, + "peak_memory_mb": 3482, "disk_read_mb": 255, "disk_write_mb": 1 } @@ -314,9 +314,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 3.1, - "cpu_pct": 280.9, - "peak_memory_mb": 5735, + "duration_sec": 2.7, + "cpu_pct": 285.8, + "peak_memory_mb": 3482, "disk_read_mb": 256, "disk_write_mb": 1 } @@ -329,8 +329,8 @@ "resources": { "exit_code": 0, "duration_sec": 1.9, - "cpu_pct": 476.5, - "peak_memory_mb": 2765, + "cpu_pct": 331.6, + "peak_memory_mb": 1639, "disk_read_mb": 201, "disk_write_mb": 1 } @@ -342,9 +342,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 6.8, - "cpu_pct": 131, - "peak_memory_mb": 5837, + "duration_sec": 3.3, + "cpu_pct": 212.3, + "peak_memory_mb": 3482, "disk_read_mb": 242, "disk_write_mb": 1 } @@ -356,9 +356,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 6.9, - "cpu_pct": 149.3, - "peak_memory_mb": 5837, + "duration_sec": 4.1, + "cpu_pct": 171.3, + "peak_memory_mb": 3380, "disk_read_mb": 241, "disk_write_mb": 1 } @@ -370,9 +370,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 3.3, - "cpu_pct": 211.6, - "peak_memory_mb": 5837, + "duration_sec": 2.8, + "cpu_pct": 271.1, + "peak_memory_mb": 3380, "disk_read_mb": 241, "disk_write_mb": 1 } @@ -384,9 +384,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 6.3, - "cpu_pct": 147.3, - "peak_memory_mb": 5735, + "duration_sec": 2.7, + "cpu_pct": 285.3, + "peak_memory_mb": 3482, "disk_read_mb": 238, "disk_write_mb": 1 } @@ -398,9 +398,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 3.5, - "cpu_pct": 224.7, - "peak_memory_mb": 5837, + "duration_sec": 4.8, + "cpu_pct": 108.8, + "peak_memory_mb": 3482, "disk_read_mb": 241, "disk_write_mb": 1 } @@ -412,9 +412,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 5.9, - "cpu_pct": 110.6, - "peak_memory_mb": 5632, + "duration_sec": 3.4, + "cpu_pct": 179.2, + "peak_memory_mb": 3380, "disk_read_mb": 190, "disk_write_mb": 1 } @@ -426,9 +426,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 10, - "cpu_pct": 89.4, - "peak_memory_mb": 6144, + "duration_sec": 5.1, + "cpu_pct": 154.3, + "peak_memory_mb": 3687, "disk_read_mb": 435, "disk_write_mb": 1 } @@ -440,9 +440,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 4, - "cpu_pct": 265.1, - "peak_memory_mb": 6042, + "duration_sec": 5.9, + "cpu_pct": 135, + "peak_memory_mb": 3687, "disk_read_mb": 434, "disk_write_mb": 1 } @@ -454,9 +454,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 4.1, - "cpu_pct": 279.5, - "peak_memory_mb": 6042, + "duration_sec": 4.4, + "cpu_pct": 147.8, + "peak_memory_mb": 3687, "disk_read_mb": 434, "disk_write_mb": 1 } @@ -468,9 +468,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 56.9, - "cpu_pct": 17.1, - "peak_memory_mb": 2765, + "duration_sec": 3.7, + "cpu_pct": 239.6, + "peak_memory_mb": 3789, "disk_read_mb": 426, "disk_write_mb": 1 } @@ -482,9 +482,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 116, - "cpu_pct": 11.3, - "peak_memory_mb": 5940, + "duration_sec": 4.1, + "cpu_pct": 181, + "peak_memory_mb": 3687, "disk_read_mb": 434, "disk_write_mb": 1 } @@ -496,9 +496,9 @@ "metric_id": "mse", "resources": { "exit_code": 0, - "duration_sec": 7.3, - "cpu_pct": 95.8, - "peak_memory_mb": 5530, + "duration_sec": 3, + "cpu_pct": 249.9, + "peak_memory_mb": 3175, "disk_read_mb": 337, "disk_write_mb": 1 } diff --git a/results/match_modalities/data/metric_info.json b/results/match_modalities/data/metric_info.json index a9b30699..0a73d15d 100644 --- a/results/match_modalities/data/metric_info.json +++ b/results/match_modalities/data/metric_info.json @@ -5,9 +5,9 @@ "metric_name": "kNN Area Under the Curve", "metric_summary": "Let $f(i) \\in F$ be the scRNA-seq measurement of cell $i$, and $g(i) \\in G$ be the scATAC- seq measurement of cell $i$. kNN-AUC calculates the average percentage overlap of neighborhoods of $f(i)$ in $F$ with neighborhoods of $g(i)$ in $G$. Higher is better.\n", "paper_reference": "lance2022multimodal", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/metrics/knn_auc/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/metrics/knn_auc/config.vsh.yaml", "code_version": null, - "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1", + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb", "maximize": true }, { @@ -16,9 +16,9 @@ "metric_name": "Mean Squared Error", "metric_summary": "Mean squared error (MSE) is the average distance between each pair of matched observations of the same cell in the learned latent space. Lower is better.\n", "paper_reference": "lance2022multimodal", - "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/0891a3c0e7611c9536e55469b5ac2eade7407ff1/src/tasks/match_modalities/metrics/mse/config.vsh.yaml", + "implementation_url": "https://github.com/openproblems-bio/openproblems-v2/tree/cf678cdaee2b5f1cc3bbae256de382ea3cc96acb/src/tasks/match_modalities/metrics/mse/config.vsh.yaml", "code_version": null, - "commit_sha": "0891a3c0e7611c9536e55469b5ac2eade7407ff1", + "commit_sha": "cf678cdaee2b5f1cc3bbae256de382ea3cc96acb", "maximize": false } ] diff --git a/results/match_modalities/data/quality_control.json b/results/match_modalities/data/quality_control.json index 1af95968..5fc3e5b2 100644 --- a/results/match_modalities/data/quality_control.json +++ b/results/match_modalities/data/quality_control.json @@ -332,32 +332,32 @@ { "task_id": "match_modalities", "category": "Raw results", - "name": "Dataset 'openproblems_v1_multimodal/scicar_cell_lines' %missing", + "name": "Dataset 'openproblems_v1_multimodal/scicar_mouse_kidney' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/scicar_cell_lines\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/scicar_mouse_kidney\n Percentage missing: 0%\n" }, { "task_id": "match_modalities", "category": "Raw results", - "name": "Dataset 'openproblems_v1_multimodal/scicar_mouse_kidney' %missing", + "name": "Dataset 'openproblems_v1_multimodal/citeseq_cbmc' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/scicar_mouse_kidney\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/citeseq_cbmc\n Percentage missing: 0%\n" }, { "task_id": "match_modalities", "category": "Raw results", - "name": "Dataset 'openproblems_v1_multimodal/citeseq_cbmc' %missing", + "name": "Dataset 'openproblems_v1_multimodal/scicar_cell_lines' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/citeseq_cbmc\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: match_modalities\n dataset id: openproblems_v1_multimodal/scicar_cell_lines\n Percentage missing: 0%\n" }, { "task_id": "match_modalities", @@ -373,21 +373,21 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Best score random_features knn_auc", - "value": 1, + "value": 0, "severity": 0, - "severity_value": 0.5, + "severity_value": 0.0, "code": "best_score <= 2", - "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 1%\n" + "message": "Method random_features performs a lot better than baselines.\n Task id: match_modalities\n Method id: random_features\n Metric id: knn_auc\n Best score: 0%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score true_features knn_auc", - "value": 0, + "value": 1, "severity": 0, - "severity_value": -0.0, + "severity_value": -1.0, "code": "worst_score >= -1", - "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Worst score: 0%\n" + "message": "Method true_features performs much worse than baselines.\n Task id: match_modalities\n Method id: true_features\n Metric id: knn_auc\n Worst score: 1%\n" }, { "task_id": "match_modalities", @@ -403,81 +403,81 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Worst score scot knn_auc", - "value": -0.1298, + "value": -0.0248, "severity": 0, - "severity_value": 0.1298, + "severity_value": 0.0248, "code": "worst_score >= -1", - "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Worst score: -0.1298%\n" + "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Worst score: -0.0248%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score scot knn_auc", - "value": 1.2458, + "value": 0.0528, "severity": 0, - "severity_value": 0.6229, + "severity_value": 0.0264, "code": "best_score <= 2", - "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Best score: 1.2458%\n" + "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: knn_auc\n Best score: 0.0528%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score harmonic_alignment knn_auc", - "value": -7.8462, - "severity": 3, - "severity_value": 7.8462, + "value": -0.0052, + "severity": 0, + "severity_value": 0.0052, "code": "worst_score >= -1", - "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Worst score: -7.8462%\n" + "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Worst score: -0.0052%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score harmonic_alignment knn_auc", - "value": 0.7456, + "value": 0.0201, "severity": 0, - "severity_value": 0.3728, + "severity_value": 0.01005, "code": "best_score <= 2", - "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Best score: 0.7456%\n" + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: knn_auc\n Best score: 0.0201%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score fastmnn knn_auc", - "value": -6.5863, - "severity": 3, - "severity_value": 6.5863, + "value": -0.0147, + "severity": 0, + "severity_value": 0.0147, "code": "worst_score >= -1", - "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Worst score: -6.5863%\n" + "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Worst score: -0.0147%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score fastmnn knn_auc", - "value": 0.6265, + "value": 0.0274, "severity": 0, - "severity_value": 0.31325, + "severity_value": 0.0137, "code": "best_score <= 2", - "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Best score: 0.6265%\n" + "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: knn_auc\n Best score: 0.0274%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score procrustes knn_auc", - "value": 2.9564, + "value": -0.011, "severity": 0, - "severity_value": -2.9564, + "severity_value": 0.011, "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: 2.9564%\n" + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Worst score: -0.011%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score procrustes knn_auc", - "value": 5.7299, - "severity": 2, - "severity_value": 2.86495, + "value": 0.0432, + "severity": 0, + "severity_value": 0.0216, "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 5.7299%\n" + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: knn_auc\n Best score: 0.0432%\n" }, { "task_id": "match_modalities", @@ -523,80 +523,80 @@ "task_id": "match_modalities", "category": "Scaling", "name": "Worst score scot mse", - "value": -0.2718, + "value": -0.2218, "severity": 0, - "severity_value": 0.2718, + "severity_value": 0.2218, "code": "worst_score >= -1", - "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Worst score: -0.2718%\n" + "message": "Method scot performs much worse than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Worst score: -0.2218%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score scot mse", - "value": 0.1444, + "value": 0.1505, "severity": 0, - "severity_value": 0.0722, + "severity_value": 0.07525, "code": "best_score <= 2", - "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Best score: 0.1444%\n" + "message": "Method scot performs a lot better than baselines.\n Task id: match_modalities\n Method id: scot\n Metric id: mse\n Best score: 0.1505%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score harmonic_alignment mse", - "value": -0.0143, + "value": -0.0029, "severity": 0, - "severity_value": 0.0143, + "severity_value": 0.0029, "code": "worst_score >= -1", - "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Worst score: -0.0143%\n" + "message": "Method harmonic_alignment performs much worse than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Worst score: -0.0029%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score harmonic_alignment mse", - "value": 0.0037, + "value": 0.0143, "severity": 0, - "severity_value": 0.00185, + "severity_value": 0.00715, "code": "best_score <= 2", - "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Best score: 0.0037%\n" + "message": "Method harmonic_alignment performs a lot better than baselines.\n Task id: match_modalities\n Method id: harmonic_alignment\n Metric id: mse\n Best score: 0.0143%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score fastmnn mse", - "value": -0.1897, + "value": -0.158, "severity": 0, - "severity_value": 0.1897, + "severity_value": 0.158, "code": "worst_score >= -1", - "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Worst score: -0.1897%\n" + "message": "Method fastmnn performs much worse than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Worst score: -0.158%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score fastmnn mse", - "value": 0.0563, + "value": 0.0514, "severity": 0, - "severity_value": 0.02815, + "severity_value": 0.0257, "code": "best_score <= 2", - "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Best score: 0.0563%\n" + "message": "Method fastmnn performs a lot better than baselines.\n Task id: match_modalities\n Method id: fastmnn\n Metric id: mse\n Best score: 0.0514%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Worst score procrustes mse", - "value": -0.0198, + "value": -0.0095, "severity": 0, - "severity_value": 0.0198, + "severity_value": 0.0095, "code": "worst_score >= -1", - "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: -0.0198%\n" + "message": "Method procrustes performs much worse than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Worst score: -0.0095%\n" }, { "task_id": "match_modalities", "category": "Scaling", "name": "Best score procrustes mse", - "value": 0.0026, + "value": 0.009, "severity": 0, - "severity_value": 0.0013, + "severity_value": 0.0045, "code": "best_score <= 2", - "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 0.0026%\n" + "message": "Method procrustes performs a lot better than baselines.\n Task id: match_modalities\n Method id: procrustes\n Metric id: mse\n Best score: 0.009%\n" } ] \ No newline at end of file diff --git a/results/match_modalities/data/results.json b/results/match_modalities/data/results.json index 21168826..32e068d4 100644 --- a/results/match_modalities/data/results.json +++ b/results/match_modalities/data/results.json @@ -3,20 +3,20 @@ "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", "method_id": "fastmnn", "metric_values": { - "knn_auc": 0.0383, - "mse": 1.173 + "knn_auc": 0.0362, + "mse": 1.175 }, "scaled_scores": { - "knn_auc": 0.6265, - "mse": -0.1897 + "knn_auc": -0.0147, + "mse": -0.158 }, - "mean_score": 0.3133, + "mean_score": 0, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 175, - "cpu_pct": 18.1, - "peak_memory_mb": 6452, + "duration_sec": 16.5, + "cpu_pct": 146.3, + "peak_memory_mb": 4096, "disk_read_mb": 238, "disk_write_mb": 58 }, @@ -26,20 +26,20 @@ "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", "method_id": "harmonic_alignment", "metric_values": { - "knn_auc": 0.0043, - "mse": 1 + "knn_auc": 0.0692, + "mse": 1.0002 }, "scaled_scores": { - "knn_auc": -0.4399, - "mse": -0.0143 + "knn_auc": 0.0201, + "mse": 0.0143 }, - "mean_score": 0, + "mean_score": 0.0172, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 239, - "cpu_pct": 401.7, - "peak_memory_mb": 6042, + "duration_sec": 64, + "cpu_pct": 530.4, + "peak_memory_mb": 3687, "disk_read_mb": 208, "disk_write_mb": 69 }, @@ -49,20 +49,20 @@ "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", "method_id": "procrustes", "metric_values": { - "knn_auc": 0.1126, - "mse": 1.0055 + "knn_auc": 0.0397, + "mse": 1.0056 }, "scaled_scores": { - "knn_auc": 2.9564, - "mse": -0.0198 + "knn_auc": -0.011, + "mse": 0.009 }, - "mean_score": 0.5, + "mean_score": 0.0045, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 14, - "cpu_pct": 117.3, - "peak_memory_mb": 5735, + "duration_sec": 6.4, + "cpu_pct": 282.3, + "peak_memory_mb": 3380, "disk_read_mb": 199, "disk_write_mb": 58 }, @@ -73,19 +73,19 @@ "method_id": "random_features", "metric_values": { "knn_auc": 0.0502, - "mse": 0.9859 + "mse": 1.0147 }, "scaled_scores": { - "knn_auc": 1, + "knn_auc": 0, "mse": 0 }, - "mean_score": 0.5, + "mean_score": 0, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 129, - "cpu_pct": 12.1, - "peak_memory_mb": 5632, + "duration_sec": 6.3, + "cpu_pct": 173.6, + "peak_memory_mb": 3380, "disk_read_mb": 198, "disk_write_mb": 57 }, @@ -95,20 +95,20 @@ "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", "method_id": "scot", "metric_values": { - "knn_auc": 0.058, - "mse": 1.2539 + "knn_auc": 0.0266, + "mse": 1.2398 }, "scaled_scores": { - "knn_auc": 1.2458, - "mse": -0.2718 + "knn_auc": -0.0248, + "mse": -0.2218 }, - "mean_score": 0.5, + "mean_score": 0, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 6711, - "cpu_pct": 4835, - "peak_memory_mb": 14848, + "duration_sec": 3174, + "cpu_pct": 2926.3, + "peak_memory_mb": 12493, "disk_read_mb": 219, "disk_write_mb": 58 }, @@ -118,20 +118,20 @@ "dataset_id": "openproblems_v1_multimodal/citeseq_cbmc", "method_id": "true_features", "metric_values": { - "knn_auc": 0.0183, + "knn_auc": 1, "mse": 0 }, "scaled_scores": { - "knn_auc": 0, + "knn_auc": 1, "mse": 1 }, - "mean_score": 0.5, + "mean_score": 1, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 8.3, - "cpu_pct": 88.4, - "peak_memory_mb": 5735, + "duration_sec": 2.3, + "cpu_pct": 322.2, + "peak_memory_mb": 3482, "disk_read_mb": 385, "disk_write_mb": 3 }, @@ -141,20 +141,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", "method_id": "fastmnn", "metric_values": { - "knn_auc": 0.0138, - "mse": 1.0807 + "knn_auc": 0.0621, + "mse": 1.0819 }, "scaled_scores": { - "knn_auc": -6.5863, - "mse": -0.0818 + "knn_auc": 0.0128, + "mse": -0.0811 }, - "mean_score": 0, + "mean_score": 0.0064, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 37.2, - "cpu_pct": 112.4, - "peak_memory_mb": 6554, + "duration_sec": 22.7, + "cpu_pct": 122.3, + "peak_memory_mb": 4199, "disk_read_mb": 217, "disk_write_mb": 65 }, @@ -164,20 +164,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", "method_id": "harmonic_alignment", "metric_values": { - "knn_auc": 0.0064, - "mse": 1 + "knn_auc": 0.045, + "mse": 1.0001 }, "scaled_scores": { - "knn_auc": -7.8462, - "mse": -0.001 + "knn_auc": -0.0052, + "mse": 0.0007 }, - "mean_score": 0, + "mean_score": 0.0004, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 80, - "cpu_pct": 1538.7, - "peak_memory_mb": 9216, + "duration_sec": 47.4, + "cpu_pct": 833.6, + "peak_memory_mb": 5735, "disk_read_mb": 187, "disk_write_mb": 64 }, @@ -187,20 +187,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", "method_id": "procrustes", "metric_values": { - "knn_auc": 0.071, - "mse": 1.0101 + "knn_auc": 0.0802, + "mse": 1.0103 }, "scaled_scores": { - "knn_auc": 3.1469, - "mse": -0.0111 + "knn_auc": 0.0317, + "mse": -0.0095 }, - "mean_score": 0.5, + "mean_score": 0.0159, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 14.9, - "cpu_pct": 143.9, - "peak_memory_mb": 5735, + "duration_sec": 6.4, + "cpu_pct": 246, + "peak_memory_mb": 3380, "disk_read_mb": 178, "disk_write_mb": 64 }, @@ -210,8 +210,8 @@ "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", "method_id": "random_features", "metric_values": { - "knn_auc": 0.0525, - "mse": 0.999 + "knn_auc": 0.05, + "mse": 1.0008 }, "scaled_scores": { "knn_auc": 0, @@ -221,9 +221,9 @@ "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 14.1, - "cpu_pct": 62.4, - "peak_memory_mb": 5632, + "duration_sec": 6.1, + "cpu_pct": 186.7, + "peak_memory_mb": 3380, "disk_read_mb": 178, "disk_write_mb": 61 }, @@ -233,20 +233,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", "method_id": "scot", "metric_values": { - "knn_auc": 0.0517, - "mse": 0.8547 + "knn_auc": 0.1002, + "mse": 0.8502 }, "scaled_scores": { - "knn_auc": -0.1298, - "mse": 0.1444 + "knn_auc": 0.0528, + "mse": 0.1505 }, - "mean_score": 0.0722, + "mean_score": 0.1017, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 1157, - "cpu_pct": 4300.4, - "peak_memory_mb": 10445, + "duration_sec": 445, + "cpu_pct": 3047.9, + "peak_memory_mb": 8090, "disk_read_mb": 198, "disk_write_mb": 64 }, @@ -256,7 +256,7 @@ "dataset_id": "openproblems_v1_multimodal/scicar_cell_lines", "method_id": "true_features", "metric_values": { - "knn_auc": 0.0583, + "knn_auc": 1, "mse": 0 }, "scaled_scores": { @@ -267,9 +267,9 @@ "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 117, - "cpu_pct": 10.1, - "peak_memory_mb": 5837, + "duration_sec": 2.5, + "cpu_pct": 279.2, + "peak_memory_mb": 3482, "disk_read_mb": 343, "disk_write_mb": 13 }, @@ -279,20 +279,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "fastmnn", "metric_values": { - "knn_auc": 0.0432, - "mse": 0.9472 + "knn_auc": 0.076, + "mse": 0.9458 }, "scaled_scores": { - "knn_auc": -0.1204, - "mse": 0.0563 + "knn_auc": 0.0274, + "mse": 0.0514 }, - "mean_score": 0.0282, + "mean_score": 0.0394, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 49, - "cpu_pct": 108.4, - "peak_memory_mb": 6861, + "duration_sec": 43.6, + "cpu_pct": 116.6, + "peak_memory_mb": 4506, "disk_read_mb": 356, "disk_write_mb": 119 }, @@ -302,20 +302,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "harmonic_alignment", "metric_values": { - "knn_auc": 0.0822, - "mse": 1.0001 + "knn_auc": 0.0492, + "mse": 1 }, "scaled_scores": { - "knn_auc": 0.7456, - "mse": 0.0037 + "knn_auc": -0.0008, + "mse": -0.0029 }, - "mean_score": 0.3746, + "mean_score": 0, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 203, - "cpu_pct": 706.8, - "peak_memory_mb": 6759, + "duration_sec": 172, + "cpu_pct": 558.4, + "peak_memory_mb": 4404, "disk_read_mb": 326, "disk_write_mb": 118 }, @@ -325,20 +325,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "procrustes", "metric_values": { - "knn_auc": 0.3064, - "mse": 1.0011 + "knn_auc": 0.091, + "mse": 1.0008 }, "scaled_scores": { - "knn_auc": 5.7299, - "mse": 0.0026 + "knn_auc": 0.0432, + "mse": -0.0038 }, - "mean_score": 0.5013, + "mean_score": 0.0216, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 10.2, - "cpu_pct": 360.5, - "peak_memory_mb": 5940, + "duration_sec": 9.7, + "cpu_pct": 200.8, + "peak_memory_mb": 3584, "disk_read_mb": 317, "disk_write_mb": 118 }, @@ -348,8 +348,8 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "random_features", "metric_values": { - "knn_auc": 0.0486, - "mse": 1.0038 + "knn_auc": 0.05, + "mse": 0.9971 }, "scaled_scores": { "knn_auc": 0, @@ -359,9 +359,9 @@ "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 16.6, - "cpu_pct": 88.9, - "peak_memory_mb": 5837, + "duration_sec": 10.3, + "cpu_pct": 141.7, + "peak_memory_mb": 3482, "disk_read_mb": 316, "disk_write_mb": 110 }, @@ -371,20 +371,20 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "scot", "metric_values": { - "knn_auc": 0.0758, - "mse": 0.9929 + "knn_auc": 0.0606, + "mse": 0.992 }, "scaled_scores": { - "knn_auc": 0.6029, - "mse": 0.0108 + "knn_auc": 0.0112, + "mse": 0.0051 }, - "mean_score": 0.3069, + "mean_score": 0.0081, "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 9586, - "cpu_pct": 4558.5, - "peak_memory_mb": 19456, + "duration_sec": 5181, + "cpu_pct": 2619.9, + "peak_memory_mb": 17101, "disk_read_mb": 337, "disk_write_mb": 118 }, @@ -394,7 +394,7 @@ "dataset_id": "openproblems_v1_multimodal/scicar_mouse_kidney", "method_id": "true_features", "metric_values": { - "knn_auc": 0.0936, + "knn_auc": 1, "mse": 0 }, "scaled_scores": { @@ -405,9 +405,9 @@ "normalization_id": "log_cp10k", "resources": { "exit_code": 0, - "duration_sec": 113, - "cpu_pct": 10.1, - "peak_memory_mb": 5530, + "duration_sec": 3.6, + "cpu_pct": 202.3, + "peak_memory_mb": 3789, "disk_read_mb": 621, "disk_write_mb": 21 },