From 67793470ab2b2bada7b287a7030271723ffed453 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 29 Oct 2024 15:04:58 +0100 Subject: [PATCH 01/52] feat(street): add street project draft fixture --- .../tests/fixtures/projectDrafts/street.json | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 mapswipe_workers/tests/fixtures/projectDrafts/street.json diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json new file mode 100644 index 000000000..6bc291e33 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -0,0 +1,23 @@ +{ + "createdBy": "test", + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { "type": "Polygon", "coordinates": [ [ [ 39.0085, -6.7811 ], [ 39.0214, -6.7908 ], [ 39.0374, -6.7918 ], [ 39.0537, -6.7968 ], [ 39.0628, -6.8525 ], [ 39.0909, -6.8757 ], [ 39.1018, -6.8747 ], [ 39.1161, -6.8781 ], [ 39.1088, -6.9151 ], [ 39.089, -6.9307 ], [ 39.0864, -6.94 ], [ 39.0596, -6.9731 ], [ 39.0716, -6.9835 ], [ 39.0499, -7.0279 ], [ 39.0236, -7.0322 ], [ 39.038, -7.0499 ], [ 39.0693, -7.0328 ], [ 39.0826, -7.0671 ], [ 39.112, -7.0728 ], [ 39.1154, -7.0921 ], [ 39.1543, -7.0183 ], [ 39.1746, -7.0101 ], [ 39.1699, -6.9971 ], [ 39.196, -6.9848 ], [ 39.2163, -6.9934 ], [ 39.2502, -6.9509 ], [ 39.2768, -6.9486 ], [ 39.2881, -6.9529 ], [ 39.3136, -6.9595 ], [ 39.3276, -6.9483 ], [ 39.3686, -6.9726 ], [ 39.3826, -6.9677 ], [ 39.393, -7.0 ], [ 39.4038, -7.0284 ], [ 39.4037, -7.0379 ], [ 39.4028, -7.0741 ], [ 39.4, -7.0914 ], [ 39.4154, -7.1286 ], [ 39.4049, -7.1595 ], [ 39.4124, -7.1661 ], [ 39.4287, -7.1546 ], [ 39.4395, -7.1618 ], [ 39.4501, -7.1827 ], [ 39.4897, -7.1692 ], [ 39.51, -7.1833 ], [ 39.5378, -7.1229 ], [ 39.5529, -7.1335 ], [ 39.5453, -7.0318 ], [ 39.5376, -6.9873 ], [ 39.4976, -6.9093 ], [ 39.4717, -6.8731 ], [ 39.3557, -6.8455 ], [ 39.3205, -6.8175 ], [ 39.3074, -6.8193 ], [ 39.3002, -6.8253 ], [ 39.293, -6.8317 ], [ 39.2944, -6.818 ], [ 39.2963, -6.8084 ], [ 39.2855, -6.7859 ], [ 39.2808, -6.7347 ], [ 39.2609, -6.7602 ], [ 39.2344, -6.715 ], [ 39.2004, -6.6376 ], [ 39.165, -6.6022 ], [ 39.1353, -6.5748 ], [ 39.1215, -6.5662 ], [ 39.0945, -6.5987 ], [ 39.086, -6.5956 ], [ 39.0803, -6.6062 ], [ 39.0578, -6.7139 ], [ 39.059, -6.723 ], [ 39.0085, -6.7811 ] ], [ [ 39.301, -6.8372 ], [ 39.3048, -6.8555 ], [ 39.2967, -6.8469 ], [ 39.301, -6.8372 ] ], [ [ 39.3048, -6.8555 ], [ 39.2949, -6.8707 ], [ 39.2921, -6.8607 ], [ 39.3048, -6.8555 ] ] ] } + } + ] + }, + "image": "", + "lookFor": "buildings", + "name": "test - Dar es Salaam (1)\ntest", + "projectDetails": "test", + "projectNumber": 1, + "projectTopic": "test", + "projectType": 7, + "requestingOrganisation": "test", + "verificationNumber": 3, + "groupSize": 25 +} From 40cdc667d06b6ba112eeb67bf406a238542e6d3d Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 29 Oct 2024 17:07:15 +0100 Subject: [PATCH 02/52] feat: add new street project --- .../mapswipe_workers/definitions.py | 3 + .../project_types/__init__.py | 2 + .../project_types/street/__init__.py | 0 .../project_types/street/process_mapillary.py | 2 + .../project_types/street/project.py | 219 ++++++++++++++++++ .../unittests/test_project_type_street.py | 37 +++ 6 files changed, 263 insertions(+) create mode 100644 mapswipe_workers/mapswipe_workers/project_types/street/__init__.py create mode 100644 mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py create mode 100644 mapswipe_workers/mapswipe_workers/project_types/street/project.py create mode 100644 mapswipe_workers/tests/unittests/test_project_type_street.py diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index bb8c7296f..2ddf22c0b 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -134,6 +134,7 @@ class ProjectType(Enum): COMPLETENESS = 4 MEDIA_CLASSIFICATION = 5 DIGITIZATION = 6 + STREET = 7 @property def constructor(self): @@ -145,6 +146,7 @@ def constructor(self): DigitizationProject, FootprintProject, MediaClassificationProject, + StreetProject, ) project_type_classes = { @@ -154,6 +156,7 @@ def constructor(self): 4: CompletenessProject, 5: MediaClassificationProject, 6: DigitizationProject, + 7: StreetProject, } return project_type_classes[self.value] diff --git a/mapswipe_workers/mapswipe_workers/project_types/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/__init__.py index a07ff38be..048eda71c 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/__init__.py +++ b/mapswipe_workers/mapswipe_workers/project_types/__init__.py @@ -8,6 +8,7 @@ from .tile_map_service.classification.tutorial import ClassificationTutorial from .tile_map_service.completeness.project import CompletenessProject from .tile_map_service.completeness.tutorial import CompletenessTutorial +from .street.project import StreetProject __all__ = [ "ClassificationProject", @@ -20,4 +21,5 @@ "FootprintProject", "FootprintTutorial", "DigitizationProject", + "StreetProject", ] diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/street/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py b/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py new file mode 100644 index 000000000..db96c3640 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py @@ -0,0 +1,2 @@ +def get_image_ids(): + return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] \ No newline at end of file diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py new file mode 100644 index 000000000..502c1d36c --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -0,0 +1,219 @@ +import json +import os +import urllib +import math + +from osgeo import ogr +from dataclasses import asdict, dataclass +from mapswipe_workers.definitions import DATA_PATH, logger +from mapswipe_workers.firebase.firebase import Firebase +from mapswipe_workers.firebase_to_postgres.transfer_results import ( + results_to_file, + save_results_to_postgres, + truncate_temp_results, +) +from mapswipe_workers.generate_stats.project_stats import ( + get_statistics_for_integer_result_project, +) +from mapswipe_workers.project_types.project import ( + BaseProject, BaseTask, BaseGroup +) +from mapswipe_workers.project_types.street.process_mapillary import get_image_ids +from mapswipe_workers.utils.api_calls import geojsonToFeatureCollection, ohsome + + +@dataclass +class StreetGroup(BaseGroup): + # todo: does client use this, or only for the implementation of project creation? + pass + + +@dataclass +class StreetTask(BaseTask): + pass + +class StreetProject(BaseProject): + def __init__(self, project_draft): + super().__init__(project_draft) + self.groups: Dict[str, MediaClassificationGroup] = {} + self.tasks: Dict[str, List[MediaClassificationTask]] = ( + {} + ) + + self.geometry = project_draft["geometry"] + self.imageList = get_image_ids() + + def save_tasks_to_firebase(self, projectId: str, tasks: dict): + firebase = Firebase() + firebase.save_tasks_to_firebase(projectId, tasks, useCompression=True) + + @staticmethod + def results_to_postgres(results: dict, project_id: str, filter_mode: bool): + """How to move the result data from firebase to postgres.""" + results_file, user_group_results_file = results_to_file(results, project_id) + truncate_temp_results() + save_results_to_postgres(results_file, project_id, filter_mode) + return user_group_results_file + + @staticmethod + def get_per_project_statistics(project_id, project_info): + """How to aggregate the project results.""" + return get_statistics_for_integer_result_project( + project_id, project_info, generate_hot_tm_geometries=False + ) + + def validate_geometries(self): + raw_input_file = ( + f"{DATA_PATH}/input_geometries/raw_input_{self.projectId}.geojson" + ) + + if not os.path.isdir(f"{DATA_PATH}/input_geometries"): + os.mkdir(f"{DATA_PATH}/input_geometries") + + valid_input_file = ( + f"{DATA_PATH}/input_geometries/valid_input_{self.projectId}.geojson" + ) + + logger.info( + f"{self.projectId}" + f" - __init__ - " + f"downloaded input geometries from url and saved as file: " + f"{raw_input_file}" + ) + self.inputGeometries = raw_input_file + + # open the raw input file and get layer + driver = ogr.GetDriverByName("GeoJSON") + datasource = driver.Open(raw_input_file, 0) + try: + layer = datasource.GetLayer() + LayerDefn = layer.GetLayerDefn() + except AttributeError: + raise CustomError("Value error in input geometries file") + + # create layer for valid_input_file to store all valid geometries + outDriver = ogr.GetDriverByName("GeoJSON") + # Remove output geojson if it already exists + if os.path.exists(valid_input_file): + outDriver.DeleteDataSource(valid_input_file) + outDataSource = outDriver.CreateDataSource(valid_input_file) + outLayer = outDataSource.CreateLayer( + "geometries", geom_type=ogr.wkbMultiPolygon + ) + for i in range(0, LayerDefn.GetFieldCount()): + fieldDefn = LayerDefn.GetFieldDefn(i) + outLayer.CreateField(fieldDefn) + outLayerDefn = outLayer.GetLayerDefn() + + # check if raw_input_file layer is empty + feature_count = layer.GetFeatureCount() + if feature_count < 1: + err = "empty file. No geometries provided" + # TODO: How to user logger and exceptions? + logger.warning(f"{self.projectId} - check_input_geometry - {err}") + raise CustomError(err) + elif feature_count > 100000: + err = f"Too many Geometries: {feature_count}" + logger.warning(f"{self.projectId} - check_input_geometry - {err}") + raise CustomError(err) + + # get geometry as wkt + # get the bounding box/ extent of the layer + extent = layer.GetExtent() + # Create a Polygon from the extent tuple + ring = ogr.Geometry(ogr.wkbLinearRing) + ring.AddPoint(extent[0], extent[2]) + ring.AddPoint(extent[1], extent[2]) + ring.AddPoint(extent[1], extent[3]) + ring.AddPoint(extent[0], extent[3]) + ring.AddPoint(extent[0], extent[2]) + poly = ogr.Geometry(ogr.wkbPolygon) + poly.AddGeometry(ring) + wkt_geometry = poly.ExportToWkt() + + # check if the input geometry is a valid polygon + for feature in layer: + feat_geom = feature.GetGeometryRef() + geom_name = feat_geom.GetGeometryName() + fid = feature.GetFID() + + if not feat_geom.IsValid(): + layer.DeleteFeature(fid) + logger.warning( + f"{self.projectId}" + f" - check_input_geometries - " + f"deleted invalid feature {fid}" + ) + + # we accept only POLYGON or MULTIPOLYGON geometries + elif geom_name not in ["POLYGON", "MULTIPOLYGON"]: + layer.DeleteFeature(fid) + logger.warning( + f"{self.projectId}" + f" - check_input_geometries - " + f"deleted non polygon feature {fid}" + ) + + else: + # Create output Feature + outFeature = ogr.Feature(outLayerDefn) + # Add field values from input Layer + for i in range(0, outLayerDefn.GetFieldCount()): + outFeature.SetField( + outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i) + ) + outFeature.SetGeometry(feat_geom) + outLayer.CreateFeature(outFeature) + outFeature = None + + # check if layer is empty + if layer.GetFeatureCount() < 1: + err = "no geometries left after checking validity and geometry type." + logger.warning(f"{self.projectId} - check_input_geometry - {err}") + raise Exception(err) + + del datasource + del outDataSource + del layer + + self.inputGeometriesFileName = valid_input_file + + logger.info( + f"{self.projectId}" + f" - check_input_geometry - " + f"filtered correct input geometries and created file: " + f"{valid_input_file}" + ) + return wkt_geometry + + def create_groups(self): + self.numberOfGroups = math.ceil(len(self.imageList) / self.groupSize) + for group_id in range(self.numberOfGroups): + self.groups[f"g{group_id}"] = StreetGroup( + projectId=self.projectId, + groupId=f"g{group_id}", + progress=0, + finishedCount=0, + requiredCount=0, + numberOfTasks=self.groupSize, + ) + + def create_tasks(self): + if len(self.groups) == 0: + raise ValueError("Groups needs to be created before tasks can be created.") + for group_id, group in self.groups.items(): + self.tasks[group_id] = [] + for i in range(self.groupSize): + task = StreetTask( + projectId=self.projectId, + groupId=group_id, + taskId=self.imageList.pop(), + ) + self.tasks[group_id].append(task) + + # list now empty? if usual group size is not reached + # the actual number of tasks for the group is updated + if not self.imageList: + group.numberOfTasks = i + 1 + break + diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py new file mode 100644 index 000000000..70c97b13d --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -0,0 +1,37 @@ +import json +import os +import unittest +from unittest.mock import patch + +from mapswipe_workers.project_types import StreetProject +from tests import fixtures + + +class TestCreateStreetProject(unittest.TestCase): + def setUp(self) -> None: + project_draft = fixtures.get_fixture( + os.path.join( + "projectDrafts", + "street.json", + ) + ) + project_draft["projectDraftId"] = "foo" + self.project = StreetProject(project_draft) + + def test_init(self): + self.assertEqual(self.project.geometry["type"], "FeatureCollection") + + def test_create_group(self): + self.project.create_groups() + self.assertTrue(self.project.groups) + + def test_create_tasks(self): + imageId = self.project.imageList[-1] + self.project.create_groups() + self.project.create_tasks() + self.assertEqual(self.project.tasks["g0"][0].taskId, imageId) + #self.assertEqual(self.project.groups["g0"].numberOfTasks, 1) + + +if __name__ == "__main__": + unittest.main() From 33bc5ece00a13d17558b656b99e4da2cfe7947a8 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 29 Oct 2024 17:10:14 +0100 Subject: [PATCH 03/52] feat: add tutorial to street project --- .../project_types/street/tutorial.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py new file mode 100644 index 000000000..cfbfc0ead --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py @@ -0,0 +1,14 @@ +from mapswipe_workers.project_types.tutorial import BaseTutorial + + +class StreetTutorial(BaseTutorial): + """The subclass for an TMS Grid based Tutorial.""" + + def save_tutorial(self): + raise NotImplementedError("Currently Street has no Tutorial") + + def create_tutorial_groups(self): + raise NotImplementedError("Currently Street has no Tutorial") + + def create_tutorial_tasks(self): + raise NotImplementedError("Currently Street has no Tutorial") From ec59ee0ff30584a4ec5dcd37732b42e03a46dc22 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 11:13:37 +0100 Subject: [PATCH 04/52] refactor: move process_mapillary.py to utils --- .../mapswipe_workers/project_types/street/process_mapillary.py | 2 -- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py create mode 100644 mapswipe_workers/mapswipe_workers/utils/process_mapillary.py diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py b/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py deleted file mode 100644 index db96c3640..000000000 --- a/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py +++ /dev/null @@ -1,2 +0,0 @@ -def get_image_ids(): - return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] \ No newline at end of file diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py new file mode 100644 index 000000000..7287a65e9 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -0,0 +1,2 @@ +def get_image_ids(): + return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] From 78f28df91e64298fda65ed1f625512b8b9b45b28 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 11:14:35 +0100 Subject: [PATCH 05/52] feat: add functions to process mapillary data --- .../utils/process_mapillary.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 7287a65e9..188781703 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -1,2 +1,126 @@ +import mercantile +import json +import requests +import os +import time +from shapely import box, Polygon, MultiPolygon, Point, LineString, MultiLineString +import pandas as pd +from vt2geojson import tools as vt2geojson_tools +from concurrent.futures import ThreadPoolExecutor, as_completed + def get_image_ids(): return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + +def create_tiles(polygon, level): + if not isinstance(polygon, (Polygon, MultiPolygon)): + return pd.DataFrame(columns=['x', 'y', 'z', 'geometry']) + if isinstance(polygon, Polygon): + polygon = MultiPolygon([polygon]) + + tiles = [] + for i, poly in enumerate(polygon.geoms): + tiles.extend(list(mercantile.tiles(*poly.bounds, level))) + + bbox_list = [mercantile.bounds(tile.x, tile.y, tile.z) for tile in tiles] + bbox_polygons = [box(*bbox) for bbox in bbox_list] + tiles = pd.DataFrame({ + 'x': [tile.x for tile in tiles], + 'y': [tile.y for tile in tiles], + 'z': [tile.z for tile in tiles], + 'geometry': bbox_polygons}) + + return tiles + + +def download_and_process_tile(row, token, attempt_limit=3): + z = row["z"] + x = row["x"] + y = row["y"] + endpoint = "mly1_computed_public" + url = f"https://tiles.mapillary.com/maps/vtp/{endpoint}/2/{z}/{x}/{y}?access_token={token}" + + attempt = 0 + while attempt < attempt_limit: + try: + r = requests.get(url) + assert r.status_code == 200, r.content + features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get('features', []) + + data = [] + for feature in features: + geometry = feature.get('geometry', {}) + properties = feature.get('properties', {}) + geometry_type = geometry.get('type', None) + coordinates = geometry.get('coordinates', []) + + element_geometry = None + if geometry_type == 'Point': + element_geometry = Point(coordinates) + elif geometry_type == 'LineString': + element_geometry = LineString(coordinates) + elif geometry_type == 'MultiLineString': + element_geometry = MultiLineString(coordinates) + elif geometry_type == 'Polygon': + element_geometry = Polygon(coordinates[0]) + elif geometry_type == 'MultiPolygon': + element_geometry = MultiPolygon(coordinates) + + # Append the dictionary with geometry and properties + row = {'geometry': element_geometry, **properties} + data.append(row) + + data = pd.DataFrame(data) + + if not data.empty: + return data, None + except Exception as e: + print(f"An exception occurred while requesting a tile: {e}") + attempt += 1 + + print(f"A tile could not be downloaded: {row}") + return None, row + + +def coordinate_download(polygon, level, token, use_concurrency=True, attempt_limit=3, workers=os.cpu_count() * 4): + tiles = create_tiles(polygon, level) + + downloaded_metadata = [] + failed_tiles = [] + + if not tiles.empty: + if not use_concurrency: + workers = 1 + + futures = [] + start_time = time.time() + with ThreadPoolExecutor(max_workers=workers) as executor: + for index, row in tiles.iterrows(): + futures.append(executor.submit(download_and_process_tile, row, token, attempt_limit)) + + for future in as_completed(futures): + if future is not None: + df, failed_row = future.result() + + if df is not None and not df.empty: + downloaded_metadata.append(df) + if failed_row is not None: + failed_tiles.append(failed_row) + + end_time = time.time() + total_time = end_time - start_time + + total_tiles = len(tiles) + average_time_per_tile = total_time / total_tiles if total_tiles > 0 else 0 + + print(f"Total time for downloading {total_tiles} tiles: {total_time:.2f} seconds") + print(f"Average time per tile: {average_time_per_tile:.2f} seconds") + + if len(downloaded_metadata): + downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) + else: + downloaded_metadata = pd.DataFrame(downloaded_metadata) + + failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index(drop=True) + + return downloaded_metadata, failed_tiles From 3a7ac21fbf48edfdc29f84b600eedc7177127c93 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 11:15:10 +0100 Subject: [PATCH 06/52] refactor: removed unused imports and classes --- .../mapswipe_workers/project_types/street/project.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index 502c1d36c..5a2f834af 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -4,7 +4,7 @@ import math from osgeo import ogr -from dataclasses import asdict, dataclass +from dataclasses import dataclass from mapswipe_workers.definitions import DATA_PATH, logger from mapswipe_workers.firebase.firebase import Firebase from mapswipe_workers.firebase_to_postgres.transfer_results import ( @@ -18,8 +18,7 @@ from mapswipe_workers.project_types.project import ( BaseProject, BaseTask, BaseGroup ) -from mapswipe_workers.project_types.street.process_mapillary import get_image_ids -from mapswipe_workers.utils.api_calls import geojsonToFeatureCollection, ohsome +from mapswipe_workers.utils.process_mapillary import get_image_ids @dataclass @@ -35,8 +34,8 @@ class StreetTask(BaseTask): class StreetProject(BaseProject): def __init__(self, project_draft): super().__init__(project_draft) - self.groups: Dict[str, MediaClassificationGroup] = {} - self.tasks: Dict[str, List[MediaClassificationTask]] = ( + self.groups: Dict[str, StreetGroup] = {} + self.tasks: Dict[str, List[StreetTask]] = ( {} ) @@ -45,7 +44,7 @@ def __init__(self, project_draft): def save_tasks_to_firebase(self, projectId: str, tasks: dict): firebase = Firebase() - firebase.save_tasks_to_firebase(projectId, tasks, useCompression=True) + firebase.save_tasks_to_firebase(projectId, tasks) @staticmethod def results_to_postgres(results: dict, project_id: str, filter_mode: bool): From 8e5c55c879516db9a42e69dad799832540646f9d Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 14:34:28 +0100 Subject: [PATCH 07/52] feat: add function to convert input geojson to polygon --- .../utils/process_mapillary.py | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 188781703..33dc537c8 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -3,13 +3,13 @@ import requests import os import time -from shapely import box, Polygon, MultiPolygon, Point, LineString, MultiLineString +from shapely import box, Polygon, MultiPolygon, Point, LineString, MultiLineString, unary_union +from shapely.geometry import shape import pandas as pd from vt2geojson import tools as vt2geojson_tools from concurrent.futures import ThreadPoolExecutor, as_completed -def get_image_ids(): - return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +import logging def create_tiles(polygon, level): @@ -46,7 +46,6 @@ def download_and_process_tile(row, token, attempt_limit=3): r = requests.get(url) assert r.status_code == 200, r.content features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get('features', []) - data = [] for feature in features: geometry = feature.get('geometry', {}) @@ -124,3 +123,27 @@ def coordinate_download(polygon, level, token, use_concurrency=True, attempt_lim failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index(drop=True) return downloaded_metadata, failed_tiles + +def geojson_to_polygon(geojson_data): + if geojson_data["type"] == "FeatureCollection": + features = geojson_data["features"] + elif geojson_data["type"] == "Feature": + features = [geojson_data] + else: + raise ValueError("Unsupported GeoJSON type.") + + polygons = [] + for feature in features: + geometry = shape(feature["geometry"]) + if isinstance(geometry, (Polygon, MultiPolygon)): + polygons.append(geometry) + else: + raise ValueError("Non-polygon geometries cannot be combined into a MultiPolygon.") + + combined_multipolygon = unary_union(polygons) + + return combined_multipolygon + + +def get_image_ids(): + return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] \ No newline at end of file From dd4e220dcee6b41b47d26e7546df37a5b55565e0 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 14:35:07 +0100 Subject: [PATCH 08/52] tests: add unittests for process_mapillary --- .../tests/unittests/test_process_mapillary.py | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 mapswipe_workers/tests/unittests/test_process_mapillary.py diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py new file mode 100644 index 000000000..a9d102e49 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -0,0 +1,178 @@ +import unittest +import os +import json +from shapely.geometry import Polygon, MultiPolygon, Point, LineString, MultiLineString, GeometryCollection +import pandas as pd +from unittest.mock import patch, MagicMock +from mapswipe_workers.utils.process_mapillary import create_tiles, download_and_process_tile, coordinate_download, geojson_to_polygon + + +# Assuming create_tiles, download_and_process_tile, and coordinate_download are imported + +class TestTileGroupingFunctions(unittest.TestCase): + + @classmethod + def setUpClass(cls): + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "feature_collection.json", + ), + "r", + ) as file: + cls.fixture_data = json.load(file) + + def setUp(self): + self.token = "test_token" + self.level = 14 + self.test_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) + self.test_multipolygon = MultiPolygon([self.test_polygon, Polygon([(2, 2), (3, 2), (3, 3), (2, 3)])]) + self.empty_polygon = Polygon() + self.empty_geometry = GeometryCollection() + + def test_create_tiles_with_valid_polygon(self): + tiles = create_tiles(self.test_polygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertFalse(tiles.empty) + + def test_create_tiles_with_multipolygon(self): + tiles = create_tiles(self.test_multipolygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertFalse(tiles.empty) + + def test_create_tiles_with_empty_polygon(self): + tiles = create_tiles(self.empty_polygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertTrue(tiles.empty) + + def test_create_tiles_with_empty_geometry(self): + tiles = create_tiles(self.empty_geometry, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertTrue(tiles.empty) + + def test_geojson_to_polygon_feature_collection_with_multiple_polygons(self): + geojson_data = { + "type": "FeatureCollection", + "features": [ + {"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]]}}, + {"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]]}} + ] + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, MultiPolygon) + self.assertEqual(len(result.geoms), 2) + + def test_geojson_to_polygon_single_feature_polygon(self): + geojson_data = { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]] + } + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, Polygon) + + def test_geojson_to_polygon_single_feature_multipolygon(self): + geojson_data = { + "type": "Feature", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]] + ] + } + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, MultiPolygon) + self.assertEqual(len(result.geoms), 2) + + def test_geojson_to_polygon_non_polygon_geometry_in_feature_collection(self): + geojson_data = { + "type": "FeatureCollection", + "features": [ + {"type": "Feature", "geometry": {"type": "LineString", "coordinates": [(0, 0), (1, 1)]}} + ] + } + with self.assertRaises(ValueError) as context: + geojson_to_polygon(geojson_data) + self.assertEqual(str(context.exception), "Non-polygon geometries cannot be combined into a MultiPolygon.") + + def test_geojson_to_polygon_empty_feature_collection(self): + geojson_data = { + "type": "FeatureCollection", + "features": [] + } + result = geojson_to_polygon(geojson_data) + self.assertTrue(result.is_empty) + + def test_geojson_to_polygon_contribution_geojson(self): + result = geojson_to_polygon(self.fixture_data) + self.assertIsInstance(result, Polygon) + + @patch('mapswipe_workers.utils.process_mapillary.vt2geojson_tools.vt_bytes_to_geojson') + @patch('mapswipe_workers.utils.process_mapillary.requests.get') + def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): + # Mock the response from requests.get + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.content = b'mock vector tile data' # Example mock data + mock_get.return_value = mock_response + + # Mock the return value of vt_bytes_to_geojson + mock_vt2geojson.return_value = { + "features": [ + {"geometry": {"type": "Point", "coordinates": [0, 0]}, "properties": {"id": 1}} + ] + } + + row = {'x': 1, 'y': 1, 'z': 14} + token = 'test_token' + + result, failed = download_and_process_tile(row, token) + + # Assertions + self.assertIsNone(failed) + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 1) + self.assertEqual(result['geometry'][0].wkt, 'POINT (0 0)') + + @patch('mapswipe_workers.utils.process_mapillary.requests.get') + def test_download_and_process_tile_failure(self, mock_get): + # Mock a failed response + mock_response = MagicMock() + mock_response.status_code = 500 + mock_get.return_value = mock_response + + row = pd.Series({'x': 1, 'y': 1, 'z': self.level}) + result, failed = download_and_process_tile(row, self.token) + + self.assertIsNone(result) + self.assertIsNotNone(failed) + + @patch('mapswipe_workers.utils.process_mapillary') + def test_coordinate_download(self, mock_download_and_process_tile): + + mock_download_and_process_tile.return_value = (pd.DataFrame([{"geometry": None}]), None) + + metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + + self.assertIsInstance(metadata, pd.DataFrame) + self.assertTrue(failed.empty) + + @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') + def test_coordinate_download_with_failures(self, mock_download_and_process_tile): + # Simulate failed tile download + mock_download_and_process_tile.return_value = (None, pd.Series({"x": 1, "y": 1, "z": self.level})) + + metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + + self.assertTrue(metadata.empty) + self.assertFalse(failed.empty) + + +if __name__ == '__main__': + unittest.main() From b274af3d917d539ad41d007d439f230cbc0c563e Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 15:16:50 +0100 Subject: [PATCH 09/52] fix: wrong indention and patch in test_process_mapillary --- .../tests/unittests/test_process_mapillary.py | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index a9d102e49..abca8657c 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -24,6 +24,17 @@ def setUpClass(cls): ) as file: cls.fixture_data = json.load(file) + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_response.csv", + ), + "r", + ) as file: + cls.fixture_df = pd.read_csv(file) + def setUp(self): self.token = "test_token" self.level = 14 @@ -140,38 +151,36 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): self.assertEqual(len(result), 1) self.assertEqual(result['geometry'][0].wkt, 'POINT (0 0)') - @patch('mapswipe_workers.utils.process_mapillary.requests.get') - def test_download_and_process_tile_failure(self, mock_get): - # Mock a failed response - mock_response = MagicMock() - mock_response.status_code = 500 - mock_get.return_value = mock_response - - row = pd.Series({'x': 1, 'y': 1, 'z': self.level}) - result, failed = download_and_process_tile(row, self.token) + @patch('mapswipe_workers.utils.process_mapillary.requests.get') + def test_download_and_process_tile_failure(self, mock_get): + # Mock a failed response + mock_response = MagicMock() + mock_response.status_code = 500 + mock_get.return_value = mock_response - self.assertIsNone(result) - self.assertIsNotNone(failed) + row = pd.Series({'x': 1, 'y': 1, 'z': self.level}) + result, failed = download_and_process_tile(row, self.token) - @patch('mapswipe_workers.utils.process_mapillary') - def test_coordinate_download(self, mock_download_and_process_tile): + self.assertIsNone(result) + self.assertIsNotNone(failed) - mock_download_and_process_tile.return_value = (pd.DataFrame([{"geometry": None}]), None) + @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') + def test_coordinate_download(self, mock_download_and_process_tile): + mock_download_and_process_tile.return_value = (pd.DataFrame([{"geometry": None}]), None) - metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) - self.assertIsInstance(metadata, pd.DataFrame) - self.assertTrue(failed.empty) + self.assertIsInstance(metadata, pd.DataFrame) + self.assertTrue(failed.empty) - @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') - def test_coordinate_download_with_failures(self, mock_download_and_process_tile): - # Simulate failed tile download - mock_download_and_process_tile.return_value = (None, pd.Series({"x": 1, "y": 1, "z": self.level})) + @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') + def test_coordinate_download_with_failures(self, mock_download_and_process_tile): + mock_download_and_process_tile.return_value = (None, pd.Series({"x": 1, "y": 1, "z": self.level})) - metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) - self.assertTrue(metadata.empty) - self.assertFalse(failed.empty) + self.assertTrue(metadata.empty) + self.assertFalse(failed.empty) if __name__ == '__main__': From 7ec92ee774bade0e8cff74e32ef459c60cacb5b2 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 15:41:45 +0100 Subject: [PATCH 10/52] feat: add filtering and handling of mapillary response --- example.env | 3 ++ .../utils/process_mapillary.py | 31 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/example.env b/example.env index f4e8c46a1..e3b856ea2 100644 --- a/example.env +++ b/example.env @@ -75,3 +75,6 @@ COMMUNITY_DASHBOARD_GRAPHQL_ENDPOINT=https://api.example.com/graphql/ COMMUNITY_DASHBOARD_SENTRY_DSN= COMMUNITY_DASHBOARD_SENTRY_TRACES_SAMPLE_RATE= COMMUNITY_DASHBOARD_MAPSWIPE_WEBSITE=https://mapswipe.org + +# Mapillary +MAPILLARY_ACCESS_TOKEN= \ No newline at end of file diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 33dc537c8..2d51a297b 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -144,6 +144,31 @@ def geojson_to_polygon(geojson_data): return combined_multipolygon - -def get_image_ids(): - return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] \ No newline at end of file +def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str = None): + df['captured_at'] = pd.to_datetime(df['captured_at'], unit='ms') + start_time = pd.Timestamp(start_time) + + if end_time is None: + end_time = pd.Timestamp.now() + + filtered_df = df[(df['captured_at'] >= start_time) & (df['captured_at'] <= end_time)] + return filtered_df + +def filter_results(results_df: pd.DataFrame, is_pano: bool = None, organization_id: str = None, start_time: str = None, end_time: str = None): + df = results_df.copy() + if is_pano is not None: + df = df[df["is_pano"] == is_pano] + if organization_id is not None: + df = df[df["organization_id"] == organization_id] + if start_time is not None: + df = filter_by_timerange(df, start_time, end_time) + + return df + + +def get_image_ids(aoi_geojson, level = 14, attempt_limit = 3, is_pano: bool = None, organization_id: str = None, start_time: str = None, end_time: str = None): + aoi_polygon = geojson_to_polygon(aoi_geojson) + token = os.getenv("MAPILLARY_ACCESS_TOKEN") + downloaded_metadata, failed_tiles = coordinate_download(aoi_polygon, level, token, attempt_limit) + downloaded_metadata = filter_results(downloaded_metadata, is_pano, organization_id, start_time, end_time) + return downloaded_metadata["image_id"].tolist() \ No newline at end of file From 97e552c907ce765600e00fc565c891951e79c6be Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 15:42:05 +0100 Subject: [PATCH 11/52] tests: add tests for filtering and handling of mapillary response --- .../tests/fixtures/mapillary_response.csv | 8 ++++ .../tests/unittests/test_process_mapillary.py | 45 ++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 mapswipe_workers/tests/fixtures/mapillary_response.csv diff --git a/mapswipe_workers/tests/fixtures/mapillary_response.csv b/mapswipe_workers/tests/fixtures/mapillary_response.csv new file mode 100644 index 000000000..2988a1dc4 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/mapillary_response.csv @@ -0,0 +1,8 @@ +geometry,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id +POINT (38.995129466056824 -6.785243670271996),1453463352000,102506575322825,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ,1 +POINT (38.99839103221893 -6.7866606090858),1679465543298,118200124520512,603013591724120,,False,104.904435758052,Tj9u08PcRnQEAU13yeNhkr +POINT (39.000306129455566 -6.787576822940906),1679465564715,118200124520512,547943597327117,,False,103.540136343809,Tj9u08PcRnQEAU13yeNhkr +POINT (38.9906769990921 -6.783315348346505),1453463400000,102506575322825,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ +POINT (38.99797797203064 -6.786490150501777),1679465534081,118200124520512,3271824289814895,,False,112.637054443359,Tj9u08PcRnQEAU13yeNhkr +POINT (39.00127172470093 -6.787981661065601),1453463294000,102506575322825,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ + diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index abca8657c..7e9413c6a 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -4,7 +4,7 @@ from shapely.geometry import Polygon, MultiPolygon, Point, LineString, MultiLineString, GeometryCollection import pandas as pd from unittest.mock import patch, MagicMock -from mapswipe_workers.utils.process_mapillary import create_tiles, download_and_process_tile, coordinate_download, geojson_to_polygon +from mapswipe_workers.utils.process_mapillary import create_tiles, download_and_process_tile, coordinate_download, geojson_to_polygon, filter_by_timerange, filter_results # Assuming create_tiles, download_and_process_tile, and coordinate_download are imported @@ -182,6 +182,49 @@ def test_coordinate_download_with_failures(self, mock_download_and_process_tile) self.assertTrue(metadata.empty) self.assertFalse(failed.empty) + def test_filter_within_time_range(self): + start_time = '2016-01-20 00:00:00' + end_time = '2022-01-21 23:59:59' + filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) + + self.assertEqual(len(filtered_df), 3) + self.assertTrue(all(filtered_df['captured_at'] >= pd.to_datetime(start_time))) + self.assertTrue(all(filtered_df['captured_at'] <= pd.to_datetime(end_time))) + + def test_filter_without_end_time(self): + start_time = '2020-01-20 00:00:00' + filtered_df = filter_by_timerange(self.fixture_df, start_time) + + self.assertEqual(len(filtered_df), 3) + self.assertTrue(all(filtered_df['captured_at'] >= pd.to_datetime(start_time))) + + def test_filter_time_no_data(self): + start_time = '2016-01-30 00:00:00' + end_time = '2016-01-31 00:00:00' + filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) + self.assertTrue(filtered_df.empty) + + def test_filter_default(self): + filtered_df = filter_results(self.fixture_df) + self.assertTrue(len(filtered_df) == len(self.fixture_df)) + + def test_filter_pano_true(self): + filtered_df = filter_results(self.fixture_df, is_pano=True) + self.assertEqual(len(filtered_df), 3) + + def test_filter_pano_false(self): + filtered_df = filter_results(self.fixture_df, is_pano=False) + self.assertEqual(len(filtered_df), 3) + + def test_filter_organization_id(self): + filtered_df = filter_results(self.fixture_df, organization_id=1) + self.assertEqual(len(filtered_df), 1) + + def test_filter_time_range(self): + start_time = '2016-01-20 00:00:00' + end_time = '2022-01-21 23:59:59' + filtered_df = filter_results(self.fixture_df, start_time=start_time, end_time=end_time) + self.assertEqual(len(filtered_df), 3) if __name__ == '__main__': unittest.main() From 53dfda5621aba90cdb08ca89d288eb738d4d8b5f Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:04:42 +0100 Subject: [PATCH 12/52] refactor: use mapillary incredentials in variables --- mapswipe_workers/mapswipe_workers/definitions.py | 2 ++ .../mapswipe_workers/utils/process_mapillary.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 2ddf22c0b..c9dec6d79 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -16,6 +16,8 @@ OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/" OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] +MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" +MAPILLARY_API_KEY = os.environ["MAPILLARY_API_KEY"] # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 2d51a297b..5e1d00c47 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -8,8 +8,7 @@ import pandas as pd from vt2geojson import tools as vt2geojson_tools from concurrent.futures import ThreadPoolExecutor, as_completed - -import logging +from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY def create_tiles(polygon, level): @@ -33,12 +32,11 @@ def create_tiles(polygon, level): return tiles -def download_and_process_tile(row, token, attempt_limit=3): +def download_and_process_tile(row, attempt_limit=3): z = row["z"] x = row["x"] y = row["y"] - endpoint = "mly1_computed_public" - url = f"https://tiles.mapillary.com/maps/vtp/{endpoint}/2/{z}/{x}/{y}?access_token={token}" + url = f"{MAPILLARY_API_LINK}{z}/{x}/{y}?access_token={MAPILLARY_API_KEY}" attempt = 0 while attempt < attempt_limit: @@ -81,7 +79,9 @@ def download_and_process_tile(row, token, attempt_limit=3): return None, row -def coordinate_download(polygon, level, token, use_concurrency=True, attempt_limit=3, workers=os.cpu_count() * 4): +def coordinate_download( + polygon, level, use_concurrency=True, attempt_limit=3, workers=os.cpu_count() * 4 +): tiles = create_tiles(polygon, level) downloaded_metadata = [] @@ -95,7 +95,9 @@ def coordinate_download(polygon, level, token, use_concurrency=True, attempt_lim start_time = time.time() with ThreadPoolExecutor(max_workers=workers) as executor: for index, row in tiles.iterrows(): - futures.append(executor.submit(download_and_process_tile, row, token, attempt_limit)) + futures.append( + executor.submit(download_and_process_tile, row, attempt_limit) + ) for future in as_completed(futures): if future is not None: From 481030c289cc99ce0fb7a6b9e638b05100bb03ab Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:06:19 +0100 Subject: [PATCH 13/52] refactor: reformat file process_mapillary.py --- .../utils/process_mapillary.py | 108 ++++++++++++------ 1 file changed, 70 insertions(+), 38 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 5e1d00c47..4f1a55084 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -3,7 +3,15 @@ import requests import os import time -from shapely import box, Polygon, MultiPolygon, Point, LineString, MultiLineString, unary_union +from shapely import ( + box, + Polygon, + MultiPolygon, + Point, + LineString, + MultiLineString, + unary_union, +) from shapely.geometry import shape import pandas as pd from vt2geojson import tools as vt2geojson_tools @@ -13,7 +21,7 @@ def create_tiles(polygon, level): if not isinstance(polygon, (Polygon, MultiPolygon)): - return pd.DataFrame(columns=['x', 'y', 'z', 'geometry']) + return pd.DataFrame(columns=["x", "y", "z", "geometry"]) if isinstance(polygon, Polygon): polygon = MultiPolygon([polygon]) @@ -23,11 +31,14 @@ def create_tiles(polygon, level): bbox_list = [mercantile.bounds(tile.x, tile.y, tile.z) for tile in tiles] bbox_polygons = [box(*bbox) for bbox in bbox_list] - tiles = pd.DataFrame({ - 'x': [tile.x for tile in tiles], - 'y': [tile.y for tile in tiles], - 'z': [tile.z for tile in tiles], - 'geometry': bbox_polygons}) + tiles = pd.DataFrame( + { + "x": [tile.x for tile in tiles], + "y": [tile.y for tile in tiles], + "z": [tile.z for tile in tiles], + "geometry": bbox_polygons, + } + ) return tiles @@ -43,28 +54,30 @@ def download_and_process_tile(row, attempt_limit=3): try: r = requests.get(url) assert r.status_code == 200, r.content - features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get('features', []) + features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get( + "features", [] + ) data = [] for feature in features: - geometry = feature.get('geometry', {}) - properties = feature.get('properties', {}) - geometry_type = geometry.get('type', None) - coordinates = geometry.get('coordinates', []) + geometry = feature.get("geometry", {}) + properties = feature.get("properties", {}) + geometry_type = geometry.get("type", None) + coordinates = geometry.get("coordinates", []) element_geometry = None - if geometry_type == 'Point': + if geometry_type == "Point": element_geometry = Point(coordinates) - elif geometry_type == 'LineString': + elif geometry_type == "LineString": element_geometry = LineString(coordinates) - elif geometry_type == 'MultiLineString': + elif geometry_type == "MultiLineString": element_geometry = MultiLineString(coordinates) - elif geometry_type == 'Polygon': + elif geometry_type == "Polygon": element_geometry = Polygon(coordinates[0]) - elif geometry_type == 'MultiPolygon': + elif geometry_type == "MultiPolygon": element_geometry = MultiPolygon(coordinates) # Append the dictionary with geometry and properties - row = {'geometry': element_geometry, **properties} + row = {"geometry": element_geometry, **properties} data.append(row) data = pd.DataFrame(data) @@ -92,7 +105,6 @@ def coordinate_download( workers = 1 futures = [] - start_time = time.time() with ThreadPoolExecutor(max_workers=workers) as executor: for index, row in tiles.iterrows(): futures.append( @@ -108,24 +120,18 @@ def coordinate_download( if failed_row is not None: failed_tiles.append(failed_row) - end_time = time.time() - total_time = end_time - start_time - - total_tiles = len(tiles) - average_time_per_tile = total_time / total_tiles if total_tiles > 0 else 0 - - print(f"Total time for downloading {total_tiles} tiles: {total_time:.2f} seconds") - print(f"Average time per tile: {average_time_per_tile:.2f} seconds") - if len(downloaded_metadata): downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: downloaded_metadata = pd.DataFrame(downloaded_metadata) - failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index(drop=True) + failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index( + drop=True + ) return downloaded_metadata, failed_tiles + def geojson_to_polygon(geojson_data): if geojson_data["type"] == "FeatureCollection": features = geojson_data["features"] @@ -140,23 +146,35 @@ def geojson_to_polygon(geojson_data): if isinstance(geometry, (Polygon, MultiPolygon)): polygons.append(geometry) else: - raise ValueError("Non-polygon geometries cannot be combined into a MultiPolygon.") + raise ValueError( + "Non-polygon geometries cannot be combined into a MultiPolygon." + ) combined_multipolygon = unary_union(polygons) return combined_multipolygon + def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str = None): - df['captured_at'] = pd.to_datetime(df['captured_at'], unit='ms') + df["captured_at"] = pd.to_datetime(df["captured_at"], unit="ms") start_time = pd.Timestamp(start_time) if end_time is None: end_time = pd.Timestamp.now() - filtered_df = df[(df['captured_at'] >= start_time) & (df['captured_at'] <= end_time)] + filtered_df = df[ + (df["captured_at"] >= start_time) & (df["captured_at"] <= end_time) + ] return filtered_df -def filter_results(results_df: pd.DataFrame, is_pano: bool = None, organization_id: str = None, start_time: str = None, end_time: str = None): + +def filter_results( + results_df: pd.DataFrame, + is_pano: bool = None, + organization_id: str = None, + start_time: str = None, + end_time: str = None, +): df = results_df.copy() if is_pano is not None: df = df[df["is_pano"] == is_pano] @@ -168,9 +186,23 @@ def filter_results(results_df: pd.DataFrame, is_pano: bool = None, organization_ return df -def get_image_ids(aoi_geojson, level = 14, attempt_limit = 3, is_pano: bool = None, organization_id: str = None, start_time: str = None, end_time: str = None): +def get_image_metadata( + aoi_geojson, + level=14, + attempt_limit=3, + is_pano: bool = None, + organization_id: str = None, + start_time: str = None, + end_time: str = None, +): aoi_polygon = geojson_to_polygon(aoi_geojson) - token = os.getenv("MAPILLARY_ACCESS_TOKEN") - downloaded_metadata, failed_tiles = coordinate_download(aoi_polygon, level, token, attempt_limit) - downloaded_metadata = filter_results(downloaded_metadata, is_pano, organization_id, start_time, end_time) - return downloaded_metadata["image_id"].tolist() \ No newline at end of file + downloaded_metadata, failed_tiles = coordinate_download( + aoi_polygon, level, attempt_limit + ) + downloaded_metadata = filter_results( + downloaded_metadata, is_pano, organization_id, start_time, end_time + ) + return { + "ids": downloaded_metadata["id"].tolist(), + "geometries": downloaded_metadata["geometry"].tolist(), + } From 1187561e96149c1cc92a9410f47c50e3c6d6e498 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:11:36 +0100 Subject: [PATCH 14/52] feat: add geometry to StreetTask and refactoring --- .../project_types/street/project.py | 157 ++++-------------- 1 file changed, 33 insertions(+), 124 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index 5a2f834af..3dd3451d5 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -15,10 +15,16 @@ from mapswipe_workers.generate_stats.project_stats import ( get_statistics_for_integer_result_project, ) -from mapswipe_workers.project_types.project import ( - BaseProject, BaseTask, BaseGroup +from mapswipe_workers.utils.validate_input import ( + check_if_layer_is_empty, + load_geojson_to_ogr, + build_multipolygon_from_layer_geometries, + check_if_layer_has_too_many_geometries, + save_geojson_to_file, + multipolygon_to_wkt ) -from mapswipe_workers.utils.process_mapillary import get_image_ids +from mapswipe_workers.project_types.project import BaseProject, BaseTask, BaseGroup +from mapswipe_workers.utils.process_mapillary import get_image_metadata @dataclass @@ -29,18 +35,22 @@ class StreetGroup(BaseGroup): @dataclass class StreetTask(BaseTask): - pass + geometry: str + data: str + class StreetProject(BaseProject): def __init__(self, project_draft): super().__init__(project_draft) self.groups: Dict[str, StreetGroup] = {} - self.tasks: Dict[str, List[StreetTask]] = ( - {} - ) + self.tasks: Dict[str, List[StreetTask]] = {} self.geometry = project_draft["geometry"] - self.imageList = get_image_ids() + ImageMetadata = get_image_metadata(self.geometry) + + + self.imageIds = ImageMetadata["ids"] + self.imageGeometries = ImageMetadata["geometries"] def save_tasks_to_firebase(self, projectId: str, tasks: dict): firebase = Firebase() @@ -62,131 +72,29 @@ def get_per_project_statistics(project_id, project_info): ) def validate_geometries(self): - raw_input_file = ( - f"{DATA_PATH}/input_geometries/raw_input_{self.projectId}.geojson" + self.inputGeometriesFileName = save_geojson_to_file( + self.projectId, self.geometry ) + layer, datasource = load_geojson_to_ogr(self.projectId, self.inputGeometriesFileName) - if not os.path.isdir(f"{DATA_PATH}/input_geometries"): - os.mkdir(f"{DATA_PATH}/input_geometries") - - valid_input_file = ( - f"{DATA_PATH}/input_geometries/valid_input_{self.projectId}.geojson" - ) + # check if inputs fit constraints + check_if_layer_is_empty(self.projectId, layer) - logger.info( - f"{self.projectId}" - f" - __init__ - " - f"downloaded input geometries from url and saved as file: " - f"{raw_input_file}" + multi_polygon, project_area = build_multipolygon_from_layer_geometries( + self.projectId, layer ) - self.inputGeometries = raw_input_file - - # open the raw input file and get layer - driver = ogr.GetDriverByName("GeoJSON") - datasource = driver.Open(raw_input_file, 0) - try: - layer = datasource.GetLayer() - LayerDefn = layer.GetLayerDefn() - except AttributeError: - raise CustomError("Value error in input geometries file") - - # create layer for valid_input_file to store all valid geometries - outDriver = ogr.GetDriverByName("GeoJSON") - # Remove output geojson if it already exists - if os.path.exists(valid_input_file): - outDriver.DeleteDataSource(valid_input_file) - outDataSource = outDriver.CreateDataSource(valid_input_file) - outLayer = outDataSource.CreateLayer( - "geometries", geom_type=ogr.wkbMultiPolygon - ) - for i in range(0, LayerDefn.GetFieldCount()): - fieldDefn = LayerDefn.GetFieldDefn(i) - outLayer.CreateField(fieldDefn) - outLayerDefn = outLayer.GetLayerDefn() - - # check if raw_input_file layer is empty - feature_count = layer.GetFeatureCount() - if feature_count < 1: - err = "empty file. No geometries provided" - # TODO: How to user logger and exceptions? - logger.warning(f"{self.projectId} - check_input_geometry - {err}") - raise CustomError(err) - elif feature_count > 100000: - err = f"Too many Geometries: {feature_count}" - logger.warning(f"{self.projectId} - check_input_geometry - {err}") - raise CustomError(err) - - # get geometry as wkt - # get the bounding box/ extent of the layer - extent = layer.GetExtent() - # Create a Polygon from the extent tuple - ring = ogr.Geometry(ogr.wkbLinearRing) - ring.AddPoint(extent[0], extent[2]) - ring.AddPoint(extent[1], extent[2]) - ring.AddPoint(extent[1], extent[3]) - ring.AddPoint(extent[0], extent[3]) - ring.AddPoint(extent[0], extent[2]) - poly = ogr.Geometry(ogr.wkbPolygon) - poly.AddGeometry(ring) - wkt_geometry = poly.ExportToWkt() - - # check if the input geometry is a valid polygon - for feature in layer: - feat_geom = feature.GetGeometryRef() - geom_name = feat_geom.GetGeometryName() - fid = feature.GetFID() - - if not feat_geom.IsValid(): - layer.DeleteFeature(fid) - logger.warning( - f"{self.projectId}" - f" - check_input_geometries - " - f"deleted invalid feature {fid}" - ) - - # we accept only POLYGON or MULTIPOLYGON geometries - elif geom_name not in ["POLYGON", "MULTIPOLYGON"]: - layer.DeleteFeature(fid) - logger.warning( - f"{self.projectId}" - f" - check_input_geometries - " - f"deleted non polygon feature {fid}" - ) - else: - # Create output Feature - outFeature = ogr.Feature(outLayerDefn) - # Add field values from input Layer - for i in range(0, outLayerDefn.GetFieldCount()): - outFeature.SetField( - outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i) - ) - outFeature.SetGeometry(feat_geom) - outLayer.CreateFeature(outFeature) - outFeature = None - - # check if layer is empty - if layer.GetFeatureCount() < 1: - err = "no geometries left after checking validity and geometry type." - logger.warning(f"{self.projectId} - check_input_geometry - {err}") - raise Exception(err) + check_if_layer_has_too_many_geometries(self.projectId, multi_polygon) del datasource - del outDataSource del layer - self.inputGeometriesFileName = valid_input_file - - logger.info( - f"{self.projectId}" - f" - check_input_geometry - " - f"filtered correct input geometries and created file: " - f"{valid_input_file}" - ) + logger.info(f"{self.projectId}" f" - validate geometry - " f"input geometry is correct.") + wkt_geometry = multipolygon_to_wkt(multi_polygon) return wkt_geometry def create_groups(self): - self.numberOfGroups = math.ceil(len(self.imageList) / self.groupSize) + self.numberOfGroups = math.ceil(len(self.imageIds) / self.groupSize) for group_id in range(self.numberOfGroups): self.groups[f"g{group_id}"] = StreetGroup( projectId=self.projectId, @@ -206,13 +114,14 @@ def create_tasks(self): task = StreetTask( projectId=self.projectId, groupId=group_id, - taskId=self.imageList.pop(), + data=str(self.imageGeometries.pop()), + geometry="", + taskId=self.imageIds.pop(), ) self.tasks[group_id].append(task) # list now empty? if usual group size is not reached # the actual number of tasks for the group is updated - if not self.imageList: + if not self.imageIds: group.numberOfTasks = i + 1 break - From 0bef2e01766f6d3b3534b2d6271236ad24f60117 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:13:24 +0100 Subject: [PATCH 15/52] tests(street): use simpler geometry and add filter parameters to testing json --- .../tests/fixtures/projectDrafts/street.json | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json index 6bc291e33..341756584 100644 --- a/mapswipe_workers/tests/fixtures/projectDrafts/street.json +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -5,8 +5,34 @@ "features": [ { "type": "Feature", - "properties": {}, - "geometry": { "type": "Polygon", "coordinates": [ [ [ 39.0085, -6.7811 ], [ 39.0214, -6.7908 ], [ 39.0374, -6.7918 ], [ 39.0537, -6.7968 ], [ 39.0628, -6.8525 ], [ 39.0909, -6.8757 ], [ 39.1018, -6.8747 ], [ 39.1161, -6.8781 ], [ 39.1088, -6.9151 ], [ 39.089, -6.9307 ], [ 39.0864, -6.94 ], [ 39.0596, -6.9731 ], [ 39.0716, -6.9835 ], [ 39.0499, -7.0279 ], [ 39.0236, -7.0322 ], [ 39.038, -7.0499 ], [ 39.0693, -7.0328 ], [ 39.0826, -7.0671 ], [ 39.112, -7.0728 ], [ 39.1154, -7.0921 ], [ 39.1543, -7.0183 ], [ 39.1746, -7.0101 ], [ 39.1699, -6.9971 ], [ 39.196, -6.9848 ], [ 39.2163, -6.9934 ], [ 39.2502, -6.9509 ], [ 39.2768, -6.9486 ], [ 39.2881, -6.9529 ], [ 39.3136, -6.9595 ], [ 39.3276, -6.9483 ], [ 39.3686, -6.9726 ], [ 39.3826, -6.9677 ], [ 39.393, -7.0 ], [ 39.4038, -7.0284 ], [ 39.4037, -7.0379 ], [ 39.4028, -7.0741 ], [ 39.4, -7.0914 ], [ 39.4154, -7.1286 ], [ 39.4049, -7.1595 ], [ 39.4124, -7.1661 ], [ 39.4287, -7.1546 ], [ 39.4395, -7.1618 ], [ 39.4501, -7.1827 ], [ 39.4897, -7.1692 ], [ 39.51, -7.1833 ], [ 39.5378, -7.1229 ], [ 39.5529, -7.1335 ], [ 39.5453, -7.0318 ], [ 39.5376, -6.9873 ], [ 39.4976, -6.9093 ], [ 39.4717, -6.8731 ], [ 39.3557, -6.8455 ], [ 39.3205, -6.8175 ], [ 39.3074, -6.8193 ], [ 39.3002, -6.8253 ], [ 39.293, -6.8317 ], [ 39.2944, -6.818 ], [ 39.2963, -6.8084 ], [ 39.2855, -6.7859 ], [ 39.2808, -6.7347 ], [ 39.2609, -6.7602 ], [ 39.2344, -6.715 ], [ 39.2004, -6.6376 ], [ 39.165, -6.6022 ], [ 39.1353, -6.5748 ], [ 39.1215, -6.5662 ], [ 39.0945, -6.5987 ], [ 39.086, -6.5956 ], [ 39.0803, -6.6062 ], [ 39.0578, -6.7139 ], [ 39.059, -6.723 ], [ 39.0085, -6.7811 ] ], [ [ 39.301, -6.8372 ], [ 39.3048, -6.8555 ], [ 39.2967, -6.8469 ], [ 39.301, -6.8372 ] ], [ [ 39.3048, -6.8555 ], [ 39.2949, -6.8707 ], [ 39.2921, -6.8607 ], [ 39.3048, -6.8555 ] ] ] } + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 39.27186980415655, + -6.818313681620424 + ], + [ + 39.27186980415655, + -6.824056026803248 + ], + [ + 39.27489288297136, + -6.823996705403303 + ], + [ + 39.27483313833096, + -6.817969613314901 + ], + [ + 39.27186980415655, + -6.818313681620424 + ] + ] + ] + } } ] }, @@ -19,5 +45,9 @@ "projectType": 7, "requestingOrganisation": "test", "verificationNumber": 3, - "groupSize": 25 + "groupSize": 25, + "isPano": false, + "startTimestamp": "2019-07-01T00:00:00.000Z", + "endTimestamp": null, + "organisationId": "1" } From 789b02769a4163e6519b2d91dc75feae05064986 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:18:27 +0100 Subject: [PATCH 16/52] refactor: tests for mapillary processing and street project --- .../tests/unittests/test_process_mapillary.py | 136 ++++++++++++------ .../unittests/test_project_type_street.py | 4 +- 2 files changed, 93 insertions(+), 47 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 7e9413c6a..18d9e3401 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -1,16 +1,30 @@ import unittest import os import json -from shapely.geometry import Polygon, MultiPolygon, Point, LineString, MultiLineString, GeometryCollection +from shapely.geometry import ( + Polygon, + MultiPolygon, + Point, + LineString, + MultiLineString, + GeometryCollection, +) import pandas as pd from unittest.mock import patch, MagicMock -from mapswipe_workers.utils.process_mapillary import create_tiles, download_and_process_tile, coordinate_download, geojson_to_polygon, filter_by_timerange, filter_results +from mapswipe_workers.utils.process_mapillary import ( + create_tiles, + download_and_process_tile, + coordinate_download, + geojson_to_polygon, + filter_by_timerange, + filter_results, +) # Assuming create_tiles, download_and_process_tile, and coordinate_download are imported -class TestTileGroupingFunctions(unittest.TestCase): +class TestTileGroupingFunctions(unittest.TestCase): @classmethod def setUpClass(cls): with open( @@ -39,7 +53,9 @@ def setUp(self): self.token = "test_token" self.level = 14 self.test_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) - self.test_multipolygon = MultiPolygon([self.test_polygon, Polygon([(2, 2), (3, 2), (3, 3), (2, 3)])]) + self.test_multipolygon = MultiPolygon( + [self.test_polygon, Polygon([(2, 2), (3, 2), (3, 3), (2, 3)])] + ) self.empty_polygon = Polygon() self.empty_geometry = GeometryCollection() @@ -67,9 +83,21 @@ def test_geojson_to_polygon_feature_collection_with_multiple_polygons(self): geojson_data = { "type": "FeatureCollection", "features": [ - {"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]]}}, - {"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]]}} - ] + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + }, + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]], + }, + }, + ], } result = geojson_to_polygon(geojson_data) self.assertIsInstance(result, MultiPolygon) @@ -80,8 +108,8 @@ def test_geojson_to_polygon_single_feature_polygon(self): "type": "Feature", "geometry": { "type": "Polygon", - "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]] - } + "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + }, } result = geojson_to_polygon(geojson_data) self.assertIsInstance(result, Polygon) @@ -93,9 +121,9 @@ def test_geojson_to_polygon_single_feature_multipolygon(self): "type": "MultiPolygon", "coordinates": [ [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], - [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]] - ] - } + [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]], + ], + }, } result = geojson_to_polygon(geojson_data) self.assertIsInstance(result, MultiPolygon) @@ -105,18 +133,18 @@ def test_geojson_to_polygon_non_polygon_geometry_in_feature_collection(self): geojson_data = { "type": "FeatureCollection", "features": [ - {"type": "Feature", "geometry": {"type": "LineString", "coordinates": [(0, 0), (1, 1)]}} - ] + { + "type": "Feature", + "geometry": {"type": "LineString", "coordinates": [(0, 0), (1, 1)]}, + } + ], } with self.assertRaises(ValueError) as context: geojson_to_polygon(geojson_data) self.assertEqual(str(context.exception), "Non-polygon geometries cannot be combined into a MultiPolygon.") def test_geojson_to_polygon_empty_feature_collection(self): - geojson_data = { - "type": "FeatureCollection", - "features": [] - } + geojson_data = {"type": "FeatureCollection", "features": []} result = geojson_to_polygon(geojson_data) self.assertTrue(result.is_empty) @@ -124,24 +152,29 @@ def test_geojson_to_polygon_contribution_geojson(self): result = geojson_to_polygon(self.fixture_data) self.assertIsInstance(result, Polygon) - @patch('mapswipe_workers.utils.process_mapillary.vt2geojson_tools.vt_bytes_to_geojson') - @patch('mapswipe_workers.utils.process_mapillary.requests.get') + @patch( + "mapswipe_workers.utils.process_mapillary.vt2geojson_tools.vt_bytes_to_geojson" + ) + @patch("mapswipe_workers.utils.process_mapillary.requests.get") def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): # Mock the response from requests.get mock_response = MagicMock() mock_response.status_code = 200 - mock_response.content = b'mock vector tile data' # Example mock data + mock_response.content = b"mock vector tile data" # Example mock data mock_get.return_value = mock_response # Mock the return value of vt_bytes_to_geojson mock_vt2geojson.return_value = { "features": [ - {"geometry": {"type": "Point", "coordinates": [0, 0]}, "properties": {"id": 1}} + { + "geometry": {"type": "Point", "coordinates": [0, 0]}, + "properties": {"id": 1}, + } ] } - row = {'x': 1, 'y': 1, 'z': 14} - token = 'test_token' + row = {"x": 1, "y": 1, "z": 14} + token = "test_token" result, failed = download_and_process_tile(row, token) @@ -149,58 +182,68 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): self.assertIsNone(failed) self.assertIsInstance(result, pd.DataFrame) self.assertEqual(len(result), 1) - self.assertEqual(result['geometry'][0].wkt, 'POINT (0 0)') + self.assertEqual(result["geometry"][0].wkt, "POINT (0 0)") - @patch('mapswipe_workers.utils.process_mapillary.requests.get') + @patch("mapswipe_workers.utils.process_mapillary.requests.get") def test_download_and_process_tile_failure(self, mock_get): # Mock a failed response mock_response = MagicMock() mock_response.status_code = 500 mock_get.return_value = mock_response - row = pd.Series({'x': 1, 'y': 1, 'z': self.level}) + row = pd.Series({"x": 1, "y": 1, "z": self.level}) result, failed = download_and_process_tile(row, self.token) self.assertIsNone(result) self.assertIsNotNone(failed) - @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') + @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download(self, mock_download_and_process_tile): - mock_download_and_process_tile.return_value = (pd.DataFrame([{"geometry": None}]), None) + mock_download_and_process_tile.return_value = ( + pd.DataFrame([{"geometry": None}]), + None, + ) - metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + metadata, failed = coordinate_download( + self.test_polygon, self.level, self.token + ) self.assertIsInstance(metadata, pd.DataFrame) self.assertTrue(failed.empty) - @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') + @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download_with_failures(self, mock_download_and_process_tile): - mock_download_and_process_tile.return_value = (None, pd.Series({"x": 1, "y": 1, "z": self.level})) + mock_download_and_process_tile.return_value = ( + None, + pd.Series({"x": 1, "y": 1, "z": self.level}), + ) - metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + metadata, failed = coordinate_download( + self.test_polygon, self.level, self.token + ) self.assertTrue(metadata.empty) self.assertFalse(failed.empty) def test_filter_within_time_range(self): - start_time = '2016-01-20 00:00:00' - end_time = '2022-01-21 23:59:59' + start_time = "2016-01-20 00:00:00" + end_time = "2022-01-21 23:59:59" filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) self.assertEqual(len(filtered_df), 3) - self.assertTrue(all(filtered_df['captured_at'] >= pd.to_datetime(start_time))) - self.assertTrue(all(filtered_df['captured_at'] <= pd.to_datetime(end_time))) + self.assertTrue(all(filtered_df["captured_at"] >= pd.to_datetime(start_time))) + self.assertTrue(all(filtered_df["captured_at"] <= pd.to_datetime(end_time))) def test_filter_without_end_time(self): - start_time = '2020-01-20 00:00:00' + start_time = "2020-01-20 00:00:00" filtered_df = filter_by_timerange(self.fixture_df, start_time) self.assertEqual(len(filtered_df), 3) - self.assertTrue(all(filtered_df['captured_at'] >= pd.to_datetime(start_time))) + self.assertTrue(all(filtered_df["captured_at"] >= pd.to_datetime(start_time))) def test_filter_time_no_data(self): - start_time = '2016-01-30 00:00:00' - end_time = '2016-01-31 00:00:00' + start_time = "2016-01-30 00:00:00" + end_time = "2016-01-31 00:00:00" filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) self.assertTrue(filtered_df.empty) @@ -221,10 +264,13 @@ def test_filter_organization_id(self): self.assertEqual(len(filtered_df), 1) def test_filter_time_range(self): - start_time = '2016-01-20 00:00:00' - end_time = '2022-01-21 23:59:59' - filtered_df = filter_results(self.fixture_df, start_time=start_time, end_time=end_time) + start_time = "2016-01-20 00:00:00" + end_time = "2022-01-21 23:59:59" + filtered_df = filter_results( + self.fixture_df, start_time=start_time, end_time=end_time + ) self.assertEqual(len(filtered_df), 3) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py index 70c97b13d..e70e10db2 100644 --- a/mapswipe_workers/tests/unittests/test_project_type_street.py +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -26,11 +26,11 @@ def test_create_group(self): self.assertTrue(self.project.groups) def test_create_tasks(self): - imageId = self.project.imageList[-1] + imageId = self.project.imageIds[-1] self.project.create_groups() self.project.create_tasks() self.assertEqual(self.project.tasks["g0"][0].taskId, imageId) - #self.assertEqual(self.project.groups["g0"].numberOfTasks, 1) + # self.assertEqual(self.project.groups["g0"].numberOfTasks, 1) if __name__ == "__main__": From c1f1a79a95da699abf496945d4f5749b2064efca Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:20:44 +0100 Subject: [PATCH 17/52] fix: add missing parameter --- .../mapswipe_workers/project_types/street/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index 3dd3451d5..9c4845d6a 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -54,7 +54,7 @@ def __init__(self, project_draft): def save_tasks_to_firebase(self, projectId: str, tasks: dict): firebase = Firebase() - firebase.save_tasks_to_firebase(projectId, tasks) + firebase.save_tasks_to_firebase(projectId, tasks, useCompression=False) @staticmethod def results_to_postgres(results: dict, project_id: str, filter_mode: bool): From a12a8033e28172af0b9d48c06a69caae81a5ff78 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:37:35 +0100 Subject: [PATCH 18/52] tests(street): add integration test for street project --- .../integration/test_create_street_project.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 mapswipe_workers/tests/integration/test_create_street_project.py diff --git a/mapswipe_workers/tests/integration/test_create_street_project.py b/mapswipe_workers/tests/integration/test_create_street_project.py new file mode 100644 index 000000000..ecf97c626 --- /dev/null +++ b/mapswipe_workers/tests/integration/test_create_street_project.py @@ -0,0 +1,60 @@ +import unittest + +from click.testing import CliRunner + +from mapswipe_workers import auth, mapswipe_workers +from mapswipe_workers.definitions import logger +from mapswipe_workers.utils.create_directories import create_directories +from tests.integration import set_up, tear_down + + +class TestCreateFootprintProject(unittest.TestCase): + def setUp(self): + self.project_id = [ + set_up.create_test_project_draft("street", "street"), + ] + create_directories() + + def tearDown(self): + for element in self.project_id: + tear_down.delete_test_data(element) + + def test_create_street_project(self): + runner = CliRunner() + result = runner.invoke( + mapswipe_workers.run_create_projects, catch_exceptions=False + ) + if result.exit_code != 0: + raise result.exception + pg_db = auth.postgresDB() + for element in self.project_id: + logger.info(f"Checking project {self.project_id}") + query = "SELECT project_id FROM projects WHERE project_id = %s" + result = pg_db.retr_query(query, [element])[0][0] + self.assertEqual(result, element) + + # check if usernames made it to postgres + query = """ + SELECT count(*) + FROM tasks + WHERE project_id = %s + """ + result = pg_db.retr_query(query, [element])[0][0] + self.assertGreater(result, 0) + + fb_db = auth.firebaseDB() + ref = fb_db.reference(f"/v2/projects/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + ref = fb_db.reference(f"/v2/groups/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + # Footprint projects have tasks in Firebase + ref = fb_db.reference(f"/v2/tasks/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + +if __name__ == "__main__": + unittest.main() From 2fb3bc74f866ac9b001edeb1357ce8e6da78f6b4 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:38:07 +0100 Subject: [PATCH 19/52] tests(street): add street.json for testing of street project --- .../fixtures/street/projectDrafts/street.json | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json diff --git a/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json new file mode 100644 index 000000000..2e510f348 --- /dev/null +++ b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json @@ -0,0 +1,54 @@ +{ + "createdBy": "test", + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 39.27186980415655, + -6.818313681620424 + ], + [ + 39.27186980415655, + -6.824056026803248 + ], + [ + 39.27489288297136, + -6.823996705403303 + ], + [ + 39.27483313833096, + -6.817969613314901 + ], + [ + 39.27186980415655, + -6.818313681620424 + ] + ] + ] + } + } + ] + }, + "image": "", + "lookFor": "buildings", + "name": "test - Dar es Salaam (1)\ntest", + "projectDetails": "test", + "projectNumber": 1, + "projectTopic": "test", + "projectType": 7, + "requestingOrganisation": "test", + "verificationNumber": 3, + "groupSize": 25, + "isPano": false, + "startTimestamp": "2019-07-01T00:00:00.000Z", + "endTimestamp": null, + "organisationId": "1", + "customOptions": [{ "color": "", "label": "", "value": -999 }, { "color": "#008000", "label": "yes", "value": 1 }, { "color": "#FF0000", "label": "no", "value": 2 }, { "color": "#FFA500", "label": "maybe", "value": 3 }] +} From 14ff2376fa134f26112c8a4f2d87d9a9d475155d Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:50:02 +0100 Subject: [PATCH 20/52] fix: test after refactoring --- .../tests/unittests/test_process_mapillary.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 18d9e3401..639295a64 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -50,7 +50,6 @@ def setUpClass(cls): cls.fixture_df = pd.read_csv(file) def setUp(self): - self.token = "test_token" self.level = 14 self.test_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) self.test_multipolygon = MultiPolygon( @@ -174,9 +173,8 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): } row = {"x": 1, "y": 1, "z": 14} - token = "test_token" - result, failed = download_and_process_tile(row, token) + result, failed = download_and_process_tile(row) # Assertions self.assertIsNone(failed) @@ -192,7 +190,7 @@ def test_download_and_process_tile_failure(self, mock_get): mock_get.return_value = mock_response row = pd.Series({"x": 1, "y": 1, "z": self.level}) - result, failed = download_and_process_tile(row, self.token) + result, failed = download_and_process_tile(row) self.assertIsNone(result) self.assertIsNotNone(failed) @@ -205,7 +203,7 @@ def test_coordinate_download(self, mock_download_and_process_tile): ) metadata, failed = coordinate_download( - self.test_polygon, self.level, self.token + self.test_polygon, self.level ) self.assertIsInstance(metadata, pd.DataFrame) @@ -219,7 +217,7 @@ def test_coordinate_download_with_failures(self, mock_download_and_process_tile) ) metadata, failed = coordinate_download( - self.test_polygon, self.level, self.token + self.test_polygon, self.level ) self.assertTrue(metadata.empty) From 80ecf282f8176cb728fd6de9cccca13f9064c3ce Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 15:49:59 +0100 Subject: [PATCH 21/52] feat: add functions for spatial sampling and tests for it --- .../project_types/street/project.py | 1 + .../utils/spatial_sampling.py | 117 ++++++++++++++++++ .../tests/fixtures/mapillary_sequence.csv | 71 +++++++++++ .../tests/unittests/test_spatial_sampling.py | 93 ++++++++++++++ 4 files changed, 282 insertions(+) create mode 100644 mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py create mode 100644 mapswipe_workers/tests/fixtures/mapillary_sequence.csv create mode 100644 mapswipe_workers/tests/unittests/test_spatial_sampling.py diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index 9c4845d6a..aa5f1ea48 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -114,6 +114,7 @@ def create_tasks(self): task = StreetTask( projectId=self.projectId, groupId=group_id, + # TODO: change when db allows point geometries data=str(self.imageGeometries.pop()), geometry="", taskId=self.imageIds.pop(), diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py new file mode 100644 index 000000000..8b057e3e6 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -0,0 +1,117 @@ +import numpy as np +import pandas as pd +from shapely import wkt +from shapely.geometry import Point + +def distance_on_sphere(p1, p2): + """ + p1 and p2 are two lists that have two elements. They are numpy arrays of the long and lat + coordinates of the points in set1 and set2 + + Calculate the distance between two points on the Earth's surface using the haversine formula. + + Args: + p1 (list): Array containing the longitude and latitude coordinates of points FROM which the distance to be calculated in degree + p2 (list): Array containing the longitude and latitude coordinates of points TO which the distance to be calculated in degree + + Returns: + numpy.ndarray: Array containing the distances between the two points on the sphere in kilometers. + + This function computes the distance between two points on the Earth's surface using the haversine formula, + which takes into account the spherical shape of the Earth. The input arrays `p1` and `p2` should contain + longitude and latitude coordinates in degrees. The function returns an array containing the distances + between corresponding pairs of points. + """ + earth_radius = 6371 # km + + p1 = np.radians(np.array(p1)) + p2 = np.radians(np.array(p2)) + + delta_lat = p2[1] - p1[1] + delta_long = p2[0] - p1[0] + + a = np.sin(delta_lat / 2) ** 2 + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(delta_long / 2) ** 2 + c = 2 * np.arcsin(np.sqrt(a)) + + distances = earth_radius * c + return distances + +"""-----------------------------------Filtering Points------------------------------------------------""" +def filter_points(df, threshold_distance): + """ + Filter points from a DataFrame based on a threshold distance. + + Args: + df (pandas.DataFrame): DataFrame containing latitude and longitude columns. + threshold_distance (float): Threshold distance for filtering points in kms. + + Returns: + pandas.DataFrame: Filtered DataFrame containing selected points. + float: Total road length calculated from the selected points. + + This function filters points from a DataFrame based on the given threshold distance. It calculates + distances between consecutive points and accumulates them until the accumulated distance surpasses + the threshold distance. It then selects those points and constructs a new DataFrame. Additionally, + it manually checks the last point to include it if it satisfies the length condition. The function + returns the filtered DataFrame along with the calculated road length. + """ + road_length = 0 + mask = np.zeros(len(df), dtype=bool) + mask[0] = True + lat = np.array([wkt.loads(point).y for point in df['data']]) + long = np.array([wkt.loads(point).x for point in df['data']]) + + df['lat'] = lat + df['long'] = long + + + distances = distance_on_sphere([long[1:],lat[1:]], + [long[:-1],lat[:-1]]) + road_length = np.sum(distances) + + #save the last point if the road segment is relavitely small (< 2*road_length) + if threshold_distance <= road_length < 2 * threshold_distance: + mask[-1] = True + + accumulated_distance = 0 + for i, distance in enumerate(distances): + accumulated_distance += distance + if accumulated_distance >= threshold_distance: + mask[i+1] = True + accumulated_distance = 0 # Reset accumulated distance + + to_be_returned_df = df[mask] + # since the last point has to be omitted in the vectorized distance calculation, it is being checked manually + p2 = to_be_returned_df.iloc[0] + distance = distance_on_sphere([float(p2["long"]),float(p2["lat"])],[long[-1],lat[-1]]) + + #last point will be added if it suffices the length condition + #last point will be added in case there is only one point returned + if distance >= threshold_distance or len(to_be_returned_df) ==1: + to_be_returned_df = pd.concat([to_be_returned_df,pd.DataFrame(df.iloc[-1],columns=to_be_returned_df.columns)],axis=0) + return to_be_returned_df + + +def calculate_spacing(df, interval_length): + """ + Calculate spacing between points in a GeoDataFrame. + + Args: + df (pandas.DataFrame): DataFrame containing points with timestamps. + interval_length (float): Interval length for filtering points in kms. + + Returns: + geopandas.GeoDataFrame: Filtered GeoDataFrame containing selected points. + float: Total road length calculated from the selected points. + + This function calculates the spacing between points in a GeoDataFrame by filtering points + based on the provided interval length. It first sorts the GeoDataFrame by timestamp and + then filters points using the filter_points function. The function returns the filtered + GeoDataFrame along with the total road length. + """ + road_length = 0 + if len(df) == 1: + return df + sorted_sub_df = df.sort_values(by=['timestamp']) + filtered_sorted_sub_df = filter_points(sorted_sub_df,interval_length) + return filtered_sorted_sub_df diff --git a/mapswipe_workers/tests/fixtures/mapillary_sequence.csv b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv new file mode 100644 index 000000000..0d8495a18 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv @@ -0,0 +1,71 @@ +,data,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id +9,POINT (38.995129466056824 -6.785243670271996),1453463352000.0,102506575322825.0,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ, +12,POINT (38.9906769990921 -6.783315348346505),1453463400000.0,102506575322825.0,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ, +14,POINT (39.00127172470093 -6.787981661065601),1453463294000.0,102506575322825.0,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ, +18,POINT (38.99769365787506 -6.786351652857817),1453463332000.0,102506575322825.0,1014398349364928,,True,288.49790876724,ywMkSP_5PaJzcbIDa5v1aQ, +19,POINT (38.99540305137634 -6.785360860858361),1453463350000.0,102506575322825.0,165035685525369,,True,293.57188365181,ywMkSP_5PaJzcbIDa5v1aQ, +20,POINT (39.00015592575073 -6.787491593818643),1453463300000.0,102506575322825.0,1139870886511452,,True,305.58569254109,ywMkSP_5PaJzcbIDa5v1aQ, +21,POINT (38.9979350566864 -6.7864262285172),1453463330000.0,102506575322825.0,921248855101166,,True,287.76018227153,ywMkSP_5PaJzcbIDa5v1aQ, +22,POINT (38.99890601634979 -6.786804433469129),1453463318000.0,102506575322825.0,233272625257058,,True,295.57372041131,ywMkSP_5PaJzcbIDa5v1aQ, +23,POINT (38.99618625640869 -6.7857071056060505),1453463344000.0,102506575322825.0,762184237820558,,True,293.88722229187,ywMkSP_5PaJzcbIDa5v1aQ, +26,POINT (38.98969531059265 -6.782905179427416),1453463424000.0,102506575322825.0,1099193250486391,,True,292.87773588969,ywMkSP_5PaJzcbIDa5v1aQ, +27,POINT (38.998627066612244 -6.786671262745287),1453463322000.0,102506575322825.0,831521697449190,,True,296.54786608582,ywMkSP_5PaJzcbIDa5v1aQ, +29,POINT (38.992629647254944 -6.784167646282242),1453463368000.0,102506575322825.0,151617226911336,,True,292.93390738544,ywMkSP_5PaJzcbIDa5v1aQ, +30,POINT (39.00003254413605 -6.7873584232849),1453463302000.0,102506575322825.0,1773698542801178,,True,296.75481871036,ywMkSP_5PaJzcbIDa5v1aQ, +32,POINT (39.000563621520996 -6.787811202949342),1453463298000.0,102506575322825.0,164116422292333,,True,305.80313236719,ywMkSP_5PaJzcbIDa5v1aQ, +37,POINT (38.9902800321579 -6.783144888578377),1453463410000.0,102506575322825.0,329957465177767,,True,293.87481737883,ywMkSP_5PaJzcbIDa5v1aQ, +38,POINT (38.99117052555084 -6.783539076700606),1453463388000.0,102506575322825.0,2948533428757015,,True,294.44274652914,ywMkSP_5PaJzcbIDa5v1aQ, +39,POINT (38.99877190589905 -6.78674051152629),1453463320000.0,102506575322825.0,479325670049740,,True,296.24340777377,ywMkSP_5PaJzcbIDa5v1aQ, +41,POINT (38.990601897239685 -6.783294040878786),1453463402000.0,102506575322825.0,501560441022559,,True,289.98678899102,ywMkSP_5PaJzcbIDa5v1aQ, +42,POINT (38.989362716674805 -6.7828785450699485),1453463432000.0,102506575322825.0,494436578424418,,True,249.25945175736,ywMkSP_5PaJzcbIDa5v1aQ, +44,POINT (38.994566202163696 -6.7850039621655895),1453463356000.0,102506575322825.0,2928848347373461,,True,295.93075138027,ywMkSP_5PaJzcbIDa5v1aQ, +45,POINT (38.993815183639526 -6.784657716912335),1453463362000.0,102506575322825.0,167884815220625,,True,290.65289338004,ywMkSP_5PaJzcbIDa5v1aQ, +47,POINT (38.991841077804565 -6.783837381011111),1453463380000.0,102506575322825.0,2783373888590755,,True,292.94668882511,ywMkSP_5PaJzcbIDa5v1aQ, +48,POINT (38.99052679538727 -6.783262079675453),1453463404000.0,102506575322825.0,500930794384261,,True,296.09431621523,ywMkSP_5PaJzcbIDa5v1aQ, +49,POINT (38.9897757768631 -6.782937140654425),1453463422000.0,102506575322825.0,473363863989539,,True,292.22088072734,ywMkSP_5PaJzcbIDa5v1aQ, +50,POINT (38.99429798126221 -6.784870790943785),1453463358000.0,102506575322825.0,792308461667709,,True,295.36479453372,ywMkSP_5PaJzcbIDa5v1aQ, +51,POINT (38.9997535943985 -6.787219925890724),1453463306000.0,102506575322825.0,1169832606865116,,True,296.51874301031,ywMkSP_5PaJzcbIDa5v1aQ, +54,POINT (38.992860317230225 -6.784268856561923),1453463364000.0,102506575322825.0,143904254368287,,True,292.87299883627,ywMkSP_5PaJzcbIDa5v1aQ, +57,POINT (38.98994743824005 -6.783006389972371),1453463418000.0,102506575322825.0,512708183243254,,True,292.58758044439,ywMkSP_5PaJzcbIDa5v1aQ, +58,POINT (38.99670124053955 -6.785941486524692),1453463340000.0,102506575322825.0,168474601828325,,True,294.32908734047,ywMkSP_5PaJzcbIDa5v1aQ, +59,POINT (38.992136120796204 -6.783959898799395),1453463376000.0,102506575322825.0,171815874817246,,True,292.47687051975,ywMkSP_5PaJzcbIDa5v1aQ, +60,POINT (38.99090766906738 -6.783405905073778),1453463394000.0,102506575322825.0,475809606904698,,True,297.28158189053,ywMkSP_5PaJzcbIDa5v1aQ, +61,POINT (38.99251699447632 -6.7841250314212544),1453463370000.0,102506575322825.0,798930200741228,,True,292.63573224675,ywMkSP_5PaJzcbIDa5v1aQ, +62,POINT (38.99019956588745 -6.7831129273651385),1453463412000.0,102506575322825.0,989719805170705,,True,292.67075887406,ywMkSP_5PaJzcbIDa5v1aQ, +63,POINT (38.991336822509766 -6.783613652795566),1453463386000.0,102506575322825.0,887351401825944,,True,294.11111203184,ywMkSP_5PaJzcbIDa5v1aQ, +64,POINT (38.99745762348175 -6.786271750352796),1453463334000.0,102506575322825.0,820185135568044,,True,292.38261450394,ywMkSP_5PaJzcbIDa5v1aQ, +69,POINT (38.99919033050537 -6.7869376041561225),1453463314000.0,102506575322825.0,1401323273568889,,True,296.07037190303,ywMkSP_5PaJzcbIDa5v1aQ, +70,POINT (38.99229168891907 -6.784023821111347),1453463374000.0,102506575322825.0,971311816939999,,True,293.93020293096,ywMkSP_5PaJzcbIDa5v1aQ, +71,POINT (38.9956659078598 -6.785483378259087),1453463348000.0,102506575322825.0,1888667171315269,,True,294.14026651816,ywMkSP_5PaJzcbIDa5v1aQ, +77,POINT (38.99095058441162 -6.783437866267576),1453463392000.0,102506575322825.0,313013763568992,,True,299.51413646126,ywMkSP_5PaJzcbIDa5v1aQ, +78,POINT (38.99240434169769 -6.784077089698172),1453463372000.0,102506575322825.0,300316491494684,,True,293.91812405268,ywMkSP_5PaJzcbIDa5v1aQ, +79,POINT (38.994094133377075 -6.784774907641307),1453463360000.0,102506575322825.0,799986720927690,,True,293.50534124709,ywMkSP_5PaJzcbIDa5v1aQ, +80,POINT (38.998122811317444 -6.786495477333432),1453463328000.0,102506575322825.0,766866250678914,,True,291.66896324881,ywMkSP_5PaJzcbIDa5v1aQ, +82,POINT (38.99715185165405 -6.786143906317193),1453463336000.0,102506575322825.0,4104687166260249,,True,294.20932475917,ywMkSP_5PaJzcbIDa5v1aQ, +84,POINT (38.99081647396088 -6.783368617011661),1453463396000.0,102506575322825.0,311476673697450,,True,292.1387426333,ywMkSP_5PaJzcbIDa5v1aQ, +85,POINT (38.999614119529724 -6.787150677178687),1453463308000.0,102506575322825.0,486545769438431,,True,296.75245798459,ywMkSP_5PaJzcbIDa5v1aQ, +86,POINT (38.99830520153046 -6.786516784659511),1453463326000.0,102506575322825.0,287818266220456,,True,296.90157475174,ywMkSP_5PaJzcbIDa5v1aQ, +89,POINT (38.991073966026306 -6.78349646178404),1453463390000.0,102506575322825.0,1104648166685558,,True,294.64381621501,ywMkSP_5PaJzcbIDa5v1aQ, +95,POINT (38.99045169353485 -6.783224791602194),1453463406000.0,102506575322825.0,1189096484862313,,True,296.52075233925,ywMkSP_5PaJzcbIDa5v1aQ, +96,POINT (38.989604115486145 -6.782867891326546),1453463426000.0,102506575322825.0,149492403718498,,True,291.35167681135,ywMkSP_5PaJzcbIDa5v1aQ, +99,POINT (38.990371227264404 -6.783187503526065),1453463408000.0,102506575322825.0,1891398304374063,,True,293.92928705821,ywMkSP_5PaJzcbIDa5v1aQ, +102,POINT (38.99695873260498 -6.786058676941238),1453463338000.0,102506575322825.0,823059255257426,,True,294.12835748694,ywMkSP_5PaJzcbIDa5v1aQ, +105,POINT (38.99198055267334 -6.7838959764789735),1453463378000.0,102506575322825.0,3696263247264941,,True,293.15037639266,ywMkSP_5PaJzcbIDa5v1aQ, +108,POINT (38.989861607551575 -6.782974428749938),1453463420000.0,102506575322825.0,862036401042035,,True,291.44770311405,ywMkSP_5PaJzcbIDa5v1aQ, +109,POINT (38.98951292037964 -6.782841256967004),1453463428000.0,102506575322825.0,304865891166877,,True,272.6464361279,ywMkSP_5PaJzcbIDa5v1aQ, +111,POINT (38.99274230003357 -6.784220914853137),1453463366000.0,102506575322825.0,794241458144562,,True,293.2506353664,ywMkSP_5PaJzcbIDa5v1aQ, +113,POINT (38.99847149848938 -6.786596687123861),1453463324000.0,102506575322825.0,939866586773160,,True,295.61868532377,ywMkSP_5PaJzcbIDa5v1aQ, +115,POINT (38.99947464466095 -6.787081428456702),1453463310000.0,102506575322825.0,1143620816058871,,True,296.46351643457,ywMkSP_5PaJzcbIDa5v1aQ, +116,POINT (38.99003326892853 -6.783043678062526),1453463416000.0,102506575322825.0,1405161323178994,,True,293.46012249789,ywMkSP_5PaJzcbIDa5v1aQ, +119,POINT (39.00104105472565 -6.78785914430064),1453463296000.0,102506575322825.0,1273957886333326,,True,279.89824953791,ywMkSP_5PaJzcbIDa5v1aQ, +120,POINT (38.98943245410919 -6.782857237582903),1453463430000.0,102506575322825.0,1171907646606759,,True,252.08147683237,ywMkSP_5PaJzcbIDa5v1aQ, +121,POINT (38.99148166179657 -6.783677575153462),1453463384000.0,102506575322825.0,164230852280713,,True,293.89449177018,ywMkSP_5PaJzcbIDa5v1aQ, +122,POINT (38.999335169792175 -6.787012179724755),1453463312000.0,102506575322825.0,510570906624460,,True,297.86598575521,ywMkSP_5PaJzcbIDa5v1aQ, +124,POINT (38.99592339992523 -6.785595241945558),1453463346000.0,102506575322825.0,136548521821574,,True,293.97664288588,ywMkSP_5PaJzcbIDa5v1aQ, +125,POINT (38.990119099617004 -6.783075639280355),1453463414000.0,102506575322825.0,1112069379291585,,True,293.19285658924,ywMkSP_5PaJzcbIDa5v1aQ, +126,POINT (38.99644374847412 -6.785829622918669),1453463342000.0,102506575322825.0,846432512639247,,True,294.78265773594,ywMkSP_5PaJzcbIDa5v1aQ, +128,POINT (38.99165332317352 -6.783752151226963),1453463382000.0,102506575322825.0,133891088754131,,True,294.07658010782,ywMkSP_5PaJzcbIDa5v1aQ, +129,POINT (38.99078965187073 -6.7833579632791015),1453463398000.0,102506575322825.0,211793933759059,,True,294.17362400652,ywMkSP_5PaJzcbIDa5v1aQ, +131,POINT (38.99904549121857 -6.786873682230961),1453463316000.0,102506575322825.0,137437058395340,,True,295.32783278431,ywMkSP_5PaJzcbIDa5v1aQ, +132,POINT (38.99989306926727 -6.787289174592786),1453463304000.0,102506575322825.0,323401522460093,,True,296.4023335068,ywMkSP_5PaJzcbIDa5v1aQ, +133,POINT (38.994882702827454 -6.785147787043741),1453463354000.0,102506575322825.0,1070660863461711,,True,294.24600460049,ywMkSP_5PaJzcbIDa5v1aQ, diff --git a/mapswipe_workers/tests/unittests/test_spatial_sampling.py b/mapswipe_workers/tests/unittests/test_spatial_sampling.py new file mode 100644 index 000000000..38322dfc8 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_spatial_sampling.py @@ -0,0 +1,93 @@ +import os + +import unittest +import numpy as np +import pandas as pd +from shapely import wkt +from shapely.geometry import Point + +from mapswipe_workers.utils.spatial_sampling import distance_on_sphere, filter_points, calculate_spacing + + +class TestDistanceCalculations(unittest.TestCase): + + @classmethod + def setUpClass(cls): + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_sequence.csv", + ), + "r", + ) as file: + cls.fixture_df = pd.read_csv(file) + + def test_distance_on_sphere(self): + p1 = Point(-74.006, 40.7128) + p2 = Point(-118.2437, 34.0522) + + distance = distance_on_sphere((p1.x, p1.y), (p2.x, p2.y)) + expected_distance = 3940 # Approximate known distance in km + + self.assertTrue(np.isclose(distance, expected_distance, atol=50)) + + def test_filter_points(self): + data = { + "data": [ + "POINT (-74.006 40.7128)", + "POINT (-75.006 41.7128)", + "POINT (-76.006 42.7128)", + "POINT (-77.006 43.7128)" + ] + } + df = pd.DataFrame(data) + + threshold_distance = 100 + filtered_df = filter_points(df, threshold_distance) + + self.assertIsInstance(filtered_df, pd.DataFrame) + self.assertLessEqual(len(filtered_df), len(df)) + + + def test_calculate_spacing(self): + data = { + "data": [ + "POINT (-74.006 40.7128)", + "POINT (-75.006 41.7128)", + "POINT (-76.006 42.7128)", + "POINT (-77.006 43.7128)" + ], + 'timestamp': [1, 2, 3, 4] + } + df = pd.DataFrame(data) + gdf = pd.DataFrame(df) + + interval_length = 100 + filtered_gdf = calculate_spacing(gdf, interval_length) + + self.assertTrue(filtered_gdf['timestamp'].is_monotonic_increasing) + + + def test_calculate_spacing_with_sequence(self): + threshold_distance = 5 + filtered_df = filter_points(self.fixture_df, threshold_distance) + self.assertIsInstance(filtered_df, pd.DataFrame) + self.assertLess(len(filtered_df), len(self.fixture_df)) + + filtered_df.reset_index(drop=True, inplace=True) + + for i in range(len(filtered_df) - 1): + geom1 = wkt.loads(filtered_df.loc[i, 'data']) + geom2 = wkt.loads(filtered_df.loc[i + 1, 'data']) + + distance = geom1.distance(geom2) + + self.assertLess(distance, threshold_distance) + + + + +if __name__ == "__main__": + unittest.main() From 984d8213927d437655fe0821222e9ee4738f3a0a Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 14 Nov 2024 17:00:55 +0100 Subject: [PATCH 22/52] feat: use filtering in street project --- .../project_types/street/project.py | 10 ++- .../utils/process_mapillary.py | 44 +++++++++--- .../tests/fixtures/projectDrafts/street.json | 4 +- .../tests/unittests/test_process_mapillary.py | 68 +++++++++++++++++-- .../unittests/test_project_type_street.py | 18 ++++- 5 files changed, 124 insertions(+), 20 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index aa5f1ea48..cb2e83763 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -46,7 +46,15 @@ def __init__(self, project_draft): self.tasks: Dict[str, List[StreetTask]] = {} self.geometry = project_draft["geometry"] - ImageMetadata = get_image_metadata(self.geometry) + + # TODO: validate inputs + ImageMetadata = get_image_metadata( + self.geometry, + is_pano=project_draft.get("isPano", None), + start_time=project_draft.get("startTimestamp", None), + end_time=project_draft.get("endTimestamp", None), + organization_id=project_draft.get("organizationId", None), + ) self.imageIds = ImageMetadata["ids"] diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 4f1a55084..5a2ba53c2 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -17,6 +17,7 @@ from vt2geojson import tools as vt2geojson_tools from concurrent.futures import ThreadPoolExecutor, as_completed from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY +from mapswipe_workers.definitions import logger def create_tiles(polygon, level): @@ -128,6 +129,12 @@ def coordinate_download( failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index( drop=True ) + target_columns = [ + "id", "geometry", "captured_at", "is_pano", "compass_angle", "sequence", "organization_id" + ] + for col in target_columns: + if col not in downloaded_metadata.columns: + downloaded_metadata[col] = None return downloaded_metadata, failed_tiles @@ -157,11 +164,11 @@ def geojson_to_polygon(geojson_data): def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str = None): df["captured_at"] = pd.to_datetime(df["captured_at"], unit="ms") - start_time = pd.Timestamp(start_time) - + start_time = pd.to_datetime(start_time).tz_localize(None) if end_time is None: - end_time = pd.Timestamp.now() - + end_time = pd.Timestamp.now().tz_localize(None) + else: + end_time = pd.to_datetime(end_time).tz_localize(None) filtered_df = df[ (df["captured_at"] >= start_time) & (df["captured_at"] <= end_time) ] @@ -177,12 +184,30 @@ def filter_results( ): df = results_df.copy() if is_pano is not None: + if df["is_pano"].isna().all(): + logger.exception( + "No Mapillary Feature in the AoI has a 'is_pano' value." + ) + return None df = df[df["is_pano"] == is_pano] + if organization_id is not None: + if df["organization_id"].isna().all(): + logger.exception( + "No Mapillary Feature in the AoI has an 'organization_id' value." + ) + return None df = df[df["organization_id"] == organization_id] + if start_time is not None: + if df["captured_at"].isna().all(): + logger.exception( + "No Mapillary Feature in the AoI has a 'captured_at' value." + ) + return None df = filter_by_timerange(df, start_time, end_time) + return df @@ -202,7 +227,10 @@ def get_image_metadata( downloaded_metadata = filter_results( downloaded_metadata, is_pano, organization_id, start_time, end_time ) - return { - "ids": downloaded_metadata["id"].tolist(), - "geometries": downloaded_metadata["geometry"].tolist(), - } + if downloaded_metadata.isna().all().all() == False: + return { + "ids": downloaded_metadata["id"].tolist(), + "geometries": downloaded_metadata["geometry"].tolist(), + } + else: + raise ValueError("No Mapillary Features in the AoI match the filter criteria.") diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json index 341756584..3093e489c 100644 --- a/mapswipe_workers/tests/fixtures/projectDrafts/street.json +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -47,7 +47,5 @@ "verificationNumber": 3, "groupSize": 25, "isPano": false, - "startTimestamp": "2019-07-01T00:00:00.000Z", - "endTimestamp": null, - "organisationId": "1" + "startTimestamp": "2019-07-01T00:00:00.000Z" } diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 639295a64..caa9e76d6 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -18,12 +18,10 @@ geojson_to_polygon, filter_by_timerange, filter_results, + get_image_metadata, ) -# Assuming create_tiles, download_and_process_tile, and coordinate_download are imported - - class TestTileGroupingFunctions(unittest.TestCase): @classmethod def setUpClass(cls): @@ -156,13 +154,11 @@ def test_geojson_to_polygon_contribution_geojson(self): ) @patch("mapswipe_workers.utils.process_mapillary.requests.get") def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): - # Mock the response from requests.get mock_response = MagicMock() mock_response.status_code = 200 mock_response.content = b"mock vector tile data" # Example mock data mock_get.return_value = mock_response - # Mock the return value of vt_bytes_to_geojson mock_vt2geojson.return_value = { "features": [ { @@ -176,7 +172,6 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): result, failed = download_and_process_tile(row) - # Assertions self.assertIsNone(failed) self.assertIsInstance(result, pd.DataFrame) self.assertEqual(len(result), 1) @@ -269,6 +264,67 @@ def test_filter_time_range(self): ) self.assertEqual(len(filtered_df), 3) + def test_filter_no_rows_after_filter(self): + filtered_df = filter_results(self.fixture_df, is_pano="False") + self.assertTrue(filtered_df.empty) + + def test_filter_missing_columns(self): + columns_to_check = ["is_pano", "organization_id", "captured_at"] # Add your column names here + for column in columns_to_check: + df_copy = self.fixture_df.copy() + df_copy[column] = None + if column == "captured_at": + column = "start_time" + + result = filter_results(df_copy, **{column: True}) + self.assertIsNone(result) + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata(self, mock_coordinate_download): + mock_coordinate_download.return_value = ( + self.fixture_df, + None, + ) + result = get_image_metadata(self.fixture_data) + self.assertIsInstance(result, dict) + self.assertIn("ids", result) + self.assertIn("geometries", result) + + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_filtering(self, mock_coordinate_download): + mock_coordinate_download.return_value = ( + self.fixture_df, + None, + ) + + params = { + "is_pano": True, + "start_time": "2016-01-20 00:00:00", + "end_time": "2022-01-21 23:59:59", + } + + result = get_image_metadata(self.fixture_data, **params) + self.assertIsInstance(result, dict) + self.assertIn("ids", result) + self.assertIn("geometries", result) + + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_no_rows(self, mock_coordinate_download): + mock_coordinate_download.return_value = ( + self.fixture_df, + None, + ) + + params = { + "is_pano": True, + "start_time": "1916-01-20 00:00:00", + "end_time": "1922-01-21 23:59:59", + } + with self.assertRaises(ValueError): + get_image_metadata(self.fixture_data, **params) + if __name__ == "__main__": unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py index e70e10db2..7cc56057c 100644 --- a/mapswipe_workers/tests/unittests/test_project_type_street.py +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -2,6 +2,7 @@ import os import unittest from unittest.mock import patch +import pandas as pd from mapswipe_workers.project_types import StreetProject from tests import fixtures @@ -16,7 +17,21 @@ def setUp(self) -> None: ) ) project_draft["projectDraftId"] = "foo" - self.project = StreetProject(project_draft) + + with patch( + "mapswipe_workers.utils.process_mapillary.coordinate_download" + ) as mock_get: + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_response.csv", + ), + "r", + ) as file: + mock_get.return_value = (pd.read_csv(file), None) + self.project = StreetProject(project_draft) def test_init(self): self.assertEqual(self.project.geometry["type"], "FeatureCollection") @@ -30,7 +45,6 @@ def test_create_tasks(self): self.project.create_groups() self.project.create_tasks() self.assertEqual(self.project.tasks["g0"][0].taskId, imageId) - # self.assertEqual(self.project.groups["g0"].numberOfTasks, 1) if __name__ == "__main__": From 4ab528dcd463f666e54669b58fb070f025b912ef Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 18 Nov 2024 15:29:11 +0100 Subject: [PATCH 23/52] feat: add spatial sampling to street project --- .../project_types/street/project.py | 1 + .../utils/process_mapillary.py | 8 ++++ .../utils/spatial_sampling.py | 30 +++++++++----- .../tests/fixtures/mapillary_sequence.csv | 2 +- .../tests/fixtures/projectDrafts/street.json | 4 +- .../fixtures/street/projectDrafts/street.json | 4 -- .../tests/unittests/test_process_mapillary.py | 5 ++- .../unittests/test_project_type_street.py | 6 ++- .../tests/unittests/test_spatial_sampling.py | 39 +++++++++++-------- 9 files changed, 64 insertions(+), 35 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index cb2e83763..f7184897c 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -54,6 +54,7 @@ def __init__(self, project_draft): start_time=project_draft.get("startTimestamp", None), end_time=project_draft.get("endTimestamp", None), organization_id=project_draft.get("organizationId", None), + sampling_threshold=project_draft.get("samplingThreshold", None), ) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 5a2ba53c2..0e8f8a4a1 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -18,6 +18,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY from mapswipe_workers.definitions import logger +from mapswipe_workers.utils.spatial_sampling import spatial_sampling def create_tiles(polygon, level): @@ -219,14 +220,21 @@ def get_image_metadata( organization_id: str = None, start_time: str = None, end_time: str = None, + sampling_threshold = None, ): aoi_polygon = geojson_to_polygon(aoi_geojson) downloaded_metadata, failed_tiles = coordinate_download( aoi_polygon, level, attempt_limit ) + downloaded_metadata = downloaded_metadata[ + downloaded_metadata['geometry'].apply(lambda geom: isinstance(geom, Point)) + ] + downloaded_metadata = filter_results( downloaded_metadata, is_pano, organization_id, start_time, end_time ) + if sampling_threshold is not None: + downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) if downloaded_metadata.isna().all().all() == False: return { "ids": downloaded_metadata["id"].tolist(), diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py index 8b057e3e6..082346b96 100644 --- a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -58,11 +58,8 @@ def filter_points(df, threshold_distance): road_length = 0 mask = np.zeros(len(df), dtype=bool) mask[0] = True - lat = np.array([wkt.loads(point).y for point in df['data']]) - long = np.array([wkt.loads(point).x for point in df['data']]) - - df['lat'] = lat - df['long'] = long + lat = df["lat"].to_numpy() + long = df["long"].to_numpy() distances = distance_on_sphere([long[1:],lat[1:]], @@ -92,7 +89,7 @@ def filter_points(df, threshold_distance): return to_be_returned_df -def calculate_spacing(df, interval_length): +def spatial_sampling(df, interval_length): """ Calculate spacing between points in a GeoDataFrame. @@ -109,9 +106,22 @@ def calculate_spacing(df, interval_length): then filters points using the filter_points function. The function returns the filtered GeoDataFrame along with the total road length. """ - road_length = 0 if len(df) == 1: return df - sorted_sub_df = df.sort_values(by=['timestamp']) - filtered_sorted_sub_df = filter_points(sorted_sub_df,interval_length) - return filtered_sorted_sub_df + + df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None) + df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None) + sorted_df = df.sort_values(by=['captured_at']) + + sampled_sequence_df = pd.DataFrame() + + # loop through each sequence + for sequence in sorted_df['sequence_id'].unique(): + sequence_df = sorted_df[sorted_df['sequence_id'] == sequence] + + filtered_sorted_sub_df = filter_points(sequence_df,interval_length) + sampled_sequence_df = pd.concat([sampled_sequence_df,filtered_sorted_sub_df],axis=0) + + + + return sampled_sequence_df diff --git a/mapswipe_workers/tests/fixtures/mapillary_sequence.csv b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv index 0d8495a18..597fa6f66 100644 --- a/mapswipe_workers/tests/fixtures/mapillary_sequence.csv +++ b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv @@ -1,4 +1,4 @@ -,data,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id +,geometry,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id 9,POINT (38.995129466056824 -6.785243670271996),1453463352000.0,102506575322825.0,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ, 12,POINT (38.9906769990921 -6.783315348346505),1453463400000.0,102506575322825.0,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ, 14,POINT (39.00127172470093 -6.787981661065601),1453463294000.0,102506575322825.0,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ, diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json index 3093e489c..1dd5b452a 100644 --- a/mapswipe_workers/tests/fixtures/projectDrafts/street.json +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -46,6 +46,6 @@ "requestingOrganisation": "test", "verificationNumber": 3, "groupSize": 25, - "isPano": false, - "startTimestamp": "2019-07-01T00:00:00.000Z" + "startTimestamp": "2019-07-01T00:00:00.000Z", + "samplingThreshold": 0.1 } diff --git a/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json index 2e510f348..f945f1661 100644 --- a/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json +++ b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json @@ -46,9 +46,5 @@ "requestingOrganisation": "test", "verificationNumber": 3, "groupSize": 25, - "isPano": false, - "startTimestamp": "2019-07-01T00:00:00.000Z", - "endTimestamp": null, - "organisationId": "1", "customOptions": [{ "color": "", "label": "", "value": -999 }, { "color": "#008000", "label": "yes", "value": 1 }, { "color": "#FF0000", "label": "no", "value": 2 }, { "color": "#FFA500", "label": "maybe", "value": 3 }] } diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index caa9e76d6..5bec093d3 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -9,6 +9,7 @@ MultiLineString, GeometryCollection, ) +from shapely import wkt import pandas as pd from unittest.mock import patch, MagicMock from mapswipe_workers.utils.process_mapillary import ( @@ -45,7 +46,9 @@ def setUpClass(cls): ), "r", ) as file: - cls.fixture_df = pd.read_csv(file) + df = pd.read_csv(file) + df['geometry'] = df['geometry'].apply(wkt.loads) + cls.fixture_df = df def setUp(self): self.level = 14 diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py index 7cc56057c..64b80b3e7 100644 --- a/mapswipe_workers/tests/unittests/test_project_type_street.py +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -2,6 +2,7 @@ import os import unittest from unittest.mock import patch +from shapely import wkt import pandas as pd from mapswipe_workers.project_types import StreetProject @@ -30,7 +31,10 @@ def setUp(self) -> None: ), "r", ) as file: - mock_get.return_value = (pd.read_csv(file), None) + df = pd.read_csv(file) + df['geometry'] = df['geometry'].apply(wkt.loads) + + mock_get.return_value = (df, None) self.project = StreetProject(project_draft) def test_init(self): diff --git a/mapswipe_workers/tests/unittests/test_spatial_sampling.py b/mapswipe_workers/tests/unittests/test_spatial_sampling.py index 38322dfc8..f43597c89 100644 --- a/mapswipe_workers/tests/unittests/test_spatial_sampling.py +++ b/mapswipe_workers/tests/unittests/test_spatial_sampling.py @@ -6,7 +6,7 @@ from shapely import wkt from shapely.geometry import Point -from mapswipe_workers.utils.spatial_sampling import distance_on_sphere, filter_points, calculate_spacing +from mapswipe_workers.utils.spatial_sampling import distance_on_sphere, filter_points, spatial_sampling class TestDistanceCalculations(unittest.TestCase): @@ -22,7 +22,10 @@ def setUpClass(cls): ), "r", ) as file: - cls.fixture_df = pd.read_csv(file) + df = pd.read_csv(file) + df['geometry'] = df['geometry'].apply(wkt.loads) + + cls.fixture_df = df def test_distance_on_sphere(self): p1 = Point(-74.006, 40.7128) @@ -35,7 +38,7 @@ def test_distance_on_sphere(self): def test_filter_points(self): data = { - "data": [ + "geometry": [ "POINT (-74.006 40.7128)", "POINT (-75.006 41.7128)", "POINT (-76.006 42.7128)", @@ -44,6 +47,10 @@ def test_filter_points(self): } df = pd.DataFrame(data) + df['geometry'] = df['geometry'].apply(wkt.loads) + + df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None) + df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None) threshold_distance = 100 filtered_df = filter_points(df, threshold_distance) @@ -51,36 +58,36 @@ def test_filter_points(self): self.assertLessEqual(len(filtered_df), len(df)) - def test_calculate_spacing(self): + def test_spatial_sampling_ordering(self): data = { - "data": [ + "geometry": [ "POINT (-74.006 40.7128)", "POINT (-75.006 41.7128)", "POINT (-76.006 42.7128)", "POINT (-77.006 43.7128)" ], - 'timestamp': [1, 2, 3, 4] + 'captured_at': [1, 2, 3, 4], + 'sequence_id': ['1', '1', '1', '1'] } df = pd.DataFrame(data) - gdf = pd.DataFrame(df) + df['geometry'] = df['geometry'].apply(wkt.loads) - interval_length = 100 - filtered_gdf = calculate_spacing(gdf, interval_length) + interval_length = 0.1 + filtered_gdf = spatial_sampling(df, interval_length) - self.assertTrue(filtered_gdf['timestamp'].is_monotonic_increasing) + self.assertTrue(filtered_gdf['captured_at'].is_monotonic_increasing) - def test_calculate_spacing_with_sequence(self): - threshold_distance = 5 - filtered_df = filter_points(self.fixture_df, threshold_distance) + def test_spatial_sampling_with_sequence(self): + threshold_distance = 0.01 + filtered_df = spatial_sampling(self.fixture_df, threshold_distance) self.assertIsInstance(filtered_df, pd.DataFrame) self.assertLess(len(filtered_df), len(self.fixture_df)) filtered_df.reset_index(drop=True, inplace=True) - for i in range(len(filtered_df) - 1): - geom1 = wkt.loads(filtered_df.loc[i, 'data']) - geom2 = wkt.loads(filtered_df.loc[i + 1, 'data']) + geom1 = filtered_df.loc[i, 'geometry'] + geom2 = filtered_df.loc[i + 1, 'geometry'] distance = geom1.distance(geom2) From f7fc5f3123f1690e562e535a7a92324c54eea55e Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 18 Nov 2024 17:42:32 +0100 Subject: [PATCH 24/52] feat: add size restriction for too many images --- .../utils/process_mapillary.py | 16 ++++++++----- .../tests/unittests/test_process_mapillary.py | 24 +++++++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 0e8f8a4a1..a762190d6 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -208,7 +208,6 @@ def filter_results( return None df = filter_by_timerange(df, start_time, end_time) - return df @@ -235,10 +234,15 @@ def get_image_metadata( ) if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) - if downloaded_metadata.isna().all().all() == False: - return { - "ids": downloaded_metadata["id"].tolist(), - "geometries": downloaded_metadata["geometry"].tolist(), - } + if downloaded_metadata.isna().all().all() == False or downloaded_metadata.empty == False: + if len(downloaded_metadata) > 100000: + err = (f"Too many Images with selected filter " + f"options for the AoI: {len(downloaded_metadata)}") + raise ValueError(err) + else: + return { + "ids": downloaded_metadata["id"].tolist(), + "geometries": downloaded_metadata["geometry"].tolist(), + } else: raise ValueError("No Mapillary Features in the AoI match the filter criteria.") diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 5bec093d3..f6a7e3fdb 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -328,6 +328,30 @@ def test_get_image_metadata_no_rows(self, mock_coordinate_download): with self.assertRaises(ValueError): get_image_metadata(self.fixture_data, **params) + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_empty_response(self, mock_coordinate_download): + df = self.fixture_df.copy() + df = df.drop(df.index) + mock_coordinate_download.return_value = ( + df, + None + ) + + with self.assertRaises(ValueError): + get_image_metadata(self.fixture_data) + + @patch("mapswipe_workers.utils.process_mapillary.filter_results") + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_size_restriction(self, mock_coordinate_download, mock_filter_results): + mock_filter_results.return_value = pd.DataFrame({'ID': range(1, 100002)}) + mock_coordinate_download.return_value = ( + self.fixture_df, + None, + ) + + with self.assertRaises(ValueError): + get_image_metadata(self.fixture_data) + if __name__ == "__main__": unittest.main() From f11f75edf271b08df2c564d7d42f58be0e711a3f Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 18 Nov 2024 17:44:23 +0100 Subject: [PATCH 25/52] breaking: change geom column in postgres to allow all geometry types --- .../v2_to_v3/08_change_geom_type_for_tasks.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql diff --git a/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql b/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql new file mode 100644 index 000000000..97ecc298f --- /dev/null +++ b/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql @@ -0,0 +1,12 @@ +/* + * This script updates the `tasks` table by adjusting the following column: + * - `geom`: Stores now not only polygon geometries but all geometry types (e.g. Point, LineString). + * + * Existing entries for `geom` are not affected by this change. + * + * The new street project type requires Point geometries to store the image location. + * + */ + + +ALTER TABLE tasks ALTER COLUMN geom SET DATA TYPE geometry(Geometry, 4326); \ No newline at end of file From 98f82c936e0143da89fe0b4bd0149eb63db06453 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 18 Nov 2024 17:46:35 +0100 Subject: [PATCH 26/52] feat: save location of image as geometry --- django/apps/existing_database/models.py | 2 +- .../mapswipe_workers/project_types/street/project.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/django/apps/existing_database/models.py b/django/apps/existing_database/models.py index 0cf6582fb..ef65496b0 100644 --- a/django/apps/existing_database/models.py +++ b/django/apps/existing_database/models.py @@ -127,7 +127,7 @@ class Task(Model): project = models.ForeignKey(Project, models.DO_NOTHING, related_name="+") group_id = models.CharField(max_length=999) task_id = models.CharField(max_length=999) - geom = gis_models.MultiPolygonField(blank=True, null=True) + geom = gis_models.GeometryField(blank=True, null=True) # Database uses JSON instead of JSONB (not supported by django) project_type_specifics = models.TextField(blank=True, null=True) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index f7184897c..1853755df 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -36,7 +36,6 @@ class StreetGroup(BaseGroup): @dataclass class StreetTask(BaseTask): geometry: str - data: str class StreetProject(BaseProject): @@ -123,9 +122,7 @@ def create_tasks(self): task = StreetTask( projectId=self.projectId, groupId=group_id, - # TODO: change when db allows point geometries - data=str(self.imageGeometries.pop()), - geometry="", + geometry=self.imageGeometries.pop(), taskId=self.imageIds.pop(), ) self.tasks[group_id].append(task) From 6a1d15e789f6dc7785a09dc7a9e3a1e15f42bd94 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 18 Nov 2024 18:11:47 +0100 Subject: [PATCH 27/52] feat: use spatial filter on downloaded images --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index a762190d6..5773792c4 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -137,6 +137,10 @@ def coordinate_download( if col not in downloaded_metadata.columns: downloaded_metadata[col] = None + downloaded_metadata = downloaded_metadata[ + downloaded_metadata['geometry'].apply(lambda point: point.within(polygon)) + ] + return downloaded_metadata, failed_tiles From 05f1449954fe85e4a645122f197f49616e2cde7d Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 12:29:37 +0100 Subject: [PATCH 28/52] fix: adapt mapillary token entry in example env --- example.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example.env b/example.env index e3b856ea2..2fa92ae33 100644 --- a/example.env +++ b/example.env @@ -77,4 +77,4 @@ COMMUNITY_DASHBOARD_SENTRY_TRACES_SAMPLE_RATE= COMMUNITY_DASHBOARD_MAPSWIPE_WEBSITE=https://mapswipe.org # Mapillary -MAPILLARY_ACCESS_TOKEN= \ No newline at end of file +MAPILLARY_API_KEY= \ No newline at end of file From 263d738adeb05e5175760333869e4cbdb707c1cc Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 28 Nov 2024 13:34:33 +0100 Subject: [PATCH 29/52] fix: do not return failed_rows and do not use functions on empty df --- .../utils/process_mapillary.py | 13 ++++---- .../tests/unittests/test_process_mapillary.py | 30 +++++-------------- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 5773792c4..5d83b89f5 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -125,7 +125,7 @@ def coordinate_download( if len(downloaded_metadata): downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: - downloaded_metadata = pd.DataFrame(downloaded_metadata) + return pd.DataFrame(downloaded_metadata) failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index( drop=True @@ -137,11 +137,12 @@ def coordinate_download( if col not in downloaded_metadata.columns: downloaded_metadata[col] = None - downloaded_metadata = downloaded_metadata[ - downloaded_metadata['geometry'].apply(lambda point: point.within(polygon)) - ] + if downloaded_metadata.isna().all().all() == False or downloaded_metadata.empty == True: + downloaded_metadata = downloaded_metadata[ + downloaded_metadata['geometry'].apply(lambda point: point.within(polygon)) + ] - return downloaded_metadata, failed_tiles + return downloaded_metadata def geojson_to_polygon(geojson_data): @@ -226,7 +227,7 @@ def get_image_metadata( sampling_threshold = None, ): aoi_polygon = geojson_to_polygon(aoi_geojson) - downloaded_metadata, failed_tiles = coordinate_download( + downloaded_metadata = coordinate_download( aoi_polygon, level, attempt_limit ) downloaded_metadata = downloaded_metadata[ diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index f6a7e3fdb..b6429417b 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -200,7 +200,7 @@ def test_coordinate_download(self, mock_download_and_process_tile): None, ) - metadata, failed = coordinate_download( + metadata = coordinate_download( self.test_polygon, self.level ) @@ -214,12 +214,11 @@ def test_coordinate_download_with_failures(self, mock_download_and_process_tile) pd.Series({"x": 1, "y": 1, "z": self.level}), ) - metadata, failed = coordinate_download( + metadata = coordinate_download( self.test_polygon, self.level ) self.assertTrue(metadata.empty) - self.assertFalse(failed.empty) def test_filter_within_time_range(self): start_time = "2016-01-20 00:00:00" @@ -284,10 +283,7 @@ def test_filter_missing_columns(self): @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata(self, mock_coordinate_download): - mock_coordinate_download.return_value = ( - self.fixture_df, - None, - ) + mock_coordinate_download.return_value = self.fixture_df result = get_image_metadata(self.fixture_data) self.assertIsInstance(result, dict) self.assertIn("ids", result) @@ -296,10 +292,7 @@ def test_get_image_metadata(self, mock_coordinate_download): @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_filtering(self, mock_coordinate_download): - mock_coordinate_download.return_value = ( - self.fixture_df, - None, - ) + mock_coordinate_download.return_value = self.fixture_df params = { "is_pano": True, @@ -315,10 +308,7 @@ def test_get_image_metadata_filtering(self, mock_coordinate_download): @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_no_rows(self, mock_coordinate_download): - mock_coordinate_download.return_value = ( - self.fixture_df, - None, - ) + mock_coordinate_download.return_value = self.fixture_df params = { "is_pano": True, @@ -332,10 +322,7 @@ def test_get_image_metadata_no_rows(self, mock_coordinate_download): def test_get_image_metadata_empty_response(self, mock_coordinate_download): df = self.fixture_df.copy() df = df.drop(df.index) - mock_coordinate_download.return_value = ( - df, - None - ) + mock_coordinate_download.return_value = df with self.assertRaises(ValueError): get_image_metadata(self.fixture_data) @@ -344,10 +331,7 @@ def test_get_image_metadata_empty_response(self, mock_coordinate_download): @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_size_restriction(self, mock_coordinate_download, mock_filter_results): mock_filter_results.return_value = pd.DataFrame({'ID': range(1, 100002)}) - mock_coordinate_download.return_value = ( - self.fixture_df, - None, - ) + mock_coordinate_download.return_value = self.fixture_df with self.assertRaises(ValueError): get_image_metadata(self.fixture_data) From df17d3c9e9b3ecab9f252f02ca11d4a8c133596f Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 28 Nov 2024 13:38:17 +0100 Subject: [PATCH 30/52] fix: testing for removed funcionality --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 4 ---- mapswipe_workers/tests/unittests/test_process_mapillary.py | 1 - 2 files changed, 5 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 5d83b89f5..3558e9b38 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -121,15 +121,11 @@ def coordinate_download( downloaded_metadata.append(df) if failed_row is not None: failed_tiles.append(failed_row) - if len(downloaded_metadata): downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: return pd.DataFrame(downloaded_metadata) - failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index( - drop=True - ) target_columns = [ "id", "geometry", "captured_at", "is_pano", "compass_angle", "sequence", "organization_id" ] diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index b6429417b..8f918f0c0 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -205,7 +205,6 @@ def test_coordinate_download(self, mock_download_and_process_tile): ) self.assertIsInstance(metadata, pd.DataFrame) - self.assertTrue(failed.empty) @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download_with_failures(self, mock_download_and_process_tile): From 9f2ac813c771f50a9e23bb9488263d388415d4e1 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 14:11:41 +0100 Subject: [PATCH 31/52] style: code formatting --- .../project_types/street/project.py | 11 ++-- .../utils/process_mapillary.py | 40 ++++++++----- .../utils/spatial_sampling.py | 59 ++++++++++++------- 3 files changed, 71 insertions(+), 39 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index 1853755df..fba87f390 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -21,7 +21,7 @@ build_multipolygon_from_layer_geometries, check_if_layer_has_too_many_geometries, save_geojson_to_file, - multipolygon_to_wkt + multipolygon_to_wkt, ) from mapswipe_workers.project_types.project import BaseProject, BaseTask, BaseGroup from mapswipe_workers.utils.process_mapillary import get_image_metadata @@ -56,7 +56,6 @@ def __init__(self, project_draft): sampling_threshold=project_draft.get("samplingThreshold", None), ) - self.imageIds = ImageMetadata["ids"] self.imageGeometries = ImageMetadata["geometries"] @@ -83,7 +82,9 @@ def validate_geometries(self): self.inputGeometriesFileName = save_geojson_to_file( self.projectId, self.geometry ) - layer, datasource = load_geojson_to_ogr(self.projectId, self.inputGeometriesFileName) + layer, datasource = load_geojson_to_ogr( + self.projectId, self.inputGeometriesFileName + ) # check if inputs fit constraints check_if_layer_is_empty(self.projectId, layer) @@ -97,7 +98,9 @@ def validate_geometries(self): del datasource del layer - logger.info(f"{self.projectId}" f" - validate geometry - " f"input geometry is correct.") + logger.info( + f"{self.projectId}" f" - validate geometry - " f"input geometry is correct." + ) wkt_geometry = multipolygon_to_wkt(multi_polygon) return wkt_geometry diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 3558e9b38..8181006e7 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -127,15 +127,26 @@ def coordinate_download( return pd.DataFrame(downloaded_metadata) target_columns = [ - "id", "geometry", "captured_at", "is_pano", "compass_angle", "sequence", "organization_id" + "id", + "geometry", + "captured_at", + "is_pano", + "compass_angle", + "sequence", + "organization_id", ] for col in target_columns: if col not in downloaded_metadata.columns: downloaded_metadata[col] = None - if downloaded_metadata.isna().all().all() == False or downloaded_metadata.empty == True: + if ( + downloaded_metadata.isna().all().all() == False + or downloaded_metadata.empty == True + ): downloaded_metadata = downloaded_metadata[ - downloaded_metadata['geometry'].apply(lambda point: point.within(polygon)) + downloaded_metadata["geometry"].apply( + lambda point: point.within(polygon) + ) ] return downloaded_metadata @@ -187,9 +198,7 @@ def filter_results( df = results_df.copy() if is_pano is not None: if df["is_pano"].isna().all(): - logger.exception( - "No Mapillary Feature in the AoI has a 'is_pano' value." - ) + logger.exception("No Mapillary Feature in the AoI has a 'is_pano' value.") return None df = df[df["is_pano"] == is_pano] @@ -220,14 +229,12 @@ def get_image_metadata( organization_id: str = None, start_time: str = None, end_time: str = None, - sampling_threshold = None, + sampling_threshold=None, ): aoi_polygon = geojson_to_polygon(aoi_geojson) - downloaded_metadata = coordinate_download( - aoi_polygon, level, attempt_limit - ) + downloaded_metadata = coordinate_download(aoi_polygon, level, attempt_limit) downloaded_metadata = downloaded_metadata[ - downloaded_metadata['geometry'].apply(lambda geom: isinstance(geom, Point)) + downloaded_metadata["geometry"].apply(lambda geom: isinstance(geom, Point)) ] downloaded_metadata = filter_results( @@ -235,10 +242,15 @@ def get_image_metadata( ) if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) - if downloaded_metadata.isna().all().all() == False or downloaded_metadata.empty == False: + if ( + downloaded_metadata.isna().all().all() == False + or downloaded_metadata.empty == False + ): if len(downloaded_metadata) > 100000: - err = (f"Too many Images with selected filter " - f"options for the AoI: {len(downloaded_metadata)}") + err = ( + f"Too many Images with selected filter " + f"options for the AoI: {len(downloaded_metadata)}" + ) raise ValueError(err) else: return { diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py index 082346b96..1d9c53dc0 100644 --- a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -3,6 +3,7 @@ from shapely import wkt from shapely.geometry import Point + def distance_on_sphere(p1, p2): """ p1 and p2 are two lists that have two elements. They are numpy arrays of the long and lat @@ -30,13 +31,19 @@ def distance_on_sphere(p1, p2): delta_lat = p2[1] - p1[1] delta_long = p2[0] - p1[0] - a = np.sin(delta_lat / 2) ** 2 + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(delta_long / 2) ** 2 + a = ( + np.sin(delta_lat / 2) ** 2 + + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(delta_long / 2) ** 2 + ) c = 2 * np.arcsin(np.sqrt(a)) distances = earth_radius * c return distances + """-----------------------------------Filtering Points------------------------------------------------""" + + def filter_points(df, threshold_distance): """ Filter points from a DataFrame based on a threshold distance. @@ -61,12 +68,10 @@ def filter_points(df, threshold_distance): lat = df["lat"].to_numpy() long = df["long"].to_numpy() - - distances = distance_on_sphere([long[1:],lat[1:]], - [long[:-1],lat[:-1]]) + distances = distance_on_sphere([long[1:], lat[1:]], [long[:-1], lat[:-1]]) road_length = np.sum(distances) - #save the last point if the road segment is relavitely small (< 2*road_length) + # save the last point if the road segment is relavitely small (< 2*road_length) if threshold_distance <= road_length < 2 * threshold_distance: mask[-1] = True @@ -74,18 +79,26 @@ def filter_points(df, threshold_distance): for i, distance in enumerate(distances): accumulated_distance += distance if accumulated_distance >= threshold_distance: - mask[i+1] = True + mask[i + 1] = True accumulated_distance = 0 # Reset accumulated distance to_be_returned_df = df[mask] # since the last point has to be omitted in the vectorized distance calculation, it is being checked manually p2 = to_be_returned_df.iloc[0] - distance = distance_on_sphere([float(p2["long"]),float(p2["lat"])],[long[-1],lat[-1]]) - - #last point will be added if it suffices the length condition - #last point will be added in case there is only one point returned - if distance >= threshold_distance or len(to_be_returned_df) ==1: - to_be_returned_df = pd.concat([to_be_returned_df,pd.DataFrame(df.iloc[-1],columns=to_be_returned_df.columns)],axis=0) + distance = distance_on_sphere( + [float(p2["long"]), float(p2["lat"])], [long[-1], lat[-1]] + ) + + # last point will be added if it suffices the length condition + # last point will be added in case there is only one point returned + if distance >= threshold_distance or len(to_be_returned_df) == 1: + to_be_returned_df = pd.concat( + [ + to_be_returned_df, + pd.DataFrame(df.iloc[-1], columns=to_be_returned_df.columns), + ], + axis=0, + ) return to_be_returned_df @@ -109,19 +122,23 @@ def spatial_sampling(df, interval_length): if len(df) == 1: return df - df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None) - df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None) - sorted_df = df.sort_values(by=['captured_at']) + df["long"] = df["geometry"].apply( + lambda geom: geom.x if geom.geom_type == "Point" else None + ) + df["lat"] = df["geometry"].apply( + lambda geom: geom.y if geom.geom_type == "Point" else None + ) + sorted_df = df.sort_values(by=["captured_at"]) sampled_sequence_df = pd.DataFrame() # loop through each sequence - for sequence in sorted_df['sequence_id'].unique(): - sequence_df = sorted_df[sorted_df['sequence_id'] == sequence] - - filtered_sorted_sub_df = filter_points(sequence_df,interval_length) - sampled_sequence_df = pd.concat([sampled_sequence_df,filtered_sorted_sub_df],axis=0) - + for sequence in sorted_df["sequence_id"].unique(): + sequence_df = sorted_df[sorted_df["sequence_id"] == sequence] + filtered_sorted_sub_df = filter_points(sequence_df, interval_length) + sampled_sequence_df = pd.concat( + [sampled_sequence_df, filtered_sorted_sub_df], axis=0 + ) return sampled_sequence_df From 8593a3c7112ea9171f7dd63b28cb6beb80d63377 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 28 Nov 2024 15:08:51 +0100 Subject: [PATCH 32/52] fix: fixed tests and removed return of failed rows for download from mapillary --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 4 ++-- mapswipe_workers/tests/unittests/test_project_type_street.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 3558e9b38..6cd77e787 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -85,13 +85,13 @@ def download_and_process_tile(row, attempt_limit=3): data = pd.DataFrame(data) if not data.empty: - return data, None + return data except Exception as e: print(f"An exception occurred while requesting a tile: {e}") attempt += 1 print(f"A tile could not be downloaded: {row}") - return None, row + return None def coordinate_download( diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py index 64b80b3e7..8ec8d0fa4 100644 --- a/mapswipe_workers/tests/unittests/test_project_type_street.py +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -34,7 +34,7 @@ def setUp(self) -> None: df = pd.read_csv(file) df['geometry'] = df['geometry'].apply(wkt.loads) - mock_get.return_value = (df, None) + mock_get.return_value = df self.project = StreetProject(project_draft) def test_init(self): From 9503a7d8c1d180f1f3a01042da55add4dffa200f Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 28 Nov 2024 15:51:45 +0100 Subject: [PATCH 33/52] fix: tests for removed failing rows --- mapswipe_workers/tests/unittests/test_process_mapillary.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 8f918f0c0..a94520a67 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -173,9 +173,8 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): row = {"x": 1, "y": 1, "z": 14} - result, failed = download_and_process_tile(row) + result = download_and_process_tile(row) - self.assertIsNone(failed) self.assertIsInstance(result, pd.DataFrame) self.assertEqual(len(result), 1) self.assertEqual(result["geometry"][0].wkt, "POINT (0 0)") @@ -188,10 +187,9 @@ def test_download_and_process_tile_failure(self, mock_get): mock_get.return_value = mock_response row = pd.Series({"x": 1, "y": 1, "z": self.level}) - result, failed = download_and_process_tile(row) + result = download_and_process_tile(row) self.assertIsNone(result) - self.assertIsNotNone(failed) @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download(self, mock_download_and_process_tile): From c0016e26dce13bd3ab25653a799d16420f150619 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 16:40:35 +0100 Subject: [PATCH 34/52] style: fix flake8 errors and isort --- .../project_types/street/project.py | 19 ++++---- .../utils/process_mapillary.py | 29 ++++++------ .../utils/spatial_sampling.py | 47 ++++++++++--------- 3 files changed, 48 insertions(+), 47 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index fba87f390..e15f42c1f 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -1,11 +1,8 @@ -import json -import os -import urllib import math - -from osgeo import ogr from dataclasses import dataclass -from mapswipe_workers.definitions import DATA_PATH, logger +from typing import Dict, List + +from mapswipe_workers.definitions import logger from mapswipe_workers.firebase.firebase import Firebase from mapswipe_workers.firebase_to_postgres.transfer_results import ( results_to_file, @@ -15,16 +12,16 @@ from mapswipe_workers.generate_stats.project_stats import ( get_statistics_for_integer_result_project, ) +from mapswipe_workers.project_types.project import BaseGroup, BaseProject, BaseTask +from mapswipe_workers.utils.process_mapillary import get_image_metadata from mapswipe_workers.utils.validate_input import ( - check_if_layer_is_empty, - load_geojson_to_ogr, build_multipolygon_from_layer_geometries, check_if_layer_has_too_many_geometries, - save_geojson_to_file, + check_if_layer_is_empty, + load_geojson_to_ogr, multipolygon_to_wkt, + save_geojson_to_file, ) -from mapswipe_workers.project_types.project import BaseProject, BaseTask, BaseGroup -from mapswipe_workers.utils.process_mapillary import get_image_metadata @dataclass diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index c55b213d9..c17839691 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -1,23 +1,22 @@ +import os +from concurrent.futures import ThreadPoolExecutor, as_completed + import mercantile -import json +import pandas as pd import requests -import os -import time from shapely import ( - box, - Polygon, - MultiPolygon, - Point, LineString, MultiLineString, + MultiPolygon, + Point, + Polygon, + box, unary_union, ) from shapely.geometry import shape -import pandas as pd from vt2geojson import tools as vt2geojson_tools -from concurrent.futures import ThreadPoolExecutor, as_completed -from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY -from mapswipe_workers.definitions import logger + +from mapswipe_workers.definitions import MAPILLARY_API_KEY, MAPILLARY_API_LINK, logger from mapswipe_workers.utils.spatial_sampling import spatial_sampling @@ -140,8 +139,8 @@ def coordinate_download( downloaded_metadata[col] = None if ( - downloaded_metadata.isna().all().all() == False - or downloaded_metadata.empty == True + downloaded_metadata.isna().all().all() is False + or downloaded_metadata.empty is True ): downloaded_metadata = downloaded_metadata[ downloaded_metadata["geometry"].apply( @@ -243,8 +242,8 @@ def get_image_metadata( if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) if ( - downloaded_metadata.isna().all().all() == False - or downloaded_metadata.empty == False + downloaded_metadata.isna().all().all() is False + or downloaded_metadata.empty is False ): if len(downloaded_metadata) > 100000: err = ( diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py index 1d9c53dc0..3cafb5362 100644 --- a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -1,26 +1,29 @@ import numpy as np import pandas as pd -from shapely import wkt -from shapely.geometry import Point def distance_on_sphere(p1, p2): """ - p1 and p2 are two lists that have two elements. They are numpy arrays of the long and lat - coordinates of the points in set1 and set2 + p1 and p2 are two lists that have two elements. They are numpy arrays of the long + and lat coordinates of the points in set1 and set2 - Calculate the distance between two points on the Earth's surface using the haversine formula. + Calculate the distance between two points on the Earth's surface using the + haversine formula. Args: - p1 (list): Array containing the longitude and latitude coordinates of points FROM which the distance to be calculated in degree - p2 (list): Array containing the longitude and latitude coordinates of points TO which the distance to be calculated in degree + p1 (list): Array containing the longitude and latitude coordinates of points + FROM which the distance to be calculated in degree + p2 (list): Array containing the longitude and latitude coordinates of points + TO which the distance to be calculated in degree Returns: - numpy.ndarray: Array containing the distances between the two points on the sphere in kilometers. + numpy.ndarray: Array containing the distances between the two points on the + sphere in kilometers. - This function computes the distance between two points on the Earth's surface using the haversine formula, - which takes into account the spherical shape of the Earth. The input arrays `p1` and `p2` should contain - longitude and latitude coordinates in degrees. The function returns an array containing the distances + This function computes the distance between two points on the Earth's surface + using the haversine formula, which takes into account the spherical shape of the + Earth. The input arrays `p1` and `p2` should contain longitude and latitude + coordinates in degrees. The function returns an array containing the distances between corresponding pairs of points. """ earth_radius = 6371 # km @@ -41,7 +44,7 @@ def distance_on_sphere(p1, p2): return distances -"""-----------------------------------Filtering Points------------------------------------------------""" +"""----------------------------Filtering Points-------------------------------""" def filter_points(df, threshold_distance): @@ -56,10 +59,11 @@ def filter_points(df, threshold_distance): pandas.DataFrame: Filtered DataFrame containing selected points. float: Total road length calculated from the selected points. - This function filters points from a DataFrame based on the given threshold distance. It calculates - distances between consecutive points and accumulates them until the accumulated distance surpasses - the threshold distance. It then selects those points and constructs a new DataFrame. Additionally, - it manually checks the last point to include it if it satisfies the length condition. The function + This function filters points from a DataFrame based on the given threshold + distance. It calculates distances between consecutive points and accumulates them + until the accumulated distance surpasses the threshold distance. It then selects + those points and constructs a new DataFrame. Additionally, it manually checks the + last point to include it if it satisfies the length condition. The function returns the filtered DataFrame along with the calculated road length. """ road_length = 0 @@ -83,7 +87,8 @@ def filter_points(df, threshold_distance): accumulated_distance = 0 # Reset accumulated distance to_be_returned_df = df[mask] - # since the last point has to be omitted in the vectorized distance calculation, it is being checked manually + # since the last point has to be omitted in the vectorized distance calculation, + # it is being checked manually p2 = to_be_returned_df.iloc[0] distance = distance_on_sphere( [float(p2["long"]), float(p2["lat"])], [long[-1], lat[-1]] @@ -114,10 +119,10 @@ def spatial_sampling(df, interval_length): geopandas.GeoDataFrame: Filtered GeoDataFrame containing selected points. float: Total road length calculated from the selected points. - This function calculates the spacing between points in a GeoDataFrame by filtering points - based on the provided interval length. It first sorts the GeoDataFrame by timestamp and - then filters points using the filter_points function. The function returns the filtered - GeoDataFrame along with the total road length. + This function calculates the spacing between points in a GeoDataFrame by filtering + points based on the provided interval length. It first sorts the GeoDataFrame by + timestamp and then filters points using the filter_points function. The function + returns the filtered GeoDataFrame along with the total road length. """ if len(df) == 1: return df From 2f3d523a03abf6b0e247e02cfb880200c56e89a1 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 16:47:12 +0100 Subject: [PATCH 35/52] style: isort --- mapswipe_workers/mapswipe_workers/project_types/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/__init__.py index 048eda71c..43013b0dc 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/__init__.py +++ b/mapswipe_workers/mapswipe_workers/project_types/__init__.py @@ -2,13 +2,13 @@ from .arbitrary_geometry.footprint.project import FootprintProject from .arbitrary_geometry.footprint.tutorial import FootprintTutorial from .media_classification.project import MediaClassificationProject +from .street.project import StreetProject from .tile_map_service.change_detection.project import ChangeDetectionProject from .tile_map_service.change_detection.tutorial import ChangeDetectionTutorial from .tile_map_service.classification.project import ClassificationProject from .tile_map_service.classification.tutorial import ClassificationTutorial from .tile_map_service.completeness.project import CompletenessProject from .tile_map_service.completeness.tutorial import CompletenessTutorial -from .street.project import StreetProject __all__ = [ "ClassificationProject", From 6ad174f052fab8e289130c48fb68d90b1c0b8fc2 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 17:17:47 +0100 Subject: [PATCH 36/52] build: add requirements --- mapswipe_workers/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mapswipe_workers/requirements.txt b/mapswipe_workers/requirements.txt index 7e125fe4b..588754060 100644 --- a/mapswipe_workers/requirements.txt +++ b/mapswipe_workers/requirements.txt @@ -14,3 +14,6 @@ sentry-sdk==0.18.0 six==1.15.0 slackclient==2.9.2 xdg==4.0.1 +shapely +mercantile +vt2geojson From 315d239dc61340f92b0a1c0144d57887851a3ba0 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 17:28:13 +0100 Subject: [PATCH 37/52] build: add dummy mapillary key to github workflow --- .github/workflows/actions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index aacc4dcd3..d66de00b8 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -80,6 +80,7 @@ jobs: POSTGRES_DB: postgres OSMCHA_API_KEY: ${{ secrets.OSMCHA_API_KEY }} DJANGO_SECRET_KEY: test-django-secret-key + MAPILLARY_API_KEY: test-mapillary-api-key COMPOSE_FILE: ../docker-compose.yaml:../docker-compose-ci.yaml run: | docker compose run --rm mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/ From 2b88cf7f491c9088cb641989b0cc8efa5b006590 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 18:15:07 +0100 Subject: [PATCH 38/52] fix: use os.getenv instead of os.environ --- mapswipe_workers/mapswipe_workers/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index c9dec6d79..b6be718ef 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -17,7 +17,7 @@ OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" -MAPILLARY_API_KEY = os.environ["MAPILLARY_API_KEY"] +MAPILLARY_API_KEY = os.getenv("MAPILLARY_API_KEY") # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 From 24c7eec528c38a22ad7f1705697fd0aacb1b50fb Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 18:44:41 +0100 Subject: [PATCH 39/52] test: rename class and correct comments --- .../tests/integration/test_create_street_project.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/tests/integration/test_create_street_project.py b/mapswipe_workers/tests/integration/test_create_street_project.py index ecf97c626..fd0608f98 100644 --- a/mapswipe_workers/tests/integration/test_create_street_project.py +++ b/mapswipe_workers/tests/integration/test_create_street_project.py @@ -8,7 +8,7 @@ from tests.integration import set_up, tear_down -class TestCreateFootprintProject(unittest.TestCase): +class TestCreateStreetProject(unittest.TestCase): def setUp(self): self.project_id = [ set_up.create_test_project_draft("street", "street"), @@ -33,7 +33,7 @@ def test_create_street_project(self): result = pg_db.retr_query(query, [element])[0][0] self.assertEqual(result, element) - # check if usernames made it to postgres + # check if tasks made it to postgres query = """ SELECT count(*) FROM tasks @@ -51,10 +51,11 @@ def test_create_street_project(self): result = ref.get(shallow=True) self.assertIsNotNone(result) - # Footprint projects have tasks in Firebase + # Street projects have tasks in Firebase ref = fb_db.reference(f"/v2/tasks/{element}") result = ref.get(shallow=True) self.assertIsNotNone(result) + if __name__ == "__main__": unittest.main() From 7f46e80d875eb39b49f77d70c6b71ca77c61b9bc Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 13:20:25 +0100 Subject: [PATCH 40/52] fix: remove left overs of failed rows --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index c55b213d9..62dd00d5a 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -100,7 +100,6 @@ def coordinate_download( tiles = create_tiles(polygon, level) downloaded_metadata = [] - failed_tiles = [] if not tiles.empty: if not use_concurrency: @@ -115,12 +114,10 @@ def coordinate_download( for future in as_completed(futures): if future is not None: - df, failed_row = future.result() + df = future.result() if df is not None and not df.empty: downloaded_metadata.append(df) - if failed_row is not None: - failed_tiles.append(failed_row) if len(downloaded_metadata): downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: From bfa0818cd8cfb367c984ba2093e7d990774a27f4 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 13:37:51 +0100 Subject: [PATCH 41/52] fix: unittests for tile download --- .../tests/unittests/test_process_mapillary.py | 62 ++++++++----------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index a94520a67..8134ad2bf 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -1,24 +1,19 @@ -import unittest -import os import json -from shapely.geometry import ( - Polygon, - MultiPolygon, - Point, - LineString, - MultiLineString, - GeometryCollection, -) -from shapely import wkt +import os +import unittest +from unittest.mock import MagicMock, patch + import pandas as pd -from unittest.mock import patch, MagicMock +from shapely import wkt +from shapely.geometry import GeometryCollection, MultiPolygon, Polygon + from mapswipe_workers.utils.process_mapillary import ( + coordinate_download, create_tiles, download_and_process_tile, - coordinate_download, - geojson_to_polygon, filter_by_timerange, filter_results, + geojson_to_polygon, get_image_metadata, ) @@ -47,7 +42,7 @@ def setUpClass(cls): "r", ) as file: df = pd.read_csv(file) - df['geometry'] = df['geometry'].apply(wkt.loads) + df["geometry"] = df["geometry"].apply(wkt.loads) cls.fixture_df = df def setUp(self): @@ -141,7 +136,10 @@ def test_geojson_to_polygon_non_polygon_geometry_in_feature_collection(self): } with self.assertRaises(ValueError) as context: geojson_to_polygon(geojson_data) - self.assertEqual(str(context.exception), "Non-polygon geometries cannot be combined into a MultiPolygon.") + self.assertEqual( + str(context.exception), + "Non-polygon geometries cannot be combined into a MultiPolygon.", + ) def test_geojson_to_polygon_empty_feature_collection(self): geojson_data = {"type": "FeatureCollection", "features": []} @@ -193,27 +191,17 @@ def test_download_and_process_tile_failure(self, mock_get): @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download(self, mock_download_and_process_tile): - mock_download_and_process_tile.return_value = ( - pd.DataFrame([{"geometry": None}]), - None, - ) + mock_download_and_process_tile.return_value = pd.DataFrame([{"geometry": None}]) - metadata = coordinate_download( - self.test_polygon, self.level - ) + metadata = coordinate_download(self.test_polygon, self.level) self.assertIsInstance(metadata, pd.DataFrame) @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download_with_failures(self, mock_download_and_process_tile): - mock_download_and_process_tile.return_value = ( - None, - pd.Series({"x": 1, "y": 1, "z": self.level}), - ) + mock_download_and_process_tile.return_value = pd.DataFrame() - metadata = coordinate_download( - self.test_polygon, self.level - ) + metadata = coordinate_download(self.test_polygon, self.level) self.assertTrue(metadata.empty) @@ -268,7 +256,11 @@ def test_filter_no_rows_after_filter(self): self.assertTrue(filtered_df.empty) def test_filter_missing_columns(self): - columns_to_check = ["is_pano", "organization_id", "captured_at"] # Add your column names here + columns_to_check = [ + "is_pano", + "organization_id", + "captured_at", + ] # Add your column names here for column in columns_to_check: df_copy = self.fixture_df.copy() df_copy[column] = None @@ -286,7 +278,6 @@ def test_get_image_metadata(self, mock_coordinate_download): self.assertIn("ids", result) self.assertIn("geometries", result) - @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_filtering(self, mock_coordinate_download): mock_coordinate_download.return_value = self.fixture_df @@ -302,7 +293,6 @@ def test_get_image_metadata_filtering(self, mock_coordinate_download): self.assertIn("ids", result) self.assertIn("geometries", result) - @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_no_rows(self, mock_coordinate_download): mock_coordinate_download.return_value = self.fixture_df @@ -326,8 +316,10 @@ def test_get_image_metadata_empty_response(self, mock_coordinate_download): @patch("mapswipe_workers.utils.process_mapillary.filter_results") @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") - def test_get_image_metadata_size_restriction(self, mock_coordinate_download, mock_filter_results): - mock_filter_results.return_value = pd.DataFrame({'ID': range(1, 100002)}) + def test_get_image_metadata_size_restriction( + self, mock_coordinate_download, mock_filter_results + ): + mock_filter_results.return_value = pd.DataFrame({"ID": range(1, 100002)}) mock_coordinate_download.return_value = self.fixture_df with self.assertRaises(ValueError): From 816b1e6f48697e5c2b6ad81c7ef8a74e5dd51f5d Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 14:54:33 +0100 Subject: [PATCH 42/52] fix: add condition to raise valueerror if no features are found in aoi --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index fa2922f93..4b6943fdf 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -229,6 +229,8 @@ def get_image_metadata( ): aoi_polygon = geojson_to_polygon(aoi_geojson) downloaded_metadata = coordinate_download(aoi_polygon, level, attempt_limit) + if downloaded_metadata.isna().all().all() or downloaded_metadata.empty: + raise ValueError("No Mapillary Features in the AoI.") downloaded_metadata = downloaded_metadata[ downloaded_metadata["geometry"].apply(lambda geom: isinstance(geom, Point)) ] From f287b3ae138f381392b289baab5c939666aa3984 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 3 Dec 2024 15:16:27 +0100 Subject: [PATCH 43/52] fix: use mapillary api key secret in github workflow --- .github/workflows/actions.yml | 2 +- mapswipe_workers/mapswipe_workers/definitions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index d66de00b8..4609eb697 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -80,7 +80,7 @@ jobs: POSTGRES_DB: postgres OSMCHA_API_KEY: ${{ secrets.OSMCHA_API_KEY }} DJANGO_SECRET_KEY: test-django-secret-key - MAPILLARY_API_KEY: test-mapillary-api-key + MAPILLARY_API_KEY: ${{ secrets.MAPILLARY_API_KEY }} COMPOSE_FILE: ../docker-compose.yaml:../docker-compose-ci.yaml run: | docker compose run --rm mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/ diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index b6be718ef..e5037675b 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -17,7 +17,7 @@ OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" -MAPILLARY_API_KEY = os.getenv("MAPILLARY_API_KEY") +MAPILLARY_API_KEY = os.environ("MAPILLARY_API_KEY") # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 From 020178d6e57cac481c000af831f2265647944b7d Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 3 Dec 2024 15:29:44 +0100 Subject: [PATCH 44/52] fix: use square brackets with os.environ --- mapswipe_workers/mapswipe_workers/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index e5037675b..c9dec6d79 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -17,7 +17,7 @@ OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" -MAPILLARY_API_KEY = os.environ("MAPILLARY_API_KEY") +MAPILLARY_API_KEY = os.environ["MAPILLARY_API_KEY"] # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 From dcaab3d045b62191bc847ebf1e8c4bc47c45ea44 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 3 Dec 2024 16:28:29 +0100 Subject: [PATCH 45/52] fix: add mapillary key to docker compose --- docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index ccbc2aa9d..d465a704c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -76,6 +76,7 @@ x-mapswipe-workers: &base_mapswipe_workers SLACK_CHANNEL: '${SLACK_CHANNEL}' SENTRY_DSN: '${SENTRY_DSN}' OSMCHA_API_KEY: '${OSMCHA_API_KEY}' + MAPILLARY_API_KEY: '${MAPILLARY_API_KEY}' depends_on: - postgres volumes: From 81284889ca7c465787f32621b51d930c3e927330 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 3 Dec 2024 17:02:12 +0100 Subject: [PATCH 46/52] fix: allow tasks with point geom in postgres --- postgres/initdb.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/initdb.sql b/postgres/initdb.sql index ce9b97197..f954d3a8c 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON, 4326), + geom geometry(Geometry, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), From 74064f835999db1fbd86155975fbac5ad3575f0b Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 17:15:17 +0100 Subject: [PATCH 47/52] fix: change geometry type in tasks table in initdb --- postgres/initdb.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/initdb.sql b/postgres/initdb.sql index ce9b97197..f954d3a8c 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON, 4326), + geom geometry(Geometry, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), From 729a4be11567c2cf82c9068d8d2f6f37c5b264cd Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 17:29:07 +0100 Subject: [PATCH 48/52] fix: change to all geometries in tasks table in setup db --- mapswipe_workers/tests/integration/set_up_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/tests/integration/set_up_db.sql b/mapswipe_workers/tests/integration/set_up_db.sql index ce9b97197..b03e76699 100644 --- a/mapswipe_workers/tests/integration/set_up_db.sql +++ b/mapswipe_workers/tests/integration/set_up_db.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON, 4326), + geom geometry(GEOMETRY, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), From 994f3983157a8a0ee980cdb61a737e38f12060c3 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 17:44:36 +0100 Subject: [PATCH 49/52] fix: use lower case in setup db --- mapswipe_workers/tests/integration/set_up_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/tests/integration/set_up_db.sql b/mapswipe_workers/tests/integration/set_up_db.sql index b03e76699..f954d3a8c 100644 --- a/mapswipe_workers/tests/integration/set_up_db.sql +++ b/mapswipe_workers/tests/integration/set_up_db.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(GEOMETRY, 4326), + geom geometry(Geometry, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), From 57dc509ee472a7db7c75ac5abf720db5a60704eb Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 11 Dec 2024 09:09:40 +0100 Subject: [PATCH 50/52] feat(django): add new project types to project type model --- django/apps/existing_database/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django/apps/existing_database/models.py b/django/apps/existing_database/models.py index ef65496b0..06fc81190 100644 --- a/django/apps/existing_database/models.py +++ b/django/apps/existing_database/models.py @@ -1,6 +1,7 @@ +from mapswipe.db import Model + from django.contrib.gis.db import models as gis_models from django.db import models -from mapswipe.db import Model # NOTE: Django model defination and existing table structure doesn't entirely matches. # This is to be used for testing only. @@ -66,6 +67,9 @@ class Type(models.IntegerChoices): FOOTPRINT = 2, "Validate" CHANGE_DETECTION = 3, "Compare" COMPLETENESS = 4, "Completeness" + MEDIA = 5, "Media" + DIGITIZATION = 6, "Digitization" + STREET = 7, "Street" project_id = models.CharField(primary_key=True, max_length=999) created = models.DateTimeField(blank=True, null=True) From 26c822c962cdf104b17f7edd154ddb9a6acad5b3 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 11 Dec 2024 09:56:12 +0100 Subject: [PATCH 51/52] fix: sort imports --- django/apps/existing_database/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django/apps/existing_database/models.py b/django/apps/existing_database/models.py index 06fc81190..5bc85e113 100644 --- a/django/apps/existing_database/models.py +++ b/django/apps/existing_database/models.py @@ -1,7 +1,6 @@ -from mapswipe.db import Model - from django.contrib.gis.db import models as gis_models from django.db import models +from mapswipe.db import Model # NOTE: Django model defination and existing table structure doesn't entirely matches. # This is to be used for testing only. From 4787dd4025ce6648261e90f234ae269aeac1b145 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 11 Dec 2024 10:18:05 +0100 Subject: [PATCH 52/52] fix: add new project types to schema.graphql --- django/schema.graphql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/django/schema.graphql b/django/schema.graphql index d0187f7f1..b5596fc46 100644 --- a/django/schema.graphql +++ b/django/schema.graphql @@ -97,6 +97,9 @@ enum ProjectTypeEnum { FOOTPRINT CHANGE_DETECTION COMPLETENESS + MEDIA + DIGITIZATION + STREET } type ProjectTypeSwipeStatsType {