diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index aacc4dcd3..4609eb697 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -80,6 +80,7 @@ jobs: POSTGRES_DB: postgres OSMCHA_API_KEY: ${{ secrets.OSMCHA_API_KEY }} DJANGO_SECRET_KEY: test-django-secret-key + MAPILLARY_API_KEY: ${{ secrets.MAPILLARY_API_KEY }} COMPOSE_FILE: ../docker-compose.yaml:../docker-compose-ci.yaml run: | docker compose run --rm mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/ diff --git a/django/apps/existing_database/models.py b/django/apps/existing_database/models.py index 0cf6582fb..5bc85e113 100644 --- a/django/apps/existing_database/models.py +++ b/django/apps/existing_database/models.py @@ -66,6 +66,9 @@ class Type(models.IntegerChoices): FOOTPRINT = 2, "Validate" CHANGE_DETECTION = 3, "Compare" COMPLETENESS = 4, "Completeness" + MEDIA = 5, "Media" + DIGITIZATION = 6, "Digitization" + STREET = 7, "Street" project_id = models.CharField(primary_key=True, max_length=999) created = models.DateTimeField(blank=True, null=True) @@ -127,7 +130,7 @@ class Task(Model): project = models.ForeignKey(Project, models.DO_NOTHING, related_name="+") group_id = models.CharField(max_length=999) task_id = models.CharField(max_length=999) - geom = gis_models.MultiPolygonField(blank=True, null=True) + geom = gis_models.GeometryField(blank=True, null=True) # Database uses JSON instead of JSONB (not supported by django) project_type_specifics = models.TextField(blank=True, null=True) diff --git a/django/schema.graphql b/django/schema.graphql index d0187f7f1..b5596fc46 100644 --- a/django/schema.graphql +++ b/django/schema.graphql @@ -97,6 +97,9 @@ enum ProjectTypeEnum { FOOTPRINT CHANGE_DETECTION COMPLETENESS + MEDIA + DIGITIZATION + STREET } type ProjectTypeSwipeStatsType { diff --git a/docker-compose.yaml b/docker-compose.yaml index ccbc2aa9d..d465a704c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -76,6 +76,7 @@ x-mapswipe-workers: &base_mapswipe_workers SLACK_CHANNEL: '${SLACK_CHANNEL}' SENTRY_DSN: '${SENTRY_DSN}' OSMCHA_API_KEY: '${OSMCHA_API_KEY}' + MAPILLARY_API_KEY: '${MAPILLARY_API_KEY}' depends_on: - postgres volumes: diff --git a/example.env b/example.env index f4e8c46a1..2fa92ae33 100644 --- a/example.env +++ b/example.env @@ -75,3 +75,6 @@ COMMUNITY_DASHBOARD_GRAPHQL_ENDPOINT=https://api.example.com/graphql/ COMMUNITY_DASHBOARD_SENTRY_DSN= COMMUNITY_DASHBOARD_SENTRY_TRACES_SAMPLE_RATE= COMMUNITY_DASHBOARD_MAPSWIPE_WEBSITE=https://mapswipe.org + +# Mapillary +MAPILLARY_API_KEY= \ No newline at end of file diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index bb8c7296f..c9dec6d79 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -16,6 +16,8 @@ OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/" OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] +MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" +MAPILLARY_API_KEY = os.environ["MAPILLARY_API_KEY"] # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 @@ -134,6 +136,7 @@ class ProjectType(Enum): COMPLETENESS = 4 MEDIA_CLASSIFICATION = 5 DIGITIZATION = 6 + STREET = 7 @property def constructor(self): @@ -145,6 +148,7 @@ def constructor(self): DigitizationProject, FootprintProject, MediaClassificationProject, + StreetProject, ) project_type_classes = { @@ -154,6 +158,7 @@ def constructor(self): 4: CompletenessProject, 5: MediaClassificationProject, 6: DigitizationProject, + 7: StreetProject, } return project_type_classes[self.value] diff --git a/mapswipe_workers/mapswipe_workers/project_types/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/__init__.py index a07ff38be..43013b0dc 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/__init__.py +++ b/mapswipe_workers/mapswipe_workers/project_types/__init__.py @@ -2,6 +2,7 @@ from .arbitrary_geometry.footprint.project import FootprintProject from .arbitrary_geometry.footprint.tutorial import FootprintTutorial from .media_classification.project import MediaClassificationProject +from .street.project import StreetProject from .tile_map_service.change_detection.project import ChangeDetectionProject from .tile_map_service.change_detection.tutorial import ChangeDetectionTutorial from .tile_map_service.classification.project import ClassificationProject @@ -20,4 +21,5 @@ "FootprintProject", "FootprintTutorial", "DigitizationProject", + "StreetProject", ] diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/street/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py new file mode 100644 index 000000000..e15f42c1f --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -0,0 +1,134 @@ +import math +from dataclasses import dataclass +from typing import Dict, List + +from mapswipe_workers.definitions import logger +from mapswipe_workers.firebase.firebase import Firebase +from mapswipe_workers.firebase_to_postgres.transfer_results import ( + results_to_file, + save_results_to_postgres, + truncate_temp_results, +) +from mapswipe_workers.generate_stats.project_stats import ( + get_statistics_for_integer_result_project, +) +from mapswipe_workers.project_types.project import BaseGroup, BaseProject, BaseTask +from mapswipe_workers.utils.process_mapillary import get_image_metadata +from mapswipe_workers.utils.validate_input import ( + build_multipolygon_from_layer_geometries, + check_if_layer_has_too_many_geometries, + check_if_layer_is_empty, + load_geojson_to_ogr, + multipolygon_to_wkt, + save_geojson_to_file, +) + + +@dataclass +class StreetGroup(BaseGroup): + # todo: does client use this, or only for the implementation of project creation? + pass + + +@dataclass +class StreetTask(BaseTask): + geometry: str + + +class StreetProject(BaseProject): + def __init__(self, project_draft): + super().__init__(project_draft) + self.groups: Dict[str, StreetGroup] = {} + self.tasks: Dict[str, List[StreetTask]] = {} + + self.geometry = project_draft["geometry"] + + # TODO: validate inputs + ImageMetadata = get_image_metadata( + self.geometry, + is_pano=project_draft.get("isPano", None), + start_time=project_draft.get("startTimestamp", None), + end_time=project_draft.get("endTimestamp", None), + organization_id=project_draft.get("organizationId", None), + sampling_threshold=project_draft.get("samplingThreshold", None), + ) + + self.imageIds = ImageMetadata["ids"] + self.imageGeometries = ImageMetadata["geometries"] + + def save_tasks_to_firebase(self, projectId: str, tasks: dict): + firebase = Firebase() + firebase.save_tasks_to_firebase(projectId, tasks, useCompression=False) + + @staticmethod + def results_to_postgres(results: dict, project_id: str, filter_mode: bool): + """How to move the result data from firebase to postgres.""" + results_file, user_group_results_file = results_to_file(results, project_id) + truncate_temp_results() + save_results_to_postgres(results_file, project_id, filter_mode) + return user_group_results_file + + @staticmethod + def get_per_project_statistics(project_id, project_info): + """How to aggregate the project results.""" + return get_statistics_for_integer_result_project( + project_id, project_info, generate_hot_tm_geometries=False + ) + + def validate_geometries(self): + self.inputGeometriesFileName = save_geojson_to_file( + self.projectId, self.geometry + ) + layer, datasource = load_geojson_to_ogr( + self.projectId, self.inputGeometriesFileName + ) + + # check if inputs fit constraints + check_if_layer_is_empty(self.projectId, layer) + + multi_polygon, project_area = build_multipolygon_from_layer_geometries( + self.projectId, layer + ) + + check_if_layer_has_too_many_geometries(self.projectId, multi_polygon) + + del datasource + del layer + + logger.info( + f"{self.projectId}" f" - validate geometry - " f"input geometry is correct." + ) + wkt_geometry = multipolygon_to_wkt(multi_polygon) + return wkt_geometry + + def create_groups(self): + self.numberOfGroups = math.ceil(len(self.imageIds) / self.groupSize) + for group_id in range(self.numberOfGroups): + self.groups[f"g{group_id}"] = StreetGroup( + projectId=self.projectId, + groupId=f"g{group_id}", + progress=0, + finishedCount=0, + requiredCount=0, + numberOfTasks=self.groupSize, + ) + + def create_tasks(self): + if len(self.groups) == 0: + raise ValueError("Groups needs to be created before tasks can be created.") + for group_id, group in self.groups.items(): + self.tasks[group_id] = [] + for i in range(self.groupSize): + task = StreetTask( + projectId=self.projectId, + groupId=group_id, + geometry=self.imageGeometries.pop(), + taskId=self.imageIds.pop(), + ) + self.tasks[group_id].append(task) + + # list now empty? if usual group size is not reached + # the actual number of tasks for the group is updated + if not self.imageIds: + group.numberOfTasks = i + 1 + break diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py new file mode 100644 index 000000000..cfbfc0ead --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py @@ -0,0 +1,14 @@ +from mapswipe_workers.project_types.tutorial import BaseTutorial + + +class StreetTutorial(BaseTutorial): + """The subclass for an TMS Grid based Tutorial.""" + + def save_tutorial(self): + raise NotImplementedError("Currently Street has no Tutorial") + + def create_tutorial_groups(self): + raise NotImplementedError("Currently Street has no Tutorial") + + def create_tutorial_tasks(self): + raise NotImplementedError("Currently Street has no Tutorial") diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py new file mode 100644 index 000000000..4b6943fdf --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -0,0 +1,259 @@ +import os +from concurrent.futures import ThreadPoolExecutor, as_completed + +import mercantile +import pandas as pd +import requests +from shapely import ( + LineString, + MultiLineString, + MultiPolygon, + Point, + Polygon, + box, + unary_union, +) +from shapely.geometry import shape +from vt2geojson import tools as vt2geojson_tools + +from mapswipe_workers.definitions import MAPILLARY_API_KEY, MAPILLARY_API_LINK, logger +from mapswipe_workers.utils.spatial_sampling import spatial_sampling + + +def create_tiles(polygon, level): + if not isinstance(polygon, (Polygon, MultiPolygon)): + return pd.DataFrame(columns=["x", "y", "z", "geometry"]) + if isinstance(polygon, Polygon): + polygon = MultiPolygon([polygon]) + + tiles = [] + for i, poly in enumerate(polygon.geoms): + tiles.extend(list(mercantile.tiles(*poly.bounds, level))) + + bbox_list = [mercantile.bounds(tile.x, tile.y, tile.z) for tile in tiles] + bbox_polygons = [box(*bbox) for bbox in bbox_list] + tiles = pd.DataFrame( + { + "x": [tile.x for tile in tiles], + "y": [tile.y for tile in tiles], + "z": [tile.z for tile in tiles], + "geometry": bbox_polygons, + } + ) + + return tiles + + +def download_and_process_tile(row, attempt_limit=3): + z = row["z"] + x = row["x"] + y = row["y"] + url = f"{MAPILLARY_API_LINK}{z}/{x}/{y}?access_token={MAPILLARY_API_KEY}" + + attempt = 0 + while attempt < attempt_limit: + try: + r = requests.get(url) + assert r.status_code == 200, r.content + features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get( + "features", [] + ) + data = [] + for feature in features: + geometry = feature.get("geometry", {}) + properties = feature.get("properties", {}) + geometry_type = geometry.get("type", None) + coordinates = geometry.get("coordinates", []) + + element_geometry = None + if geometry_type == "Point": + element_geometry = Point(coordinates) + elif geometry_type == "LineString": + element_geometry = LineString(coordinates) + elif geometry_type == "MultiLineString": + element_geometry = MultiLineString(coordinates) + elif geometry_type == "Polygon": + element_geometry = Polygon(coordinates[0]) + elif geometry_type == "MultiPolygon": + element_geometry = MultiPolygon(coordinates) + + # Append the dictionary with geometry and properties + row = {"geometry": element_geometry, **properties} + data.append(row) + + data = pd.DataFrame(data) + + if not data.empty: + return data + except Exception as e: + print(f"An exception occurred while requesting a tile: {e}") + attempt += 1 + + print(f"A tile could not be downloaded: {row}") + return None + + +def coordinate_download( + polygon, level, use_concurrency=True, attempt_limit=3, workers=os.cpu_count() * 4 +): + tiles = create_tiles(polygon, level) + + downloaded_metadata = [] + + if not tiles.empty: + if not use_concurrency: + workers = 1 + + futures = [] + with ThreadPoolExecutor(max_workers=workers) as executor: + for index, row in tiles.iterrows(): + futures.append( + executor.submit(download_and_process_tile, row, attempt_limit) + ) + + for future in as_completed(futures): + if future is not None: + df = future.result() + + if df is not None and not df.empty: + downloaded_metadata.append(df) + if len(downloaded_metadata): + downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) + else: + return pd.DataFrame(downloaded_metadata) + + target_columns = [ + "id", + "geometry", + "captured_at", + "is_pano", + "compass_angle", + "sequence", + "organization_id", + ] + for col in target_columns: + if col not in downloaded_metadata.columns: + downloaded_metadata[col] = None + + if ( + downloaded_metadata.isna().all().all() is False + or downloaded_metadata.empty is True + ): + downloaded_metadata = downloaded_metadata[ + downloaded_metadata["geometry"].apply( + lambda point: point.within(polygon) + ) + ] + + return downloaded_metadata + + +def geojson_to_polygon(geojson_data): + if geojson_data["type"] == "FeatureCollection": + features = geojson_data["features"] + elif geojson_data["type"] == "Feature": + features = [geojson_data] + else: + raise ValueError("Unsupported GeoJSON type.") + + polygons = [] + for feature in features: + geometry = shape(feature["geometry"]) + if isinstance(geometry, (Polygon, MultiPolygon)): + polygons.append(geometry) + else: + raise ValueError( + "Non-polygon geometries cannot be combined into a MultiPolygon." + ) + + combined_multipolygon = unary_union(polygons) + + return combined_multipolygon + + +def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str = None): + df["captured_at"] = pd.to_datetime(df["captured_at"], unit="ms") + start_time = pd.to_datetime(start_time).tz_localize(None) + if end_time is None: + end_time = pd.Timestamp.now().tz_localize(None) + else: + end_time = pd.to_datetime(end_time).tz_localize(None) + filtered_df = df[ + (df["captured_at"] >= start_time) & (df["captured_at"] <= end_time) + ] + return filtered_df + + +def filter_results( + results_df: pd.DataFrame, + is_pano: bool = None, + organization_id: str = None, + start_time: str = None, + end_time: str = None, +): + df = results_df.copy() + if is_pano is not None: + if df["is_pano"].isna().all(): + logger.exception("No Mapillary Feature in the AoI has a 'is_pano' value.") + return None + df = df[df["is_pano"] == is_pano] + + if organization_id is not None: + if df["organization_id"].isna().all(): + logger.exception( + "No Mapillary Feature in the AoI has an 'organization_id' value." + ) + return None + df = df[df["organization_id"] == organization_id] + + if start_time is not None: + if df["captured_at"].isna().all(): + logger.exception( + "No Mapillary Feature in the AoI has a 'captured_at' value." + ) + return None + df = filter_by_timerange(df, start_time, end_time) + + return df + + +def get_image_metadata( + aoi_geojson, + level=14, + attempt_limit=3, + is_pano: bool = None, + organization_id: str = None, + start_time: str = None, + end_time: str = None, + sampling_threshold=None, +): + aoi_polygon = geojson_to_polygon(aoi_geojson) + downloaded_metadata = coordinate_download(aoi_polygon, level, attempt_limit) + if downloaded_metadata.isna().all().all() or downloaded_metadata.empty: + raise ValueError("No Mapillary Features in the AoI.") + downloaded_metadata = downloaded_metadata[ + downloaded_metadata["geometry"].apply(lambda geom: isinstance(geom, Point)) + ] + + downloaded_metadata = filter_results( + downloaded_metadata, is_pano, organization_id, start_time, end_time + ) + if sampling_threshold is not None: + downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) + if ( + downloaded_metadata.isna().all().all() is False + or downloaded_metadata.empty is False + ): + if len(downloaded_metadata) > 100000: + err = ( + f"Too many Images with selected filter " + f"options for the AoI: {len(downloaded_metadata)}" + ) + raise ValueError(err) + else: + return { + "ids": downloaded_metadata["id"].tolist(), + "geometries": downloaded_metadata["geometry"].tolist(), + } + else: + raise ValueError("No Mapillary Features in the AoI match the filter criteria.") diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py new file mode 100644 index 000000000..3cafb5362 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -0,0 +1,149 @@ +import numpy as np +import pandas as pd + + +def distance_on_sphere(p1, p2): + """ + p1 and p2 are two lists that have two elements. They are numpy arrays of the long + and lat coordinates of the points in set1 and set2 + + Calculate the distance between two points on the Earth's surface using the + haversine formula. + + Args: + p1 (list): Array containing the longitude and latitude coordinates of points + FROM which the distance to be calculated in degree + p2 (list): Array containing the longitude and latitude coordinates of points + TO which the distance to be calculated in degree + + Returns: + numpy.ndarray: Array containing the distances between the two points on the + sphere in kilometers. + + This function computes the distance between two points on the Earth's surface + using the haversine formula, which takes into account the spherical shape of the + Earth. The input arrays `p1` and `p2` should contain longitude and latitude + coordinates in degrees. The function returns an array containing the distances + between corresponding pairs of points. + """ + earth_radius = 6371 # km + + p1 = np.radians(np.array(p1)) + p2 = np.radians(np.array(p2)) + + delta_lat = p2[1] - p1[1] + delta_long = p2[0] - p1[0] + + a = ( + np.sin(delta_lat / 2) ** 2 + + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(delta_long / 2) ** 2 + ) + c = 2 * np.arcsin(np.sqrt(a)) + + distances = earth_radius * c + return distances + + +"""----------------------------Filtering Points-------------------------------""" + + +def filter_points(df, threshold_distance): + """ + Filter points from a DataFrame based on a threshold distance. + + Args: + df (pandas.DataFrame): DataFrame containing latitude and longitude columns. + threshold_distance (float): Threshold distance for filtering points in kms. + + Returns: + pandas.DataFrame: Filtered DataFrame containing selected points. + float: Total road length calculated from the selected points. + + This function filters points from a DataFrame based on the given threshold + distance. It calculates distances between consecutive points and accumulates them + until the accumulated distance surpasses the threshold distance. It then selects + those points and constructs a new DataFrame. Additionally, it manually checks the + last point to include it if it satisfies the length condition. The function + returns the filtered DataFrame along with the calculated road length. + """ + road_length = 0 + mask = np.zeros(len(df), dtype=bool) + mask[0] = True + lat = df["lat"].to_numpy() + long = df["long"].to_numpy() + + distances = distance_on_sphere([long[1:], lat[1:]], [long[:-1], lat[:-1]]) + road_length = np.sum(distances) + + # save the last point if the road segment is relavitely small (< 2*road_length) + if threshold_distance <= road_length < 2 * threshold_distance: + mask[-1] = True + + accumulated_distance = 0 + for i, distance in enumerate(distances): + accumulated_distance += distance + if accumulated_distance >= threshold_distance: + mask[i + 1] = True + accumulated_distance = 0 # Reset accumulated distance + + to_be_returned_df = df[mask] + # since the last point has to be omitted in the vectorized distance calculation, + # it is being checked manually + p2 = to_be_returned_df.iloc[0] + distance = distance_on_sphere( + [float(p2["long"]), float(p2["lat"])], [long[-1], lat[-1]] + ) + + # last point will be added if it suffices the length condition + # last point will be added in case there is only one point returned + if distance >= threshold_distance or len(to_be_returned_df) == 1: + to_be_returned_df = pd.concat( + [ + to_be_returned_df, + pd.DataFrame(df.iloc[-1], columns=to_be_returned_df.columns), + ], + axis=0, + ) + return to_be_returned_df + + +def spatial_sampling(df, interval_length): + """ + Calculate spacing between points in a GeoDataFrame. + + Args: + df (pandas.DataFrame): DataFrame containing points with timestamps. + interval_length (float): Interval length for filtering points in kms. + + Returns: + geopandas.GeoDataFrame: Filtered GeoDataFrame containing selected points. + float: Total road length calculated from the selected points. + + This function calculates the spacing between points in a GeoDataFrame by filtering + points based on the provided interval length. It first sorts the GeoDataFrame by + timestamp and then filters points using the filter_points function. The function + returns the filtered GeoDataFrame along with the total road length. + """ + if len(df) == 1: + return df + + df["long"] = df["geometry"].apply( + lambda geom: geom.x if geom.geom_type == "Point" else None + ) + df["lat"] = df["geometry"].apply( + lambda geom: geom.y if geom.geom_type == "Point" else None + ) + sorted_df = df.sort_values(by=["captured_at"]) + + sampled_sequence_df = pd.DataFrame() + + # loop through each sequence + for sequence in sorted_df["sequence_id"].unique(): + sequence_df = sorted_df[sorted_df["sequence_id"] == sequence] + + filtered_sorted_sub_df = filter_points(sequence_df, interval_length) + sampled_sequence_df = pd.concat( + [sampled_sequence_df, filtered_sorted_sub_df], axis=0 + ) + + return sampled_sequence_df diff --git a/mapswipe_workers/requirements.txt b/mapswipe_workers/requirements.txt index 7e125fe4b..588754060 100644 --- a/mapswipe_workers/requirements.txt +++ b/mapswipe_workers/requirements.txt @@ -14,3 +14,6 @@ sentry-sdk==0.18.0 six==1.15.0 slackclient==2.9.2 xdg==4.0.1 +shapely +mercantile +vt2geojson diff --git a/mapswipe_workers/tests/fixtures/mapillary_response.csv b/mapswipe_workers/tests/fixtures/mapillary_response.csv new file mode 100644 index 000000000..2988a1dc4 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/mapillary_response.csv @@ -0,0 +1,8 @@ +geometry,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id +POINT (38.995129466056824 -6.785243670271996),1453463352000,102506575322825,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ,1 +POINT (38.99839103221893 -6.7866606090858),1679465543298,118200124520512,603013591724120,,False,104.904435758052,Tj9u08PcRnQEAU13yeNhkr +POINT (39.000306129455566 -6.787576822940906),1679465564715,118200124520512,547943597327117,,False,103.540136343809,Tj9u08PcRnQEAU13yeNhkr +POINT (38.9906769990921 -6.783315348346505),1453463400000,102506575322825,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ +POINT (38.99797797203064 -6.786490150501777),1679465534081,118200124520512,3271824289814895,,False,112.637054443359,Tj9u08PcRnQEAU13yeNhkr +POINT (39.00127172470093 -6.787981661065601),1453463294000,102506575322825,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ + diff --git a/mapswipe_workers/tests/fixtures/mapillary_sequence.csv b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv new file mode 100644 index 000000000..597fa6f66 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv @@ -0,0 +1,71 @@ +,geometry,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id +9,POINT (38.995129466056824 -6.785243670271996),1453463352000.0,102506575322825.0,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ, +12,POINT (38.9906769990921 -6.783315348346505),1453463400000.0,102506575322825.0,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ, +14,POINT (39.00127172470093 -6.787981661065601),1453463294000.0,102506575322825.0,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ, +18,POINT (38.99769365787506 -6.786351652857817),1453463332000.0,102506575322825.0,1014398349364928,,True,288.49790876724,ywMkSP_5PaJzcbIDa5v1aQ, +19,POINT (38.99540305137634 -6.785360860858361),1453463350000.0,102506575322825.0,165035685525369,,True,293.57188365181,ywMkSP_5PaJzcbIDa5v1aQ, +20,POINT (39.00015592575073 -6.787491593818643),1453463300000.0,102506575322825.0,1139870886511452,,True,305.58569254109,ywMkSP_5PaJzcbIDa5v1aQ, +21,POINT (38.9979350566864 -6.7864262285172),1453463330000.0,102506575322825.0,921248855101166,,True,287.76018227153,ywMkSP_5PaJzcbIDa5v1aQ, +22,POINT (38.99890601634979 -6.786804433469129),1453463318000.0,102506575322825.0,233272625257058,,True,295.57372041131,ywMkSP_5PaJzcbIDa5v1aQ, +23,POINT (38.99618625640869 -6.7857071056060505),1453463344000.0,102506575322825.0,762184237820558,,True,293.88722229187,ywMkSP_5PaJzcbIDa5v1aQ, +26,POINT (38.98969531059265 -6.782905179427416),1453463424000.0,102506575322825.0,1099193250486391,,True,292.87773588969,ywMkSP_5PaJzcbIDa5v1aQ, +27,POINT (38.998627066612244 -6.786671262745287),1453463322000.0,102506575322825.0,831521697449190,,True,296.54786608582,ywMkSP_5PaJzcbIDa5v1aQ, +29,POINT (38.992629647254944 -6.784167646282242),1453463368000.0,102506575322825.0,151617226911336,,True,292.93390738544,ywMkSP_5PaJzcbIDa5v1aQ, +30,POINT (39.00003254413605 -6.7873584232849),1453463302000.0,102506575322825.0,1773698542801178,,True,296.75481871036,ywMkSP_5PaJzcbIDa5v1aQ, +32,POINT (39.000563621520996 -6.787811202949342),1453463298000.0,102506575322825.0,164116422292333,,True,305.80313236719,ywMkSP_5PaJzcbIDa5v1aQ, +37,POINT (38.9902800321579 -6.783144888578377),1453463410000.0,102506575322825.0,329957465177767,,True,293.87481737883,ywMkSP_5PaJzcbIDa5v1aQ, +38,POINT (38.99117052555084 -6.783539076700606),1453463388000.0,102506575322825.0,2948533428757015,,True,294.44274652914,ywMkSP_5PaJzcbIDa5v1aQ, +39,POINT (38.99877190589905 -6.78674051152629),1453463320000.0,102506575322825.0,479325670049740,,True,296.24340777377,ywMkSP_5PaJzcbIDa5v1aQ, +41,POINT (38.990601897239685 -6.783294040878786),1453463402000.0,102506575322825.0,501560441022559,,True,289.98678899102,ywMkSP_5PaJzcbIDa5v1aQ, +42,POINT (38.989362716674805 -6.7828785450699485),1453463432000.0,102506575322825.0,494436578424418,,True,249.25945175736,ywMkSP_5PaJzcbIDa5v1aQ, +44,POINT (38.994566202163696 -6.7850039621655895),1453463356000.0,102506575322825.0,2928848347373461,,True,295.93075138027,ywMkSP_5PaJzcbIDa5v1aQ, +45,POINT (38.993815183639526 -6.784657716912335),1453463362000.0,102506575322825.0,167884815220625,,True,290.65289338004,ywMkSP_5PaJzcbIDa5v1aQ, +47,POINT (38.991841077804565 -6.783837381011111),1453463380000.0,102506575322825.0,2783373888590755,,True,292.94668882511,ywMkSP_5PaJzcbIDa5v1aQ, +48,POINT (38.99052679538727 -6.783262079675453),1453463404000.0,102506575322825.0,500930794384261,,True,296.09431621523,ywMkSP_5PaJzcbIDa5v1aQ, +49,POINT (38.9897757768631 -6.782937140654425),1453463422000.0,102506575322825.0,473363863989539,,True,292.22088072734,ywMkSP_5PaJzcbIDa5v1aQ, +50,POINT (38.99429798126221 -6.784870790943785),1453463358000.0,102506575322825.0,792308461667709,,True,295.36479453372,ywMkSP_5PaJzcbIDa5v1aQ, +51,POINT (38.9997535943985 -6.787219925890724),1453463306000.0,102506575322825.0,1169832606865116,,True,296.51874301031,ywMkSP_5PaJzcbIDa5v1aQ, +54,POINT (38.992860317230225 -6.784268856561923),1453463364000.0,102506575322825.0,143904254368287,,True,292.87299883627,ywMkSP_5PaJzcbIDa5v1aQ, +57,POINT (38.98994743824005 -6.783006389972371),1453463418000.0,102506575322825.0,512708183243254,,True,292.58758044439,ywMkSP_5PaJzcbIDa5v1aQ, +58,POINT (38.99670124053955 -6.785941486524692),1453463340000.0,102506575322825.0,168474601828325,,True,294.32908734047,ywMkSP_5PaJzcbIDa5v1aQ, +59,POINT (38.992136120796204 -6.783959898799395),1453463376000.0,102506575322825.0,171815874817246,,True,292.47687051975,ywMkSP_5PaJzcbIDa5v1aQ, +60,POINT (38.99090766906738 -6.783405905073778),1453463394000.0,102506575322825.0,475809606904698,,True,297.28158189053,ywMkSP_5PaJzcbIDa5v1aQ, +61,POINT (38.99251699447632 -6.7841250314212544),1453463370000.0,102506575322825.0,798930200741228,,True,292.63573224675,ywMkSP_5PaJzcbIDa5v1aQ, +62,POINT (38.99019956588745 -6.7831129273651385),1453463412000.0,102506575322825.0,989719805170705,,True,292.67075887406,ywMkSP_5PaJzcbIDa5v1aQ, +63,POINT (38.991336822509766 -6.783613652795566),1453463386000.0,102506575322825.0,887351401825944,,True,294.11111203184,ywMkSP_5PaJzcbIDa5v1aQ, +64,POINT (38.99745762348175 -6.786271750352796),1453463334000.0,102506575322825.0,820185135568044,,True,292.38261450394,ywMkSP_5PaJzcbIDa5v1aQ, +69,POINT (38.99919033050537 -6.7869376041561225),1453463314000.0,102506575322825.0,1401323273568889,,True,296.07037190303,ywMkSP_5PaJzcbIDa5v1aQ, +70,POINT (38.99229168891907 -6.784023821111347),1453463374000.0,102506575322825.0,971311816939999,,True,293.93020293096,ywMkSP_5PaJzcbIDa5v1aQ, +71,POINT (38.9956659078598 -6.785483378259087),1453463348000.0,102506575322825.0,1888667171315269,,True,294.14026651816,ywMkSP_5PaJzcbIDa5v1aQ, +77,POINT (38.99095058441162 -6.783437866267576),1453463392000.0,102506575322825.0,313013763568992,,True,299.51413646126,ywMkSP_5PaJzcbIDa5v1aQ, +78,POINT (38.99240434169769 -6.784077089698172),1453463372000.0,102506575322825.0,300316491494684,,True,293.91812405268,ywMkSP_5PaJzcbIDa5v1aQ, +79,POINT (38.994094133377075 -6.784774907641307),1453463360000.0,102506575322825.0,799986720927690,,True,293.50534124709,ywMkSP_5PaJzcbIDa5v1aQ, +80,POINT (38.998122811317444 -6.786495477333432),1453463328000.0,102506575322825.0,766866250678914,,True,291.66896324881,ywMkSP_5PaJzcbIDa5v1aQ, +82,POINT (38.99715185165405 -6.786143906317193),1453463336000.0,102506575322825.0,4104687166260249,,True,294.20932475917,ywMkSP_5PaJzcbIDa5v1aQ, +84,POINT (38.99081647396088 -6.783368617011661),1453463396000.0,102506575322825.0,311476673697450,,True,292.1387426333,ywMkSP_5PaJzcbIDa5v1aQ, +85,POINT (38.999614119529724 -6.787150677178687),1453463308000.0,102506575322825.0,486545769438431,,True,296.75245798459,ywMkSP_5PaJzcbIDa5v1aQ, +86,POINT (38.99830520153046 -6.786516784659511),1453463326000.0,102506575322825.0,287818266220456,,True,296.90157475174,ywMkSP_5PaJzcbIDa5v1aQ, +89,POINT (38.991073966026306 -6.78349646178404),1453463390000.0,102506575322825.0,1104648166685558,,True,294.64381621501,ywMkSP_5PaJzcbIDa5v1aQ, +95,POINT (38.99045169353485 -6.783224791602194),1453463406000.0,102506575322825.0,1189096484862313,,True,296.52075233925,ywMkSP_5PaJzcbIDa5v1aQ, +96,POINT (38.989604115486145 -6.782867891326546),1453463426000.0,102506575322825.0,149492403718498,,True,291.35167681135,ywMkSP_5PaJzcbIDa5v1aQ, +99,POINT (38.990371227264404 -6.783187503526065),1453463408000.0,102506575322825.0,1891398304374063,,True,293.92928705821,ywMkSP_5PaJzcbIDa5v1aQ, +102,POINT (38.99695873260498 -6.786058676941238),1453463338000.0,102506575322825.0,823059255257426,,True,294.12835748694,ywMkSP_5PaJzcbIDa5v1aQ, +105,POINT (38.99198055267334 -6.7838959764789735),1453463378000.0,102506575322825.0,3696263247264941,,True,293.15037639266,ywMkSP_5PaJzcbIDa5v1aQ, +108,POINT (38.989861607551575 -6.782974428749938),1453463420000.0,102506575322825.0,862036401042035,,True,291.44770311405,ywMkSP_5PaJzcbIDa5v1aQ, +109,POINT (38.98951292037964 -6.782841256967004),1453463428000.0,102506575322825.0,304865891166877,,True,272.6464361279,ywMkSP_5PaJzcbIDa5v1aQ, +111,POINT (38.99274230003357 -6.784220914853137),1453463366000.0,102506575322825.0,794241458144562,,True,293.2506353664,ywMkSP_5PaJzcbIDa5v1aQ, +113,POINT (38.99847149848938 -6.786596687123861),1453463324000.0,102506575322825.0,939866586773160,,True,295.61868532377,ywMkSP_5PaJzcbIDa5v1aQ, +115,POINT (38.99947464466095 -6.787081428456702),1453463310000.0,102506575322825.0,1143620816058871,,True,296.46351643457,ywMkSP_5PaJzcbIDa5v1aQ, +116,POINT (38.99003326892853 -6.783043678062526),1453463416000.0,102506575322825.0,1405161323178994,,True,293.46012249789,ywMkSP_5PaJzcbIDa5v1aQ, +119,POINT (39.00104105472565 -6.78785914430064),1453463296000.0,102506575322825.0,1273957886333326,,True,279.89824953791,ywMkSP_5PaJzcbIDa5v1aQ, +120,POINT (38.98943245410919 -6.782857237582903),1453463430000.0,102506575322825.0,1171907646606759,,True,252.08147683237,ywMkSP_5PaJzcbIDa5v1aQ, +121,POINT (38.99148166179657 -6.783677575153462),1453463384000.0,102506575322825.0,164230852280713,,True,293.89449177018,ywMkSP_5PaJzcbIDa5v1aQ, +122,POINT (38.999335169792175 -6.787012179724755),1453463312000.0,102506575322825.0,510570906624460,,True,297.86598575521,ywMkSP_5PaJzcbIDa5v1aQ, +124,POINT (38.99592339992523 -6.785595241945558),1453463346000.0,102506575322825.0,136548521821574,,True,293.97664288588,ywMkSP_5PaJzcbIDa5v1aQ, +125,POINT (38.990119099617004 -6.783075639280355),1453463414000.0,102506575322825.0,1112069379291585,,True,293.19285658924,ywMkSP_5PaJzcbIDa5v1aQ, +126,POINT (38.99644374847412 -6.785829622918669),1453463342000.0,102506575322825.0,846432512639247,,True,294.78265773594,ywMkSP_5PaJzcbIDa5v1aQ, +128,POINT (38.99165332317352 -6.783752151226963),1453463382000.0,102506575322825.0,133891088754131,,True,294.07658010782,ywMkSP_5PaJzcbIDa5v1aQ, +129,POINT (38.99078965187073 -6.7833579632791015),1453463398000.0,102506575322825.0,211793933759059,,True,294.17362400652,ywMkSP_5PaJzcbIDa5v1aQ, +131,POINT (38.99904549121857 -6.786873682230961),1453463316000.0,102506575322825.0,137437058395340,,True,295.32783278431,ywMkSP_5PaJzcbIDa5v1aQ, +132,POINT (38.99989306926727 -6.787289174592786),1453463304000.0,102506575322825.0,323401522460093,,True,296.4023335068,ywMkSP_5PaJzcbIDa5v1aQ, +133,POINT (38.994882702827454 -6.785147787043741),1453463354000.0,102506575322825.0,1070660863461711,,True,294.24600460049,ywMkSP_5PaJzcbIDa5v1aQ, diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json new file mode 100644 index 000000000..1dd5b452a --- /dev/null +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -0,0 +1,51 @@ +{ + "createdBy": "test", + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 39.27186980415655, + -6.818313681620424 + ], + [ + 39.27186980415655, + -6.824056026803248 + ], + [ + 39.27489288297136, + -6.823996705403303 + ], + [ + 39.27483313833096, + -6.817969613314901 + ], + [ + 39.27186980415655, + -6.818313681620424 + ] + ] + ] + } + } + ] + }, + "image": "", + "lookFor": "buildings", + "name": "test - Dar es Salaam (1)\ntest", + "projectDetails": "test", + "projectNumber": 1, + "projectTopic": "test", + "projectType": 7, + "requestingOrganisation": "test", + "verificationNumber": 3, + "groupSize": 25, + "startTimestamp": "2019-07-01T00:00:00.000Z", + "samplingThreshold": 0.1 +} diff --git a/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json new file mode 100644 index 000000000..f945f1661 --- /dev/null +++ b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json @@ -0,0 +1,50 @@ +{ + "createdBy": "test", + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 39.27186980415655, + -6.818313681620424 + ], + [ + 39.27186980415655, + -6.824056026803248 + ], + [ + 39.27489288297136, + -6.823996705403303 + ], + [ + 39.27483313833096, + -6.817969613314901 + ], + [ + 39.27186980415655, + -6.818313681620424 + ] + ] + ] + } + } + ] + }, + "image": "", + "lookFor": "buildings", + "name": "test - Dar es Salaam (1)\ntest", + "projectDetails": "test", + "projectNumber": 1, + "projectTopic": "test", + "projectType": 7, + "requestingOrganisation": "test", + "verificationNumber": 3, + "groupSize": 25, + "customOptions": [{ "color": "", "label": "", "value": -999 }, { "color": "#008000", "label": "yes", "value": 1 }, { "color": "#FF0000", "label": "no", "value": 2 }, { "color": "#FFA500", "label": "maybe", "value": 3 }] +} diff --git a/mapswipe_workers/tests/integration/set_up_db.sql b/mapswipe_workers/tests/integration/set_up_db.sql index ce9b97197..f954d3a8c 100644 --- a/mapswipe_workers/tests/integration/set_up_db.sql +++ b/mapswipe_workers/tests/integration/set_up_db.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON, 4326), + geom geometry(Geometry, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), diff --git a/mapswipe_workers/tests/integration/test_create_street_project.py b/mapswipe_workers/tests/integration/test_create_street_project.py new file mode 100644 index 000000000..fd0608f98 --- /dev/null +++ b/mapswipe_workers/tests/integration/test_create_street_project.py @@ -0,0 +1,61 @@ +import unittest + +from click.testing import CliRunner + +from mapswipe_workers import auth, mapswipe_workers +from mapswipe_workers.definitions import logger +from mapswipe_workers.utils.create_directories import create_directories +from tests.integration import set_up, tear_down + + +class TestCreateStreetProject(unittest.TestCase): + def setUp(self): + self.project_id = [ + set_up.create_test_project_draft("street", "street"), + ] + create_directories() + + def tearDown(self): + for element in self.project_id: + tear_down.delete_test_data(element) + + def test_create_street_project(self): + runner = CliRunner() + result = runner.invoke( + mapswipe_workers.run_create_projects, catch_exceptions=False + ) + if result.exit_code != 0: + raise result.exception + pg_db = auth.postgresDB() + for element in self.project_id: + logger.info(f"Checking project {self.project_id}") + query = "SELECT project_id FROM projects WHERE project_id = %s" + result = pg_db.retr_query(query, [element])[0][0] + self.assertEqual(result, element) + + # check if tasks made it to postgres + query = """ + SELECT count(*) + FROM tasks + WHERE project_id = %s + """ + result = pg_db.retr_query(query, [element])[0][0] + self.assertGreater(result, 0) + + fb_db = auth.firebaseDB() + ref = fb_db.reference(f"/v2/projects/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + ref = fb_db.reference(f"/v2/groups/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + # Street projects have tasks in Firebase + ref = fb_db.reference(f"/v2/tasks/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py new file mode 100644 index 000000000..8134ad2bf --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -0,0 +1,330 @@ +import json +import os +import unittest +from unittest.mock import MagicMock, patch + +import pandas as pd +from shapely import wkt +from shapely.geometry import GeometryCollection, MultiPolygon, Polygon + +from mapswipe_workers.utils.process_mapillary import ( + coordinate_download, + create_tiles, + download_and_process_tile, + filter_by_timerange, + filter_results, + geojson_to_polygon, + get_image_metadata, +) + + +class TestTileGroupingFunctions(unittest.TestCase): + @classmethod + def setUpClass(cls): + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "feature_collection.json", + ), + "r", + ) as file: + cls.fixture_data = json.load(file) + + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_response.csv", + ), + "r", + ) as file: + df = pd.read_csv(file) + df["geometry"] = df["geometry"].apply(wkt.loads) + cls.fixture_df = df + + def setUp(self): + self.level = 14 + self.test_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) + self.test_multipolygon = MultiPolygon( + [self.test_polygon, Polygon([(2, 2), (3, 2), (3, 3), (2, 3)])] + ) + self.empty_polygon = Polygon() + self.empty_geometry = GeometryCollection() + + def test_create_tiles_with_valid_polygon(self): + tiles = create_tiles(self.test_polygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertFalse(tiles.empty) + + def test_create_tiles_with_multipolygon(self): + tiles = create_tiles(self.test_multipolygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertFalse(tiles.empty) + + def test_create_tiles_with_empty_polygon(self): + tiles = create_tiles(self.empty_polygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertTrue(tiles.empty) + + def test_create_tiles_with_empty_geometry(self): + tiles = create_tiles(self.empty_geometry, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertTrue(tiles.empty) + + def test_geojson_to_polygon_feature_collection_with_multiple_polygons(self): + geojson_data = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + }, + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]], + }, + }, + ], + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, MultiPolygon) + self.assertEqual(len(result.geoms), 2) + + def test_geojson_to_polygon_single_feature_polygon(self): + geojson_data = { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + }, + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, Polygon) + + def test_geojson_to_polygon_single_feature_multipolygon(self): + geojson_data = { + "type": "Feature", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]], + ], + }, + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, MultiPolygon) + self.assertEqual(len(result.geoms), 2) + + def test_geojson_to_polygon_non_polygon_geometry_in_feature_collection(self): + geojson_data = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "LineString", "coordinates": [(0, 0), (1, 1)]}, + } + ], + } + with self.assertRaises(ValueError) as context: + geojson_to_polygon(geojson_data) + self.assertEqual( + str(context.exception), + "Non-polygon geometries cannot be combined into a MultiPolygon.", + ) + + def test_geojson_to_polygon_empty_feature_collection(self): + geojson_data = {"type": "FeatureCollection", "features": []} + result = geojson_to_polygon(geojson_data) + self.assertTrue(result.is_empty) + + def test_geojson_to_polygon_contribution_geojson(self): + result = geojson_to_polygon(self.fixture_data) + self.assertIsInstance(result, Polygon) + + @patch( + "mapswipe_workers.utils.process_mapillary.vt2geojson_tools.vt_bytes_to_geojson" + ) + @patch("mapswipe_workers.utils.process_mapillary.requests.get") + def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.content = b"mock vector tile data" # Example mock data + mock_get.return_value = mock_response + + mock_vt2geojson.return_value = { + "features": [ + { + "geometry": {"type": "Point", "coordinates": [0, 0]}, + "properties": {"id": 1}, + } + ] + } + + row = {"x": 1, "y": 1, "z": 14} + + result = download_and_process_tile(row) + + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 1) + self.assertEqual(result["geometry"][0].wkt, "POINT (0 0)") + + @patch("mapswipe_workers.utils.process_mapillary.requests.get") + def test_download_and_process_tile_failure(self, mock_get): + # Mock a failed response + mock_response = MagicMock() + mock_response.status_code = 500 + mock_get.return_value = mock_response + + row = pd.Series({"x": 1, "y": 1, "z": self.level}) + result = download_and_process_tile(row) + + self.assertIsNone(result) + + @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") + def test_coordinate_download(self, mock_download_and_process_tile): + mock_download_and_process_tile.return_value = pd.DataFrame([{"geometry": None}]) + + metadata = coordinate_download(self.test_polygon, self.level) + + self.assertIsInstance(metadata, pd.DataFrame) + + @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") + def test_coordinate_download_with_failures(self, mock_download_and_process_tile): + mock_download_and_process_tile.return_value = pd.DataFrame() + + metadata = coordinate_download(self.test_polygon, self.level) + + self.assertTrue(metadata.empty) + + def test_filter_within_time_range(self): + start_time = "2016-01-20 00:00:00" + end_time = "2022-01-21 23:59:59" + filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) + + self.assertEqual(len(filtered_df), 3) + self.assertTrue(all(filtered_df["captured_at"] >= pd.to_datetime(start_time))) + self.assertTrue(all(filtered_df["captured_at"] <= pd.to_datetime(end_time))) + + def test_filter_without_end_time(self): + start_time = "2020-01-20 00:00:00" + filtered_df = filter_by_timerange(self.fixture_df, start_time) + + self.assertEqual(len(filtered_df), 3) + self.assertTrue(all(filtered_df["captured_at"] >= pd.to_datetime(start_time))) + + def test_filter_time_no_data(self): + start_time = "2016-01-30 00:00:00" + end_time = "2016-01-31 00:00:00" + filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) + self.assertTrue(filtered_df.empty) + + def test_filter_default(self): + filtered_df = filter_results(self.fixture_df) + self.assertTrue(len(filtered_df) == len(self.fixture_df)) + + def test_filter_pano_true(self): + filtered_df = filter_results(self.fixture_df, is_pano=True) + self.assertEqual(len(filtered_df), 3) + + def test_filter_pano_false(self): + filtered_df = filter_results(self.fixture_df, is_pano=False) + self.assertEqual(len(filtered_df), 3) + + def test_filter_organization_id(self): + filtered_df = filter_results(self.fixture_df, organization_id=1) + self.assertEqual(len(filtered_df), 1) + + def test_filter_time_range(self): + start_time = "2016-01-20 00:00:00" + end_time = "2022-01-21 23:59:59" + filtered_df = filter_results( + self.fixture_df, start_time=start_time, end_time=end_time + ) + self.assertEqual(len(filtered_df), 3) + + def test_filter_no_rows_after_filter(self): + filtered_df = filter_results(self.fixture_df, is_pano="False") + self.assertTrue(filtered_df.empty) + + def test_filter_missing_columns(self): + columns_to_check = [ + "is_pano", + "organization_id", + "captured_at", + ] # Add your column names here + for column in columns_to_check: + df_copy = self.fixture_df.copy() + df_copy[column] = None + if column == "captured_at": + column = "start_time" + + result = filter_results(df_copy, **{column: True}) + self.assertIsNone(result) + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata(self, mock_coordinate_download): + mock_coordinate_download.return_value = self.fixture_df + result = get_image_metadata(self.fixture_data) + self.assertIsInstance(result, dict) + self.assertIn("ids", result) + self.assertIn("geometries", result) + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_filtering(self, mock_coordinate_download): + mock_coordinate_download.return_value = self.fixture_df + + params = { + "is_pano": True, + "start_time": "2016-01-20 00:00:00", + "end_time": "2022-01-21 23:59:59", + } + + result = get_image_metadata(self.fixture_data, **params) + self.assertIsInstance(result, dict) + self.assertIn("ids", result) + self.assertIn("geometries", result) + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_no_rows(self, mock_coordinate_download): + mock_coordinate_download.return_value = self.fixture_df + + params = { + "is_pano": True, + "start_time": "1916-01-20 00:00:00", + "end_time": "1922-01-21 23:59:59", + } + with self.assertRaises(ValueError): + get_image_metadata(self.fixture_data, **params) + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_empty_response(self, mock_coordinate_download): + df = self.fixture_df.copy() + df = df.drop(df.index) + mock_coordinate_download.return_value = df + + with self.assertRaises(ValueError): + get_image_metadata(self.fixture_data) + + @patch("mapswipe_workers.utils.process_mapillary.filter_results") + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_size_restriction( + self, mock_coordinate_download, mock_filter_results + ): + mock_filter_results.return_value = pd.DataFrame({"ID": range(1, 100002)}) + mock_coordinate_download.return_value = self.fixture_df + + with self.assertRaises(ValueError): + get_image_metadata(self.fixture_data) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py new file mode 100644 index 000000000..8ec8d0fa4 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -0,0 +1,55 @@ +import json +import os +import unittest +from unittest.mock import patch +from shapely import wkt +import pandas as pd + +from mapswipe_workers.project_types import StreetProject +from tests import fixtures + + +class TestCreateStreetProject(unittest.TestCase): + def setUp(self) -> None: + project_draft = fixtures.get_fixture( + os.path.join( + "projectDrafts", + "street.json", + ) + ) + project_draft["projectDraftId"] = "foo" + + with patch( + "mapswipe_workers.utils.process_mapillary.coordinate_download" + ) as mock_get: + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_response.csv", + ), + "r", + ) as file: + df = pd.read_csv(file) + df['geometry'] = df['geometry'].apply(wkt.loads) + + mock_get.return_value = df + self.project = StreetProject(project_draft) + + def test_init(self): + self.assertEqual(self.project.geometry["type"], "FeatureCollection") + + def test_create_group(self): + self.project.create_groups() + self.assertTrue(self.project.groups) + + def test_create_tasks(self): + imageId = self.project.imageIds[-1] + self.project.create_groups() + self.project.create_tasks() + self.assertEqual(self.project.tasks["g0"][0].taskId, imageId) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_spatial_sampling.py b/mapswipe_workers/tests/unittests/test_spatial_sampling.py new file mode 100644 index 000000000..f43597c89 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_spatial_sampling.py @@ -0,0 +1,100 @@ +import os + +import unittest +import numpy as np +import pandas as pd +from shapely import wkt +from shapely.geometry import Point + +from mapswipe_workers.utils.spatial_sampling import distance_on_sphere, filter_points, spatial_sampling + + +class TestDistanceCalculations(unittest.TestCase): + + @classmethod + def setUpClass(cls): + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_sequence.csv", + ), + "r", + ) as file: + df = pd.read_csv(file) + df['geometry'] = df['geometry'].apply(wkt.loads) + + cls.fixture_df = df + + def test_distance_on_sphere(self): + p1 = Point(-74.006, 40.7128) + p2 = Point(-118.2437, 34.0522) + + distance = distance_on_sphere((p1.x, p1.y), (p2.x, p2.y)) + expected_distance = 3940 # Approximate known distance in km + + self.assertTrue(np.isclose(distance, expected_distance, atol=50)) + + def test_filter_points(self): + data = { + "geometry": [ + "POINT (-74.006 40.7128)", + "POINT (-75.006 41.7128)", + "POINT (-76.006 42.7128)", + "POINT (-77.006 43.7128)" + ] + } + df = pd.DataFrame(data) + + df['geometry'] = df['geometry'].apply(wkt.loads) + + df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None) + df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None) + threshold_distance = 100 + filtered_df = filter_points(df, threshold_distance) + + self.assertIsInstance(filtered_df, pd.DataFrame) + self.assertLessEqual(len(filtered_df), len(df)) + + + def test_spatial_sampling_ordering(self): + data = { + "geometry": [ + "POINT (-74.006 40.7128)", + "POINT (-75.006 41.7128)", + "POINT (-76.006 42.7128)", + "POINT (-77.006 43.7128)" + ], + 'captured_at': [1, 2, 3, 4], + 'sequence_id': ['1', '1', '1', '1'] + } + df = pd.DataFrame(data) + df['geometry'] = df['geometry'].apply(wkt.loads) + + interval_length = 0.1 + filtered_gdf = spatial_sampling(df, interval_length) + + self.assertTrue(filtered_gdf['captured_at'].is_monotonic_increasing) + + + def test_spatial_sampling_with_sequence(self): + threshold_distance = 0.01 + filtered_df = spatial_sampling(self.fixture_df, threshold_distance) + self.assertIsInstance(filtered_df, pd.DataFrame) + self.assertLess(len(filtered_df), len(self.fixture_df)) + + filtered_df.reset_index(drop=True, inplace=True) + for i in range(len(filtered_df) - 1): + geom1 = filtered_df.loc[i, 'geometry'] + geom2 = filtered_df.loc[i + 1, 'geometry'] + + distance = geom1.distance(geom2) + + self.assertLess(distance, threshold_distance) + + + + +if __name__ == "__main__": + unittest.main() diff --git a/postgres/initdb.sql b/postgres/initdb.sql index ce9b97197..f954d3a8c 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON, 4326), + geom geometry(Geometry, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), diff --git a/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql b/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql new file mode 100644 index 000000000..97ecc298f --- /dev/null +++ b/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql @@ -0,0 +1,12 @@ +/* + * This script updates the `tasks` table by adjusting the following column: + * - `geom`: Stores now not only polygon geometries but all geometry types (e.g. Point, LineString). + * + * Existing entries for `geom` are not affected by this change. + * + * The new street project type requires Point geometries to store the image location. + * + */ + + +ALTER TABLE tasks ALTER COLUMN geom SET DATA TYPE geometry(Geometry, 4326); \ No newline at end of file