diff --git a/specifyweb/backend/trees/tree_mutations.py b/specifyweb/backend/trees/tree_mutations.py new file mode 100644 index 00000000000..1014c46cdf1 --- /dev/null +++ b/specifyweb/backend/trees/tree_mutations.py @@ -0,0 +1,66 @@ + +from specifyweb.backend.permissions.permissions import PermissionTarget, PermissionTargetAction + + +class TaxonMutationPT(PermissionTarget): + resource = "/tree/edit/taxon" + merge = PermissionTargetAction() + move = PermissionTargetAction() + synonymize = PermissionTargetAction() + desynonymize = PermissionTargetAction() + repair = PermissionTargetAction() + + +class GeographyMutationPT(PermissionTarget): + resource = "/tree/edit/geography" + merge = PermissionTargetAction() + move = PermissionTargetAction() + synonymize = PermissionTargetAction() + desynonymize = PermissionTargetAction() + repair = PermissionTargetAction() + + +class StorageMutationPT(PermissionTarget): + resource = "/tree/edit/storage" + merge = PermissionTargetAction() + move = PermissionTargetAction() + bulk_move = PermissionTargetAction() + synonymize = PermissionTargetAction() + desynonymize = PermissionTargetAction() + repair = PermissionTargetAction() + + +class GeologictimeperiodMutationPT(PermissionTarget): + resource = "/tree/edit/geologictimeperiod" + merge = PermissionTargetAction() + move = PermissionTargetAction() + synonymize = PermissionTargetAction() + desynonymize = PermissionTargetAction() + repair = PermissionTargetAction() + + +class LithostratMutationPT(PermissionTarget): + resource = "/tree/edit/lithostrat" + merge = PermissionTargetAction() + move = PermissionTargetAction() + synonymize = PermissionTargetAction() + desynonymize = PermissionTargetAction() + repair = PermissionTargetAction() + +class TectonicunitMutationPT(PermissionTarget): + resource = "/tree/edit/tectonicunit" + merge = PermissionTargetAction() + move = PermissionTargetAction() + synonymize = PermissionTargetAction() + desynonymize = PermissionTargetAction() + repair = PermissionTargetAction() + +def perm_target(tree): + return { + 'taxon': TaxonMutationPT, + 'geography': GeographyMutationPT, + 'storage': StorageMutationPT, + 'geologictimeperiod': GeologictimeperiodMutationPT, + 'lithostrat': LithostratMutationPT, + 'tectonicunit':TectonicunitMutationPT + }[tree] \ No newline at end of file diff --git a/specifyweb/backend/trees/urls.py b/specifyweb/backend/trees/urls.py index 825068ed747..4c6ac547345 100644 --- a/specifyweb/backend/trees/urls.py +++ b/specifyweb/backend/trees/urls.py @@ -20,4 +20,9 @@ re_path(r'^(?P\d+)/(?P\w+)/(?P\w+)/$', views.tree_view), path('repair/', views.repair_tree), ])), + + # Create new trees + path('create_default_tree/', views.create_default_tree_view), + re_path(r'^create_default_tree/status/(?P[^/]+)/$', views.default_tree_upload_status), + re_path(r'^create_default_tree/abort/(?P[^/]+)/$', views.abort_default_tree_creation), ] \ No newline at end of file diff --git a/specifyweb/backend/trees/utils.py b/specifyweb/backend/trees/utils.py index 24e64724355..dbe57d8fc03 100644 --- a/specifyweb/backend/trees/utils.py +++ b/specifyweb/backend/trees/utils.py @@ -1,13 +1,34 @@ -from typing import Tuple, List +from typing import Any, Callable, Dict, Iterator, Optional, TypedDict, NotRequired +import json +import requests +import csv +import time +from requests.exceptions import ChunkedEncodingError, ConnectionError + +from django.db import transaction from django.db.models import Q, Count, Model +from specifyweb.backend.notifications.models import Message +from specifyweb.celery_tasks import LogErrorsTask, app import specifyweb.specify.models as spmodels from specifyweb.specify.datamodel import datamodel, Table +import logging +logger = logging.getLogger(__name__) + lookup = lambda tree: (tree.lower() + 'treedef') SPECIFY_TREES = {"taxon", "storage", "geography", "geologictimeperiod", "lithostrat", 'tectonicunit'} +TREE_ROOT_NODES = { + 'taxon': 'Life', + 'storage': 'Root', + 'geography': "Earth", + 'geologictimeperiod': 'Time', + 'lithostrat': 'Earth', + 'tectonicunit': 'Root' +} + def get_search_filters(collection: spmodels.Collection, tree: str): tree_name = tree.lower() if tree_name == 'storage': @@ -84,4 +105,291 @@ def get_models(name: str): tree_rank_model = get_treedefitem_model(name) tree_node_model = getattr(spmodels, name.lower().title()) - return tree_def_model, tree_rank_model, tree_node_model \ No newline at end of file + return tree_def_model, tree_rank_model, tree_node_model + +class RankConfiguration(TypedDict): + name: str + title: NotRequired[str] + enforced: bool + infullname: bool + fullnameseparator: NotRequired[str] + rank: int # rank id + +def initialize_default_tree(tree_type: str, discipline, tree_name: str, rank_cfg: list[RankConfiguration], full_name_direction: int=1): + """Creates an initial empty tree.""" + with transaction.atomic(): + tree_def_model, tree_rank_model, tree_node_model = get_models(tree_type) + + # Uniquify name + tree_def = None + unique_tree_name = tree_name + if tree_def_model.objects.filter(name=tree_name).exists(): + i = 1 + while tree_def_model.objects.filter(name=f"{tree_name}_{i}").exists(): + i += 1 + unique_tree_name = f"{tree_name}_{i}" + + # Create tree definition + tree_def, _ = tree_def_model.objects.get_or_create( + name=unique_tree_name, + discipline=discipline, + fullnamedirection=full_name_direction + ) + + # Create tree ranks + treedefitems_bulk = [] + rank_id = 0 + for rank in rank_cfg: + treedefitems_bulk.append( + tree_rank_model( + treedef=tree_def, + name=rank.get('name'), + title=rank.get('title', rank['name'].title()), + rankid=int(rank.get('rank', rank_id)), + isenforced=rank.get('enforced', True), + isinfullname=rank.get('infullname', False), + fullnameseparator=rank.get('fullnameseparator', ' ') + ) + ) + rank_id += 10 + if treedefitems_bulk: + tree_rank_model.objects.bulk_create(treedefitems_bulk, ignore_conflicts=False) + + # Create root node + # TODO: Avoid having duplicated code from add_root endpoint + root_rank = tree_rank_model.objects.get(treedef=tree_def, rankid=0) + tree_node, _ = tree_node_model.objects.get_or_create( + name=TREE_ROOT_NODES.get(tree_type, "Root"), + fullname=TREE_ROOT_NODES.get(tree_type, "Root"), + nodenumber=1, + definition=tree_def, + definitionitem=root_rank, + parent=None + ) + + return tree_def.name + +class RankMappingConfiguration(TypedDict): + name: str + column: str + enforced: NotRequired[bool] + rank: NotRequired[int] + infullname: NotRequired[bool] + fullnameseparator: NotRequired[str] + fields: Dict[str, str] + +def add_default_tree_record(tree_type: str, row: dict, tree_name: str, tree_cfg: dict[str, RankMappingConfiguration]): + """ + Given one CSV row and a column mapping / rank configuration dictionary, + walk through the 'ranks' in order, creating or updating each tree record and linking + it to its parent. + """ + tree_def_model, tree_rank_model, tree_node_model = get_models(tree_type) + tree_def = tree_def_model.objects.get(name=tree_name) + parent = tree_node_model.objects.filter(definitionitem__rankid=0, definition=tree_def).first() + rank_id = 10 + + for rank_map in tree_cfg['ranks']: + rank_name = rank_map['name'] + fields_map = rank_map['fields'] + + record_name = row.get(rank_map.get('column', rank_name)) # Record's name is in the column. + + if not record_name: + continue # This row doesn't contain a record for this rank. + + defaults = {} + for model_field, csv_col in fields_map.items(): + if model_field == 'name': + continue + v = row.get(csv_col) + if v: + defaults[model_field] = v + + rank_title = rank_map.get('title', rank_name.capitalize()) + + # Get the rank by the column name. + # It should already exist by this point, but worst case it will be generated here. + treedef_item, _ = tree_rank_model.objects.get_or_create( + name=rank_name, + treedef=tree_def, + defaults={ + 'title': rank_title, + 'rankid': rank_id + } + ) + + # Create the record at this rank if it isn't already there. + obj = tree_node_model.objects.filter( + name=record_name, + fullname=record_name, + definition=tree_def, + definitionitem=treedef_item, + parent=parent, + ).first() + if obj is None: + data = { + 'name': record_name, + 'fullname': record_name, + 'definition': tree_def, + 'definitionitem': treedef_item, + 'parent': parent, + 'rankid': treedef_item.rankid, + **defaults + } + obj = tree_node_model(**data) + obj.save(skip_tree_extras=True) + + # if not taxon_obj and defaults: + # for f, v in defaults.items(): + # setattr(taxon_obj, f, v) + # taxon_obj.save() + + parent = obj + rank_id += 10 + +@app.task(base=LogErrorsTask, bind=True) +def create_default_tree_task(self, url: str, discipline_id: int, tree_discipline_name: str, specify_collection_id: int, + specify_user_id: int, tree_cfg: dict, row_count: Optional[int], initial_tree_name: str): + logger.info(f'starting task {str(self.request.id)}') + + specify_user = spmodels.Specifyuser.objects.get(id=specify_user_id) + discipline = spmodels.Discipline.objects.get(id=discipline_id) + tree_name = initial_tree_name # Name will be uniquified on tree creation + + Message.objects.create( + user=specify_user, + content=json.dumps({ + 'type': 'create-default-tree-running', + 'name': initial_tree_name, + 'taskid': str(self.request.id), + 'collection_id': specify_collection_id, + }) + ) + + current = 0 + total = 1 + def progress(cur: int, additional_total: int=0) -> None: + nonlocal current, total + current += cur + total += additional_total + if current > total: + current = total + self.update_state(state='RUNNING', meta={'current': current, 'total': total}) + + try: + with transaction.atomic(): + tree_type = 'taxon' + if tree_discipline_name in SPECIFY_TREES: + # non-taxon tree + tree_type = tree_discipline_name + + # Create a new empty tree. Get rank configuration from the mapping. + full_name_direction = 1 + if tree_type in ('geologictimeperiod'): + full_name_direction = -1 + + rank_cfg = [{ + 'name': 'Root', + 'enforced': True, + 'rank': 0, + **tree_cfg.get('root', {}) + }] + auto_rank_id = 10 + for rank in tree_cfg['ranks']: + rank_cfg.append({ + 'name': rank['name'], + 'enforced': rank.get('enforced', True), + 'infullname': rank.get('infullname', False), + 'fullnameseparator': rank.get('fullnameseparator', ' '), + 'rank': rank.get('rank', auto_rank_id) + }) + auto_rank_id += 10 + tree_name = initialize_default_tree(tree_type, discipline, initial_tree_name, rank_cfg, full_name_direction) + + # Start importing CSV data + total_rows = 0 + if row_count: + total_rows = row_count-2 + progress(0, total_rows) + for row in stream_csv_from_url(url): + add_default_tree_record(tree_type, row, tree_name, tree_cfg) + progress(1, 0) + except Exception as e: + Message.objects.create( + user=specify_user, + content=json.dumps({ + 'type': 'create-default-tree-failed', + 'name': tree_name, + 'taskid': str(self.request.id), + 'collection_id': specify_collection_id, + # 'error': str(e) + }) + ) + raise + + Message.objects.create( + user=specify_user, + content=json.dumps({ + 'type': 'create-default-tree-completed', + 'name': tree_name, + 'taskid': str(self.request.id), + 'collection_id': specify_collection_id, + }) + ) + +def stream_csv_from_url(url: str) -> Iterator[Dict[str, str]]: + """ + Streams a taxon CSV from a URL. Yields each row. + """ + chunk_size = 8192 + max_retries = 10 + + def lines_iter() -> Iterator[str]: + # Streams data from the server in -chunks-, yields -lines-. + buffer = b"" + bytes_downloaded = 0 + retries = 0 + + headers = {} + while True: + # Request data starting from the last downloaded bytes + if bytes_downloaded > 0: + headers['Range'] = f'bytes={bytes_downloaded}-' + + try: + with requests.get(url, stream=True, timeout=(5, 30), headers=headers) as resp: + resp.raise_for_status() + for chunk in resp.iter_content(chunk_size=chunk_size): + chunk_length = len(chunk) + if chunk_length == 0: + continue + buffer += chunk + bytes_downloaded += chunk_length + + # Extract all lines from chunk + while True: + new_line_index = buffer.find(b'\n') + if new_line_index == -1: break + line = buffer[:new_line_index + 1] # extract line + buffer = buffer[new_line_index + 1 :] # clear read buffer + yield line.decode('utf-8-sig', errors='replace') + + if buffer: + # yield last line + yield buffer.decode('utf-8-sig', errors='replace') + return + except (ChunkedEncodingError, ConnectionError) as e: + # Trigger retry + if retries < max_retries: + retries += 1 + time.sleep(2 ** retries) + continue + raise + except Exception: + raise + + reader = csv.DictReader(lines_iter()) + + for row in reader: + yield row \ No newline at end of file diff --git a/specifyweb/backend/trees/views.py b/specifyweb/backend/trees/views.py index 2a503ead2f0..7ad515ebc19 100644 --- a/specifyweb/backend/trees/views.py +++ b/specifyweb/backend/trees/views.py @@ -1,29 +1,39 @@ +import csv from functools import wraps +import json +from uuid import uuid4 from django import http -from typing import Literal, TypedDict, Any +from typing import Iterator, Literal, TypedDict, Any, Dict from django.db import connection, transaction from django.http import HttpResponse from django.views.decorators.http import require_POST +import requests +from specifyweb.backend.trees.tree_mutations import perm_target +from specifyweb.specify.views import login_maybe_required, openapi +from sqlalchemy import distinct from specifyweb.specify import models as spmodels from specifyweb.specify.api.crud import get_object_or_404 from specifyweb.specify.api.serializers import obj_to_data, toJson from sqlalchemy import select, func, distinct, literal from sqlalchemy.orm import aliased +from jsonschema import validate # type: ignore +from jsonschema.exceptions import ValidationError # type: ignore from specifyweb.middleware.general import require_GET from specifyweb.backend.businessrules.exceptions import BusinessRuleException -from specifyweb.backend.permissions.permissions import PermissionTarget, PermissionTargetAction, check_permission_targets, has_table_permission +from specifyweb.backend.permissions.permissions import check_permission_targets, has_table_permission from specifyweb.backend.stored_queries import models as sqlmodels from specifyweb.backend.stored_queries.execution import set_group_concat_max_len from specifyweb.backend.stored_queries.group_concat import group_concat -from specifyweb.backend.trees.utils import get_search_filters +from specifyweb.backend.notifications.models import Message + +from specifyweb.backend.trees.utils import add_default_tree_record, create_default_tree_task, get_search_filters, stream_csv_from_url from specifyweb.specify.utils.field_change_info import FieldChangeInfo from specifyweb.backend.trees.ranks import tree_rank_count from . import extras from specifyweb.backend.workbench.upload.auditcodes import TREE_MOVE from specifyweb.backend.trees.stats import get_tree_stats -from specifyweb.specify.views import login_maybe_required, openapi import logging @@ -543,65 +553,340 @@ def has_tree_read_permission(tree: TREE_TABLE) -> bool: return result -class TaxonMutationPT(PermissionTarget): - resource = "/tree/edit/taxon" - merge = PermissionTargetAction() - move = PermissionTargetAction() - synonymize = PermissionTargetAction() - desynonymize = PermissionTargetAction() - repair = PermissionTargetAction() - - -class GeographyMutationPT(PermissionTarget): - resource = "/tree/edit/geography" - merge = PermissionTargetAction() - move = PermissionTargetAction() - synonymize = PermissionTargetAction() - desynonymize = PermissionTargetAction() - repair = PermissionTargetAction() - - -class StorageMutationPT(PermissionTarget): - resource = "/tree/edit/storage" - merge = PermissionTargetAction() - move = PermissionTargetAction() - bulk_move = PermissionTargetAction() - synonymize = PermissionTargetAction() - desynonymize = PermissionTargetAction() - repair = PermissionTargetAction() - - -class GeologictimeperiodMutationPT(PermissionTarget): - resource = "/tree/edit/geologictimeperiod" - merge = PermissionTargetAction() - move = PermissionTargetAction() - synonymize = PermissionTargetAction() - desynonymize = PermissionTargetAction() - repair = PermissionTargetAction() - - -class LithostratMutationPT(PermissionTarget): - resource = "/tree/edit/lithostrat" - merge = PermissionTargetAction() - move = PermissionTargetAction() - synonymize = PermissionTargetAction() - desynonymize = PermissionTargetAction() - repair = PermissionTargetAction() - -class TectonicunitMutationPT(PermissionTarget): - resource = "/tree/edit/tectonicunit" - merge = PermissionTargetAction() - move = PermissionTargetAction() - synonymize = PermissionTargetAction() - desynonymize = PermissionTargetAction() - repair = PermissionTargetAction() - -def perm_target(tree): - return { - 'taxon': TaxonMutationPT, - 'geography': GeographyMutationPT, - 'storage': StorageMutationPT, - 'geologictimeperiod': GeologictimeperiodMutationPT, - 'lithostrat': LithostratMutationPT, - 'tectonicunit':TectonicunitMutationPT - }[tree] \ No newline at end of file +# class DefaultTreePT(PermissionTarget): +# resource = "/tree/default" +# update = PermissionTargetAction() +# delete = PermissionTargetAction() + +# Schema definition for the mapping file that is used in default tree creation. +DEFAULT_TREE_MAPPING_SCHEMA = { + "title": "Tree column mapping for default trees", + "description": "The mapping of the CSV columns for default tree creation.", + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "all_columns": { + "description": "A list of all the column header names contained in the CSV. The first columns names should correspond the number of ranks defined in this schema.", + "type": "array", + "items": { + "type": "string" + } + }, + "ranks": { + "description": "An ordered list containing all the ranks to be created.", + "type": "array", + "minItems": 1, + "items": { + "description": "A rank's mapping definition.", + "type": "object", + "properties": { + "name": {"type": "string", "description": "Display name for the rank"}, + "enforced": {"type": "boolean", "description": "isEnforced"}, + "infullname": {"type": "boolean", "description": "isInFullName"}, + "fullnameseparator": {"type": "string", "description": "fullNameSeparator"}, + "rank": {"type": "integer", "description": "Rank's rankid"}, + "column": {"type": "string", "description": "The CSV column corresponding to this rank"}, + "fields": { + "type": "object", + "description": "Mapping of the rank's field names to the CSV columns containing the values.", + "additionalProperties": {"type": "string"} + } + }, + "required": ["name", "column", "fields"], + "additionalProperties": False + } + }, + "root": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "fullnameseparator": {"type": "string"} + } + } + }, + "required": ["ranks"] +} +@openapi(schema={ + "post": { + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/TreeCreationRequest"} + } + } + }, + "responses": { + "201": { + "description": 'Default tree created.', + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Success"} + } + } + }, + "202": { + "description": 'Default tree creation started in the background.', + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/SuccessBackground"} + } + } + } + } + }}, + components={ + "schemas": { + "TreeCreationRequest": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL of the tree CSV file." + }, + "treeName": { + "type": "string", + "description": "The name to be used by the new tree.", + }, + "disciplineName": { + "type": "string", + "description": "Name of the disicpline the tree belongs to." + }, + "collectionName": { + "type": "string", + "description": "The name of the destination collection. The logged in colleciton will be used otherwise." + }, + "rowCount": { + "type": "integer", + "description": "The total number of rows contained in the CSV file. Only used for progress tracking." + }, + }, + "required": ["url", "disciplineName"], + "oneOf": [ + {"required": ["mappingUrl"], + "properties": { + "mappingUrl": { + "type": "string", + "description": "The URL of a JSON file describing the column mapping of the CSV data." + }, + }}, + {"required": ["mapping"], + "properties": { + "mapping": { + "type": "object", + "description": "An object describing the column mapping of the CSV data." + }, + }} + ] + }, + "SuccessBackground": { + "type": "object", + "properties": { + "message": {"type": "string"}, + "task_id": {"type": "string"} + }, + "required": ["message", "task_id"] + }, + "Success": { + "type": "object", + "properties": {"message": {"type": "string"}}, + "required": ["message"] + } + } + }) +@login_maybe_required +@require_POST +@transaction.atomic +def create_default_tree_view(request): + """Creates or populates a tree with default records from a CSV file. + """ + # Check permissions in the normal case and for the case of intial database setup. + # check_permission_targets(request.specify_collection.id, request.specify_user.id, [DefaultTreePT.create]) + + data = json.loads(request.body) + + tree_discipline_name = data.get('disciplineName', None) + if not tree_discipline_name: + return http.JsonResponse({'error': 'Discipline name was not provided.'}, status=400) + + collection_name = data.get('collectionName', None) + if collection_name is not None: + try: + collection = spmodels.Collection.objects.get(collectionname=collection_name) + except: + return http.JsonResponse({'error': 'Collection was not found.'}, status=404) + else: + collection = request.specify_collection + + logged_in_discipline_name = collection.discipline.name + # logged_in_discipline_name = request.user.logindisciplinename + + discipline = spmodels.Discipline.objects.filter(name=tree_discipline_name).first() + if not discipline: + discipline = spmodels.Discipline.objects.filter(name=logged_in_discipline_name).first() + if not discipline: + discipline = spmodels.Discipline.objects.all().first() + + url = data.get('url', None) + + tree_rank_model_name = tree_discipline_name.capitalize() + tree_name = data.get('treeName', tree_rank_model_name) + + row_count = data.get('rowCount', None) + + if not url: + return http.JsonResponse({'error': 'Tree not found.'}, status=404) + + # CSV mapping. Accept the mapping directly or a url to a JSON file containing the mapping. + tree_cfg = data.get('mapping', None) + mapping_url = data.get('mappingUrl', None) + if mapping_url: + try: + resp = requests.get(mapping_url) + resp.raise_for_status() + tree_cfg = resp.json() + except Exception: + return http.JsonResponse({'error': f'Could not retrieve default tree mapping from {mapping_url}.'}, status=404) + try: + validate(tree_cfg, DEFAULT_TREE_MAPPING_SCHEMA) + except ValidationError as e: + return http.JsonResponse({'error': f'Default tree mapping is invalid: {e}'}, status=400) + + Message.objects.create(user=request.specify_user, content=json.dumps({ + 'type': 'create-default-tree-starting', + 'name': tree_name, + 'collection_id': request.specify_collection.id, + })) + + task_id = str(uuid4()) + async_result = create_default_tree_task.apply_async( + args=[url, discipline.id, tree_discipline_name, request.specify_collection.id, request.specify_user.id, tree_cfg, row_count, tree_name], + task_id=f"create_default_tree_{tree_discipline_name}_{task_id}", + taskid=task_id + ) + return http.JsonResponse({ + 'message': 'Trees creation started in the background.', + 'task_id': async_result.id + }, status=202) + +@openapi(schema={ + "get": { + "parameters": [ + { + "name": "task_id", + "in": "path", + "required": True, + "schema": {"type": "string"}, + "description": "ID of the default tree creation task" + } + ], + "responses": { + "200": { + "description": 'Status of the tree creation task', + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Progress"} + } + } + }, + } + }}, + components={ + "schemas": { + "Progress": { + "type": "object", + "properties": { + "taskstatus": { + "type": "string", + "description": "Celery task status (PENDING, STARTED, RUNNING, SUCCESS, FAILURE, REVOKED)" + }, + "taskprogress": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "description": "Progress info for the task.", + "properties": { + "current": {"type": "integer", "minimum": 0}, + "total": {"type": "integer", "minimum": 0} + }, + "required": ["current", "total"] + } + ], + "description": "Info returned by the task" + }, + "taskid": { + "type": "string", + "description": "The id of the task you queried" + } + }, + "required": ["taskstatus", "taskid"] + } + } + }) +@require_GET +def default_tree_upload_status(request, task_id: str) -> http.HttpResponse: + """Returns the task status for the default tree upload celery task""" + + result = create_default_tree_task.AsyncResult(task_id) + + status = { + 'taskstatus': result.status, + 'taskprogress': result.info if isinstance(result.info, dict) else repr(result.info), + 'taskid': task_id + } + + return http.JsonResponse(status) + +@openapi(schema={ + "post": { + "parameters": [ + { + "name": "task_id", + "in": "path", + "required": True, + "schema": {"type": "string"}, + "description": "ID of the default tree creation task to abort" + } + ], + "responses": { + "200": { + "description": "Task aborted successfully" + }, + "400": { + "description": "Error aborting the task", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "description": "Error message" + } + }, + "required": ["error"] + } + } + } + } + } + }, +}) +@require_POST +def abort_default_tree_creation(request, task_id: str) -> http.HttpResponse: + """Stops a default tree upload celery task""" + try: + task = create_default_tree_task.AsyncResult(task_id) + task.revoke(terminate=True) + + Message.objects.create(user=request.specify_user, content=json.dumps({ + 'type': 'create-default-tree-failed', + 'name': 'Aborted', + 'taskid': task_id, + 'collection_id': request.specify_collection.id, + })) + + return http.HttpResponse('', status=204) + except Exception as e: + return http.JsonResponse({'error': str(e)}, status=400) \ No newline at end of file diff --git a/specifyweb/frontend/js_src/lib/components/Notifications/NotificationRenderers.tsx b/specifyweb/frontend/js_src/lib/components/Notifications/NotificationRenderers.tsx index 58300d772f8..5db4b83aadb 100644 --- a/specifyweb/frontend/js_src/lib/components/Notifications/NotificationRenderers.tsx +++ b/specifyweb/frontend/js_src/lib/components/Notifications/NotificationRenderers.tsx @@ -3,10 +3,13 @@ import type { LocalizedString } from 'typesafe-i18n'; import { useBooleanState } from '../../hooks/useBooleanState'; import { backupText } from '../../localization/backup'; +import { commonText } from '../../localization/common'; import { localityText } from '../../localization/locality'; import { mergingText } from '../../localization/merging'; import { notificationsText } from '../../localization/notifications'; +import { treeText } from '../../localization/tree'; import { StringToJsx } from '../../localization/utils'; +import { ping } from '../../utils/ajax/ping'; import type { IR, RA } from '../../utils/types'; import { Button } from '../Atoms/Button'; import { Link } from '../Atoms/Link'; @@ -360,6 +363,59 @@ export const notificationRenderers: IR< ); }, + 'create-default-tree-starting'(notification) { + return ( + <> +

{treeText.defaultTreeTaskStarting()}

+ {notification.payload.name} + + ) + }, + 'create-default-tree-running'(notification) { + return ( + <> +

{treeText.defaultTreeTaskRunning()}

+ {notification.payload.name} + { + ping(`/trees/create_default_tree/abort/${notification.payload.taskid}/`, { + method: 'POST', + body: {}, + errorMode: 'dismissible', + }) + } + }>{commonText.cancel()} +
+ {localityText.taskId()} + {notification.payload.taskid} +
+ + ) + }, + 'create-default-tree-failed'(notification) { + return ( + <> +

{treeText.defaultTreeTaskFailed()}

+ {notification.payload.name} +
+ {localityText.taskId()} + {notification.payload.taskid} +
+ + ) + }, + 'create-default-tree-completed'(notification) { + return ( + <> +

{treeText.defaultTreeTaskCompleted()}

+ {notification.payload.name} +
+ {localityText.taskId()} + {notification.payload.taskid} +
+ + ) + }, default(notification) { console.error('Unknown notification type', { notification }); return 
{JSON.stringify(notification, null, 2)}
; diff --git a/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx b/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx index c24855839dc..25a95345a54 100644 --- a/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx +++ b/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx @@ -2,12 +2,17 @@ import React from 'react'; import { commonText } from '../../localization/common'; import { treeText } from '../../localization/tree'; -import type { DeepPartial } from '../../utils/types'; +import { ajax } from '../../utils/ajax'; +import { Http } from '../../utils/ajax/definitions'; +import { ping } from '../../utils/ajax/ping'; +import type { DeepPartial, RA } from '../../utils/types'; import { localized } from '../../utils/types'; import { getUniqueName } from '../../utils/uniquifyName'; -import { Ul } from '../Atoms'; +import { H2, Ul } from '../Atoms'; +import { Progress } from '../Atoms'; import { Button } from '../Atoms/Button'; import { className } from '../Atoms/className'; +import { LoadingContext } from '../Core/Contexts'; import type { AnySchema, AnyTree, @@ -17,11 +22,34 @@ import type { SpecifyResource } from '../DataModel/legacyTypes'; import { deserializeResource } from '../DataModel/serializers'; import type { TaxonTreeDef } from '../DataModel/types'; import { ResourceView } from '../Forms/ResourceView'; +import { getSystemInfo } from '../InitialContext/systemInfo'; import type { TreeInformation } from '../InitialContext/treeRanks'; import { userInformation } from '../InitialContext/userInformation'; import { Dialog } from '../Molecules/Dialog'; import { defaultTreeDefs } from './defaults'; +type TaxonFileDefaultDefinition = { + readonly discipline: string; + readonly title: string; + readonly coverage: string; + readonly file: string; + readonly mappingFile: string; + readonly src: string; + readonly size: number; + readonly rows: number; + readonly description: string; +}; +type TaxonFileDefaultList = RA; +type TreeCreationInfo = { + readonly message: string; + readonly task_id?: string; +} +type TreeCreationProgressInfo = { + readonly taskstatus: string; + readonly taskprogress: any; + readonly taskid: string; +} + export function CreateTree< SCHEMA extends AnyTree, TREE_NAME extends AnyTree['tableName'], @@ -34,13 +62,64 @@ export function CreateTree< }): JSX.Element { const treeNameArray = treeDefinitions.map((tree) => tree.definition.name); + const loading = React.useContext(LoadingContext); const [isActive, setIsActive] = React.useState(0); + const [isTreeCreationStarted, setIsTreeCreationStarted] = React.useState(false); + const [treeCreationTaskId, setTreeCreationTaskId] = React.useState(undefined); const [selectedResource, setSelectedResource] = React.useState< SpecifyResource | undefined >(undefined); - const handleClick = ( + const [treeOptions, setTreeOptions] = React.useState< + TaxonFileDefaultList | undefined + >(undefined); + + // Fetch list of available default trees. + React.useEffect(() => { + fetch('https://files.specifysoftware.org/taxonfiles/taxonfiles.json') + .then(async (response) => response.json()) + .then((data: TaxonFileDefaultList) => { + setTreeOptions(data); + }) + .catch((error) => { + console.error('Failed to fetch tree options:', error); + }); + }, []); + + const connectedCollection = getSystemInfo().collection; + + // Start default tree creation + const handleClick = async (resourceUrl: string, mappingUrl: string, disciplineName: string, rowCount: number, treeName: string): Promise => { + setIsTreeCreationStarted(true); + return ajax('/trees/create_default_tree/', { + method: 'POST', + headers: { Accept: 'application/json' }, + body: { + url: resourceUrl, + mappingUrl, + collection: connectedCollection, + disciplineName, + rowCount, + treeName, + }, + }) + .then(({ data, status }) => { + if (status === Http.OK) { + console.log(`${disciplineName} tree created successfully:`, data); + } else if (status === Http.ACCEPTED) { + // Tree is being created in the background. + console.log(`${disciplineName} tree creation started successfully:`, data); + setTreeCreationTaskId(data.task_id); + } + }) + .catch((error) => { + console.error(`Request failed for ${resourceUrl}:`, error); + throw error; + }); + } + + const handleClickEmptyTree = ( resource: DeepPartial> ) => { const uniqueName = getUniqueName( @@ -78,14 +157,54 @@ export function CreateTree< onClose={() => setIsActive(0)} >
    +

    {treeText.populatedTrees()}

    + {treeOptions === undefined + ? undefined + : treeOptions.map((resource, index) => ( +
  • + { + loading( + handleClick(resource.file, resource.mappingFile, resource.discipline, resource.rows, resource.title).catch(console.error) + ); + }} + > + {localized(resource.title)} + +
    + {resource.description} +
    +
    + {`Source: ${resource.src}`} +
    +
    • + ))} +

    {treeText.emptyTrees()}

    {defaultTreeDefs.map((resource, index) => (
  • - handleClick(resource)}> + handleClickEmptyTree(resource)} + > {localized(resource.name)}
    • ))}
+ <> + {isTreeCreationStarted && treeCreationTaskId ? ( + { + setIsTreeCreationStarted(false); + setTreeCreationTaskId(undefined); + }} + onStopped={() => { + setIsTreeCreationStarted(false); + setTreeCreationTaskId(undefined); + }} + /> + ) : undefined} + ) : null} {isActive === 2 && selectedResource !== undefined ? ( @@ -103,3 +222,78 @@ export function CreateTree< ); } + +export function TreeCreationProgressDialog({ + taskId, + onClose, + onStopped, +}: { + readonly taskId: string; + readonly onClose: () => void; + readonly onStopped: () => void; +}): JSX.Element | null { + const loading = React.useContext(LoadingContext); + const [progress, setProgress] = React.useState(undefined); + const [progressTotal, setProgressTotal] = React.useState(1); + + const handleStop = async (): Promise => { + ping( + `/trees/create_default_tree/abort/${taskId}/`, + { + method: 'POST', + body: {}, + } + ) + .then((status) => { + if (status === Http.NO_CONTENT) { + onStopped(); + } + }) + } + + // Poll for tree creation progress + React.useEffect(() => { + const interval = setInterval( + async () => + ajax(`/trees/create_default_tree/status/${taskId}/`, { + method: 'GET', + headers: { Accept: 'application/json' }, + errorMode: 'silent', + }) + .then(({ data }) => { + if (data.taskstatus === 'RUNNING') { + setProgress(data.taskprogress.current ?? 0); + setProgressTotal(data.taskprogress.total ?? 1); + } else if (data.taskstatus === 'FAILURE') { + onStopped(); + throw data.taskprogress; + } else if (data.taskstatus === 'SUCCESS') { + globalThis.location.reload(); + } + }), + 5000 + ); + return () => clearInterval(interval); + }, [taskId]); + + return ( + {loading(handleStop())}}>{commonText.cancel()} + } + header={treeText.defaultTreeTaskStarting()} + onClose={onClose} + > + <> + {progress === undefined + ? null + : treeText.defaultTreeCreationProgress({ + current: progress, + total: progressTotal, + })} + + + {treeText.defaultTreeTaskStartingDescription()} + + ); +} \ No newline at end of file diff --git a/specifyweb/frontend/js_src/lib/localization/tree.ts b/specifyweb/frontend/js_src/lib/localization/tree.ts index 57c0ec99021..f742fd8c48f 100644 --- a/specifyweb/frontend/js_src/lib/localization/tree.ts +++ b/specifyweb/frontend/js_src/lib/localization/tree.ts @@ -706,6 +706,33 @@ export const treeText = createDictionary({ 'ru-ru': 'Метеориты', 'uk-ua': 'Метеорити', }, + emptyTrees: { + 'en-us': 'Empty Trees', + }, + populatedTrees: { + 'en-us': 'Populated Trees', + }, + defaultTreeTaskStarting: { + 'en-us': 'The Default Tree creation process has started.', + }, + defaultTreeTaskStartingDescription: { + 'en-us': + 'The tree will be created in the background. You will be notified once it is completed.', + }, + defaultTreeTaskRunning: { + 'en-us': 'Default Tree creation in progress.', + }, + defaultTreeTaskFailed: { + 'en-us': 'Default Tree creation failure.', + }, + defaultTreeTaskCompleted: { + 'en-us': 'Default Tree creation completed successfully.', + }, + defaultTreeCreationProgress: { + comment: 'E.x, Creating tree record 999/1,000', + 'en-us': + 'Creating tree record {current:number|formatted}/{total:number|formatted}', + }, treeManagement: { 'en-us': 'Tree Management', 'de-ch': 'Baumpflege', diff --git a/specifyweb/frontend/js_src/lib/utils/ajax/definitions.ts b/specifyweb/frontend/js_src/lib/utils/ajax/definitions.ts index a2a4024bfd4..a59be940cfc 100644 --- a/specifyweb/frontend/js_src/lib/utils/ajax/definitions.ts +++ b/specifyweb/frontend/js_src/lib/utils/ajax/definitions.ts @@ -5,6 +5,7 @@ export const Http = { // You may add other codes as needed OK: 200, CREATED: 201, + ACCEPTED: 202, NO_CONTENT: 204, BAD_REQUEST: 400, NOT_FOUND: 404, @@ -30,6 +31,8 @@ export const httpCodeToErrorMessage: RR, string> = { 'This error is likely caused by a bug in Specify. Please report it.', [Http.CREATED]: 'This error is likely caused by a bug in Specify. Please report it.', + [Http.ACCEPTED]: + 'This error is likely caused by a bug in Specify. Please report it.', [Http.NO_CONTENT]: 'This error is likely caused by a bug in Specify. Please report it.', [Http.BAD_REQUEST]: