From 0cd5286b17dfad74a7c30b0e3cb8524d2dacee22 Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 17:23:34 +0100 Subject: [PATCH 01/12] simplify all updates for entities --- server/kitsu/utils.py | 68 +++++++++++-------------------------------- 1 file changed, 17 insertions(+), 51 deletions(-) diff --git a/server/kitsu/utils.py b/server/kitsu/utils.py index 7f97ea0..d3016e9 100644 --- a/server/kitsu/utils.py +++ b/server/kitsu/utils.py @@ -138,32 +138,14 @@ async def update_folder( **kwargs, ) -> bool: folder = await FolderEntity.load(project_name, folder_id) - changed = False + kwargs: dict[str, Any] = {**kwargs, **create_name_and_label(name)} - payload: dict[str, Any] = {**kwargs, **create_name_and_label(name)} - - for key in ["name", "label"]: - if key in payload and getattr(folder, key) != payload[key]: - setattr(folder, key, payload[key]) - changed = True - - for key, value in payload["attrib"].items(): - if getattr(folder.attrib, key) != value: - setattr(folder.attrib, key, value) - if key not in folder.own_attrib: - folder.own_attrib.append(key) - changed = True - if changed: - await folder.save() - event = { - "topic": "entity.folder.updated", - "description": f"Folder {folder.name} updated", - "summary": {"entityId": folder.id, "parentId": folder.parent_id}, - "project": project_name, - } - await dispatch_event(**event) - - return changed + return await update_entity( + project_name, + folder, + kwargs, + attr_whitelist=["name", "label"], + ) async def delete_folder( @@ -216,32 +198,14 @@ async def update_task( **kwargs, ) -> bool: task = await TaskEntity.load(project_name, task_id) - changed = False + kwargs = {**kwargs, **create_name_and_label(name)} - payload = {**kwargs, **create_name_and_label(name)} - - # keys that can be updated - for key in ["name", "label", "status", "task_type", "assignees"]: - if key in payload and getattr(task, key) != payload[key]: - setattr(task, key, payload[key]) - changed = True - if "attrib" in payload: - for key, value in payload["attrib"].items(): - if getattr(task.attrib, key) != value: - setattr(task.attrib, key, value) - if key not in task.own_attrib: - task.own_attrib.append(key) - changed = True - if changed: - await task.save() - event = { - "topic": "entity.task.updated", - "description": f"Task {task.name} updated", - "summary": {"entityId": task.id, "parentId": task.parent_id}, - "project": project_name, - } - await dispatch_event(**event) - return changed + return await update_entity( + project_name, + task, + kwargs, + attr_whitelist=["name", "label", "status", "task_type", "assignees"], + ) async def delete_task( @@ -281,7 +245,9 @@ async def update_project( ) -async def update_entity(project_name, entity, kwargs, attr_whitelist: list[str] | None = None): +async def update_entity( + project_name, entity, kwargs, attr_whitelist: list[str] | None = None +): """updates the entity for given attribute whitelist, saves changes and dispatches an update event""" if attr_whitelist is None: From 6d5df2917dd48192a8ee4fe4a51f708668ad3c10 Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 17:32:24 +0100 Subject: [PATCH 02/12] bigfix for update with no changes --- server/kitsu/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/kitsu/utils.py b/server/kitsu/utils.py index d3016e9..0420c67 100644 --- a/server/kitsu/utils.py +++ b/server/kitsu/utils.py @@ -249,7 +249,7 @@ async def update_entity( project_name, entity, kwargs, attr_whitelist: list[str] | None = None ): """updates the entity for given attribute whitelist, saves changes and dispatches an update event""" - + changed = False if attr_whitelist is None: attr_whitelist = [] From 563a60b6f343b3816e1301fd03b72ef33a31bf7f Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 17:35:03 +0100 Subject: [PATCH 03/12] set logs to debug level --- server/kitsu/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/kitsu/utils.py b/server/kitsu/utils.py index 0420c67..b671821 100644 --- a/server/kitsu/utils.py +++ b/server/kitsu/utils.py @@ -257,7 +257,7 @@ async def update_entity( for key in attr_whitelist: if key in kwargs and getattr(entity, key) != kwargs[key]: setattr(entity, key, kwargs[key]) - logging.info(f"setattr {key}") + logging.debug(f"setattr {key} {getattr(entity, key)} => {kwargs[key]}") changed = True if "attrib" in kwargs: for key, value in kwargs["attrib"].items(): @@ -265,7 +265,7 @@ async def update_entity( setattr(entity.attrib, key, value) if key not in entity.own_attrib: entity.own_attrib.append(key) - logging.info( + logging.debug( f"setattr attrib.{key} {getattr(entity.attrib, key)} => {value}" ) changed = True @@ -286,6 +286,6 @@ async def update_entity( "summary": summary, "project": project_name, } - logging.info(f"dispatch_event: {event}") + logging.debug(f"dispatch_event: {event}") await dispatch_event(**event) return changed From 99c06ced82e06a43273aa87e51f077e3cbbf2814 Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 17:47:07 +0100 Subject: [PATCH 04/12] simplify all entity deletes --- server/kitsu/utils.py | 59 +++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/server/kitsu/utils.py b/server/kitsu/utils.py index b671821..8958e59 100644 --- a/server/kitsu/utils.py +++ b/server/kitsu/utils.py @@ -155,18 +155,7 @@ async def delete_folder( **kwargs, ) -> None: folder = await FolderEntity.load(project_name, folder_id) - - # do we need this? - await folder.ensure_delete_access(user) - - await folder.delete() - event = { - "topic": "entity.folder.deleted", - "description": f"Folder {folder.name} deleted", - "summary": {"entityId": folder.id, "parentId": folder.parent_id}, - "project": project_name, - } - await dispatch_event(**event) + delete_entity(project_name, folder, user) async def create_task( @@ -215,18 +204,7 @@ async def delete_task( **kwargs, ) -> None: task = await TaskEntity.load(project_name, task_id) - - # do we need this? - await task.ensure_delete_access(user) - - await task.delete() - event = { - "topic": "entity.task.deleted", - "description": f"Task {task.name} deleted", - "summary": {"entityId": task.id, "parentId": task.parent_id}, - "project": project_name, - } - await dispatch_event(**event) + delete_entity(project_name, task, user) async def update_project( @@ -289,3 +267,36 @@ async def update_entity( logging.debug(f"dispatch_event: {event}") await dispatch_event(**event) return changed + + +async def delete_entity( + project_name: str, + entity, + user: "UserEntity", +) -> None: + """delete the given entity after checking user permission, dispatches a delete event""" + + # check user permission to delete this entity + if hasattr(entity, "ensure_delete_access") and callable( + entity.ensure_delete_access + ): + await entity.ensure_delete_access(user) + + await entity.delete() + + summary = {} + if hasattr(entity, "id"): + summary["id"] = entity.id + if hasattr(entity, "parent_id"): + summary["parent_id"] = entity.parent_id + if hasattr(entity, "name"): + summary["name"] = entity.name + + event = { + "topic": f"entity.{entity.entity_type}.deleted", + "description": f"{entity.entity_type} {entity.name} deleted", + "summary": summary, + "project": project_name, + } + logging.debug(f"dispatch_event: {event}") + await dispatch_event(**event) From 295c0eab8776e2ad409b52af5496c8dfe395b92b Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 17:55:20 +0100 Subject: [PATCH 05/12] simplify entity create --- server/kitsu/utils.py | 49 ++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/server/kitsu/utils.py b/server/kitsu/utils.py index 8958e59..31214ae 100644 --- a/server/kitsu/utils.py +++ b/server/kitsu/utils.py @@ -119,16 +119,7 @@ async def create_folder( project_name=project_name, payload=payload, ) - await folder.save() - event = { - "topic": "entity.folder.created", - "description": f"Folder {folder.name} created", - "summary": {"entityId": folder.id, "parentId": folder.parent_id}, - "project": project_name, - } - - await dispatch_event(**event) - return folder + return create_entity(project_name, folder) async def update_folder( @@ -168,16 +159,7 @@ async def create_task( project_name=project_name, payload=payload, ) - - await task.save() - event = { - "topic": "entity.task.created", - "description": f"Task {task.name} created", - "summary": {"entityId": task.id, "parentId": task.parent_id}, - "project": project_name, - } - await dispatch_event(**event) - return task + return create_entity(project_name, task) async def update_task( @@ -223,9 +205,34 @@ async def update_project( ) +## ==================================================== + + +async def create_entity(project_name: str, entity): + """create a new entity and dispatch a create event, returns the entity""" + await entity.save() + + summary = {} + if hasattr(entity, "id"): + summary["id"] = entity.id + if hasattr(entity, "parent_id"): + summary["parent_id"] = entity.parent_id + if hasattr(entity, "name"): + summary["name"] = entity.name + + event = { + "topic": f"entity.{entity.entity_type}.created", + "description": f"{entity.entity_type} {entity.name} created", + "summary": summary, + "project": project_name, + } + await dispatch_event(**event) + return entity + + async def update_entity( project_name, entity, kwargs, attr_whitelist: list[str] | None = None -): +) -> bool: """updates the entity for given attribute whitelist, saves changes and dispatches an update event""" changed = False if attr_whitelist is None: From aa2633e7eac68af2980da406ad2261461aec1d60 Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 18:02:05 +0100 Subject: [PATCH 06/12] await for async methods --- server/kitsu/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/kitsu/utils.py b/server/kitsu/utils.py index 31214ae..abfd9d4 100644 --- a/server/kitsu/utils.py +++ b/server/kitsu/utils.py @@ -119,7 +119,7 @@ async def create_folder( project_name=project_name, payload=payload, ) - return create_entity(project_name, folder) + return await create_entity(project_name, folder) async def update_folder( @@ -146,7 +146,7 @@ async def delete_folder( **kwargs, ) -> None: folder = await FolderEntity.load(project_name, folder_id) - delete_entity(project_name, folder, user) + await delete_entity(project_name, folder, user) async def create_task( @@ -159,7 +159,7 @@ async def create_task( project_name=project_name, payload=payload, ) - return create_entity(project_name, task) + return await create_entity(project_name, task) async def update_task( @@ -186,7 +186,7 @@ async def delete_task( **kwargs, ) -> None: task = await TaskEntity.load(project_name, task_id) - delete_entity(project_name, task, user) + await delete_entity(project_name, task, user) async def update_project( From 4e79fe8224c4b6e0c2d79a5bbbaa04ef360328c3 Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 21:37:59 +0100 Subject: [PATCH 07/12] fix param name --- server/kitsu/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/kitsu/utils.py b/server/kitsu/utils.py index abfd9d4..b94a6c8 100644 --- a/server/kitsu/utils.py +++ b/server/kitsu/utils.py @@ -86,7 +86,7 @@ async def get_task_by_kitsu_id( """Get an Ayon TaskEntity by its Kitsu ID""" if existing_tasks and (kitsu_id in existing_tasks): - folder_id = existing_tasks[kitsu_id] + task_id = existing_tasks[kitsu_id] else: res = await Postgres.fetch( @@ -98,9 +98,9 @@ async def get_task_by_kitsu_id( ) if not res: return None - folder_id = res[0]["id"] + task_id = res[0]["id"] - return await TaskEntity.load(project_name, folder_id) + return await TaskEntity.load(project_name, task_id) async def create_folder( From 994dbac31877e42461893e88741454533af679cb Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 22:35:05 +0100 Subject: [PATCH 08/12] move delete_project to utils with the rest of the entity calls --- server/kitsu/utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/kitsu/utils.py b/server/kitsu/utils.py index b94a6c8..de0bde1 100644 --- a/server/kitsu/utils.py +++ b/server/kitsu/utils.py @@ -9,6 +9,7 @@ UserEntity, ) from ayon_server.events import dispatch_event +from ayon_server.exceptions import ForbiddenException from ayon_server.lib.postgres import Postgres @@ -205,6 +206,14 @@ async def update_project( ) +async def delete_project(project_name: str, user: "UserEntity"): + project = await ProjectEntity.load(project_name) + if not user.is_manager: + raise ForbiddenException("You need to be a manager in order to delete projects") + + return await delete_entity(project_name, project, user) + + ## ==================================================== From 15a80422277144e32125340b3c9276d0d1e510c8 Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 22:35:57 +0100 Subject: [PATCH 09/12] tests for deleting projects if the setting to allow project deletion is enabled and if it is disabled --- tests/tests/fixtures.py | 51 ++++++++++++++++++++++++++++++-- tests/tests/mock_data.py | 4 +-- tests/tests/test_push_project.py | 34 +++++++++++++++++++++ 3 files changed, 85 insertions(+), 4 deletions(-) diff --git a/tests/tests/fixtures.py b/tests/tests/fixtures.py index 5ef33cf..ac9fd92 100644 --- a/tests/tests/fixtures.py +++ b/tests/tests/fixtures.py @@ -154,7 +154,6 @@ def get_paired_ayon_project(self, kitsu_project_id): return MockProcessor() - # ======= Studio Settings Fixtures ========== @pytest.fixture() def ensure_kitsu_server_setting(api, kitsu_url): @@ -162,7 +161,7 @@ def ensure_kitsu_server_setting(api, kitsu_url): res = api.get(f"{kitsu_url}/settings") assert res.status_code == 200 settings = res.data - + value = settings["server"] # set settings for tests @@ -176,6 +175,7 @@ def ensure_kitsu_server_setting(api, kitsu_url): settings["server"] = "" res = api.post(f"{kitsu_url}/settings", **settings) + @pytest.fixture() def users_enabled(api, kitsu_url): """update kitsu addon settings.sync_settings.sync_users.enabled""" @@ -250,3 +250,50 @@ def access_group(api, kitsu_url): settings["sync_settings"]["sync_users"]["access_group"] = value res = api.post(f"{kitsu_url}/settings", **settings) + +@pytest.fixture() +def delete_projects_enabled(api, kitsu_url): + """update kitsu addon settings.sync_settings.delete_projects""" + # lets get the settings for the addon + res = api.get(f"{kitsu_url}/settings") + assert res.status_code == 200 + settings = res.data + + # get original values + value = settings["sync_settings"]["delete_projects"] + + # set settings for tests + if not value: + settings["sync_settings"]["delete_projects"] = True + res = api.post(f"{kitsu_url}/settings", **settings) + + yield + + # set settings back to orginal values + if not value: + settings["sync_settings"]["delete_projects"] = False + res = api.post(f"{kitsu_url}/settings", **settings) + + +@pytest.fixture() +def delete_projects_disabled(api, kitsu_url): + """update kitsu addon settings.sync_settings.delete_projects""" + # lets get the settings for the addon + res = api.get(f"{kitsu_url}/settings") + assert res.status_code == 200 + settings = res.data + + # get original values + value = settings["sync_settings"]["delete_projects"] + + # set settings for tests + if value: + settings["sync_settings"]["delete_projects"] = False + res = api.post(f"{kitsu_url}/settings", **settings) + + yield + + # set settings back to orginal values + if value: + settings["sync_settings"]["delete_projects"] = True + res = api.post(f"{kitsu_url}/settings", **settings) diff --git a/tests/tests/mock_data.py b/tests/tests/mock_data.py index 562b75a..5a48354 100644 --- a/tests/tests/mock_data.py +++ b/tests/tests/mock_data.py @@ -2,13 +2,13 @@ projects = [ { - "name": "TestProject1", + "name": "test_kitsu_project", "code": "TP1", "id": "kitsu-project-id-1", "type": "Project", }, { - "name": "TestProject2", + "name": "another_test_kitsu_project", "code": "TP2", "id": "kitsu-project-id-2", "type": "Project", diff --git a/tests/tests/test_push_project.py b/tests/tests/test_push_project.py index 156d65b..6686464 100644 --- a/tests/tests/test_push_project.py +++ b/tests/tests/test_push_project.py @@ -18,6 +18,8 @@ api, kitsu_url, ensure_kitsu_server_setting, + delete_projects_enabled, + delete_projects_disabled, ) from kitsu_mock import KitsuMock @@ -216,3 +218,35 @@ def test_push_unsynced_project(api, kitsu_url): # no project changes as project is not synced target_project = api.get_project(entity["name"]) assert project == target_project + + +def test_delete_project_disabled(api, kitsu_url, delete_projects_disabled): + """testing attempting to remove a project + when delete_projects is False in the kitsu settings""" + + entity = mock_data.projects[0] + + # check the project exists + assert api.get_project(entity["name"]) + + res = api.post( + f"{kitsu_url}/remove", project_name=entity["name"], entities=[entity] + ) + assert res.status_code == 200 + + # project should still exist + assert api.get_project(entity["name"]) + + +def test_delete_project_enabled(api, kitsu_url, delete_projects_enabled): + """testing attempting to remove a project + when delete_projects is True in the kitsu settings""" + entity = mock_data.projects[0] + + res = api.post( + f"{kitsu_url}/remove", project_name=entity["name"], entities=[entity] + ) + assert res.status_code == 200 + + # project should be deleted + assert not api.get_project(entity["name"]) From 1c82c669dafee1558daa704adb83cb0495817112 Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 22:36:34 +0100 Subject: [PATCH 10/12] move delete to utils --- server/kitsu/push.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/server/kitsu/push.py b/server/kitsu/push.py index 66b47c7..0b4e658 100644 --- a/server/kitsu/push.py +++ b/server/kitsu/push.py @@ -330,23 +330,6 @@ async def sync_project( await update_project(project.name, **anatomy_data) -async def delete_project( - addon: "KitsuAddon", - user: "UserEntity", - project: "ProjectEntity", - entity_dict: "EntityDict", -): - logging.info("delete_project") - session = await Session.create(user) - headers = {"Authorization": f"Bearer {session.token}"} - # Check if group already exists - async with httpx.AsyncClient() as client: - await client.delete( - f"{entity_dict['ayon_server_url']}/api/projects/{project.name}", - headers=headers, - ) - - async def sync_folder( addon: "KitsuAddon", user: "UserEntity", From d2a46944ab04e3f6aef109e53aa5480cbfdd7b56 Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 22:37:02 +0100 Subject: [PATCH 11/12] formatting and imports --- server/kitsu/push.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/server/kitsu/push.py b/server/kitsu/push.py index 0b4e658..928c6a0 100644 --- a/server/kitsu/push.py +++ b/server/kitsu/push.py @@ -21,11 +21,11 @@ create_task, delete_folder, delete_task, + delete_project, get_folder_by_kitsu_id, get_task_by_kitsu_id, get_user_by_kitsu_id, update_project, - update_folder, update_task, ) @@ -230,16 +230,13 @@ async def sync_person( existing_users: dict[str, Any], entity_dict: "EntityDict", ): - - first_name, entity_id= required_values(entity_dict, ["first_name", "id"]) - last_name = entity_dict.get("last_name", '') + first_name, entity_id = required_values(entity_dict, ["first_name", "id"]) + last_name = entity_dict.get("last_name", "") # == check should Person entity be synced == # do not sync Kitsu API bots if entity_dict.get("is_bot"): - logging.info( - f"skipping sync_person for Kitsu Bot: {first_name} {last_name}" - ) + logging.info(f"skipping sync_person for Kitsu Bot: {first_name} {last_name}") return logging.info(f"sync_person: {first_name} {last_name}") From 86c1aba7584ef28c236f42f5e6658826964a803c Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jun 2024 22:37:33 +0100 Subject: [PATCH 12/12] BUGFIX - setting path fix --- server/kitsu/push.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/kitsu/push.py b/server/kitsu/push.py index 928c6a0..879ace7 100644 --- a/server/kitsu/push.py +++ b/server/kitsu/push.py @@ -625,7 +625,7 @@ async def remove_entities( continue if entity_dict["type"] == "Project": - if settings.delete_ayon_projects.enabled: + if settings.sync_settings.delete_projects: await update_project( addon, user,