diff --git a/src/api-service/__app__/requirements.txt b/src/api-service/__app__/requirements.txt index 46ce15a21e..21cbcc18b0 100644 --- a/src/api-service/__app__/requirements.txt +++ b/src/api-service/__app__/requirements.txt @@ -31,4 +31,4 @@ jsonpatch==1.32 semver==2.13.0 base58==2.1.0 # onefuzz types version is set during build -onefuzztypes==0.0.0 +onefuzztypes==2.30.0 diff --git a/src/api-service/__app__/timer_retention/__init__.py b/src/api-service/__app__/timer_retention/__init__.py new file mode 100644 index 0000000000..cb86f85436 --- /dev/null +++ b/src/api-service/__app__/timer_retention/__init__.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import datetime +import logging + +import azure.functions as func +from onefuzztypes.enums import JobState, TaskState + +from ..onefuzzlib.events import get_events +from ..onefuzzlib.jobs import Job +from ..onefuzzlib.notifications.main import Notification +from ..onefuzzlib.repro import Repro +from ..onefuzzlib.tasks.main import Task + +RETENTION_POLICY = datetime.timedelta(minutes=(20)) +SEARCH_EXTENT = datetime.timedelta(minutes=(120)) + + +def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None: # noqa: F841 + + now = datetime.datetime.now(tz=datetime.timezone.utc) + + time_retained_older = now - RETENTION_POLICY + time_retained_newer = now - SEARCH_EXTENT + + time_filter = ( + f"Timestamp lt datetime'{time_retained_older.isoformat()}' " + f"and Timestamp gt datetime'{time_retained_newer.isoformat()}'" + ) + time_filter_newer = f"Timestamp gt datetime'{time_retained_older.isoformat()}'" + + # Collecting 'still relevant' task containers. + # NOTE: This must be done before potentially modifying tasks otherwise + # the task timestamps will not be useful. + used_containers = set() + for task in Task.search(raw_unchecked_filter=time_filter_newer): + task_containers = {x.name for x in task.config.containers} + used_containers.update(task_containers) + + for notification in Notification.search(raw_unchecked_filter=time_filter): + logging.debug( + "checking expired notification for removal: %s", + notification.notification_id, + ) + container = notification.container + if container not in used_containers: + logging.info( + "deleting expired notification: %s", notification.notification_id + ) + notification.delete() + + for job in Job.search( + query={"state": [JobState.stopped]}, raw_unchecked_filter=time_filter + ): + if job.user_info is not None and job.user_info.upn is not None: + logging.info("removing PII from job: %s", job.job_id) + job.user_info.upn = None + job.save() + + for task in Task.search( + query={"state": [TaskState.stopped]}, raw_unchecked_filter=time_filter + ): + if task.user_info is not None and task.user_info.upn is not None: + logging.info("removing PII from task: %s", task.task_id) + task.user_info.upn = None + task.save() + + for repro in Repro.search(raw_unchecked_filter=time_filter): + if repro.user_info is not None and repro.user_info.upn is not None: + logging.info("removing PII from repro: %s", repro.vm_id) + repro.user_info.upn = None + repro.save() + + events = get_events() + if events: + dashboard.set(events) diff --git a/src/api-service/__app__/timer_retention/function.json b/src/api-service/__app__/timer_retention/function.json new file mode 100644 index 0000000000..f377ecad7b --- /dev/null +++ b/src/api-service/__app__/timer_retention/function.json @@ -0,0 +1,17 @@ +{ + "bindings": [ + { + "direction": "in", + "name": "mytimer", + "schedule": "20:00:00", + "type": "timerTrigger" + }, + { + "type": "signalR", + "direction": "out", + "name": "dashboard", + "hubName": "dashboard" + } + ], + "scriptFile": "__init__.py" +}