From 3620ed06e4b900e1ade3759dcaf3e051cefb2b4f Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Sat, 14 Jun 2025 13:26:12 +1000 Subject: [PATCH 1/4] Fixed lint issues --- gitopsconfig.yaml | 12 ------------ src/configuration/gitops_config.py | 10 +++++----- src/configuration/gitops_config_operator.py | 8 +------- src/configuration/gitops_connector.py | 5 +++-- src/configuration/gitops_connector_manager.py | 1 + src/gitops_event_handler.py | 10 ++++------ 6 files changed, 14 insertions(+), 32 deletions(-) delete mode 100644 gitopsconfig.yaml diff --git a/gitopsconfig.yaml b/gitopsconfig.yaml deleted file mode 100644 index 27f553e..0000000 --- a/gitopsconfig.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: example.com/v1 -kind: GitOpsConfig -metadata: - name: ezievent-identityservice-gitops-stage-dev -spec: - gitRepositoryType: "AZDO" - ciCdOrchestratorType: "AZDO" - gitOpsOperatorType: "ARGOCD" - gitOpsAppURL: "https://dev.azure.com/ezievent/EziEvent/_git/ezievent-identityservice-gitops" - azdoGitOpsRepoName: "ezievent-identityservice-gitops" - azdoPrRepoName: "ezievent-identityservice-gitops" - azdoOrgUrl: "https://dev.azure.com/ezievent/EziEvent" diff --git a/src/configuration/gitops_config.py b/src/configuration/gitops_config.py index 57b5023..de045d9 100644 --- a/src/configuration/gitops_config.py +++ b/src/configuration/gitops_config.py @@ -1,10 +1,10 @@ class GitOpsConfig: def __init__(self, - name, - git_repository_type, - cicd_orchestrator_type, - gitops_operator_type, - gitops_app_url, + name, + git_repository_type, + cicd_orchestrator_type, + gitops_operator_type, + gitops_app_url, azdo_gitops_repo_name=None, azdo_pr_repo_name=None, azdo_org_url=None, diff --git a/src/configuration/gitops_config_operator.py b/src/configuration/gitops_config_operator.py index 69ee2f8..99bf989 100644 --- a/src/configuration/gitops_config_operator.py +++ b/src/configuration/gitops_config_operator.py @@ -1,9 +1,7 @@ import kopf import logging from kubernetes import config as k8s_config -from threading import Thread from configuration.gitops_config import GitOpsConfig -from typing import Callable, Optional, List from configuration.gitops_connector_manager import GitOpsConnectorManager class GitOpsConfigOperator: @@ -61,10 +59,6 @@ def parse_config(self, spec, name): def get_configuration(self, name): """Get the configuration object by name.""" return self.configurations.get(name) - - # def get_gitops_connector(self, name): - # """Get the gitops_connector object by name.""" - # return self.connector_manager.connectors.get(name) def stop_all(self): - self.connector_manager.stop_all() \ No newline at end of file + self.connector_manager.stop_all() diff --git a/src/configuration/gitops_connector.py b/src/configuration/gitops_connector.py index cd2acb3..1411842 100644 --- a/src/configuration/gitops_connector.py +++ b/src/configuration/gitops_connector.py @@ -17,6 +17,7 @@ PR_CLEANUP_INTERVAL = 1 * 30 DISABLE_POLLING_PR_TASK = False + # Instance is shared across threads. class GitopsConnector: @@ -29,7 +30,7 @@ def __init__(self, gitops_config: GitOpsConfig): self.status_thread = None self.status_thread_running = False - + self.cleanup_task = Timeloop() self.cleanup_task_running = False @@ -47,7 +48,7 @@ def pr_polling_thread_worker(): def is_supported_message(self, payload): return self._gitops_operator.is_supported_message(payload) - + def start_background_work(self): self._start_status_thread() self._start_cleanup_task() diff --git a/src/configuration/gitops_connector_manager.py b/src/configuration/gitops_connector_manager.py index c616e04..7abf441 100644 --- a/src/configuration/gitops_connector_manager.py +++ b/src/configuration/gitops_connector_manager.py @@ -2,6 +2,7 @@ from configuration.gitops_connector import GitopsConnector from configuration.gitops_config import GitOpsConfig + class GitOpsConnectorManager: """Manages configurations and the lifecycle of GitOpsConnector instances.""" def __init__(self): diff --git a/src/gitops_event_handler.py b/src/gitops_event_handler.py index 366198e..0b6aa7c 100644 --- a/src/gitops_event_handler.py +++ b/src/gitops_event_handler.py @@ -5,7 +5,6 @@ import logging import kopf from kubernetes import client, config -from kubernetes.client.rest import ApiException import atexit import time import utils @@ -42,11 +41,11 @@ else: logging.debug('Detected no ENV configuration data. Running in multiple instance configuration mode via gitopsconfig resources.') try: - cluster_domain=utils.getenv('CLUSTER_DOMAIN') + cluster_domain = utils.getenv('CLUSTER_DOMAIN') logging.debug(f"cluster domain: '{cluster_domain}'") config.load_incluster_config() # In-cluster Kubernetes config api_instance = client.CustomObjectsApi() - instances = api_instance.list_cluster_custom_object(cluster_domain, "v1", "gitopsconfigs") + instances = api_instance.list_cluster_custom_object(cluster_domain, "v1", "gitopsconfigs") for instance in instances.get("items"): config_name = instance.get("metadata").get("name") config_namespace = instance.get("metadata").get("namespace") @@ -76,8 +75,6 @@ def run_kopf_operator(): kopf_thread = Thread(target=run_kopf_operator) kopf_thread.start() - - @application.route("/gitopsphase", methods=['POST']) def gitopsphase(): # Use per process timer to stash the time we got the request @@ -99,7 +96,7 @@ def gitopsphase(): logging.debug(f'GitOps phase: {payload}') gitops_connector = connector_manager.get_supported_gitops_connector(payload) - if gitops_connector != None: + if gitops_connector is not None: gitops_connector.process_gitops_phase(payload, req_time) return f'GitOps phase: {payload}', 200 @@ -108,6 +105,7 @@ def gitopsphase(): def interrupt(): connector_manager.stop_all() + atexit.register(interrupt) From 87823da995b16de2090cc4d5d81542de1415185b Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Sat, 14 Jun 2025 13:34:33 +1000 Subject: [PATCH 2/4] Added license headers --- src/configuration/gitops_config.py | 3 +++ src/configuration/gitops_config_operator.py | 3 +++ src/configuration/gitops_connector_manager.py | 3 +++ src/gitops_event_handler.py | 1 + src/orchestrators/cicd_orchestrator_factory.py | 1 - 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/configuration/gitops_config.py b/src/configuration/gitops_config.py index de045d9..366d23a 100644 --- a/src/configuration/gitops_config.py +++ b/src/configuration/gitops_config.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + class GitOpsConfig: def __init__(self, name, diff --git a/src/configuration/gitops_config_operator.py b/src/configuration/gitops_config_operator.py index 99bf989..ad42035 100644 --- a/src/configuration/gitops_config_operator.py +++ b/src/configuration/gitops_config_operator.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + import kopf import logging from kubernetes import config as k8s_config diff --git a/src/configuration/gitops_connector_manager.py b/src/configuration/gitops_connector_manager.py index 7abf441..71f519e 100644 --- a/src/configuration/gitops_connector_manager.py +++ b/src/configuration/gitops_connector_manager.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + import logging from configuration.gitops_connector import GitopsConnector from configuration.gitops_config import GitOpsConfig diff --git a/src/gitops_event_handler.py b/src/gitops_event_handler.py index 0b6aa7c..bba4c1f 100644 --- a/src/gitops_event_handler.py +++ b/src/gitops_event_handler.py @@ -75,6 +75,7 @@ def run_kopf_operator(): kopf_thread = Thread(target=run_kopf_operator) kopf_thread.start() + @application.route("/gitopsphase", methods=['POST']) def gitopsphase(): # Use per process timer to stash the time we got the request diff --git a/src/orchestrators/cicd_orchestrator_factory.py b/src/orchestrators/cicd_orchestrator_factory.py index d6fadd1..d5daa63 100644 --- a/src/orchestrators/cicd_orchestrator_factory.py +++ b/src/orchestrators/cicd_orchestrator_factory.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import utils from orchestrators.cicd_orchestrator import CicdOrchestratorInterface from repositories.git_repository import GitRepositoryInterface from orchestrators.azdo_cicd_orchestrator import AzdoCicdOrchestrator From 3d247dfbd3489cc9430802e943daf7cf5513a981 Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Sat, 14 Jun 2025 13:44:33 +1000 Subject: [PATCH 3/4] Fixed more lint errors --- src/clients/azdo_client.py | 4 ++-- src/clients/github_client.py | 2 +- src/configuration/gitops_config.py | 4 ++-- src/configuration/gitops_connector.py | 1 - src/operators/argo_gitops_operator.py | 5 +++-- src/operators/flux_gitops_operator.py | 6 +++--- src/operators/gitops_operator.py | 4 ++-- src/operators/gitops_operator_factory.py | 3 +-- src/orchestrators/azdo_cicd_orchestrator.py | 4 ++-- src/orchestrators/cicd_orchestrator_factory.py | 2 +- src/orchestrators/github_cicd_orchestrator.py | 3 +-- src/repositories/azdo_git_repository.py | 6 ++---- src/repositories/git_repository_factory.py | 5 ++--- src/repositories/github_git_repository.py | 4 ++-- 14 files changed, 24 insertions(+), 29 deletions(-) diff --git a/src/clients/azdo_client.py b/src/clients/azdo_client.py index 99246c2..abd0fd7 100644 --- a/src/clients/azdo_client.py +++ b/src/clients/azdo_client.py @@ -6,11 +6,11 @@ import logging from configuration.gitops_config import GitOpsConfig + class AzdoClient: def __init__(self, gitops_config: GitOpsConfig): - # https://dev.azure.com/csedevops/GitOps - self.org_url = gitops_config.azdo_org_url # utils.getenv("AZDO_ORG_URL") + self.org_url = gitops_config.azdo_org_url # token is supposed to be stored in a secret without any transformations token = base64.b64encode(f':{utils.getenv("PAT")}'.encode("ascii")).decode("ascii") diff --git a/src/clients/github_client.py b/src/clients/github_client.py index abda2b2..e07bd85 100644 --- a/src/clients/github_client.py +++ b/src/clients/github_client.py @@ -8,7 +8,7 @@ class GitHubClient: def __init__(self, gitops_config: GitOpsConfig): - self.org_url = gitops_config.github_org_url # utils.getenv("GITHUB_ORG_URL") # https://api.github.com/repos/kaizentm + self.org_url = gitops_config.github_org_url # token is supposed to be stored in a secret without any transformations self.token = utils.getenv("PAT") self.headers = {'Authorization': f'token {self.token}'} diff --git a/src/configuration/gitops_config.py b/src/configuration/gitops_config.py index 366d23a..9bd72c1 100644 --- a/src/configuration/gitops_config.py +++ b/src/configuration/gitops_config.py @@ -8,8 +8,8 @@ def __init__(self, cicd_orchestrator_type, gitops_operator_type, gitops_app_url, - azdo_gitops_repo_name=None, - azdo_pr_repo_name=None, + azdo_gitops_repo_name=None, + azdo_pr_repo_name=None, azdo_org_url=None, github_gitops_repo_name=None, github_gitops_manifests_repo_name=None, diff --git a/src/configuration/gitops_connector.py b/src/configuration/gitops_connector.py index 1411842..f3650a4 100644 --- a/src/configuration/gitops_connector.py +++ b/src/configuration/gitops_connector.py @@ -143,4 +143,3 @@ def drain_commit_status_queue(self): except Exception as e: logging.error(f'Unexpected exception in the message queue draining thread: {e}') - diff --git a/src/operators/argo_gitops_operator.py b/src/operators/argo_gitops_operator.py index b8c9489..7185643 100644 --- a/src/operators/argo_gitops_operator.py +++ b/src/operators/argo_gitops_operator.py @@ -7,6 +7,7 @@ from operators.git_commit_status import GitCommitStatus from configuration.gitops_config import GitOpsConfig + class ArgoGitopsOperator(GitopsOperatorInterface): def __init__(self, gitops_config: GitOpsConfig): @@ -48,7 +49,7 @@ def is_finished(self, phase_data): logging.debug(f'is_finished called. phase_data: {json.dumps(phase_data, indent=2)}') phase_status, _, health_status = self._get_statuses(phase_data) logging.debug(f'is_finished: phase_status: {phase_status}, health_status: {health_status}') - + is_finished = \ phase_status != 'Inconclusive' \ and phase_status != 'Running' \ @@ -80,7 +81,7 @@ def is_supported_operator(self, phase_data) -> bool: def is_supported_message(self, phase_data) -> bool: if ((not self.is_supported_operator(phase_data)) or - phase_data['commitid'] == "" or + phase_data['commitid'] == "" or phase_data['resources'] == None): return False return True diff --git a/src/operators/flux_gitops_operator.py b/src/operators/flux_gitops_operator.py index 8b2886b..9efdb06 100644 --- a/src/operators/flux_gitops_operator.py +++ b/src/operators/flux_gitops_operator.py @@ -132,11 +132,11 @@ def get_commit_id(self, phase_data) -> str: revision = phase_data['metadata']['source.toolkit.fluxcd.io/revision'] return parse_commit_id(revision) - + def is_supported_operator(self, phase_data) -> bool: return (self.gitops_config.name == 'singleInstance' or - (self.gitops_config.name != 'singleInstance' and - 'gitops_connector_config_name' in phase_data['metadata'] and + (self.gitops_config.name != 'singleInstance' and + 'gitops_connector_config_name' in phase_data['metadata'] and phase_data['metadata']['gitops_connector_config_name'] == self.gitops_config.name)) def is_supported_message(self, phase_data) -> bool: diff --git a/src/operators/gitops_operator.py b/src/operators/gitops_operator.py index d243ea8..a767f6c 100644 --- a/src/operators/gitops_operator.py +++ b/src/operators/gitops_operator.py @@ -1,15 +1,15 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import utils from abc import ABC, abstractmethod from configuration.gitops_config import GitOpsConfig + class GitopsOperatorInterface(ABC): def __init__(self, gitops_config: GitOpsConfig): self.gitops_config = gitops_config - self.callback_url = gitops_config.gitops_app_url # utils.getenv("GITOPS_APP_URL") + self.callback_url = gitops_config.gitops_app_url @abstractmethod def extract_commit_statuses(self, phase_data): diff --git a/src/operators/gitops_operator_factory.py b/src/operators/gitops_operator_factory.py index a40f389..10fa3df 100644 --- a/src/operators/gitops_operator_factory.py +++ b/src/operators/gitops_operator_factory.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import utils from operators.argo_gitops_operator import ArgoGitopsOperator from operators.flux_gitops_operator import FluxGitopsOperator from operators.gitops_operator import GitopsOperatorInterface @@ -16,7 +15,7 @@ class GitopsOperatorFactory: @staticmethod def new_gitops_operator(gitops_config: GitOpsConfig) -> GitopsOperatorInterface: - gitops_operator_type = gitops_config.gitops_operator_type # utils.getenv("GITOPS_OPERATOR_TYPE", FLUX_TYPE) + gitops_operator_type = gitops_config.gitops_operator_type if gitops_operator_type == FLUX_TYPE: return FluxGitopsOperator(gitops_config) diff --git a/src/orchestrators/azdo_cicd_orchestrator.py b/src/orchestrators/azdo_cicd_orchestrator.py index 7f5fcd6..07e532e 100644 --- a/src/orchestrators/azdo_cicd_orchestrator.py +++ b/src/orchestrators/azdo_cicd_orchestrator.py @@ -77,7 +77,7 @@ def _update_pr_task(self, is_successful, pr_num, is_alive=True): def _get_pr_task_data(self, pr_num, is_alive=True): logging.debug(f'_get_pr_task_data called. pr_num: {pr_num}, is_alive: {is_alive}') return self.git_repository.get_pr_metadata(pr_num) - + # Given a PR task, check if it's parent plan has already completed. # Note: Completed does not necessarily mean it succeeded. def _plan_already_completed(self, pr_task): @@ -120,7 +120,7 @@ def _build_job_already_completed(self, pr_task, plan_info): logging.debug(f'Check if job {job_id} already completed: state = {job_state}') job_state_completed = job_state == 'completed' return job_state_completed - + def notify_abandoned_pr_tasks(self): logging.debug('notify_abandoned_pr_tasks called') update_count = 0 diff --git a/src/orchestrators/cicd_orchestrator_factory.py b/src/orchestrators/cicd_orchestrator_factory.py index d5daa63..d0e8ca2 100644 --- a/src/orchestrators/cicd_orchestrator_factory.py +++ b/src/orchestrators/cicd_orchestrator_factory.py @@ -16,7 +16,7 @@ class CicdOrchestratorFactory: @staticmethod def new_cicd_orchestrator(git_repository: GitRepositoryInterface, gitops_config: GitOpsConfig) -> CicdOrchestratorInterface: - cicd_orchestrator_type = gitops_config.cicd_orchestrator_type # utils.getenv("CICD_ORCHESTRATOR_TYPE", AZDO_TYPE) + cicd_orchestrator_type = gitops_config.cicd_orchestrator_type if cicd_orchestrator_type == AZDO_TYPE: return AzdoCicdOrchestrator(git_repository, gitops_config) diff --git a/src/orchestrators/github_cicd_orchestrator.py b/src/orchestrators/github_cicd_orchestrator.py index ed3d364..a9ae90c 100644 --- a/src/orchestrators/github_cicd_orchestrator.py +++ b/src/orchestrators/github_cicd_orchestrator.py @@ -2,7 +2,6 @@ # Licensed under the MIT License. import logging -import utils import requests from orchestrators.cicd_orchestrator import CicdOrchestratorInterface from repositories.git_repository import GitRepositoryInterface @@ -14,7 +13,7 @@ class GitHubCicdOrchestrator(CicdOrchestratorInterface): def __init__(self, git_repository: GitRepositoryInterface, gitops_config: GitOpsConfig): super().__init__(git_repository) - self.gitops_repo_name = gitops_config.github_gitops_repo_name # utils.getenv("GITHUB_GITOPS_REPO_NAME") # cloud-native-ops + self.gitops_repo_name = gitops_config.github_gitops_repo_name self.github_client = GitHubClient(gitops_config) self.headers = self.github_client.get_rest_api_headers() self.rest_api_url = self.github_client.get_rest_api_url() diff --git a/src/repositories/azdo_git_repository.py b/src/repositories/azdo_git_repository.py index ee7ab24..d1b7279 100644 --- a/src/repositories/azdo_git_repository.py +++ b/src/repositories/azdo_git_repository.py @@ -1,11 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import os import logging import json import requests -import utils from clients.azdo_client import AzdoClient from repositories.git_repository import GitRepositoryInterface from configuration.gitops_config import GitOpsConfig @@ -16,8 +14,8 @@ class AzdoGitRepository(GitRepositoryInterface): def __init__(self, gitops_config: GitOpsConfig): - self.gitops_repo_name = gitops_config.azdo_gitops_repo_name # utils.getenv("AZDO_GITOPS_REPO_NAME") - self.pr_repo_name = gitops_config.azdo_pr_repo_name # os.getenv("AZDO_PR_REPO_NAME", self.gitops_repo_name) + self.gitops_repo_name = gitops_config.azdo_gitops_repo_name + self.pr_repo_name = gitops_config.azdo_pr_repo_name self.azdo_client = AzdoClient(gitops_config) self.repository_api = f'{self.azdo_client.get_rest_api_url()}/_apis/git/repositories/{self.gitops_repo_name}' self.pr_repository_api = f'{self.azdo_client.get_rest_api_url()}/_apis/git/repositories/{self.pr_repo_name}' diff --git a/src/repositories/git_repository_factory.py b/src/repositories/git_repository_factory.py index 023391d..43ebe0f 100644 --- a/src/repositories/git_repository_factory.py +++ b/src/repositories/git_repository_factory.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import os from repositories.git_repository import GitRepositoryInterface from repositories.azdo_git_repository import AzdoGitRepository from repositories.github_git_repository import GitHubGitRepository @@ -15,8 +14,8 @@ class GitRepositoryFactory: @staticmethod - def new_git_repository(gitops_config:GitOpsConfig) -> GitRepositoryInterface: - git_repository_type = gitops_config.git_repository_type # os.getenv("GIT_REPOSITORY_TYPE", AZDO_TYPE) + def new_git_repository(gitops_config: GitOpsConfig) -> GitRepositoryInterface: + git_repository_type = gitops_config.git_repository_type if git_repository_type == AZDO_TYPE: return AzdoGitRepository(gitops_config) diff --git a/src/repositories/github_git_repository.py b/src/repositories/github_git_repository.py index d2a06c7..adca2bd 100644 --- a/src/repositories/github_git_repository.py +++ b/src/repositories/github_git_repository.py @@ -2,18 +2,18 @@ # Licensed under the MIT License. import requests -import utils import logging from clients.github_client import GitHubClient from repositories.git_repository import GitRepositoryInterface from configuration.gitops_config import GitOpsConfig + class GitHubGitRepository(GitRepositoryInterface): MAX_DESCR_LENGTH = 140 def __init__(self, gitops_config: GitOpsConfig): - self.gitops_repo_name = gitops_config.github_gitops_manifests_repo_name # utils.getenv("GITHUB_GITOPS_MANIFEST_REPO_NAME") # gitops-manifests + self.gitops_repo_name = gitops_config.github_gitops_manifests_repo_name self.github_client = GitHubClient(gitops_config) self.headers = self.github_client.get_rest_api_headers() self.rest_api_url = self.github_client.get_rest_api_url() From 70a7f36e21da87de81e33c7f37b11da755e67cbe Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Sat, 14 Jun 2025 13:58:11 +1000 Subject: [PATCH 4/4] Cleaner condition logic --- src/configuration/gitops_config_operator.py | 1 + src/operators/argo_gitops_operator.py | 9 ++------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/configuration/gitops_config_operator.py b/src/configuration/gitops_config_operator.py index ad42035..fee30cf 100644 --- a/src/configuration/gitops_config_operator.py +++ b/src/configuration/gitops_config_operator.py @@ -7,6 +7,7 @@ from configuration.gitops_config import GitOpsConfig from configuration.gitops_connector_manager import GitOpsConnectorManager + class GitOpsConfigOperator: def __init__(self, connector_manager: GitOpsConnectorManager): self.configurations = {} # Store configuration objects indexed by resource name diff --git a/src/operators/argo_gitops_operator.py b/src/operators/argo_gitops_operator.py index 7185643..a279a91 100644 --- a/src/operators/argo_gitops_operator.py +++ b/src/operators/argo_gitops_operator.py @@ -76,15 +76,10 @@ def _new_git_commit_status(self, commit_id, status_name, state, message: str): genre='ArgoCD') def is_supported_operator(self, phase_data) -> bool: - return (self.gitops_config.name == 'singleInstance' or - self.gitops_config.name != 'singleInstance' and phase_data.get('gitops_connector_config_name') == self.gitops_config.name) + return self.gitops_config.name == 'singleInstance' or phase_data.get('gitops_connector_config_name') == self.gitops_config.name def is_supported_message(self, phase_data) -> bool: - if ((not self.is_supported_operator(phase_data)) or - phase_data['commitid'] == "" or - phase_data['resources'] == None): - return False - return True + return self.is_supported_operator(phase_data) and phase_data.get('commitid') != "" and phase_data.get('resources') is not None def _get_deployment_status_summary(self, resources): total = len(resources)