diff --git a/tools/c7n_azure/c7n_azure/container_host/host.py b/tools/c7n_azure/c7n_azure/container_host/host.py index bc4b0ec9189..192a13dec79 100644 --- a/tools/c7n_azure/c7n_azure/container_host/host.py +++ b/tools/c7n_azure/c7n_azure/container_host/host.py @@ -159,7 +159,7 @@ def update_policies(self): client.get_blob_to_path(container, blob.name, policy_path) self.load_policy(policy_path, policies_copy) - self.blob_cache.update({blob.name: blob.properties.content_settings.content_md5}) + self.blob_cache.update({blob.name: blob.content_settings.content_md5}) # Assign our copy back over the original self.policies = policies_copy @@ -167,7 +167,7 @@ def update_policies(self): def _get_new_blobs(self, blobs): new_blobs = [] for blob in blobs: - md5_hash = blob.properties.content_settings.content_md5 + md5_hash = blob.content_settings.content_md5 if not md5_hash: blob, md5_hash = self._try_create_md5_content_hash(blob) if blob and md5_hash and md5_hash != self.blob_cache.get(blob.name): @@ -191,7 +191,7 @@ def _try_create_md5_content_hash(self, blob): # Re-fetch the blob with the new hash hashed_blob = client.get_blob_properties(container, blob.name) - return hashed_blob, hashed_blob.properties.content_settings.content_md5 + return hashed_blob, hashed_blob.content_settings.content_md5 except AzureHttpError as e: log.warning("Failed to apply a md5 content hash to policy {}. " "This policy will be skipped.".format(blob.name)) diff --git a/tools/c7n_azure/c7n_azure/handler.py b/tools/c7n_azure/c7n_azure/handler.py index 0303da2d917..cd8b397cdb2 100644 --- a/tools/c7n_azure/c7n_azure/handler.py +++ b/tools/c7n_azure/c7n_azure/handler.py @@ -8,6 +8,7 @@ from azure.common import AzureHttpError from msrestazure.azure_exceptions import CloudError +from azure.core.exceptions import AzureError from c7n.utils import reset_session_cache from c7n.config import Config @@ -53,7 +54,7 @@ def run(event, context, subscription_id=None): for p in policies: try: p.push(event, context) - except (CloudError, AzureHttpError) as error: + except (CloudError, AzureHttpError, AzureError) as error: log.error("Unable to process policy: %s :: %s" % (p.name, error)) reset_session_cache() diff --git a/tools/c7n_azure/c7n_azure/provisioning/app_insights.py b/tools/c7n_azure/c7n_azure/provisioning/app_insights.py index 5e8cc3f112d..f42cdb7d6f3 100644 --- a/tools/c7n_azure/c7n_azure/provisioning/app_insights.py +++ b/tools/c7n_azure/c7n_azure/provisioning/app_insights.py @@ -1,6 +1,6 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 -from msrestazure.azure_exceptions import CloudError +from azure.core.exceptions import ResourceNotFoundError from c7n_azure.provisioning.deployment_unit import DeploymentUnit from c7n_azure.provisioning.resource_group import ResourceGroupUnit @@ -16,7 +16,7 @@ def __init__(self): def _get(self, params): try: return self.client.components.get(params['resource_group_name'], params['name']) - except CloudError: + except ResourceNotFoundError: return None def _provision(self, params): diff --git a/tools/c7n_azure/c7n_azure/provisioning/resource_group.py b/tools/c7n_azure/c7n_azure/provisioning/resource_group.py index a574716effb..8fc6d6d0b53 100644 --- a/tools/c7n_azure/c7n_azure/provisioning/resource_group.py +++ b/tools/c7n_azure/c7n_azure/provisioning/resource_group.py @@ -1,6 +1,6 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 -from msrestazure.azure_exceptions import CloudError +from azure.core.exceptions import ResourceNotFoundError from c7n_azure.provisioning.deployment_unit import DeploymentUnit @@ -18,7 +18,7 @@ def verify_params(self, params): def _get(self, params): try: return self.client.resource_groups.get(params['name']) - except CloudError: + except ResourceNotFoundError: return None def _provision(self, params): diff --git a/tools/c7n_azure/c7n_azure/provisioning/storage_account.py b/tools/c7n_azure/c7n_azure/provisioning/storage_account.py index cd0e600610c..aaa2f247630 100644 --- a/tools/c7n_azure/c7n_azure/provisioning/storage_account.py +++ b/tools/c7n_azure/c7n_azure/provisioning/storage_account.py @@ -1,6 +1,6 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 -from msrestazure.azure_exceptions import CloudError +from azure.core.exceptions import ResourceNotFoundError from c7n_azure.provisioning.deployment_unit import DeploymentUnit from c7n_azure.provisioning.resource_group import ResourceGroupUnit @@ -17,7 +17,7 @@ def _get(self, params): try: return self.client.storage_accounts.get_properties(params['resource_group_name'], params['name']) - except CloudError: + except ResourceNotFoundError: return None def _provision(self, params): diff --git a/tools/c7n_azure/c7n_azure/resources/network_security_group.py b/tools/c7n_azure/c7n_azure/resources/network_security_group.py index e5ce49b61c6..2c1e2c2dee1 100644 --- a/tools/c7n_azure/c7n_azure/resources/network_security_group.py +++ b/tools/c7n_azure/c7n_azure/resources/network_security_group.py @@ -6,7 +6,7 @@ from c7n_azure.provider import resources from c7n_azure.resources.arm import ArmResourceManager from c7n_azure.utils import StringUtils, PortsRangeHelper -from msrestazure.azure_exceptions import CloudError +from azure.core.exceptions import AzureError from c7n.actions import BaseAction from c7n.filters import Filter, FilterValidationError @@ -295,7 +295,7 @@ def process(self, network_security_groups): rule_name, new_rule ) - except CloudError as e: + except AzureError as e: self.manager.log.error('Failed to create or update security rule for %s NSG.', nsg_name) self.manager.log.error(e) diff --git a/tools/c7n_azure/c7n_azure/resources/sqldatabase.py b/tools/c7n_azure/c7n_azure/resources/sqldatabase.py index 93371735f5a..8a994dfa521 100644 --- a/tools/c7n_azure/c7n_azure/resources/sqldatabase.py +++ b/tools/c7n_azure/c7n_azure/resources/sqldatabase.py @@ -17,6 +17,7 @@ import enum import logging +from azure.core.exceptions import AzureError, ResourceNotFoundError from azure.mgmt.sql.models import (BackupLongTermRetentionPolicy, BackupShortTermRetentionPolicy, DatabaseUpdate, Sku) @@ -29,7 +30,6 @@ from c7n_azure.query import ChildTypeInfo from c7n_azure.resources.arm import ChildArmResourceManager from c7n_azure.utils import ResourceIdParser, RetentionPeriod, ThreadHelper -from msrestazure.azure_exceptions import CloudError log = logging.getLogger('custodian.azure.sqldatabase') @@ -119,17 +119,16 @@ def get_backup_retention_policy(database, get_operation, cache_key): try: response = get_operation(resource_group_name, server_name, database_name, policy) - except CloudError as e: - if e.status_code == 404: - return None - else: - log.error( - "Unable to get backup retention policy. " - "(resourceGroup: {}, sqlserver: {}, sqldatabase: {})".format( - resource_group_name, server_name, database_name - ) + except ResourceNotFoundError: + return None + except AzureError as e: + log.error( + "Unable to get backup retention policy. " + "(resourceGroup: {}, sqlserver: {}, sqldatabase: {})".format( + resource_group_name, server_name, database_name ) - raise e + ) + raise e retention_policy = response.as_dict() database[policy_key] = retention_policy diff --git a/tools/c7n_azure/c7n_azure/session.py b/tools/c7n_azure/c7n_azure/session.py index 0333fd36aa8..a4e27087296 100644 --- a/tools/c7n_azure/c7n_azure/session.py +++ b/tools/c7n_azure/c7n_azure/session.py @@ -75,31 +75,39 @@ def __init__(self, cloud_endpoints, authorization_file=None, subscription_id_ove self._credential = None self._subscription_id = self._auth_params['subscription_id'] if self._auth_params.get('access_token') is not None: + auth_name = 'Access Token' pass - elif (self._auth_params['client_id'] and - self._auth_params['client_secret'] and - self._auth_params['tenant_id'] + elif (self._auth_params.get('client_id') and + self._auth_params.get('client_secret') and + self._auth_params.get('tenant_id') ): + auth_name = 'Principal' self._credential = ClientSecretCredential( client_id=self._auth_params['client_id'], client_secret=self._auth_params['client_secret'], tenant_id=self._auth_params['tenant_id'], authority=self._auth_params['authority']) elif self._auth_params.get('use_msi'): + auth_name = 'MSI' self._credential = ManagedIdentityCredential( client_id=self._auth_params.get('client_id')) elif self._auth_params.get('enable_cli_auth'): + auth_name = 'Azure CLI' self._credential = AzureCliCredential() self._subscription_id, error = _run_command('az account show --output tsv --query id') self._subscription_id = self._subscription_id.strip('\n') if error is not None: raise Exception('Unable to query SubscriptionId') + log.info('Authenticated [%s | %s%s]', + auth_name, self.subscription_id, + ' | Authorization File' if authorization_file else '') + def get_token(self, *scopes, **kwargs): # Access Token is used only in tests realistically because # KeyVault, Storage and mgmt plane requires separate tokens. # TODO: Should we scope this to tests only? - if (self._auth_params['access_token']): + if (self._auth_params.get('access_token')): return AccessToken(self._auth_params['access_token'], expires_on=0) try: return self._credential.get_token(*scopes, **kwargs) diff --git a/tools/c7n_azure/c7n_azure/storage_utils.py b/tools/c7n_azure/c7n_azure/storage_utils.py index 269664e9e34..b08185caab0 100644 --- a/tools/c7n_azure/c7n_azure/storage_utils.py +++ b/tools/c7n_azure/c7n_azure/storage_utils.py @@ -19,6 +19,12 @@ def get_blob_to_bytes(self, container_name, blob_name): client = self.get_blob_client(container_name, blob_name) return client.download_blob().content_as_bytes() + def get_blob_to_path(self, container_name, blob_name, path): + client = self.get_blob_client(container_name, blob_name) + with open(path, "wb") as f: + download_stream = client.download_blob() + f.write(download_stream.readall()) + def create_blob_from_bytes(self, container_name, blob_name, content, validate_content): client = self.get_blob_client(container_name, blob_name) client.upload_blob(content, overwrite=True) @@ -27,6 +33,10 @@ def get_blob_properties(self, container_name, blob_name): client = self.get_blob_client(container_name, blob_name) return client.get_blob_properties() + def list_blobs(self, container_name): + client = self.get_container_client(container_name) + return client.list_blobs() + class OldQueueService: diff --git a/tools/c7n_azure/c7n_azure/utils.py b/tools/c7n_azure/c7n_azure/utils.py index 3fa5c911c36..04eccb6f82c 100644 --- a/tools/c7n_azure/c7n_azure/utils.py +++ b/tools/c7n_azure/c7n_azure/utils.py @@ -633,7 +633,8 @@ def log_response_data(response): send_logger.debug(http_response.status_code) for k, v in http_response.headers.items(): if k.startswith('x-ms-ratelimit'): - send_logger.info(k + ':' + v) + send_logger.debug(k + ':' + v) + # This workaround will replace used api-version for costmanagement requests diff --git a/tools/c7n_azure/poetry.lock b/tools/c7n_azure/poetry.lock index b9a6c27c606..dd98dcbc413 100644 --- a/tools/c7n_azure/poetry.lock +++ b/tools/c7n_azure/poetry.lock @@ -547,6 +547,19 @@ azure-common = ">=1.1,<2.0" azure-mgmt-core = ">=1.2.0,<2.0.0" msrest = ">=0.5.0" +[[package]] +name = "azure-mgmt-msi" +version = "1.0.0" +description = "Microsoft Azure MSI Management Client Library for Python" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +azure-common = ">=1.1,<2.0" +msrest = ">=0.5.0" +msrestazure = ">=0.4.32,<2.0.0" + [[package]] name = "azure-mgmt-network" version = "17.1.0" @@ -1297,7 +1310,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "9ac507c9c0fa482c21c2e91c688c027ac9585328dd369157412a3b9dc0cacf29" +content-hash = "1f75519ef4af8cfffd36ad9d6d141fc8f7a3c619c8c097ba00bc6d9b958dfd20" [metadata.files] adal = [ @@ -1468,6 +1481,10 @@ azure-mgmt-monitor = [ {file = "azure-mgmt-monitor-2.0.0.zip", hash = "sha256:e7f7943fe8f0efe98b3b1996cdec47c709765257a6e09e7940f7838a0f829e82"}, {file = "azure_mgmt_monitor-2.0.0-py2.py3-none-any.whl", hash = "sha256:af4917df2fe685e3daf25750f3586f11ccd2e7c2da68df392ca093fc3b7b8089"}, ] +azure-mgmt-msi = [ + {file = "azure-mgmt-msi-1.0.0.zip", hash = "sha256:d46f3aab25db3dad520e4055c1d67afe4fcc6d66335c762134e60f82265f8f58"}, + {file = "azure_mgmt_msi-1.0.0-py2.py3-none-any.whl", hash = "sha256:e75175af21f9a471c1e8d7a538c11905d65083b86d661b9a759578fb65a1dbcc"}, +] azure-mgmt-network = [ {file = "azure-mgmt-network-17.1.0.zip", hash = "sha256:f47852836a5960447ab534784a9285696969f007744ba030828da2eab92621ab"}, {file = "azure_mgmt_network-17.1.0-py2.py3-none-any.whl", hash = "sha256:43a1896c4d674ab28c46e2261f128d3ab7a30bcb19cf806f20ff5bccf95187d9"}, diff --git a/tools/c7n_azure/pyproject.toml b/tools/c7n_azure/pyproject.toml index 52207d4f371..95201960cb0 100644 --- a/tools/c7n_azure/pyproject.toml +++ b/tools/c7n_azure/pyproject.toml @@ -76,6 +76,8 @@ azure-identity = "^1.5.0" azure-keyvault = "^4.1.0" azure-storage-file-share = "^12.4.1" cryptography = "^3.4.6" +azure-mgmt-msi = "^1.0.0" +jmespath = "^0.10.0" [tool.poetry.dev-dependencies] # setup custodian as a dev dependency diff --git a/tools/c7n_azure/requirements.txt b/tools/c7n_azure/requirements.txt index a0a496d1b3d..7a1495c3b92 100644 --- a/tools/c7n_azure/requirements.txt +++ b/tools/c7n_azure/requirements.txt @@ -38,6 +38,7 @@ azure-mgmt-keyvault==8.0.0 azure-mgmt-logic==9.0.0 azure-mgmt-managementgroups==1.0.0b1 azure-mgmt-monitor==2.0.0 +azure-mgmt-msi==1.0.0 azure-mgmt-network==17.1.0 azure-mgmt-policyinsights==1.0.0 azure-mgmt-rdbms==8.0.0 @@ -64,6 +65,7 @@ distlib==0.3.1 idna==2.10; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" importlib-resources==5.1.2; python_version >= "3.6" and python_version < "3.7" isodate==0.6.0 +jmespath==0.10.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") jsonpickle==1.3 msal-extensions==0.3.0 msal==1.10.0 diff --git a/tools/c7n_azure/setup.py b/tools/c7n_azure/setup.py index a54e29d8727..d2e01ba26e4 100644 --- a/tools/c7n_azure/setup.py +++ b/tools/c7n_azure/setup.py @@ -52,6 +52,7 @@ 'azure-mgmt-logic>=9.0.0,<10.0.0', 'azure-mgmt-managementgroups==1.0.0b1', 'azure-mgmt-monitor>=2.0.0,<3.0.0', + 'azure-mgmt-msi>=1.0.0,<2.0.0', 'azure-mgmt-network>=17.1.0,<18.0.0', 'azure-mgmt-policyinsights>=1.0.0,<2.0.0', 'azure-mgmt-rdbms>=8.0.0,<9.0.0', @@ -75,6 +76,7 @@ 'distlib>=0.3.0,<0.4.0', 'importlib-metadata (>=3.7.2,<4.0.0)', 'jmespath (>=0.10.0,<0.11.0)', + 'jmespath>=0.10.0,<0.11.0', 'jsonpickle (>=1.3,<2.0)', 'jsonpickle==1.3', 'jsonschema (>=3.2.0,<4.0.0)', diff --git a/tools/c7n_azure/tests_azure/azure_common.py b/tools/c7n_azure/tests_azure/azure_common.py index d7448f03c10..969be8f9a67 100644 --- a/tools/c7n_azure/tests_azure/azure_common.py +++ b/tools/c7n_azure/tests_azure/azure_common.py @@ -473,7 +473,7 @@ def lro_init(self, client, initial_response, deserialization_callback, polling_m issubclass(deserialization_callback, Model): deserialization_callback = deserialization_callback.deserialize - # Might raise a CloudError + # Might raise a AzureError self._polling_method.initialize(self._client, self._response, deserialization_callback) self._thread = None diff --git a/tools/c7n_azure/tests_azure/tests_resources/test_container_host.py b/tools/c7n_azure/tests_azure/tests_resources/test_container_host.py index d34c007f9ee..d709e0c5c99 100644 --- a/tools/c7n_azure/tests_azure/tests_resources/test_container_host.py +++ b/tools/c7n_azure/tests_azure/tests_resources/test_container_host.py @@ -630,7 +630,7 @@ def download_missing_mode_policy_blob(_, name, path): def get_mock_blob(name, md5): new_blob = Mock() new_blob.name = name - new_blob.properties.content_settings.content_md5 = md5 + new_blob.content_settings.content_md5 = md5 return new_blob @staticmethod diff --git a/tools/c7n_azure/tests_azure/tests_resources/test_keyvault.py b/tools/c7n_azure/tests_azure/tests_resources/test_keyvault.py index dbe3ace697b..ff6ca89ef05 100644 --- a/tools/c7n_azure/tests_azure/tests_resources/test_keyvault.py +++ b/tools/c7n_azure/tests_azure/tests_resources/test_keyvault.py @@ -7,7 +7,7 @@ from c7n_azure.session import Session from c7n_azure.utils import GraphHelper from mock import patch, Mock -from msrestazure.azure_exceptions import CloudError +from azure.core.exceptions import HttpResponseError from netaddr import IPSet from parameterized import parameterized import pytest @@ -138,8 +138,8 @@ def test_whitelist_not_authorized(self, get_principal_dictionary): mock_response = Mock(spec=Response) mock_response.status_code = 403 - mock_response.text = 'forbidden' - get_principal_dictionary.side_effect = CloudError(mock_response) + mock_response.reason = 'forbidden' + get_principal_dictionary.side_effect = HttpResponseError(response=mock_response) p = self.load_policy({ 'name': 'test-key-vault', @@ -158,7 +158,7 @@ def test_whitelist_not_authorized(self, get_principal_dictionary): ] }) - with self.assertRaises(CloudError) as e: + with self.assertRaises(HttpResponseError) as e: p.run() self.assertEqual(403, e.exception.status_code) diff --git a/tools/c7n_mailer/c7n_mailer/azure_mailer/utils.py b/tools/c7n_mailer/c7n_mailer/azure_mailer/utils.py index 283da7ff2a8..f5255a9ffe2 100644 --- a/tools/c7n_mailer/c7n_mailer/azure_mailer/utils.py +++ b/tools/c7n_mailer/c7n_mailer/azure_mailer/utils.py @@ -1,16 +1,15 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 -from c7n_azure.constants import VAULT_AUTH_ENDPOINT -from azure.keyvault import KeyVaultId +from azure.keyvault.secrets import SecretProperties def azure_decrypt(config, logger, session, encrypted_field): data = config[encrypted_field] # type: str if type(data) is dict: - kv_session = session.get_session_for_resource(resource=VAULT_AUTH_ENDPOINT) - secret_id = KeyVaultId.parse_secret_id(data['secret']) - kv_client = kv_session.client('azure.keyvault.KeyVaultClient') - return kv_client.get_secret(secret_id.vault, secret_id.name, secret_id.version).value + secret_id = SecretProperties(attributes=None, vault_id=data['secret']) + kv_client = session.client('azure.keyvault.secrets.SecretClient', + vault_url=secret_id.vault_url) + return kv_client.get_secret(secret_id.name, secret_id.version).value return data diff --git a/tools/c7n_mailer/tests/test_azure.py b/tools/c7n_mailer/tests/test_azure.py index 97c437dd883..6e5e18f26d6 100644 --- a/tools/c7n_mailer/tests/test_azure.py +++ b/tools/c7n_mailer/tests/test_azure.py @@ -106,7 +106,8 @@ def test_sendgrid_handler_multiple_to_addrs(self, mock_send): def test_azure_mailer_requirements(self): reqs = deploy.get_mailer_requirements() self.assertIn('adal', reqs) - self.assertIn('azure-storage-common', reqs) + self.assertIn('azure-storage-blob', reqs) + self.assertIn('azure-storage-queue', reqs) self.assertIn('azure-common', reqs) self.assertIn('azure-mgmt-managementgroups', reqs) self.assertIn('azure-mgmt-web', reqs)