Skip to content

Manual validation fixesli #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
logachev wants to merge 8 commits into
base: kirill/azure_sdk_update/lint
Choose a base branch
from
Open

Manual validation fixesli #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6198a7b
Fix Azure Functions run
Mar 17, 2021
5a75e52
Update exception
Mar 17, 2021
9e03c3c
Container Host fix
Mar 17, 2021
d626136
mailer_tests
Mar 17, 2021
470b38f
Containerhost test
Mar 17, 2021
d3d7dd3
lint
Mar 17, 2021
5d338e8
[Azure SDK integration branch PR] Lint changes (#6540)
Mar 17, 2021
089af9c
Merge remote-tracking branch 'upstream/azure_sdk_update' into kirill/…
Mar 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions tools/c7n_azure/c7n_azure/container_host/host.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,15 +159,15 @@ def update_policies(self):

client.get_blob_to_path(container, blob.name, policy_path)
self.load_policy(policy_path, policies_copy)
self.blob_cache.update({blob.name: blob.properties.content_settings.content_md5})
self.blob_cache.update({blob.name: blob.content_settings.content_md5})

# Assign our copy back over the original
self.policies = policies_copy

def _get_new_blobs(self, blobs):
new_blobs = []
for blob in blobs:
md5_hash = blob.properties.content_settings.content_md5
md5_hash = blob.content_settings.content_md5
if not md5_hash:
blob, md5_hash = self._try_create_md5_content_hash(blob)
if blob and md5_hash and md5_hash != self.blob_cache.get(blob.name):
Expand All @@ -191,7 +191,7 @@ def _try_create_md5_content_hash(self, blob):
# Re-fetch the blob with the new hash
hashed_blob = client.get_blob_properties(container, blob.name)

return hashed_blob, hashed_blob.properties.content_settings.content_md5
return hashed_blob, hashed_blob.content_settings.content_md5
except AzureHttpError as e:
log.warning("Failed to apply a md5 content hash to policy {}. "
"This policy will be skipped.".format(blob.name))
Expand Down
3 changes: 2 additions & 1 deletion tools/c7n_azure/c7n_azure/handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from azure.common import AzureHttpError
from msrestazure.azure_exceptions import CloudError
from azure.core.exceptions import AzureError

from c7n.utils import reset_session_cache
from c7n.config import Config
Expand Down Expand Up @@ -53,7 +54,7 @@ def run(event, context, subscription_id=None):
for p in policies:
try:
p.push(event, context)
except (CloudError, AzureHttpError) as error:
except (CloudError, AzureHttpError, AzureError) as error:
log.error("Unable to process policy: %s :: %s" % (p.name, error))

reset_session_cache()
Expand Down
4 changes: 2 additions & 2 deletions tools/c7n_azure/c7n_azure/provisioning/app_insights.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Copyright The Cloud Custodian Authors.
# SPDX-License-Identifier: Apache-2.0
from msrestazure.azure_exceptions import CloudError
from azure.core.exceptions import ResourceNotFoundError

from c7n_azure.provisioning.deployment_unit import DeploymentUnit
from c7n_azure.provisioning.resource_group import ResourceGroupUnit
Expand All @@ -16,7 +16,7 @@ def __init__(self):
def _get(self, params):
try:
return self.client.components.get(params['resource_group_name'], params['name'])
except CloudError:
except ResourceNotFoundError:
return None

def _provision(self, params):
Expand Down
4 changes: 2 additions & 2 deletions tools/c7n_azure/c7n_azure/provisioning/resource_group.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Copyright The Cloud Custodian Authors.
# SPDX-License-Identifier: Apache-2.0
from msrestazure.azure_exceptions import CloudError
from azure.core.exceptions import ResourceNotFoundError

from c7n_azure.provisioning.deployment_unit import DeploymentUnit

Expand All @@ -18,7 +18,7 @@ def verify_params(self, params):
def _get(self, params):
try:
return self.client.resource_groups.get(params['name'])
except CloudError:
except ResourceNotFoundError:
return None

def _provision(self, params):
Expand Down
4 changes: 2 additions & 2 deletions tools/c7n_azure/c7n_azure/provisioning/storage_account.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Copyright The Cloud Custodian Authors.
# SPDX-License-Identifier: Apache-2.0
from msrestazure.azure_exceptions import CloudError
from azure.core.exceptions import ResourceNotFoundError

from c7n_azure.provisioning.deployment_unit import DeploymentUnit
from c7n_azure.provisioning.resource_group import ResourceGroupUnit
Expand All @@ -17,7 +17,7 @@ def _get(self, params):
try:
return self.client.storage_accounts.get_properties(params['resource_group_name'],
params['name'])
except CloudError:
except ResourceNotFoundError:
return None

def _provision(self, params):
Expand Down
4 changes: 2 additions & 2 deletions tools/c7n_azure/c7n_azure/resources/network_security_group.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from c7n_azure.provider import resources
from c7n_azure.resources.arm import ArmResourceManager
from c7n_azure.utils import StringUtils, PortsRangeHelper
from msrestazure.azure_exceptions import CloudError
from azure.core.exceptions import AzureError

from c7n.actions import BaseAction
from c7n.filters import Filter, FilterValidationError
Expand Down Expand Up @@ -295,7 +295,7 @@ def process(self, network_security_groups):
rule_name,
new_rule
)
except CloudError as e:
except AzureError as e:
self.manager.log.error('Failed to create or update security rule for %s NSG.',
nsg_name)
self.manager.log.error(e)
Expand Down
21 changes: 10 additions & 11 deletions tools/c7n_azure/c7n_azure/resources/sqldatabase.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import enum
import logging

from azure.core.exceptions import AzureError, ResourceNotFoundError
from azure.mgmt.sql.models import (BackupLongTermRetentionPolicy,
BackupShortTermRetentionPolicy,
DatabaseUpdate, Sku)
Expand All @@ -29,7 +30,6 @@
from c7n_azure.query import ChildTypeInfo
from c7n_azure.resources.arm import ChildArmResourceManager
from c7n_azure.utils import ResourceIdParser, RetentionPeriod, ThreadHelper
from msrestazure.azure_exceptions import CloudError

log = logging.getLogger('custodian.azure.sqldatabase')

Expand Down Expand Up @@ -119,17 +119,16 @@ def get_backup_retention_policy(database, get_operation, cache_key):

try:
response = get_operation(resource_group_name, server_name, database_name, policy)
except CloudError as e:
if e.status_code == 404:
return None
else:
log.error(
"Unable to get backup retention policy. "
"(resourceGroup: {}, sqlserver: {}, sqldatabase: {})".format(
resource_group_name, server_name, database_name
)
except ResourceNotFoundError:
return None
except AzureError as e:
log.error(
"Unable to get backup retention policy. "
"(resourceGroup: {}, sqlserver: {}, sqldatabase: {})".format(
resource_group_name, server_name, database_name
)
raise e
)
raise e

retention_policy = response.as_dict()
database[policy_key] = retention_policy
Expand Down
16 changes: 12 additions & 4 deletions tools/c7n_azure/c7n_azure/session.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,31 +75,39 @@ def __init__(self, cloud_endpoints, authorization_file=None, subscription_id_ove
self._credential = None
self._subscription_id = self._auth_params['subscription_id']
if self._auth_params.get('access_token') is not None:
auth_name = 'Access Token'
pass
elif (self._auth_params['client_id'] and
self._auth_params['client_secret'] and
self._auth_params['tenant_id']
elif (self._auth_params.get('client_id') and
self._auth_params.get('client_secret') and
self._auth_params.get('tenant_id')
):
auth_name = 'Principal'
self._credential = ClientSecretCredential(
client_id=self._auth_params['client_id'],
client_secret=self._auth_params['client_secret'],
tenant_id=self._auth_params['tenant_id'],
authority=self._auth_params['authority'])
elif self._auth_params.get('use_msi'):
auth_name = 'MSI'
self._credential = ManagedIdentityCredential(
client_id=self._auth_params.get('client_id'))
elif self._auth_params.get('enable_cli_auth'):
auth_name = 'Azure CLI'
self._credential = AzureCliCredential()
self._subscription_id, error = _run_command('az account show --output tsv --query id')
self._subscription_id = self._subscription_id.strip('\n')
if error is not None:
raise Exception('Unable to query SubscriptionId')

log.info('Authenticated [%s | %s%s]',
auth_name, self.subscription_id,
' | Authorization File' if authorization_file else '')

def get_token(self, *scopes, **kwargs):
# Access Token is used only in tests realistically because
# KeyVault, Storage and mgmt plane requires separate tokens.
# TODO: Should we scope this to tests only?
if (self._auth_params['access_token']):
if (self._auth_params.get('access_token')):
return AccessToken(self._auth_params['access_token'], expires_on=0)
try:
return self._credential.get_token(*scopes, **kwargs)
Expand Down
10 changes: 10 additions & 0 deletions tools/c7n_azure/c7n_azure/storage_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ def get_blob_to_bytes(self, container_name, blob_name):
client = self.get_blob_client(container_name, blob_name)
return client.download_blob().content_as_bytes()

def get_blob_to_path(self, container_name, blob_name, path):
client = self.get_blob_client(container_name, blob_name)
with open(path, "wb") as f:
download_stream = client.download_blob()
f.write(download_stream.readall())

def create_blob_from_bytes(self, container_name, blob_name, content, validate_content):
client = self.get_blob_client(container_name, blob_name)
client.upload_blob(content, overwrite=True)
Expand All @@ -27,6 +33,10 @@ def get_blob_properties(self, container_name, blob_name):
client = self.get_blob_client(container_name, blob_name)
return client.get_blob_properties()

def list_blobs(self, container_name):
client = self.get_container_client(container_name)
return client.list_blobs()


class OldQueueService:

Expand Down
3 changes: 2 additions & 1 deletion tools/c7n_azure/c7n_azure/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -633,7 +633,8 @@ def log_response_data(response):
send_logger.debug(http_response.status_code)
for k, v in http_response.headers.items():
if k.startswith('x-ms-ratelimit'):
send_logger.info(k + ':' + v)
send_logger.debug(k + ':' + v)



# This workaround will replace used api-version for costmanagement requests
Expand Down
19 changes: 18 additions & 1 deletion tools/c7n_azure/poetry.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions tools/c7n_azure/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ azure-identity = "^1.5.0"
azure-keyvault = "^4.1.0"
azure-storage-file-share = "^12.4.1"
cryptography = "^3.4.6"
azure-mgmt-msi = "^1.0.0"
jmespath = "^0.10.0"

[tool.poetry.dev-dependencies]
# setup custodian as a dev dependency
Expand Down
2 changes: 2 additions & 0 deletions tools/c7n_azure/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ azure-mgmt-keyvault==8.0.0
azure-mgmt-logic==9.0.0
azure-mgmt-managementgroups==1.0.0b1
azure-mgmt-monitor==2.0.0
azure-mgmt-msi==1.0.0
azure-mgmt-network==17.1.0
azure-mgmt-policyinsights==1.0.0
azure-mgmt-rdbms==8.0.0
Expand All @@ -64,6 +65,7 @@ distlib==0.3.1
idna==2.10; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
importlib-resources==5.1.2; python_version >= "3.6" and python_version < "3.7"
isodate==0.6.0
jmespath==0.10.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
jsonpickle==1.3
msal-extensions==0.3.0
msal==1.10.0
Expand Down
2 changes: 2 additions & 0 deletions tools/c7n_azure/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
'azure-mgmt-logic>=9.0.0,<10.0.0',
'azure-mgmt-managementgroups==1.0.0b1',
'azure-mgmt-monitor>=2.0.0,<3.0.0',
'azure-mgmt-msi>=1.0.0,<2.0.0',
'azure-mgmt-network>=17.1.0,<18.0.0',
'azure-mgmt-policyinsights>=1.0.0,<2.0.0',
'azure-mgmt-rdbms>=8.0.0,<9.0.0',
Expand All @@ -75,6 +76,7 @@
'distlib>=0.3.0,<0.4.0',
'importlib-metadata (>=3.7.2,<4.0.0)',
'jmespath (>=0.10.0,<0.11.0)',
'jmespath>=0.10.0,<0.11.0',
'jsonpickle (>=1.3,<2.0)',
'jsonpickle==1.3',
'jsonschema (>=3.2.0,<4.0.0)',
Expand Down
2 changes: 1 addition & 1 deletion tools/c7n_azure/tests_azure/azure_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,7 @@ def lro_init(self, client, initial_response, deserialization_callback, polling_m
issubclass(deserialization_callback, Model):
deserialization_callback = deserialization_callback.deserialize

# Might raise a CloudError
# Might raise a AzureError
self._polling_method.initialize(self._client, self._response, deserialization_callback)

self._thread = None
Expand Down
2 changes: 1 addition & 1 deletion tools/c7n_azure/tests_azure/tests_resources/test_container_host.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -630,7 +630,7 @@ def download_missing_mode_policy_blob(_, name, path):
def get_mock_blob(name, md5):
new_blob = Mock()
new_blob.name = name
new_blob.properties.content_settings.content_md5 = md5
new_blob.content_settings.content_md5 = md5
return new_blob

@staticmethod
Expand Down
8 changes: 4 additions & 4 deletions tools/c7n_azure/tests_azure/tests_resources/test_keyvault.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from c7n_azure.session import Session
from c7n_azure.utils import GraphHelper
from mock import patch, Mock
from msrestazure.azure_exceptions import CloudError
from azure.core.exceptions import HttpResponseError
from netaddr import IPSet
from parameterized import parameterized
import pytest
Expand Down Expand Up @@ -138,8 +138,8 @@ def test_whitelist_not_authorized(self, get_principal_dictionary):

mock_response = Mock(spec=Response)
mock_response.status_code = 403
mock_response.text = 'forbidden'
get_principal_dictionary.side_effect = CloudError(mock_response)
mock_response.reason = 'forbidden'
get_principal_dictionary.side_effect = HttpResponseError(response=mock_response)

p = self.load_policy({
'name': 'test-key-vault',
Expand All @@ -158,7 +158,7 @@ def test_whitelist_not_authorized(self, get_principal_dictionary):
]
})

with self.assertRaises(CloudError) as e:
with self.assertRaises(HttpResponseError) as e:
p.run()

self.assertEqual(403, e.exception.status_code)
Expand Down
11 changes: 5 additions & 6 deletions tools/c7n_mailer/c7n_mailer/azure_mailer/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
# Copyright The Cloud Custodian Authors.
# SPDX-License-Identifier: Apache-2.0

from c7n_azure.constants import VAULT_AUTH_ENDPOINT
from azure.keyvault import KeyVaultId
from azure.keyvault.secrets import SecretProperties


def azure_decrypt(config, logger, session, encrypted_field):
data = config[encrypted_field] # type: str
if type(data) is dict:
kv_session = session.get_session_for_resource(resource=VAULT_AUTH_ENDPOINT)
secret_id = KeyVaultId.parse_secret_id(data['secret'])
kv_client = kv_session.client('azure.keyvault.KeyVaultClient')
return kv_client.get_secret(secret_id.vault, secret_id.name, secret_id.version).value
secret_id = SecretProperties(attributes=None, vault_id=data['secret'])
kv_client = session.client('azure.keyvault.secrets.SecretClient',
vault_url=secret_id.vault_url)
return kv_client.get_secret(secret_id.name, secret_id.version).value

return data
3 changes: 2 additions & 1 deletion tools/c7n_mailer/tests/test_azure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ def test_sendgrid_handler_multiple_to_addrs(self, mock_send):
def test_azure_mailer_requirements(self):
reqs = deploy.get_mailer_requirements()
self.assertIn('adal', reqs)
self.assertIn('azure-storage-common', reqs)
self.assertIn('azure-storage-blob', reqs)
self.assertIn('azure-storage-queue', reqs)
self.assertIn('azure-common', reqs)
self.assertIn('azure-mgmt-managementgroups', reqs)
self.assertIn('azure-mgmt-web', reqs)
Expand Down