From 20530e55ee9431e0ad87274d04bb1580a3ea1d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=B8jland?= Date: Tue, 12 Oct 2021 10:52:26 +0200 Subject: [PATCH 1/5] patch: handling typeerrors using default value using enforce flag --- dataclass_type_validator/__init__.py | 91 ++++++++++++++-------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/dataclass_type_validator/__init__.py b/dataclass_type_validator/__init__.py index 200111e..e4cf232 100644 --- a/dataclass_type_validator/__init__.py +++ b/dataclass_type_validator/__init__.py @@ -1,13 +1,12 @@ import dataclasses -import typing import functools -from typing import Any -from typing import Optional +import typing +import warnings +from typing import Any, Optional class TypeValidationError(Exception): - """Exception raised on type validation errors. - """ + """Exception raised on type validation errors.""" def __init__(self, *args, target: dataclasses.dataclass, errors: dict): super(TypeValidationError, self).__init__(*args) @@ -16,28 +15,20 @@ def __init__(self, *args, target: dataclasses.dataclass, errors: dict): def __repr__(self): cls = self.class_ - cls_name = ( - f"{cls.__module__}.{cls.__name__}" - if cls.__module__ != "__main__" - else cls.__name__ - ) + cls_name = f"{cls.__module__}.{cls.__name__}" if cls.__module__ != "__main__" else cls.__name__ attrs = ", ".join([repr(v) for v in self.args]) return f"{cls_name}({attrs}, errors={repr(self.errors)})" def __str__(self): cls = self.class_ - cls_name = ( - f"{cls.__module__}.{cls.__name__}" - if cls.__module__ != "__main__" - else cls.__name__ - ) + cls_name = f"{cls.__module__}.{cls.__name__}" if cls.__module__ != "__main__" else cls.__name__ s = cls_name return f"{s} (errors = {self.errors})" def _validate_type(expected_type: type, value: Any) -> Optional[str]: if not isinstance(value, expected_type): - return f'must be an instance of {expected_type}, but received {type(value)}' + return f"must be an instance of {expected_type}, but received {type(value)}" def _validate_iterable_items(expected_type: type, value: Any, strict: bool) -> Optional[str]: @@ -45,30 +36,30 @@ def _validate_iterable_items(expected_type: type, value: Any, strict: bool) -> O errors = [_validate_types(expected_type=expected_item_type, value=v, strict=strict) for v in value] errors = [x for x in errors if x] if len(errors) > 0: - return f'must be an instance of {expected_type}, but there are some errors: {errors}' + return f"must be an instance of {expected_type}, but there are some errors: {errors}" def _validate_typing_list(expected_type: type, value: Any, strict: bool) -> Optional[str]: if not isinstance(value, list): - return f'must be an instance of list, but received {type(value)}' + return f"must be an instance of list, but received {type(value)}" return _validate_iterable_items(expected_type, value, strict) def _validate_typing_tuple(expected_type: type, value: Any, strict: bool) -> Optional[str]: if not isinstance(value, tuple): - return f'must be an instance of tuple, but received {type(value)}' + return f"must be an instance of tuple, but received {type(value)}" return _validate_iterable_items(expected_type, value, strict) def _validate_typing_frozenset(expected_type: type, value: Any, strict: bool) -> Optional[str]: if not isinstance(value, frozenset): - return f'must be an instance of frozenset, but received {type(value)}' + return f"must be an instance of frozenset, but received {type(value)}" return _validate_iterable_items(expected_type, value, strict) def _validate_typing_dict(expected_type: type, value: Any, strict: bool) -> Optional[str]: if not isinstance(value, dict): - return f'must be an instance of dict, but received {type(value)}' + return f"must be an instance of dict, but received {type(value)}" expected_key_type = expected_type.__args__[0] expected_value_type = expected_type.__args__[1] @@ -80,18 +71,20 @@ def _validate_typing_dict(expected_type: type, value: Any, strict: bool) -> Opti val_errors = [v for v in val_errors if v] if len(key_errors) > 0 and len(val_errors) > 0: - return f'must be an instance of {expected_type}, but there are some errors in keys and values. '\ - f'key errors: {key_errors}, value errors: {val_errors}' + return ( + f"must be an instance of {expected_type}, but there are some errors in keys and values. " + f"key errors: {key_errors}, value errors: {val_errors}" + ) elif len(key_errors) > 0: - return f'must be an instance of {expected_type}, but there are some errors in keys: {key_errors}' + return f"must be an instance of {expected_type}, but there are some errors in keys: {key_errors}" elif len(val_errors) > 0: - return f'must be an instance of {expected_type}, but there are some errors in values: {val_errors}' + return f"must be an instance of {expected_type}, but there are some errors in values: {val_errors}" def _validate_typing_callable(expected_type: type, value: Any, strict: bool) -> Optional[str]: _ = strict if not isinstance(value, type(lambda a: a)): - return f'must be an instance of {expected_type._name}, but received {type(value)}' + return f"must be an instance of {expected_type._name}, but received {type(value)}" def _validate_typing_literal(expected_type: type, value: Any, strict: bool) -> Optional[str]: @@ -101,11 +94,11 @@ def _validate_typing_literal(expected_type: type, value: Any, strict: bool) -> O _validate_typing_mappings = { - 'List': _validate_typing_list, - 'Tuple': _validate_typing_tuple, - 'FrozenSet': _validate_typing_frozenset, - 'Dict': _validate_typing_dict, - 'Callable': _validate_typing_callable, + "List": _validate_typing_list, + "Tuple": _validate_typing_tuple, + "FrozenSet": _validate_typing_frozenset, + "Dict": _validate_typing_dict, + "Callable": _validate_typing_callable, } @@ -114,18 +107,17 @@ def _validate_sequential_types(expected_type: type, value: Any, strict: bool) -> if validate_func is not None: return validate_func(expected_type, value, strict) - if str(expected_type).startswith('typing.Literal'): + if str(expected_type).startswith("typing.Literal"): return _validate_typing_literal(expected_type, value, strict) - if str(expected_type).startswith('typing.Union'): - is_valid = any(_validate_types(expected_type=t, value=value, strict=strict) is None - for t in expected_type.__args__) + if str(expected_type).startswith("typing.Union"): + is_valid = any(_validate_types(expected_type=t, value=value, strict=strict) is None for t in expected_type.__args__) if not is_valid: - return f'must be an instance of {expected_type}, but received {value}' + return f"must be an instance of {expected_type}, but received {value}" return if strict: - raise RuntimeError(f'Unknown type of {expected_type} (_name = {expected_type._name})') + raise RuntimeError(f"Unknown type of {expected_type} (_name = {expected_type._name})") def _validate_types(expected_type: type, value: Any, strict: bool) -> Optional[str]: @@ -136,7 +128,7 @@ def _validate_types(expected_type: type, value: Any, strict: bool) -> Optional[s return _validate_sequential_types(expected_type=expected_type, value=value, strict=strict) -def dataclass_type_validator(target, strict: bool = False): +def dataclass_type_validator(target, strict: bool = False, enforce: bool = False): fields = dataclasses.fields(target) errors = {} @@ -148,14 +140,19 @@ def dataclass_type_validator(target, strict: bool = False): err = _validate_types(expected_type=expected_type, value=value, strict=strict) if err is not None: errors[field_name] = err + if enforce: + target[field_name] = field.default - if len(errors) > 0: - raise TypeValidationError( - "Dataclass Type Validation Error", target=target, errors=errors - ) + if len(errors) > 0 and not enforce: + raise TypeValidationError("Dataclass Type Validation Error", target=target, errors=errors) + + elif len(errors) > 0 and enforce: + cls = target.__class__ + cls_name = f"{cls.__module__}.{cls.__name__}" if cls.__module__ != "__main__" else cls.__name__ + warnings.warn(f"Dataclass type validation failed, types are enforced. {cls_name} errors={repr(errors)})") -def dataclass_validate(cls=None, *, strict: bool = False, before_post_init: bool = False): +def dataclass_validate(cls=None, *, strict: bool = False, before_post_init: bool = False, enforce: bool = False): """Dataclass decorator to automatically add validation to a dataclass. So you don't have to add a __post_init__ method, or if you have one, you don't have @@ -170,7 +167,7 @@ def dataclass_validate(cls=None, *, strict: bool = False, before_post_init: bool validation. Default: False. """ if cls is None: - return functools.partial(dataclass_validate, strict=strict, before_post_init=before_post_init) + return functools.partial(dataclass_validate, strict=strict, before_post_init=before_post_init, enforce=enforce) if not hasattr(cls, "__post_init__"): # No post-init method, so no processing. Wrap the constructor instead. @@ -186,15 +183,17 @@ def dataclass_validate(cls=None, *, strict: bool = False, before_post_init: bool # before the wrapped function. @functools.wraps(orig_method) def method_wrapper(self, *args, **kwargs): - dataclass_type_validator(self, strict=strict) + dataclass_type_validator(self, strict=strict, enforce=enforce) return orig_method(self, *args, **kwargs) + else: # Normal case - call validator at the end of __init__ or __post_init__. @functools.wraps(orig_method) def method_wrapper(self, *args, **kwargs): x = orig_method(self, *args, **kwargs) - dataclass_type_validator(self, strict=strict) + dataclass_type_validator(self, strict=strict, enforce=enforce) return x + setattr(cls, wrapped_method_name, method_wrapper) return cls From 24990d9c5d66ddd79294c7db05cf1666b5e525a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=B8jland?= Date: Wed, 13 Oct 2021 11:53:36 +0200 Subject: [PATCH 2/5] patch: handling default factory defaults in enforce --- dataclass_type_validator/__init__.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/dataclass_type_validator/__init__.py b/dataclass_type_validator/__init__.py index e4cf232..a780300 100644 --- a/dataclass_type_validator/__init__.py +++ b/dataclass_type_validator/__init__.py @@ -1,9 +1,11 @@ import dataclasses import functools +import logging import typing -import warnings from typing import Any, Optional +logger = logging.getLogger(__name__) + class TypeValidationError(Exception): """Exception raised on type validation errors.""" @@ -26,6 +28,14 @@ def __str__(self): return f"{s} (errors = {self.errors})" +class EnforceError(Exception): + """Exception raised on enforcing validation errors.""" + + def __init__(self, *args): + super(EnforceError, self).__init__(*args) + pass + + def _validate_type(expected_type: type, value: Any) -> Optional[str]: if not isinstance(value, expected_type): return f"must be an instance of {expected_type}, but received {type(value)}" @@ -141,7 +151,10 @@ def dataclass_type_validator(target, strict: bool = False, enforce: bool = False if err is not None: errors[field_name] = err if enforce: - target[field_name] = field.default + val = field.default if not isinstance(field.default, dataclasses._MISSING_TYPE) else field.default_factory() + if isinstance(val, dataclasses._MISSING_TYPE): + raise EnforceError("Can't enforce values as there is no default") + target[field_name] = val if len(errors) > 0 and not enforce: raise TypeValidationError("Dataclass Type Validation Error", target=target, errors=errors) @@ -149,7 +162,7 @@ def dataclass_type_validator(target, strict: bool = False, enforce: bool = False elif len(errors) > 0 and enforce: cls = target.__class__ cls_name = f"{cls.__module__}.{cls.__name__}" if cls.__module__ != "__main__" else cls.__name__ - warnings.warn(f"Dataclass type validation failed, types are enforced. {cls_name} errors={repr(errors)})") + logger.warning(f"Dataclass type validation failed, types are enforced. {cls_name} errors={repr(errors)})") def dataclass_validate(cls=None, *, strict: bool = False, before_post_init: bool = False, enforce: bool = False): From 93f69cfe6d484f1ce122de24af8b8e0c823047db Mon Sep 17 00:00:00 2001 From: GA-Ramlov <74347745+GA-Ramlov@users.noreply.github.com> Date: Wed, 10 Nov 2021 17:34:02 +0100 Subject: [PATCH 3/5] Update of dataclass_type_validator Added functionality to work with pydantic BaseModel --- dataclass_type_validator/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dataclass_type_validator/__init__.py b/dataclass_type_validator/__init__.py index a780300..5bacd1f 100644 --- a/dataclass_type_validator/__init__.py +++ b/dataclass_type_validator/__init__.py @@ -139,7 +139,10 @@ def _validate_types(expected_type: type, value: Any, strict: bool) -> Optional[s def dataclass_type_validator(target, strict: bool = False, enforce: bool = False): - fields = dataclasses.fields(target) + if isinstance(target, BaseModel): + fields = target.fields.values() + else: + fields = dataclasses.fields(target) errors = {} for field in fields: @@ -151,8 +154,8 @@ def dataclass_type_validator(target, strict: bool = False, enforce: bool = False if err is not None: errors[field_name] = err if enforce: - val = field.default if not isinstance(field.default, dataclasses._MISSING_TYPE) else field.default_factory() - if isinstance(val, dataclasses._MISSING_TYPE): + val = field.default if not isinstance(val, (dataclasses._MISSING_TYPE, type(None)) else field.default_factory() + if isinstance(val, (dataclasses._MISSING_TYPE, type(None))): raise EnforceError("Can't enforce values as there is no default") target[field_name] = val From b16913b834dd47e2949179ad29c26cf7f00e7f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=B8jland?= Date: Thu, 11 Nov 2021 10:56:57 +0100 Subject: [PATCH 4/5] feat: pydantic typechecking --- dataclass_type_validator/__init__.py | 79 ++++++++++++++++++++++++++-- requirements.in | 1 + requirements.txt | 1 + 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/dataclass_type_validator/__init__.py b/dataclass_type_validator/__init__.py index 5bacd1f..2e56944 100644 --- a/dataclass_type_validator/__init__.py +++ b/dataclass_type_validator/__init__.py @@ -3,10 +3,9 @@ import logging import typing from typing import Any, Optional - +from pydantic import BaseModel logger = logging.getLogger(__name__) - class TypeValidationError(Exception): """Exception raised on type validation errors.""" @@ -154,10 +153,10 @@ def dataclass_type_validator(target, strict: bool = False, enforce: bool = False if err is not None: errors[field_name] = err if enforce: - val = field.default if not isinstance(val, (dataclasses._MISSING_TYPE, type(None)) else field.default_factory() + val = field.default if not isinstance(field.default, (dataclasses._MISSING_TYPE, type(None))) else field.default_factory() if isinstance(val, (dataclasses._MISSING_TYPE, type(None))): raise EnforceError("Can't enforce values as there is no default") - target[field_name] = val + setattr(target, field_name, val) if len(errors) > 0 and not enforce: raise TypeValidationError("Dataclass Type Validation Error", target=target, errors=errors) @@ -167,6 +166,34 @@ def dataclass_type_validator(target, strict: bool = False, enforce: bool = False cls_name = f"{cls.__module__}.{cls.__name__}" if cls.__module__ != "__main__" else cls.__name__ logger.warning(f"Dataclass type validation failed, types are enforced. {cls_name} errors={repr(errors)})") +def pydantic_type_validator(cls, values: dict, strict: bool = False, enforce: bool = False): + fields = cls.__fields__.values() + errors = {} + for field in fields: + field_name = field.name + expected_type = field.type_ + value = values[field_name] + + err = _validate_types(expected_type=expected_type, value=value, strict=strict) + new_values = values + if err is not None: + errors[field_name] = err + if enforce: + val = field.default if not isinstance(field.default, type(None)) else None + if val is None: + val = field.default_factory() if not isinstance(field.default_factory, type(None)) else None + if val is None: + raise EnforceError("Can't enforce values as there is no default") + new_values[field_name] = val + + if len(errors) > 0 and not enforce: + raise TypeValidationError("Pydantic Type Validation Error", target=cls, errors=errors) + + elif len(errors) > 0 and enforce: + cls_name = cls.__name__ + logger.warning(f"Pydantic type validation failed, types are enforced. {cls_name} errors={repr(errors)})") + return new_values + def dataclass_validate(cls=None, *, strict: bool = False, before_post_init: bool = False, enforce: bool = False): """Dataclass decorator to automatically add validation to a dataclass. @@ -213,3 +240,47 @@ def method_wrapper(self, *args, **kwargs): setattr(cls, wrapped_method_name, method_wrapper) return cls + +if __name__ == "__main__": + #@dataclasses.dataclass + #class TestClass: + # k: str = "key" + # v: float = 1.2 + + #test_class = TestClass(k=1.2, v="key") + + #@dataclasses.dataclass + #class TestClass: + # k: str = "key" + # v: float = 1.2 + + # def __post_init__(self): + # dataclass_type_validator(self, enforce=True) + + #test_class = TestClass(k=1.2, v="key") + from pydantic import root_validator + class TestClass(BaseModel): + k: str = "key" + v: float = 1.2 + + @root_validator(pre=True) + def enforce_validator(cls, values): + values = pydantic_type_validator(cls, values, enforce=True) + return values + + def validate_class(self): + from pydantic import validate_model + object_setattr = object.__setattr__ + values, fields_set, validation_error = validate_model(self.__class__, self.dict()) + if validation_error: + raise validation_error + object_setattr(self, '__dict__', values) + object_setattr(self, '__fields_set__', fields_set) + self._init_private_attributes() + + test_class = TestClass(k=1.2, v="key") + print(test_class) + setattr(test_class, "v", "key") + print(test_class) + test_class.validate_class() + print(test_class) \ No newline at end of file diff --git a/requirements.in b/requirements.in index e079f8a..1cc0826 100644 --- a/requirements.in +++ b/requirements.in @@ -1 +1,2 @@ pytest +pydantic \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d76d9c4..0314205 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ typing-extensions==3.10.0.0 # via importlib-metadata zipp==3.4.1 # via importlib-metadata +pydantic \ No newline at end of file From 11daecb43f0d4288694f27969bad7a1e49440870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=B8jland?= Date: Mon, 29 Nov 2021 15:12:39 +0100 Subject: [PATCH 5/5] fix: fix error when there is no input, but there is a default --- dataclass_type_validator/__init__.py | 49 ++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/dataclass_type_validator/__init__.py b/dataclass_type_validator/__init__.py index 2e56944..40eac09 100644 --- a/dataclass_type_validator/__init__.py +++ b/dataclass_type_validator/__init__.py @@ -3,9 +3,12 @@ import logging import typing from typing import Any, Optional + from pydantic import BaseModel + logger = logging.getLogger(__name__) + class TypeValidationError(Exception): """Exception raised on type validation errors.""" @@ -153,7 +156,11 @@ def dataclass_type_validator(target, strict: bool = False, enforce: bool = False if err is not None: errors[field_name] = err if enforce: - val = field.default if not isinstance(field.default, (dataclasses._MISSING_TYPE, type(None))) else field.default_factory() + val = ( + field.default + if not isinstance(field.default, (dataclasses._MISSING_TYPE, type(None))) + else field.default_factory() + ) if isinstance(val, (dataclasses._MISSING_TYPE, type(None))): raise EnforceError("Can't enforce values as there is no default") setattr(target, field_name, val) @@ -166,13 +173,14 @@ def dataclass_type_validator(target, strict: bool = False, enforce: bool = False cls_name = f"{cls.__module__}.{cls.__name__}" if cls.__module__ != "__main__" else cls.__name__ logger.warning(f"Dataclass type validation failed, types are enforced. {cls_name} errors={repr(errors)})") + def pydantic_type_validator(cls, values: dict, strict: bool = False, enforce: bool = False): fields = cls.__fields__.values() errors = {} for field in fields: field_name = field.name expected_type = field.type_ - value = values[field_name] + value = values[field_name] if field_name in values.keys() else None err = _validate_types(expected_type=expected_type, value=value, strict=strict) new_values = values @@ -195,7 +203,13 @@ def pydantic_type_validator(cls, values: dict, strict: bool = False, enforce: bo return new_values -def dataclass_validate(cls=None, *, strict: bool = False, before_post_init: bool = False, enforce: bool = False): +def dataclass_validate( + cls=None, + *, + strict: bool = False, + before_post_init: bool = False, + enforce: bool = False, +): """Dataclass decorator to automatically add validation to a dataclass. So you don't have to add a __post_init__ method, or if you have one, you don't have @@ -210,7 +224,12 @@ def dataclass_validate(cls=None, *, strict: bool = False, before_post_init: bool validation. Default: False. """ if cls is None: - return functools.partial(dataclass_validate, strict=strict, before_post_init=before_post_init, enforce=enforce) + return functools.partial( + dataclass_validate, + strict=strict, + before_post_init=before_post_init, + enforce=enforce, + ) if not hasattr(cls, "__post_init__"): # No post-init method, so no processing. Wrap the constructor instead. @@ -241,24 +260,26 @@ def method_wrapper(self, *args, **kwargs): return cls + if __name__ == "__main__": - #@dataclasses.dataclass - #class TestClass: + # @dataclasses.dataclass + # class TestClass: # k: str = "key" # v: float = 1.2 - #test_class = TestClass(k=1.2, v="key") + # test_class = TestClass(k=1.2, v="key") - #@dataclasses.dataclass - #class TestClass: + # @dataclasses.dataclass + # class TestClass: # k: str = "key" # v: float = 1.2 # def __post_init__(self): # dataclass_type_validator(self, enforce=True) - #test_class = TestClass(k=1.2, v="key") + # test_class = TestClass(k=1.2, v="key") from pydantic import root_validator + class TestClass(BaseModel): k: str = "key" v: float = 1.2 @@ -270,12 +291,13 @@ def enforce_validator(cls, values): def validate_class(self): from pydantic import validate_model + object_setattr = object.__setattr__ values, fields_set, validation_error = validate_model(self.__class__, self.dict()) if validation_error: raise validation_error - object_setattr(self, '__dict__', values) - object_setattr(self, '__fields_set__', fields_set) + object_setattr(self, "__dict__", values) + object_setattr(self, "__fields_set__", fields_set) self._init_private_attributes() test_class = TestClass(k=1.2, v="key") @@ -283,4 +305,5 @@ def validate_class(self): setattr(test_class, "v", "key") print(test_class) test_class.validate_class() - print(test_class) \ No newline at end of file + print(test_class) + TestClass()