From e3586caba0212f2bacf6cc8711662fb1c17ae9e7 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 2 Dec 2020 14:25:59 +0100 Subject: [PATCH 01/43] Add assertions settings public api methods --- dataikuapi/dss/ml.py | 249 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index fe1407cb..8abb4af4 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -431,6 +431,23 @@ def get_split_params(self): """ return PredictionSplitParamsHandler(self.mltask_settings) + @property + def assertions_params(self): + """ + Retrieves the assertion params for this ml task + + :rtype: :class:`DSSMLAssertionsParams` + """ + return self.get_assertions_params() + + def get_assertions_params(self): + """ + Retrieves the assertion params for this ml task + + :rtype: :class:`DSSMLAssertionsParams` + """ + return DSSMLAssertionsParams(self.mltask_settings["assertionParams"]) + def split_ordered_by(self, feature_name, ascending=True): """ Deprecated. Use split_params.set_time_ordering() @@ -643,6 +660,238 @@ def __repr__(self): msg=self._internal_dict["message"]) +class DSSMLAssertionsParams(object): + """ + Object that represents parameters for all assertions of a ml task + Do not create this object directly, use :meth:`DSSPredictionMLTaskSettings.get_assertion_params()` instead + """ + + @classmethod + def check_assertion_names_are_uniq(cls, assertion_params_list): + _ = {} + for assertion_dict in assertion_params_list: + if 'name' not in assertion_dict: + raise ValueError('No name provided for assertion') + if assertion_dict['name'] in _: + raise ValueError('Assertion names should be uniq. {} is multiple times in the data'.format( + assertion_dict['name'])) + _[assertion_dict['name']] = True + + def __init__(self, data): + self._internal_dict = data + self.check_assertion_names_are_uniq(data["assertions"]) + + def get_raw(self): + """ + Gets the raw dictionary of the assertions params + :rtype: dict + """ + return self._internal_dict + + def get_assertion(self, assertion_name): + """ + Gets a :class:`DSSMLAssertionParams` representing the params of the assertion with the provided name + (or None) + """ + for assertion_dict in self._internal_dict["assertions"]: + if assertion_dict["name"] == assertion_name: + return DSSMLAssertionParams(assertion_dict) + return None + + def add_assertion(self, assertion_params): + """ + Adds a :class:`DSSMLAssertionParams` to the `DSSMLAssertionsParams` of the ml task. Raises a ValueError if an assertion + with the same name already exists + """ + if not isinstance(assertion_params, DSSMLAssertionParams): + raise ValueError('Assertion params should be of type: {} not {}'.format(DSSMLAssertionParams.__name__, type(assertion_params))) + self.check_assertion_names_are_uniq(self._internal_dict["assertions"] + [assertion_params._internal_dict]) + self._internal_dict["assertions"].append(assertion_params._internal_dict) + + def delete_assertion(self, assertion_name): + """ + Deletes the assertion params of the assertion with the provided name from the `DSSMLAssertionsParams` + Raises a ValueError if no assertion with the provided name was found + """ + for idx, assertion_dict in enumerate(self._internal_dict["assertions"]): + if assertion_dict["name"] == assertion_name: + del self._internal_dict["assertions"][idx] + return + raise ValueError('No assertion name: {} was found'.format(assertion_name)) + + +class DSSMLAssertionParams(object): + """ + Object that represents parameters for one assertion + Do not create this object directly, use :meth:`DSSMLAssertionsParams.get_assertion(assertion_name)` or + `create_from_parts(name, a_filter, condition)` instead + """ + def __init__(self, data): + self._internal_dict = data + + @staticmethod + def create_from_parts(name, a_filter, condition): + """ + Creates a `DSSMLAssertionParams` from name, filter and condition + + :param str name: Name of the assertion + :param ~dataikuapi.dss.utils.DSSFilter a_filter: Filter to select assertion population + :param DSSMLAssertionCondition condition: Condition for the assertion to be successful + + :rtype: DSSMLAssertionParams + """ + assertion_params = DSSMLAssertionParams({}) + assertion_params.name = name + assertion_params.filter = a_filter + assertion_params.condition = condition + return assertion_params + + def get_raw(self): + """ + Gets the raw dictionary of the assertion params + :rtype: dict + """ + return self._internal_dict + + @property + def name(self): + """ + Returns the assertion name + :rtype: str + """ + return self._internal_dict["name"] + + @name.setter + def name(self, name): + self._internal_dict["name"] = name + + @property + def filter(self): + """ + Returns the assertion filter + :rtype: ~dataikuapi.dss.utils.DSSFilter + """ + return self._internal_dict["filter"] + + @filter.setter + def filter(self, filter): + self._internal_dict["filter"] = filter + + @property + def condition(self): + """ + Returns the assertion condition + :rtype: DSSMLAssertionCondition + """ + + return DSSMLAssertionCondition(self._internal_dict["assertionCondition"]) + + @condition.setter + def condition(self, condition): + if not isinstance(condition, DSSMLAssertionCondition): + raise ValueError('Condition should be of type: {} not {}'.format(DSSMLAssertionCondition.__name__, type(condition))) + self._internal_dict["assertionCondition"] = condition._internal_dict + + +class DSSMLAssertionCondition(object): + """ + Object that represents an assertion condition + Do not create this object directly, use :meth:`DSSMLAssertionParams.condition`, `create_from_expected_class(expected_valid_ratio, expected_class)` + or `create_from_expected_class(expected_valid_ratio, expected_class)` instead + """ + def __init__(self, data): + self._internal_dict = data + + @staticmethod + def create_from_expected_class(expected_valid_ratio, expected_class): + """ + Creates a `DSSMLAssertionCondition` from an expected valid ratio and an expected class + + :param float expected_valid_ratio: Ratio of valid rows needed for the assertion to pass + :param str expected_class: Class on which the `expected_valid_ratio` will be calculated + + :rtype: DSSMLAssertionCondition + """ + assertion_condition = DSSMLAssertionCondition({}) + assertion_condition.expected_valid_ratio = expected_valid_ratio + assertion_condition.expected_class = expected_class + return assertion_condition + + @staticmethod + def create_from_expected_range(expected_valid_ratio, expected_range): + """ + Creates a `DSSMLAssertionCondition` from an expected valid ratio and an expected range + + :param float expected_valid_ratio: Ratio of valid rows to exceed for the assertion to pass + :param tuple(float,float) expected_range: Range of values (min, max) where the prediction will be considered as + valid for the `expected_valid_ratio` + + :rtype: DSSMLAssertionCondition + """ + assertion_condition = DSSMLAssertionCondition({}) + assertion_condition.expected_valid_ratio = expected_valid_ratio + assertion_condition.expected_range = expected_range + return assertion_condition + + def get_raw(self): + """ + Gets the raw dictionary of the condition + :rtype: dict + """ + return self._internal_dict + + @property + def expected_class(self): + """ + Returns the expected class on which the valid ratio will be calculated + :rtype: str + """ + if "expectedClass" in self._internal_dict: + return self._internal_dict["expectedClass"] + else: + return None + + @expected_class.setter + def expected_class(self, expected_class): + if self.expected_range is not None: + raise ValueError("Expected class and expected range can't be both set") + self._internal_dict["expectedClass"] = expected_class + + @property + def expected_valid_ratio(self): + """ + Returns the ratio of valid rows to exceed for the assertion to pass + :rtype: str + """ + return self._internal_dict["successRatio"] + + @expected_valid_ratio.setter + def expected_valid_ratio(self, expected_valid_ratio): + self._internal_dict["successRatio"] = expected_valid_ratio + + @property + def expected_range(self): + """ + Returns the expected range on which the valid ratio will be calculated + :rtype: str + """ + if "expectedMinValue" in self._internal_dict and "expectedMaxValue" in self._internal_dict: + return self._internal_dict["expectedMinValue"], self._internal_dict["expectedMaxValue"] + else: + return None + + @expected_range.setter + def expected_range(self, expected_range): + if not isinstance(expected_range, tuple): + raise ValueError("Expected range needs to be a tuple") + if self.expected_class is not None: + raise ValueError("Expected class and expected range can't be both set") + if expected_range[0] > expected_range: + raise ValueError("Expected range needs to be sorted in ascending order. (min value, max value)") + self._internal_dict["expectedMinValue"] = expected_range[0] + self._internal_dict["expectedMaxValue"] = expected_range[1] + + class DSSTreeNode(object): def __init__(self, tree, i): self.tree = tree From 067a9a5af0c22c7bf0204488406bb24b33c6f40c Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 2 Dec 2020 16:46:57 +0100 Subject: [PATCH 02/43] Fix typo --- dataikuapi/dss/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 8abb4af4..551407f5 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -797,7 +797,7 @@ class DSSMLAssertionCondition(object): """ Object that represents an assertion condition Do not create this object directly, use :meth:`DSSMLAssertionParams.condition`, `create_from_expected_class(expected_valid_ratio, expected_class)` - or `create_from_expected_class(expected_valid_ratio, expected_class)` instead + or `create_from_expected_range(expected_valid_ratio, expected_range)` instead """ def __init__(self, data): self._internal_dict = data From 7013c5f1578879d2ba3d36b003aee98375e9eb02 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 2 Dec 2020 18:26:43 +0100 Subject: [PATCH 03/43] Add assertion results methods --- dataikuapi/dss/ml.py | 107 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 551407f5..8966b496 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -892,6 +892,97 @@ def expected_range(self, expected_range): self._internal_dict["expectedMaxValue"] = expected_range[1] +class DSSMLAssertionsMetrics(object): + """ + Object that represents the per assertion metrics for all assertions on a trained model + Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_per_assertion_metrics()` instead + """ + def __init__(self, data): + self._internal_dict = data + + def get_raw(self): + """ + Gets the raw dictionary of the assertions metrics + :rtype: dict + """ + return self._internal_dict + + def get_per_assertion_metric(self, assertion_name): + """ + Retrieves the metric computed for this trained model for the assertion with the provided name (or None) + :returns: an object representing assertion metric + :rtype: `dataikuapi.dss.ml.DSSMLAssertionMetric` + """ + for assertion_metric_dict in self._internal_dict["perAssertion"]: + if assertion_name == assertion_metric_dict["name"]: + return DSSMLAssertionMetric(assertion_metric_dict) + return None + + @property + def positive_assertion_ratio(self): + """ + Returns the ratio of passing assertions + :rtype: float + """ + return self._internal_dict['positiveAssertionsRatio'] + + +class DSSMLAssertionMetric(object): + """ + Object that represents the result of an assertion on a trained model + Do not create this object directly, use :meth:`DSSMLPerAssertionMetrics.get_assertion_metric(self, assertion_name)` instead + """ + def __init__(self, data): + self._internal_dict = data + + def get_raw(self): + """ + Gets the raw dictionary of metrics of one assertion + :rtype: dict + """ + return self._internal_dict + + @property + def name(self): + """ + Returns the assertion name + :rtype: str + """ + return self._internal_dict["name"] + + @property + def result(self): + """ + Returns whether the assertion pass + :rtype: bool + """ + return self._internal_dict["result"] + + @property + def valid_ratio(self): + """ + Returns the ratio of passing rows in the assertion population + :rtype: float + """ + return self._internal_dict["validRatio"] + + @property + def nb_matching_rows(self): + """ + Returns the number of rows matching filter + :rtype: int + """ + return self._internal_dict["nbMatchingRows"] + + @property + def nb_dropped_rows(self): + """ + Returns the number of rows dropped by the preprocessing + :rtype: int + """ + return self._internal_dict["nbDroppedRows"] + + class DSSTreeNode(object): def __init__(self, tree, i): self.tree = tree @@ -1050,6 +1141,22 @@ def get_performance_metrics(self): del clean_snippet[x] return clean_snippet + @property + def assertions_metrics(self): + """ + Retrieves assertions metrics computed for this trained model + :returns: an object representing assertion metrics + :rtype: `dataikuapi.dss.ml.DSSMLAssertionsMetrics` + """ + return self.get_assertions_metrics() + + def get_assertions_metrics(self): + """ + Retrieves assertions metrics computed for this trained model + :returns: an object representing assertion metrics + :rtype: `dataikuapi.dss.ml.DSSMLAssertionsMetrics` + """ + return DSSMLAssertionsMetrics(self.snippet["assertionsMetrics"]) def get_hyperparameter_search_points(self): """ From 4d599c5130fd66001175ce1a93750f1ad9518a79 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Thu, 3 Dec 2020 09:56:23 +0100 Subject: [PATCH 04/43] Clean up --- dataikuapi/dss/ml.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 8966b496..d3ddd717 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -436,7 +436,7 @@ def assertions_params(self): """ Retrieves the assertion params for this ml task - :rtype: :class:`DSSMLAssertionsParams` + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsParams` """ return self.get_assertions_params() @@ -444,7 +444,7 @@ def get_assertions_params(self): """ Retrieves the assertion params for this ml task - :rtype: :class:`DSSMLAssertionsParams` + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsParams` """ return DSSMLAssertionsParams(self.mltask_settings["assertionParams"]) @@ -690,8 +690,10 @@ def get_raw(self): def get_assertion(self, assertion_name): """ - Gets a :class:`DSSMLAssertionParams` representing the params of the assertion with the provided name - (or None) + Gets a :class:`dataikuapi.dss.ml.DSSMLAssertionParams` representing the params of the assertion with the + provided name (or None) + :param str assertion_name: Name of the assertion + :rtype: `dataikuapi.dss.ml.DSSMLAssertionParams` or None """ for assertion_dict in self._internal_dict["assertions"]: if assertion_dict["name"] == assertion_name: @@ -700,8 +702,9 @@ def get_assertion(self, assertion_name): def add_assertion(self, assertion_params): """ - Adds a :class:`DSSMLAssertionParams` to the `DSSMLAssertionsParams` of the ml task. Raises a ValueError if an assertion + Adds params of an assertion to the assertion params of the ml task. Raises a ValueError if an assertion with the same name already exists + :param `dataikuapi.dss.ml.DSSMLAssertionParams` assertion_params: Parameters of the assertion """ if not isinstance(assertion_params, DSSMLAssertionParams): raise ValueError('Assertion params should be of type: {} not {}'.format(DSSMLAssertionParams.__name__, type(assertion_params))) @@ -710,8 +713,9 @@ def add_assertion(self, assertion_params): def delete_assertion(self, assertion_name): """ - Deletes the assertion params of the assertion with the provided name from the `DSSMLAssertionsParams` + Deletes the assertion params of the assertion with the provided name from the `dataikuapi.dss.ml.DSSMLAssertionsParams` Raises a ValueError if no assertion with the provided name was found + :param str assertion_name: Name of the assertion """ for idx, assertion_dict in enumerate(self._internal_dict["assertions"]): if assertion_dict["name"] == assertion_name: @@ -723,7 +727,7 @@ def delete_assertion(self, assertion_name): class DSSMLAssertionParams(object): """ Object that represents parameters for one assertion - Do not create this object directly, use :meth:`DSSMLAssertionsParams.get_assertion(assertion_name)` or + Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionsParams.get_assertion(assertion_name)` or `create_from_parts(name, a_filter, condition)` instead """ def __init__(self, data): @@ -732,13 +736,13 @@ def __init__(self, data): @staticmethod def create_from_parts(name, a_filter, condition): """ - Creates a `DSSMLAssertionParams` from name, filter and condition + Creates a `dataikuapi.dss.ml.DSSMLAssertionParams` from name, filter and condition :param str name: Name of the assertion :param ~dataikuapi.dss.utils.DSSFilter a_filter: Filter to select assertion population :param DSSMLAssertionCondition condition: Condition for the assertion to be successful - :rtype: DSSMLAssertionParams + :rtype: `dataikuapi.dss.ml.DSSMLAssertionParams` """ assertion_params = DSSMLAssertionParams({}) assertion_params.name = name @@ -796,7 +800,7 @@ def condition(self, condition): class DSSMLAssertionCondition(object): """ Object that represents an assertion condition - Do not create this object directly, use :meth:`DSSMLAssertionParams.condition`, `create_from_expected_class(expected_valid_ratio, expected_class)` + Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionParams.condition`, `create_from_expected_class(expected_valid_ratio, expected_class)` or `create_from_expected_range(expected_valid_ratio, expected_range)` instead """ def __init__(self, data): @@ -805,12 +809,12 @@ def __init__(self, data): @staticmethod def create_from_expected_class(expected_valid_ratio, expected_class): """ - Creates a `DSSMLAssertionCondition` from an expected valid ratio and an expected class + Creates a `dataikuapi.dss.ml.DSSMLAssertionCondition` from an expected valid ratio and an expected class :param float expected_valid_ratio: Ratio of valid rows needed for the assertion to pass :param str expected_class: Class on which the `expected_valid_ratio` will be calculated - :rtype: DSSMLAssertionCondition + :rtype: `dataikuapi.dss.ml.DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio @@ -820,13 +824,13 @@ def create_from_expected_class(expected_valid_ratio, expected_class): @staticmethod def create_from_expected_range(expected_valid_ratio, expected_range): """ - Creates a `DSSMLAssertionCondition` from an expected valid ratio and an expected range + Creates a `dataikuapi.dss.ml.DSSMLAssertionCondition` from an expected valid ratio and an expected range :param float expected_valid_ratio: Ratio of valid rows to exceed for the assertion to pass :param tuple(float,float) expected_range: Range of values (min, max) where the prediction will be considered as valid for the `expected_valid_ratio` - :rtype: DSSMLAssertionCondition + :rtype: `dataikuapi.dss.ml.DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio @@ -873,7 +877,7 @@ def expected_valid_ratio(self, expected_valid_ratio): def expected_range(self): """ Returns the expected range on which the valid ratio will be calculated - :rtype: str + :rtype: tuple(float,float) """ if "expectedMinValue" in self._internal_dict and "expectedMaxValue" in self._internal_dict: return self._internal_dict["expectedMinValue"], self._internal_dict["expectedMaxValue"] @@ -910,7 +914,9 @@ def get_raw(self): def get_per_assertion_metric(self, assertion_name): """ Retrieves the metric computed for this trained model for the assertion with the provided name (or None) - :returns: an object representing assertion metric + + :param str assertion_name: Name of the assertion + :returns: an object representing assertion metrics :rtype: `dataikuapi.dss.ml.DSSMLAssertionMetric` """ for assertion_metric_dict in self._internal_dict["perAssertion"]: From e276feb253c489bad0343368aa3ba42b27dcf9fe Mon Sep 17 00:00:00 2001 From: arnaudde Date: Thu, 3 Dec 2020 10:01:29 +0100 Subject: [PATCH 05/43] Clean up --- dataikuapi/dss/ml.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index d3ddd717..cc439226 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -916,6 +916,7 @@ def get_per_assertion_metric(self, assertion_name): Retrieves the metric computed for this trained model for the assertion with the provided name (or None) :param str assertion_name: Name of the assertion + :returns: an object representing assertion metrics :rtype: `dataikuapi.dss.ml.DSSMLAssertionMetric` """ From ab5f5dc4aafec6659cc7f4cc0245cf2628c422d9 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Thu, 3 Dec 2020 16:10:13 +0100 Subject: [PATCH 06/43] Clean up --- dataikuapi/dss/ml.py | 53 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index cc439226..7c502d57 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -434,7 +434,7 @@ def get_split_params(self): @property def assertions_params(self): """ - Retrieves the assertion params for this ml task + Retrieves the assertions params for this ml task :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsParams` """ @@ -442,7 +442,7 @@ def assertions_params(self): def get_assertions_params(self): """ - Retrieves the assertion params for this ml task + Retrieves the assertions params for this ml task :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsParams` """ @@ -663,11 +663,11 @@ def __repr__(self): class DSSMLAssertionsParams(object): """ Object that represents parameters for all assertions of a ml task - Do not create this object directly, use :meth:`DSSPredictionMLTaskSettings.get_assertion_params()` instead + Do not create this object directly, use :meth:`DSSPredictionMLTaskSettings.get_assertions_params()` instead """ - @classmethod - def check_assertion_names_are_uniq(cls, assertion_params_list): + @staticmethod + def check_assertion_names_are_uniq(assertion_params_list): _ = {} for assertion_dict in assertion_params_list: if 'name' not in assertion_dict: @@ -693,7 +693,7 @@ def get_assertion(self, assertion_name): Gets a :class:`dataikuapi.dss.ml.DSSMLAssertionParams` representing the params of the assertion with the provided name (or None) :param str assertion_name: Name of the assertion - :rtype: `dataikuapi.dss.ml.DSSMLAssertionParams` or None + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionParams` or None """ for assertion_dict in self._internal_dict["assertions"]: if assertion_dict["name"] == assertion_name: @@ -702,9 +702,9 @@ def get_assertion(self, assertion_name): def add_assertion(self, assertion_params): """ - Adds params of an assertion to the assertion params of the ml task. Raises a ValueError if an assertion - with the same name already exists - :param `dataikuapi.dss.ml.DSSMLAssertionParams` assertion_params: Parameters of the assertion + Adds params of an assertion to the assertions params of the ml task. + Raises a ValueError if an assertion with the same name already exists + :param object assertion_params: A :class:`~dataikuapi.dss.utils.DSSMLAssertionParams` representing parameters of the assertion """ if not isinstance(assertion_params, DSSMLAssertionParams): raise ValueError('Assertion params should be of type: {} not {}'.format(DSSMLAssertionParams.__name__, type(assertion_params))) @@ -736,13 +736,13 @@ def __init__(self, data): @staticmethod def create_from_parts(name, a_filter, condition): """ - Creates a `dataikuapi.dss.ml.DSSMLAssertionParams` from name, filter and condition + Creates assertion params from name, filter and condition :param str name: Name of the assertion - :param ~dataikuapi.dss.utils.DSSFilter a_filter: Filter to select assertion population - :param DSSMLAssertionCondition condition: Condition for the assertion to be successful + :param object a_filter: A :class:`~dataikuapi.dss.utils.DSSFilter` to select assertion population + :param object condition: A :class:`~dataikuapi.dss.ml.DSSMLAssertionCondition` for the assertion to be successful - :rtype: `dataikuapi.dss.ml.DSSMLAssertionParams` + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionParams` """ assertion_params = DSSMLAssertionParams({}) assertion_params.name = name @@ -773,7 +773,7 @@ def name(self, name): def filter(self): """ Returns the assertion filter - :rtype: ~dataikuapi.dss.utils.DSSFilter + :rtype: :class:`dataikuapi.dss.utils.DSSFilter` """ return self._internal_dict["filter"] @@ -785,9 +785,8 @@ def filter(self, filter): def condition(self): """ Returns the assertion condition - :rtype: DSSMLAssertionCondition + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionCondition` """ - return DSSMLAssertionCondition(self._internal_dict["assertionCondition"]) @condition.setter @@ -800,8 +799,8 @@ def condition(self, condition): class DSSMLAssertionCondition(object): """ Object that represents an assertion condition - Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionParams.condition`, `create_from_expected_class(expected_valid_ratio, expected_class)` - or `create_from_expected_range(expected_valid_ratio, expected_range)` instead + Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionParams.condition`, :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.create_from_expected_class(expected_valid_ratio, expected_class)` + or :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.create_from_expected_range(expected_valid_ratio, expected_range)` instead """ def __init__(self, data): self._internal_dict = data @@ -809,12 +808,12 @@ def __init__(self, data): @staticmethod def create_from_expected_class(expected_valid_ratio, expected_class): """ - Creates a `dataikuapi.dss.ml.DSSMLAssertionCondition` from an expected valid ratio and an expected class + Creates an assertion condition from an expected valid ratio and an expected class :param float expected_valid_ratio: Ratio of valid rows needed for the assertion to pass :param str expected_class: Class on which the `expected_valid_ratio` will be calculated - :rtype: `dataikuapi.dss.ml.DSSMLAssertionCondition` + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio @@ -824,13 +823,13 @@ def create_from_expected_class(expected_valid_ratio, expected_class): @staticmethod def create_from_expected_range(expected_valid_ratio, expected_range): """ - Creates a `dataikuapi.dss.ml.DSSMLAssertionCondition` from an expected valid ratio and an expected range + Creates an assertion condition from an expected valid ratio and an expected range :param float expected_valid_ratio: Ratio of valid rows to exceed for the assertion to pass :param tuple(float,float) expected_range: Range of values (min, max) where the prediction will be considered as valid for the `expected_valid_ratio` - :rtype: `dataikuapi.dss.ml.DSSMLAssertionCondition` + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio @@ -899,7 +898,7 @@ def expected_range(self, expected_range): class DSSMLAssertionsMetrics(object): """ Object that represents the per assertion metrics for all assertions on a trained model - Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_per_assertion_metrics()` instead + Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSTrainedPredictionModelDetails.get_per_assertion_metrics()` instead """ def __init__(self, data): self._internal_dict = data @@ -918,7 +917,7 @@ def get_per_assertion_metric(self, assertion_name): :param str assertion_name: Name of the assertion :returns: an object representing assertion metrics - :rtype: `dataikuapi.dss.ml.DSSMLAssertionMetric` + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionMetric` """ for assertion_metric_dict in self._internal_dict["perAssertion"]: if assertion_name == assertion_metric_dict["name"]: @@ -937,7 +936,7 @@ def positive_assertion_ratio(self): class DSSMLAssertionMetric(object): """ Object that represents the result of an assertion on a trained model - Do not create this object directly, use :meth:`DSSMLPerAssertionMetrics.get_assertion_metric(self, assertion_name)` instead + Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLPerAssertionMetrics.get_assertion_metric(self, assertion_name)` instead """ def __init__(self, data): self._internal_dict = data @@ -1153,7 +1152,7 @@ def assertions_metrics(self): """ Retrieves assertions metrics computed for this trained model :returns: an object representing assertion metrics - :rtype: `dataikuapi.dss.ml.DSSMLAssertionsMetrics` + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsMetrics` """ return self.get_assertions_metrics() @@ -1161,7 +1160,7 @@ def get_assertions_metrics(self): """ Retrieves assertions metrics computed for this trained model :returns: an object representing assertion metrics - :rtype: `dataikuapi.dss.ml.DSSMLAssertionsMetrics` + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsMetrics` """ return DSSMLAssertionsMetrics(self.snippet["assertionsMetrics"]) From 190676c2e8ddf61434905cad8514041ec201dd9e Mon Sep 17 00:00:00 2001 From: Arnaud dE Date: Fri, 4 Dec 2020 14:25:07 +0100 Subject: [PATCH 07/43] Wording Co-authored-by: Adrien Lavoillotte --- dataikuapi/dss/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 7c502d57..c863a9e1 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -983,7 +983,7 @@ def nb_matching_rows(self): @property def nb_dropped_rows(self): """ - Returns the number of rows dropped by the preprocessing + Returns the number of rows dropped by the model's preprocessing :rtype: int """ return self._internal_dict["nbDroppedRows"] From fe840399e217868f4ec8645b40a20e14eebe4d69 Mon Sep 17 00:00:00 2001 From: Arnaud dE Date: Fri, 4 Dec 2020 14:28:26 +0100 Subject: [PATCH 08/43] Clarify expected_valid_ratio parameter docstring Co-authored-by: Adrien Lavoillotte --- dataikuapi/dss/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index c863a9e1..d9bb9a28 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -825,7 +825,7 @@ def create_from_expected_range(expected_valid_ratio, expected_range): """ Creates an assertion condition from an expected valid ratio and an expected range - :param float expected_valid_ratio: Ratio of valid rows to exceed for the assertion to pass + :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted between expected_min and expected_max is attained :param tuple(float,float) expected_range: Range of values (min, max) where the prediction will be considered as valid for the `expected_valid_ratio` From da7a645e860a64c600e99c0155216fb9f79fde51 Mon Sep 17 00:00:00 2001 From: Arnaud dE Date: Fri, 4 Dec 2020 14:30:24 +0100 Subject: [PATCH 09/43] Wording Co-authored-by: Adrien Lavoillotte --- dataikuapi/dss/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index d9bb9a28..7e487fc6 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -808,7 +808,7 @@ def __init__(self, data): @staticmethod def create_from_expected_class(expected_valid_ratio, expected_class): """ - Creates an assertion condition from an expected valid ratio and an expected class + Creates an assertion condition from the expected valid ratio and class :param float expected_valid_ratio: Ratio of valid rows needed for the assertion to pass :param str expected_class: Class on which the `expected_valid_ratio` will be calculated From 7b56a083cfc45315cf81acc69fa595991f3e85f3 Mon Sep 17 00:00:00 2001 From: Arnaud dE Date: Fri, 4 Dec 2020 14:31:45 +0100 Subject: [PATCH 10/43] Simplify exception message Co-authored-by: Adrien Lavoillotte --- dataikuapi/dss/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 7e487fc6..9d92022c 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -707,7 +707,7 @@ def add_assertion(self, assertion_params): :param object assertion_params: A :class:`~dataikuapi.dss.utils.DSSMLAssertionParams` representing parameters of the assertion """ if not isinstance(assertion_params, DSSMLAssertionParams): - raise ValueError('Assertion params should be of type: {} not {}'.format(DSSMLAssertionParams.__name__, type(assertion_params))) + raise ValueError('Wrong type for assertion params: {}'.format(type(assertion_params))) self.check_assertion_names_are_uniq(self._internal_dict["assertions"] + [assertion_params._internal_dict]) self._internal_dict["assertions"].append(assertion_params._internal_dict) From 4bfcf0d2eaa236d7a37bfccb66caea5b0f6d3606 Mon Sep 17 00:00:00 2001 From: Arnaud dE Date: Fri, 4 Dec 2020 14:41:11 +0100 Subject: [PATCH 11/43] Clarify exception message Co-authored-by: Adrien Lavoillotte --- dataikuapi/dss/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 9d92022c..815574aa 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -673,7 +673,7 @@ def check_assertion_names_are_uniq(assertion_params_list): if 'name' not in assertion_dict: raise ValueError('No name provided for assertion') if assertion_dict['name'] in _: - raise ValueError('Assertion names should be uniq. {} is multiple times in the data'.format( + raise ValueError('Assertion names must be unique, but got multiple instances of: {}'.format( assertion_dict['name'])) _[assertion_dict['name']] = True From 537cd678395021d3680e3576e2ca2f70c4011b2c Mon Sep 17 00:00:00 2001 From: Arnaud dE Date: Fri, 4 Dec 2020 14:43:19 +0100 Subject: [PATCH 12/43] Wording Co-authored-by: Adrien Lavoillotte --- dataikuapi/dss/ml.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 815574aa..b2fe3d65 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -691,7 +691,7 @@ def get_raw(self): def get_assertion(self, assertion_name): """ Gets a :class:`dataikuapi.dss.ml.DSSMLAssertionParams` representing the params of the assertion with the - provided name (or None) + provided name (or None if no assertion has that name) :param str assertion_name: Name of the assertion :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionParams` or None """ @@ -721,7 +721,7 @@ def delete_assertion(self, assertion_name): if assertion_dict["name"] == assertion_name: del self._internal_dict["assertions"][idx] return - raise ValueError('No assertion name: {} was found'.format(assertion_name)) + raise ValueError('No assertion found with name: {}'.format(assertion_name)) class DSSMLAssertionParams(object): @@ -792,7 +792,7 @@ def condition(self): @condition.setter def condition(self, condition): if not isinstance(condition, DSSMLAssertionCondition): - raise ValueError('Condition should be of type: {} not {}'.format(DSSMLAssertionCondition.__name__, type(condition))) + raise ValueError('Wrong type for assertion condition: {}.format(type(condition))) self._internal_dict["assertionCondition"] = condition._internal_dict From 15343ea9d87d72caee7bd6ad6e45267df223e1ca Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 7 Dec 2020 12:07:25 +0100 Subject: [PATCH 13/43] Remove checks for assertion names uniqueness and reanme param to parameters in new code --- dataikuapi/dss/ml.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index b2fe3d65..42694ee0 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -434,7 +434,7 @@ def get_split_params(self): @property def assertions_params(self): """ - Retrieves the assertions params for this ml task + Retrieves the assertions parameters for this ml task :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsParams` """ @@ -442,7 +442,7 @@ def assertions_params(self): def get_assertions_params(self): """ - Retrieves the assertions params for this ml task + Retrieves the assertions parameters for this ml task :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsParams` """ @@ -666,31 +666,19 @@ class DSSMLAssertionsParams(object): Do not create this object directly, use :meth:`DSSPredictionMLTaskSettings.get_assertions_params()` instead """ - @staticmethod - def check_assertion_names_are_uniq(assertion_params_list): - _ = {} - for assertion_dict in assertion_params_list: - if 'name' not in assertion_dict: - raise ValueError('No name provided for assertion') - if assertion_dict['name'] in _: - raise ValueError('Assertion names must be unique, but got multiple instances of: {}'.format( - assertion_dict['name'])) - _[assertion_dict['name']] = True - def __init__(self, data): self._internal_dict = data - self.check_assertion_names_are_uniq(data["assertions"]) def get_raw(self): """ - Gets the raw dictionary of the assertions params + Gets the raw dictionary of the assertions parameters :rtype: dict """ return self._internal_dict def get_assertion(self, assertion_name): """ - Gets a :class:`dataikuapi.dss.ml.DSSMLAssertionParams` representing the params of the assertion with the + Gets a :class:`dataikuapi.dss.ml.DSSMLAssertionParams` representing the parameters of the assertion with the provided name (or None if no assertion has that name) :param str assertion_name: Name of the assertion :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionParams` or None @@ -702,18 +690,19 @@ def get_assertion(self, assertion_name): def add_assertion(self, assertion_params): """ - Adds params of an assertion to the assertions params of the ml task. + Adds parameters of an assertion to the assertions parameters of the ml task. Raises a ValueError if an assertion with the same name already exists :param object assertion_params: A :class:`~dataikuapi.dss.utils.DSSMLAssertionParams` representing parameters of the assertion """ if not isinstance(assertion_params, DSSMLAssertionParams): - raise ValueError('Wrong type for assertion params: {}'.format(type(assertion_params))) + raise ValueError('Wrong type for assertion parameters: {}'.format(type(assertion_params))) self.check_assertion_names_are_uniq(self._internal_dict["assertions"] + [assertion_params._internal_dict]) + self._internal_dict["assertions"].append(assertion_params._internal_dict) def delete_assertion(self, assertion_name): """ - Deletes the assertion params of the assertion with the provided name from the `dataikuapi.dss.ml.DSSMLAssertionsParams` + Deletes the assertion parameters of the assertion with the provided name from the `dataikuapi.dss.ml.DSSMLAssertionsParams` Raises a ValueError if no assertion with the provided name was found :param str assertion_name: Name of the assertion """ @@ -736,7 +725,7 @@ def __init__(self, data): @staticmethod def create_from_parts(name, a_filter, condition): """ - Creates assertion params from name, filter and condition + Creates assertion parameters from name, filter and condition :param str name: Name of the assertion :param object a_filter: A :class:`~dataikuapi.dss.utils.DSSFilter` to select assertion population @@ -752,7 +741,7 @@ def create_from_parts(name, a_filter, condition): def get_raw(self): """ - Gets the raw dictionary of the assertion params + Gets the raw dictionary of the assertion parameters :rtype: dict """ return self._internal_dict @@ -792,7 +781,7 @@ def condition(self): @condition.setter def condition(self, condition): if not isinstance(condition, DSSMLAssertionCondition): - raise ValueError('Wrong type for assertion condition: {}.format(type(condition))) + raise ValueError('Wrong type for assertion condition: {}.format(type(condition))') self._internal_dict["assertionCondition"] = condition._internal_dict From 5d2ab2da8ae9bd4346d7b5098cf2e7bbcc2c7f12 Mon Sep 17 00:00:00 2001 From: Arnaud dE Date: Mon, 7 Dec 2020 16:23:25 +0100 Subject: [PATCH 14/43] Fix typo Co-authored-by: Adrien Lavoillotte --- dataikuapi/dss/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 42694ee0..7db55083 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -781,7 +781,7 @@ def condition(self): @condition.setter def condition(self, condition): if not isinstance(condition, DSSMLAssertionCondition): - raise ValueError('Wrong type for assertion condition: {}.format(type(condition))') + raise ValueError('Wrong type for assertion condition: {}'.format(type(condition))) self._internal_dict["assertionCondition"] = condition._internal_dict From 56ae94174ed08adb71f9972f9591a5dac8eff841 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 7 Dec 2020 12:17:13 +0100 Subject: [PATCH 15/43] Remove leftover check --- dataikuapi/dss/ml.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 7db55083..2184e98c 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -696,7 +696,6 @@ def add_assertion(self, assertion_params): """ if not isinstance(assertion_params, DSSMLAssertionParams): raise ValueError('Wrong type for assertion parameters: {}'.format(type(assertion_params))) - self.check_assertion_names_are_uniq(self._internal_dict["assertions"] + [assertion_params._internal_dict]) self._internal_dict["assertions"].append(assertion_params._internal_dict) From 66041367f016694aaf6ba9445f19e8f8bf512795 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 7 Dec 2020 12:32:51 +0100 Subject: [PATCH 16/43] Rename create_from_* methods to from_* --- dataikuapi/dss/ml.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 2184e98c..023580cb 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -716,13 +716,13 @@ class DSSMLAssertionParams(object): """ Object that represents parameters for one assertion Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionsParams.get_assertion(assertion_name)` or - `create_from_parts(name, a_filter, condition)` instead + `from_parts(name, a_filter, condition)` instead """ def __init__(self, data): self._internal_dict = data @staticmethod - def create_from_parts(name, a_filter, condition): + def from_parts(name, a_filter, condition): """ Creates assertion parameters from name, filter and condition @@ -787,14 +787,14 @@ def condition(self, condition): class DSSMLAssertionCondition(object): """ Object that represents an assertion condition - Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionParams.condition`, :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.create_from_expected_class(expected_valid_ratio, expected_class)` - or :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.create_from_expected_range(expected_valid_ratio, expected_range)` instead + Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionParams.condition`, :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_class(expected_valid_ratio, expected_class)` + or :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_range(expected_valid_ratio, expected_range)` instead """ def __init__(self, data): self._internal_dict = data @staticmethod - def create_from_expected_class(expected_valid_ratio, expected_class): + def from_expected_class(expected_valid_ratio, expected_class): """ Creates an assertion condition from the expected valid ratio and class @@ -809,7 +809,7 @@ def create_from_expected_class(expected_valid_ratio, expected_class): return assertion_condition @staticmethod - def create_from_expected_range(expected_valid_ratio, expected_range): + def from_expected_range(expected_valid_ratio, expected_range): """ Creates an assertion condition from an expected valid ratio and an expected range From 86a7c9a3689798f1780714e3fe8a55a06fd8a711 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 7 Dec 2020 16:10:06 +0100 Subject: [PATCH 17/43] Change expected_range to expected_min and expected_max --- dataikuapi/dss/ml.py | 57 +++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 023580cb..63822bb3 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -788,7 +788,7 @@ class DSSMLAssertionCondition(object): """ Object that represents an assertion condition Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionParams.condition`, :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_class(expected_valid_ratio, expected_class)` - or :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_range(expected_valid_ratio, expected_range)` instead + or :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_range(expected_valid_ratio, expected_min, expected_max)` instead """ def __init__(self, data): self._internal_dict = data @@ -809,19 +809,22 @@ def from_expected_class(expected_valid_ratio, expected_class): return assertion_condition @staticmethod - def from_expected_range(expected_valid_ratio, expected_range): + def from_expected_range(expected_valid_ratio, expected_min, expected_max): """ - Creates an assertion condition from an expected valid ratio and an expected range + Creates an assertion condition from an expected valid ratio and an expected range. The expected range is the + interval between expected_min and expected_max where the predictions and therefore the rows will be considered + valid. :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted between expected_min and expected_max is attained - :param tuple(float,float) expected_range: Range of values (min, max) where the prediction will be considered as - valid for the `expected_valid_ratio` + :param float expected_min: Min value of the expected range + :param float expected_max: Max value of the expected range :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio - assertion_condition.expected_range = expected_range + assertion_condition.expected_min = expected_min + assertion_condition.expected_max = expected_max return assertion_condition def get_raw(self): @@ -834,7 +837,7 @@ def get_raw(self): @property def expected_class(self): """ - Returns the expected class on which the valid ratio will be calculated + Returns the expected class on which the valid ratio will be calculated or None if it is not defined :rtype: str """ if "expectedClass" in self._internal_dict: @@ -844,8 +847,6 @@ def expected_class(self): @expected_class.setter def expected_class(self, expected_class): - if self.expected_range is not None: - raise ValueError("Expected class and expected range can't be both set") self._internal_dict["expectedClass"] = expected_class @property @@ -861,26 +862,34 @@ def expected_valid_ratio(self, expected_valid_ratio): self._internal_dict["successRatio"] = expected_valid_ratio @property - def expected_range(self): + def expected_min(self): """ - Returns the expected range on which the valid ratio will be calculated - :rtype: tuple(float,float) + Returns the min (included) of the expected range or None if it is not defined + :rtype: float + """ + if "expectedMinValue" in self._internal_dict: + return self._internal_dict["expectedMinValue"] + else: + return None + + @expected_min.setter + def expected_min(self, expected_min): + self._internal_dict["expectedMinValue"] = expected_min + + @property + def expected_max(self): + """ + Returns the max (included) of the expected range + :rtype: float """ - if "expectedMinValue" in self._internal_dict and "expectedMaxValue" in self._internal_dict: - return self._internal_dict["expectedMinValue"], self._internal_dict["expectedMaxValue"] + if "expectedMaxValue" in self._internal_dict: + return self._internal_dict["expectedMaxValue"] else: return None - @expected_range.setter - def expected_range(self, expected_range): - if not isinstance(expected_range, tuple): - raise ValueError("Expected range needs to be a tuple") - if self.expected_class is not None: - raise ValueError("Expected class and expected range can't be both set") - if expected_range[0] > expected_range: - raise ValueError("Expected range needs to be sorted in ascending order. (min value, max value)") - self._internal_dict["expectedMinValue"] = expected_range[0] - self._internal_dict["expectedMaxValue"] = expected_range[1] + @expected_max.setter + def expected_max(self, expected_max): + self._internal_dict["expectedMaxValue"] = expected_max class DSSMLAssertionsMetrics(object): From 7aeafa27ad7649c12b02bc9c5c7e1c16c439b9f3 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 7 Dec 2020 17:06:52 +0100 Subject: [PATCH 18/43] Wording --- dataikuapi/dss/ml.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 63822bb3..48b2fe47 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -798,8 +798,8 @@ def from_expected_class(expected_valid_ratio, expected_class): """ Creates an assertion condition from the expected valid ratio and class - :param float expected_valid_ratio: Ratio of valid rows needed for the assertion to pass - :param str expected_class: Class on which the `expected_valid_ratio` will be calculated + :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted as expected_class is attained + :param str expected_class: Assertion passes if the ratio of rows predicted as expected_class is attained :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionCondition` """ @@ -811,7 +811,7 @@ def from_expected_class(expected_valid_ratio, expected_class): @staticmethod def from_expected_range(expected_valid_ratio, expected_min, expected_max): """ - Creates an assertion condition from an expected valid ratio and an expected range. The expected range is the + Creates an assertion condition from expected valid ratio and range. The expected range is the interval between expected_min and expected_max where the predictions and therefore the rows will be considered valid. @@ -837,7 +837,8 @@ def get_raw(self): @property def expected_class(self): """ - Returns the expected class on which the valid ratio will be calculated or None if it is not defined + Returns the expected class or None if it is not defined. Assertion passes if the ratio of rows predicted + as expected_class is attained :rtype: str """ if "expectedClass" in self._internal_dict: @@ -852,7 +853,8 @@ def expected_class(self, expected_class): @property def expected_valid_ratio(self): """ - Returns the ratio of valid rows to exceed for the assertion to pass + Returns the ratio of valid rows to exceed for the assertion to pass. A row is considered valid if the prediction + is equal to the `expected_class` for classification or in the expected range for regresion :rtype: str """ return self._internal_dict["successRatio"] @@ -864,7 +866,8 @@ def expected_valid_ratio(self, expected_valid_ratio): @property def expected_min(self): """ - Returns the min (included) of the expected range or None if it is not defined + Returns the min (included) of the expected range or None if it is not defined. Assertion passes if the ratio of rows predicted + between expected_min and expected_max is attained :rtype: float """ if "expectedMinValue" in self._internal_dict: @@ -879,7 +882,8 @@ def expected_min(self, expected_min): @property def expected_max(self): """ - Returns the max (included) of the expected range + Returns the max (included) of the expected range or None if it is not defined. Assertion passes if the ratio of rows predicted + between expected_min and expected_max is attained :rtype: float """ if "expectedMaxValue" in self._internal_dict: From c740435fc8e17c4fb8f6364dda2cb6c97ddd181f Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 7 Dec 2020 18:30:33 +0100 Subject: [PATCH 19/43] Rename functions and clean up --- dataikuapi/dss/ml.py | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 48b2fe47..4a027d37 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -431,15 +431,6 @@ def get_split_params(self): """ return PredictionSplitParamsHandler(self.mltask_settings) - @property - def assertions_params(self): - """ - Retrieves the assertions parameters for this ml task - - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsParams` - """ - return self.get_assertions_params() - def get_assertions_params(self): """ Retrieves the assertions parameters for this ml task @@ -899,7 +890,7 @@ def expected_max(self, expected_max): class DSSMLAssertionsMetrics(object): """ Object that represents the per assertion metrics for all assertions on a trained model - Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSTrainedPredictionModelDetails.get_per_assertion_metrics()` instead + Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSTrainedPredictionModelDetails.get_assertions_metrics()` instead """ def __init__(self, data): self._internal_dict = data @@ -911,13 +902,14 @@ def get_raw(self): """ return self._internal_dict - def get_per_assertion_metric(self, assertion_name): + def get_metric(self, assertion_name): """ - Retrieves the metric computed for this trained model for the assertion with the provided name (or None) + Retrieves the metric computed for this trained model for the assertion with the provided name (or None if no + assertion with that name exists) :param str assertion_name: Name of the assertion - :returns: an object representing assertion metrics + :returns: an object representing assertion metrics or None if if no assertion with that name exists :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionMetric` """ for assertion_metric_dict in self._internal_dict["perAssertion"]: @@ -937,7 +929,7 @@ def positive_assertion_ratio(self): class DSSMLAssertionMetric(object): """ Object that represents the result of an assertion on a trained model - Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLPerAssertionMetrics.get_assertion_metric(self, assertion_name)` instead + Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionMetrics.get_metric(self, assertion_name)` instead """ def __init__(self, data): self._internal_dict = data @@ -968,7 +960,8 @@ def result(self): @property def valid_ratio(self): """ - Returns the ratio of passing rows in the assertion population + Returns the ratio of rows in the assertion population with prediction equals to the expected class + for classification or in the expected range for regression :rtype: float """ return self._internal_dict["validRatio"] @@ -1148,15 +1141,6 @@ def get_performance_metrics(self): del clean_snippet[x] return clean_snippet - @property - def assertions_metrics(self): - """ - Retrieves assertions metrics computed for this trained model - :returns: an object representing assertion metrics - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsMetrics` - """ - return self.get_assertions_metrics() - def get_assertions_metrics(self): """ Retrieves assertions metrics computed for this trained model From 314cbd2d73ab9f40b194fdef13a658acc6dc6fdc Mon Sep 17 00:00:00 2001 From: Arnaud dE Date: Tue, 8 Dec 2020 11:00:20 +0100 Subject: [PATCH 20/43] Fix some wording and indent Co-authored-by: Adrien Lavoillotte --- dataikuapi/dss/ml.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 4a027d37..d1a8d0f0 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -802,9 +802,9 @@ def from_expected_class(expected_valid_ratio, expected_class): @staticmethod def from_expected_range(expected_valid_ratio, expected_min, expected_max): """ - Creates an assertion condition from expected valid ratio and range. The expected range is the - interval between expected_min and expected_max where the predictions and therefore the rows will be considered - valid. + Creates an assertion condition from expected valid ratio and range. + The expected range is the interval between expected_min and expected_max (included) + for the predictions in which the rows will be considered valid. :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted between expected_min and expected_max is attained :param float expected_min: Min value of the expected range @@ -828,8 +828,8 @@ def get_raw(self): @property def expected_class(self): """ - Returns the expected class or None if it is not defined. Assertion passes if the ratio of rows predicted - as expected_class is attained + Returns the expected class or None if it is not defined. Assertion passes if the expected_valid_ratio + of rows predicted as expected_class is attained. :rtype: str """ if "expectedClass" in self._internal_dict: @@ -845,7 +845,7 @@ def expected_class(self, expected_class): def expected_valid_ratio(self): """ Returns the ratio of valid rows to exceed for the assertion to pass. A row is considered valid if the prediction - is equal to the `expected_class` for classification or in the expected range for regresion + is equal to the `expected_class` for classification or in the expected range for regression :rtype: str """ return self._internal_dict["successRatio"] @@ -857,8 +857,8 @@ def expected_valid_ratio(self, expected_valid_ratio): @property def expected_min(self): """ - Returns the min (included) of the expected range or None if it is not defined. Assertion passes if the ratio of rows predicted - between expected_min and expected_max is attained + Returns the min (included) of the expected range or None if it is not defined. + Assertion passes if the ratio of rows predicted between expected_min and expected_max is attained. :rtype: float """ if "expectedMinValue" in self._internal_dict: @@ -873,8 +873,8 @@ def expected_min(self, expected_min): @property def expected_max(self): """ - Returns the max (included) of the expected range or None if it is not defined. Assertion passes if the ratio of rows predicted - between expected_min and expected_max is attained + Returns the max (included) of the expected range or None if it is not defined. + Assertion passes if the ratio of rows predicted between expected_min and expected_max is attained. :rtype: float """ if "expectedMaxValue" in self._internal_dict: From 0a5d7c631f890855c850a1f9d4511f78d14f7095 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Tue, 8 Dec 2020 11:31:27 +0100 Subject: [PATCH 21/43] Clean up --- dataikuapi/dss/ml.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index d1a8d0f0..25038273 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -672,7 +672,7 @@ def get_assertion(self, assertion_name): Gets a :class:`dataikuapi.dss.ml.DSSMLAssertionParams` representing the parameters of the assertion with the provided name (or None if no assertion has that name) :param str assertion_name: Name of the assertion - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionParams` or None + :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionParams` or None if no assertion has that name """ for assertion_dict in self._internal_dict["assertions"]: if assertion_dict["name"] == assertion_name: @@ -682,7 +682,6 @@ def get_assertion(self, assertion_name): def add_assertion(self, assertion_params): """ Adds parameters of an assertion to the assertions parameters of the ml task. - Raises a ValueError if an assertion with the same name already exists :param object assertion_params: A :class:`~dataikuapi.dss.utils.DSSMLAssertionParams` representing parameters of the assertion """ if not isinstance(assertion_params, DSSMLAssertionParams): @@ -692,7 +691,7 @@ def add_assertion(self, assertion_params): def delete_assertion(self, assertion_name): """ - Deletes the assertion parameters of the assertion with the provided name from the `dataikuapi.dss.ml.DSSMLAssertionsParams` + Deletes the assertion parameters of the assertion with the provided name from the assertions parameters of the ml task. Raises a ValueError if no assertion with the provided name was found :param str assertion_name: Name of the assertion """ @@ -845,7 +844,7 @@ def expected_class(self, expected_class): def expected_valid_ratio(self): """ Returns the ratio of valid rows to exceed for the assertion to pass. A row is considered valid if the prediction - is equal to the `expected_class` for classification or in the expected range for regression + is equal to the expected class for classification or in the expected range for regression :rtype: str """ return self._internal_dict["successRatio"] @@ -952,7 +951,7 @@ def name(self): @property def result(self): """ - Returns whether the assertion pass + Returns whether the assertion passes :rtype: bool """ return self._internal_dict["result"] From 852eb31d9d2b0e6a57af26195e6230c9acb3a56a Mon Sep 17 00:00:00 2001 From: arnaudde Date: Tue, 8 Dec 2020 11:35:36 +0100 Subject: [PATCH 22/43] Clean up --- dataikuapi/dss/ml.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 25038273..e2b4b419 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -776,9 +776,9 @@ def condition(self, condition): class DSSMLAssertionCondition(object): """ - Object that represents an assertion condition - Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionParams.condition`, :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_class(expected_valid_ratio, expected_class)` - or :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_range(expected_valid_ratio, expected_min, expected_max)` instead + Object that represents an assertion condition + Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionParams.condition`, :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_class(expected_valid_ratio, expected_class)` + or :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_range(expected_valid_ratio, expected_min, expected_max)` instead """ def __init__(self, data): self._internal_dict = data @@ -844,7 +844,7 @@ def expected_class(self, expected_class): def expected_valid_ratio(self): """ Returns the ratio of valid rows to exceed for the assertion to pass. A row is considered valid if the prediction - is equal to the expected class for classification or in the expected range for regression + is equal to the expected class for classification or in the expected range for regression. :rtype: str """ return self._internal_dict["successRatio"] @@ -908,7 +908,7 @@ def get_metric(self, assertion_name): :param str assertion_name: Name of the assertion - :returns: an object representing assertion metrics or None if if no assertion with that name exists + :returns: an object representing assertion metrics or None if no assertion with that name exists :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionMetric` """ for assertion_metric_dict in self._internal_dict["perAssertion"]: From 1d541a3352bbb789628505d3b460c00ee2792b4a Mon Sep 17 00:00:00 2001 From: arnaudde Date: Tue, 8 Dec 2020 11:47:29 +0100 Subject: [PATCH 23/43] Clean up --- dataikuapi/dss/ml.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index e2b4b419..049a890d 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -717,7 +717,7 @@ def from_parts(name, a_filter, condition): Creates assertion parameters from name, filter and condition :param str name: Name of the assertion - :param object a_filter: A :class:`~dataikuapi.dss.utils.DSSFilter` to select assertion population + :param object a_filter: A :class:`~dataikuapi.dss.utils.DSSFilter` to select assertion population :param object condition: A :class:`~dataikuapi.dss.ml.DSSMLAssertionCondition` for the assertion to be successful :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionParams` @@ -805,7 +805,7 @@ def from_expected_range(expected_valid_ratio, expected_min, expected_max): The expected range is the interval between expected_min and expected_max (included) for the predictions in which the rows will be considered valid. - :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted between expected_min and expected_max is attained + :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted between expected_min and expected_max (included) is attained :param float expected_min: Min value of the expected range :param float expected_max: Max value of the expected range @@ -857,7 +857,7 @@ def expected_valid_ratio(self, expected_valid_ratio): def expected_min(self): """ Returns the min (included) of the expected range or None if it is not defined. - Assertion passes if the ratio of rows predicted between expected_min and expected_max is attained. + Assertion passes if the expected_valid_ratio of rows predicted between expected_min and expected_max (included) is attained. :rtype: float """ if "expectedMinValue" in self._internal_dict: @@ -873,7 +873,7 @@ def expected_min(self, expected_min): def expected_max(self): """ Returns the max (included) of the expected range or None if it is not defined. - Assertion passes if the ratio of rows predicted between expected_min and expected_max is attained. + Assertion passes if the ratio of rows predicted between expected_min and expected_max (included) is attained. :rtype: float """ if "expectedMaxValue" in self._internal_dict: From 0f883c522ed3bdb978725e2d6d1b417fec7962ef Mon Sep 17 00:00:00 2001 From: arnaudde Date: Fri, 11 Dec 2020 12:28:35 +0100 Subject: [PATCH 24/43] Update variables names after dip variables renaming --- dataikuapi/dss/ml.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 049a890d..eb552862 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -437,7 +437,7 @@ def get_assertions_params(self): :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsParams` """ - return DSSMLAssertionsParams(self.mltask_settings["assertionParams"]) + return DSSMLAssertionsParams(self.mltask_settings["assertionsParams"]) def split_ordered_by(self, feature_name, ascending=True): """ @@ -847,11 +847,11 @@ def expected_valid_ratio(self): is equal to the expected class for classification or in the expected range for regression. :rtype: str """ - return self._internal_dict["successRatio"] + return self._internal_dict["expectedValidRatio"] @expected_valid_ratio.setter def expected_valid_ratio(self, expected_valid_ratio): - self._internal_dict["successRatio"] = expected_valid_ratio + self._internal_dict["expectedValidRatio"] = expected_valid_ratio @property def expected_min(self): @@ -917,12 +917,12 @@ def get_metric(self, assertion_name): return None @property - def positive_assertion_ratio(self): + def passing_assertions_ratio(self): """ Returns the ratio of passing assertions :rtype: float """ - return self._internal_dict['positiveAssertionsRatio'] + return self._internal_dict['passingAssertionsRatio'] class DSSMLAssertionMetric(object): From 39981e9f3764e145bb64218d1f40aca7ece2613e Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 14 Dec 2020 09:43:50 +0100 Subject: [PATCH 25/43] Rename from_parts to from_params and fix return types --- dataikuapi/dss/ml.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index eb552862..9fd0ca2a 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -712,12 +712,12 @@ def __init__(self, data): self._internal_dict = data @staticmethod - def from_parts(name, a_filter, condition): + def from_params(name, a_filter, condition): """ Creates assertion parameters from name, filter and condition :param str name: Name of the assertion - :param object a_filter: A :class:`~dataikuapi.dss.utils.DSSFilter` to select assertion population + :param object a_filter: A dict representing the filter to select assertion population :param object condition: A :class:`~dataikuapi.dss.ml.DSSMLAssertionCondition` for the assertion to be successful :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionParams` @@ -751,7 +751,7 @@ def name(self, name): def filter(self): """ Returns the assertion filter - :rtype: :class:`dataikuapi.dss.utils.DSSFilter` + :rtype: dict """ return self._internal_dict["filter"] From 62a47bd30f8a85ca5838f8fa7c29a53e44fc6183 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 14 Dec 2020 11:57:49 +0100 Subject: [PATCH 26/43] Add repr to conditon and assertionParam --- dataikuapi/dss/ml.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 9fd0ca2a..0f20a237 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -5,6 +5,7 @@ from ..utils import DataikuStreamedHttpUTF8CSVReader import json, warnings import time +import pprint from .metrics import ComputedMetrics from .utils import DSSDatasetSelectionBuilder, DSSFilterBuilder from .future import DSSFuture @@ -711,6 +712,10 @@ class DSSMLAssertionParams(object): def __init__(self, data): self._internal_dict = data + def __repr__(self): + return u"DSSMLAssertionParams(\n name= '{name}',\n condition= {condition},\n filter= {filter}\n)".format( + name=self.name, filter=pprint.pformat(self.filter).replace('\n', '\n '), condition=self.condition) + @staticmethod def from_params(name, a_filter, condition): """ @@ -783,6 +788,12 @@ class DSSMLAssertionCondition(object): def __init__(self, data): self._internal_dict = data + def __repr__(self): + return u"DSSMLAssertionCondition(expected_valid_ratio = {}, expected_class= {}," \ + u" expected_min={}, expected_max={})".format( + self.expected_valid_ratio, self.expected_class, self.expected_min, self.expected_max) + + @staticmethod def from_expected_class(expected_valid_ratio, expected_class): """ From be2dbd20fbb09ada0f86be397163ff43f1b1f532 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 14 Dec 2020 12:21:52 +0100 Subject: [PATCH 27/43] Add repr to assertionMetrics --- dataikuapi/dss/ml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 0f20a237..dadec99d 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -944,6 +944,10 @@ class DSSMLAssertionMetric(object): def __init__(self, data): self._internal_dict = data + def __repr__(self): + return u"DSSMLAssertionParams(\n name='{}',\n result={},\n valid_ratio={},\n nb_matching_rows={}," \ + u"\n nb_dropped_rows={}\n)".format(self.name, self.result, self.valid_ratio, self.nb_matching_rows, + self.nb_dropped_rows) def get_raw(self): """ Gets the raw dictionary of metrics of one assertion From 7942ac6d20d19cb930afd95865230d8aa2dc3b20 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 14 Dec 2020 14:51:08 +0100 Subject: [PATCH 28/43] Unify class references --- dataikuapi/dss/ml.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index dadec99d..0c82eb29 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -436,7 +436,7 @@ def get_assertions_params(self): """ Retrieves the assertions parameters for this ml task - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsParams` + :rtype: :class:`DSSMLAssertionsParams` """ return DSSMLAssertionsParams(self.mltask_settings["assertionsParams"]) @@ -670,10 +670,10 @@ def get_raw(self): def get_assertion(self, assertion_name): """ - Gets a :class:`dataikuapi.dss.ml.DSSMLAssertionParams` representing the parameters of the assertion with the + Gets a :class:`DSSMLAssertionParams` representing the parameters of the assertion with the provided name (or None if no assertion has that name) :param str assertion_name: Name of the assertion - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionParams` or None if no assertion has that name + :rtype: :class:`DSSMLAssertionParams` or None if no assertion has that name """ for assertion_dict in self._internal_dict["assertions"]: if assertion_dict["name"] == assertion_name: @@ -683,7 +683,7 @@ def get_assertion(self, assertion_name): def add_assertion(self, assertion_params): """ Adds parameters of an assertion to the assertions parameters of the ml task. - :param object assertion_params: A :class:`~dataikuapi.dss.utils.DSSMLAssertionParams` representing parameters of the assertion + :param object assertion_params: A :class:`DSSMLAssertionParams` representing parameters of the assertion """ if not isinstance(assertion_params, DSSMLAssertionParams): raise ValueError('Wrong type for assertion parameters: {}'.format(type(assertion_params))) @@ -723,9 +723,9 @@ def from_params(name, a_filter, condition): :param str name: Name of the assertion :param object a_filter: A dict representing the filter to select assertion population - :param object condition: A :class:`~dataikuapi.dss.ml.DSSMLAssertionCondition` for the assertion to be successful + :param object condition: A :class:`DSSMLAssertionCondition` for the assertion to be successful - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionParams` + :rtype: :class:`DSSMLAssertionParams` """ assertion_params = DSSMLAssertionParams({}) assertion_params.name = name @@ -768,7 +768,7 @@ def filter(self, filter): def condition(self): """ Returns the assertion condition - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionCondition` + :rtype: :class:`DSSMLAssertionCondition` """ return DSSMLAssertionCondition(self._internal_dict["assertionCondition"]) @@ -802,7 +802,7 @@ def from_expected_class(expected_valid_ratio, expected_class): :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted as expected_class is attained :param str expected_class: Assertion passes if the ratio of rows predicted as expected_class is attained - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionCondition` + :rtype: :class:`DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio @@ -820,7 +820,7 @@ def from_expected_range(expected_valid_ratio, expected_min, expected_max): :param float expected_min: Min value of the expected range :param float expected_max: Max value of the expected range - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionCondition` + :rtype: :class:`DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio @@ -920,7 +920,7 @@ def get_metric(self, assertion_name): :param str assertion_name: Name of the assertion :returns: an object representing assertion metrics or None if no assertion with that name exists - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionMetric` + :rtype: :class:`DSSMLAssertionMetric` """ for assertion_metric_dict in self._internal_dict["perAssertion"]: if assertion_name == assertion_metric_dict["name"]: @@ -1159,7 +1159,7 @@ def get_assertions_metrics(self): """ Retrieves assertions metrics computed for this trained model :returns: an object representing assertion metrics - :rtype: :class:`dataikuapi.dss.ml.DSSMLAssertionsMetrics` + :rtype: :class:`DSSMLAssertionsMetrics` """ return DSSMLAssertionsMetrics(self.snippet["assertionsMetrics"]) From 5c79555198a4be453ffcbd37f9d3760a2ae46546 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Mon, 14 Dec 2020 15:00:57 +0100 Subject: [PATCH 29/43] Unify method references --- dataikuapi/dss/ml.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 0c82eb29..c3551e6d 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -706,7 +706,7 @@ def delete_assertion(self, assertion_name): class DSSMLAssertionParams(object): """ Object that represents parameters for one assertion - Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionsParams.get_assertion(assertion_name)` or + Do not create this object directly, use :meth:`DSSMLAssertionsParams.get_assertion(assertion_name)` or `from_parts(name, a_filter, condition)` instead """ def __init__(self, data): @@ -782,8 +782,8 @@ def condition(self, condition): class DSSMLAssertionCondition(object): """ Object that represents an assertion condition - Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionParams.condition`, :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_class(expected_valid_ratio, expected_class)` - or :meth:`dataikuapi.dss.ml.DSSMLAssertionCondition.from_expected_range(expected_valid_ratio, expected_min, expected_max)` instead + Do not create this object directly, use :meth:`DSSMLAssertionParams.condition`, :meth:`DSSMLAssertionCondition.from_expected_class(expected_valid_ratio, expected_class)` + or :meth:`DSSMLAssertionCondition.from_expected_range(expected_valid_ratio, expected_min, expected_max)` instead """ def __init__(self, data): self._internal_dict = data @@ -900,7 +900,7 @@ def expected_max(self, expected_max): class DSSMLAssertionsMetrics(object): """ Object that represents the per assertion metrics for all assertions on a trained model - Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSTrainedPredictionModelDetails.get_assertions_metrics()` instead + Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_assertions_metrics()` instead """ def __init__(self, data): self._internal_dict = data @@ -939,7 +939,7 @@ def passing_assertions_ratio(self): class DSSMLAssertionMetric(object): """ Object that represents the result of an assertion on a trained model - Do not create this object directly, use :meth:`dataikuapi.dss.ml.DSSMLAssertionMetrics.get_metric(self, assertion_name)` instead + Do not create this object directly, use :meth:`DSSMLAssertionMetrics.get_metric(self, assertion_name)` instead """ def __init__(self, data): self._internal_dict = data From a04634f57d717e741881569b2c29552395381f0e Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 15:53:47 +0100 Subject: [PATCH 30/43] Add blank lines before :param --- dataikuapi/dss/ml.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index c3551e6d..8567e85b 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -672,6 +672,7 @@ def get_assertion(self, assertion_name): """ Gets a :class:`DSSMLAssertionParams` representing the parameters of the assertion with the provided name (or None if no assertion has that name) + :param str assertion_name: Name of the assertion :rtype: :class:`DSSMLAssertionParams` or None if no assertion has that name """ @@ -683,6 +684,7 @@ def get_assertion(self, assertion_name): def add_assertion(self, assertion_params): """ Adds parameters of an assertion to the assertions parameters of the ml task. + :param object assertion_params: A :class:`DSSMLAssertionParams` representing parameters of the assertion """ if not isinstance(assertion_params, DSSMLAssertionParams): @@ -694,6 +696,7 @@ def delete_assertion(self, assertion_name): """ Deletes the assertion parameters of the assertion with the provided name from the assertions parameters of the ml task. Raises a ValueError if no assertion with the provided name was found + :param str assertion_name: Name of the assertion """ for idx, assertion_dict in enumerate(self._internal_dict["assertions"]): From cc2b382dfdaf04dc3f4eb0167c7161d1a90a06a5 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 15:59:55 +0100 Subject: [PATCH 31/43] Add blank lines before :rtype when no other keyword --- dataikuapi/dss/ml.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 8567e85b..94aaa965 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -628,6 +628,7 @@ def get_raw(self): def get_type(self): """ Returns the base Diagnostic type + :rtype: str """ return self._internal_dict["type"] @@ -635,6 +636,7 @@ def get_type(self): def get_type_pretty(self): """ Returns the Diagnostic type as displayed in the UI + :rtype: str """ return self._internal_dict["displayableType"] @@ -642,6 +644,7 @@ def get_type_pretty(self): def get_message(self): """ Returns the message as displayed in the UI + :rtype: str """ return self._internal_dict["message"] @@ -664,6 +667,7 @@ def __init__(self, data): def get_raw(self): """ Gets the raw dictionary of the assertions parameters + :rtype: dict """ return self._internal_dict @@ -739,6 +743,7 @@ def from_params(name, a_filter, condition): def get_raw(self): """ Gets the raw dictionary of the assertion parameters + :rtype: dict """ return self._internal_dict @@ -747,6 +752,7 @@ def get_raw(self): def name(self): """ Returns the assertion name + :rtype: str """ return self._internal_dict["name"] @@ -759,6 +765,7 @@ def name(self, name): def filter(self): """ Returns the assertion filter + :rtype: dict """ return self._internal_dict["filter"] @@ -771,6 +778,7 @@ def filter(self, filter): def condition(self): """ Returns the assertion condition + :rtype: :class:`DSSMLAssertionCondition` """ return DSSMLAssertionCondition(self._internal_dict["assertionCondition"]) @@ -834,6 +842,7 @@ def from_expected_range(expected_valid_ratio, expected_min, expected_max): def get_raw(self): """ Gets the raw dictionary of the condition + :rtype: dict """ return self._internal_dict @@ -843,6 +852,7 @@ def expected_class(self): """ Returns the expected class or None if it is not defined. Assertion passes if the expected_valid_ratio of rows predicted as expected_class is attained. + :rtype: str """ if "expectedClass" in self._internal_dict: @@ -859,6 +869,7 @@ def expected_valid_ratio(self): """ Returns the ratio of valid rows to exceed for the assertion to pass. A row is considered valid if the prediction is equal to the expected class for classification or in the expected range for regression. + :rtype: str """ return self._internal_dict["expectedValidRatio"] @@ -872,6 +883,7 @@ def expected_min(self): """ Returns the min (included) of the expected range or None if it is not defined. Assertion passes if the expected_valid_ratio of rows predicted between expected_min and expected_max (included) is attained. + :rtype: float """ if "expectedMinValue" in self._internal_dict: @@ -888,6 +900,7 @@ def expected_max(self): """ Returns the max (included) of the expected range or None if it is not defined. Assertion passes if the ratio of rows predicted between expected_min and expected_max (included) is attained. + :rtype: float """ if "expectedMaxValue" in self._internal_dict: @@ -911,6 +924,7 @@ def __init__(self, data): def get_raw(self): """ Gets the raw dictionary of the assertions metrics + :rtype: dict """ return self._internal_dict @@ -934,6 +948,7 @@ def get_metric(self, assertion_name): def passing_assertions_ratio(self): """ Returns the ratio of passing assertions + :rtype: float """ return self._internal_dict['passingAssertionsRatio'] @@ -954,6 +969,7 @@ def __repr__(self): def get_raw(self): """ Gets the raw dictionary of metrics of one assertion + :rtype: dict """ return self._internal_dict @@ -962,6 +978,7 @@ def get_raw(self): def name(self): """ Returns the assertion name + :rtype: str """ return self._internal_dict["name"] @@ -970,6 +987,7 @@ def name(self): def result(self): """ Returns whether the assertion passes + :rtype: bool """ return self._internal_dict["result"] @@ -979,6 +997,7 @@ def valid_ratio(self): """ Returns the ratio of rows in the assertion population with prediction equals to the expected class for classification or in the expected range for regression + :rtype: float """ return self._internal_dict["validRatio"] @@ -987,6 +1006,7 @@ def valid_ratio(self): def nb_matching_rows(self): """ Returns the number of rows matching filter + :rtype: int """ return self._internal_dict["nbMatchingRows"] @@ -995,6 +1015,7 @@ def nb_matching_rows(self): def nb_dropped_rows(self): """ Returns the number of rows dropped by the model's preprocessing + :rtype: int """ return self._internal_dict["nbDroppedRows"] @@ -1161,6 +1182,7 @@ def get_performance_metrics(self): def get_assertions_metrics(self): """ Retrieves assertions metrics computed for this trained model + :returns: an object representing assertion metrics :rtype: :class:`DSSMLAssertionsMetrics` """ @@ -1858,6 +1880,7 @@ def get_facts(self): def get_performance_metrics(self): """ Returns all performance metrics for this clustering model. + :returns: a dict of performance metrics values :rtype: dict """ @@ -1890,6 +1913,7 @@ def get_modeling_settings(self): def get_actual_modeling_params(self): """ Gets the actual / resolved parameters that were used to train this model. + :return: A dictionary, which contains at least a "resolved" key :rtype: dict """ From 0599c6040eee23f9b61a196639ebe9b41a5d74a6 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 16:39:03 +0100 Subject: [PATCH 32/43] Remove breaklines in reprs --- dataikuapi/dss/ml.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 94aaa965..573b7957 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -5,7 +5,6 @@ from ..utils import DataikuStreamedHttpUTF8CSVReader import json, warnings import time -import pprint from .metrics import ComputedMetrics from .utils import DSSDatasetSelectionBuilder, DSSFilterBuilder from .future import DSSFuture @@ -720,8 +719,8 @@ def __init__(self, data): self._internal_dict = data def __repr__(self): - return u"DSSMLAssertionParams(\n name= '{name}',\n condition= {condition},\n filter= {filter}\n)".format( - name=self.name, filter=pprint.pformat(self.filter).replace('\n', '\n '), condition=self.condition) + return u"DSSMLAssertionParams(name= '{name}', condition= {condition}, filter= {filter})".format( + name=self.name, filter=self.filter, condition=self.condition) @staticmethod def from_params(name, a_filter, condition): @@ -963,8 +962,8 @@ def __init__(self, data): self._internal_dict = data def __repr__(self): - return u"DSSMLAssertionParams(\n name='{}',\n result={},\n valid_ratio={},\n nb_matching_rows={}," \ - u"\n nb_dropped_rows={}\n)".format(self.name, self.result, self.valid_ratio, self.nb_matching_rows, + return u"DSSMLAssertionParams(name='{}', result={}, valid_ratio={}, nb_matching_rows={}," \ + u" nb_dropped_rows={})".format(self.name, self.result, self.valid_ratio, self.nb_matching_rows, self.nb_dropped_rows) def get_raw(self): """ From c8f36791f1f2f415fbaa524dd93bd1bcb3ca8c7f Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 16:43:36 +0100 Subject: [PATCH 33/43] Change metric to metrics --- dataikuapi/dss/ml.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 573b7957..a3061929 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -928,19 +928,19 @@ def get_raw(self): """ return self._internal_dict - def get_metric(self, assertion_name): + def get_metrics(self, assertion_name): """ - Retrieves the metric computed for this trained model for the assertion with the provided name (or None if no + Retrieves the metrics computed for this trained model for the assertion with the provided name (or None if no assertion with that name exists) :param str assertion_name: Name of the assertion :returns: an object representing assertion metrics or None if no assertion with that name exists - :rtype: :class:`DSSMLAssertionMetric` + :rtype: :class:`DSSMLAssertionMetrics` """ - for assertion_metric_dict in self._internal_dict["perAssertion"]: - if assertion_name == assertion_metric_dict["name"]: - return DSSMLAssertionMetric(assertion_metric_dict) + for assertion_metrics_dict in self._internal_dict["perAssertion"]: + if assertion_name == assertion_metrics_dict["name"]: + return DSSMLAssertionMetrics(assertion_metrics_dict) return None @property @@ -953,10 +953,10 @@ def passing_assertions_ratio(self): return self._internal_dict['passingAssertionsRatio'] -class DSSMLAssertionMetric(object): +class DSSMLAssertionMetrics(object): """ Object that represents the result of an assertion on a trained model - Do not create this object directly, use :meth:`DSSMLAssertionMetrics.get_metric(self, assertion_name)` instead + Do not create this object directly, use :meth:`DSSMLAssertionMetrics.get_metrics(self, assertion_name)` instead """ def __init__(self, data): self._internal_dict = data From e020979a70c21c5da58578d1095364ffac1135c0 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 16:47:21 +0100 Subject: [PATCH 34/43] Replace access to internal dict by get raw --- dataikuapi/dss/ml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index a3061929..289c2169 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -693,7 +693,7 @@ def add_assertion(self, assertion_params): if not isinstance(assertion_params, DSSMLAssertionParams): raise ValueError('Wrong type for assertion parameters: {}'.format(type(assertion_params))) - self._internal_dict["assertions"].append(assertion_params._internal_dict) + self._internal_dict["assertions"].append(assertion_params.get_raw()) def delete_assertion(self, assertion_name): """ @@ -786,7 +786,7 @@ def condition(self): def condition(self, condition): if not isinstance(condition, DSSMLAssertionCondition): raise ValueError('Wrong type for assertion condition: {}'.format(type(condition))) - self._internal_dict["assertionCondition"] = condition._internal_dict + self._internal_dict["assertionCondition"] = condition.get_raw() class DSSMLAssertionCondition(object): From 8317ada0245042d2037e0b953b5f317f28e91e47 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 17:03:53 +0100 Subject: [PATCH 35/43] Add two missing reprs and fix one --- dataikuapi/dss/ml.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 289c2169..4b614ea7 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -663,6 +663,9 @@ class DSSMLAssertionsParams(object): def __init__(self, data): self._internal_dict = data + def __repr__(self): + return u"{}({})".format(self.__class__.__name__, self.get_raw()) + def get_raw(self): """ Gets the raw dictionary of the assertions parameters @@ -920,6 +923,9 @@ class DSSMLAssertionsMetrics(object): def __init__(self, data): self._internal_dict = data + def __repr__(self): + return u"{}({})".format(self.__class__.__name__, self.get_raw()) + def get_raw(self): """ Gets the raw dictionary of the assertions metrics @@ -962,9 +968,10 @@ def __init__(self, data): self._internal_dict = data def __repr__(self): - return u"DSSMLAssertionParams(name='{}', result={}, valid_ratio={}, nb_matching_rows={}," \ - u" nb_dropped_rows={})".format(self.name, self.result, self.valid_ratio, self.nb_matching_rows, - self.nb_dropped_rows) + return u"{}(name='{}', result={}, valid_ratio={}, nb_matching_rows={}," \ + u" nb_dropped_rows={})".format(self.__class__.__name__, self.name, self.result, self.valid_ratio, + self.nb_matching_rows, self.nb_dropped_rows) + def get_raw(self): """ Gets the raw dictionary of metrics of one assertion From 591d6aa878f238a7291b97ac05613ae325b7cb66 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 17:05:42 +0100 Subject: [PATCH 36/43] Add comment to help build condition filter --- dataikuapi/dss/ml.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 4b614ea7..dbcc841c 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -731,7 +731,8 @@ def from_params(name, a_filter, condition): Creates assertion parameters from name, filter and condition :param str name: Name of the assertion - :param object a_filter: A dict representing the filter to select assertion population + :param dict a_filter: A dict representing the filter to select assertion population. You can use + a :class:`~dataikuapi.dss.utils.DSSFilterBuilder` to build the settings of the filter :param object condition: A :class:`DSSMLAssertionCondition` for the assertion to be successful :rtype: :class:`DSSMLAssertionParams` From b10aa6fa2cdec22876340ec65e0452d17a277d6a Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 17:08:42 +0100 Subject: [PATCH 37/43] Fix typos --- dataikuapi/dss/ml.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index dbcc841c..18bca272 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -774,8 +774,8 @@ def filter(self): return self._internal_dict["filter"] @filter.setter - def filter(self, filter): - self._internal_dict["filter"] = filter + def filter(self, a_filter): + self._internal_dict["filter"] = a_filter @property def condition(self): @@ -902,7 +902,7 @@ def expected_min(self, expected_min): def expected_max(self): """ Returns the max (included) of the expected range or None if it is not defined. - Assertion passes if the ratio of rows predicted between expected_min and expected_max (included) is attained. + Assertion passes if the expected_valid_ratio of rows predicted between expected_min and expected_max (included) is attained. :rtype: float """ @@ -1021,7 +1021,7 @@ def nb_matching_rows(self): @property def nb_dropped_rows(self): """ - Returns the number of rows dropped by the model's preprocessing + Returns the number of rows matching filter and dropped by the model's preprocessing :rtype: int """ From 75a755e6a61086ec0b38b933eb18d0e05715cef3 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 17:22:17 +0100 Subject: [PATCH 38/43] Add get_assertions_names --- dataikuapi/dss/ml.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 18bca272..823458da 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -687,6 +687,15 @@ def get_assertion(self, assertion_name): return DSSMLAssertionParams(assertion_dict) return None + def get_assertions_names(self): + """ + Gets the list of all assertion's names + + :return: A list of all assertion's names + :rtype: list + """ + return [assertion_dict["name"] for assertion_dict in self._internal_dict["assertions"]] + def add_assertion(self, assertion_params): """ Adds parameters of an assertion to the assertions parameters of the ml task. From 339cbc93aec871bd6c5d15b4efd1a47f0e231d56 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 17:29:56 +0100 Subject: [PATCH 39/43] Fix typo --- dataikuapi/dss/ml.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 823458da..e3fc593b 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -689,9 +689,9 @@ def get_assertion(self, assertion_name): def get_assertions_names(self): """ - Gets the list of all assertion's names + Gets the list of all assertions' names - :return: A list of all assertion's names + :return: A list of all assertions' names :rtype: list """ return [assertion_dict["name"] for assertion_dict in self._internal_dict["assertions"]] @@ -725,7 +725,7 @@ class DSSMLAssertionParams(object): """ Object that represents parameters for one assertion Do not create this object directly, use :meth:`DSSMLAssertionsParams.get_assertion(assertion_name)` or - `from_parts(name, a_filter, condition)` instead + `from_params(name, a_filter, condition)` instead """ def __init__(self, data): self._internal_dict = data From c655c56bb70df79e75ad028533bb05beef3f11fe Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 17:34:40 +0100 Subject: [PATCH 40/43] Unify reprs --- dataikuapi/dss/ml.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index e3fc593b..35ef2b4d 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -731,8 +731,7 @@ def __init__(self, data): self._internal_dict = data def __repr__(self): - return u"DSSMLAssertionParams(name= '{name}', condition= {condition}, filter= {filter})".format( - name=self.name, filter=self.filter, condition=self.condition) + return u"{}({})".format(self.__class__.__name__, self.get_raw()) @staticmethod def from_params(name, a_filter, condition): @@ -812,9 +811,7 @@ def __init__(self, data): self._internal_dict = data def __repr__(self): - return u"DSSMLAssertionCondition(expected_valid_ratio = {}, expected_class= {}," \ - u" expected_min={}, expected_max={})".format( - self.expected_valid_ratio, self.expected_class, self.expected_min, self.expected_max) + return u"{}({})".format(self.__class__.__name__, self.get_raw()) @staticmethod From 018bed273e8c7f1075b0546de4c280f4fccaed37 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Wed, 16 Dec 2020 17:37:47 +0100 Subject: [PATCH 41/43] Wording --- dataikuapi/dss/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 35ef2b4d..93ec94dd 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -924,7 +924,7 @@ def expected_max(self, expected_max): class DSSMLAssertionsMetrics(object): """ - Object that represents the per assertion metrics for all assertions on a trained model + Object that represents the assertions metrics for all assertions on a trained model Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_assertions_metrics()` instead """ def __init__(self, data): From f2a18865936589334bc1a24d9b0d68288f75179e Mon Sep 17 00:00:00 2001 From: arnaudde Date: Fri, 18 Dec 2020 09:18:49 +0100 Subject: [PATCH 42/43] Nitpicks --- dataikuapi/dss/ml.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 93ec94dd..8c5e62eb 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -657,7 +657,7 @@ def __repr__(self): class DSSMLAssertionsParams(object): """ Object that represents parameters for all assertions of a ml task - Do not create this object directly, use :meth:`DSSPredictionMLTaskSettings.get_assertions_params()` instead + Do not create this object directly, use :meth:`DSSPredictionMLTaskSettings.get_assertions_params` instead """ def __init__(self, data): @@ -724,8 +724,8 @@ def delete_assertion(self, assertion_name): class DSSMLAssertionParams(object): """ Object that represents parameters for one assertion - Do not create this object directly, use :meth:`DSSMLAssertionsParams.get_assertion(assertion_name)` or - `from_params(name, a_filter, condition)` instead + Do not create this object directly, use :meth:`DSSMLAssertionsParams.get_assertion` or + `from_params` instead """ def __init__(self, data): self._internal_dict = data @@ -804,8 +804,8 @@ def condition(self, condition): class DSSMLAssertionCondition(object): """ Object that represents an assertion condition - Do not create this object directly, use :meth:`DSSMLAssertionParams.condition`, :meth:`DSSMLAssertionCondition.from_expected_class(expected_valid_ratio, expected_class)` - or :meth:`DSSMLAssertionCondition.from_expected_range(expected_valid_ratio, expected_min, expected_max)` instead + Do not create this object directly, use :meth:`DSSMLAssertionParams.condition`, + :meth:`DSSMLAssertionCondition.from_expected_class` or :meth:`DSSMLAssertionCondition.from_expected_range` instead """ def __init__(self, data): self._internal_dict = data @@ -813,7 +813,6 @@ def __init__(self, data): def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.get_raw()) - @staticmethod def from_expected_class(expected_valid_ratio, expected_class): """ @@ -925,7 +924,7 @@ def expected_max(self, expected_max): class DSSMLAssertionsMetrics(object): """ Object that represents the assertions metrics for all assertions on a trained model - Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_assertions_metrics()` instead + Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_assertions_metrics` instead """ def __init__(self, data): self._internal_dict = data @@ -969,7 +968,7 @@ def passing_assertions_ratio(self): class DSSMLAssertionMetrics(object): """ Object that represents the result of an assertion on a trained model - Do not create this object directly, use :meth:`DSSMLAssertionMetrics.get_metrics(self, assertion_name)` instead + Do not create this object directly, use :meth:`DSSMLAssertionMetrics.get_metrics` instead """ def __init__(self, data): self._internal_dict = data From 5eecb2280c5c0bca015c3ed14d8e8b744cefc6f2 Mon Sep 17 00:00:00 2001 From: arnaudde Date: Fri, 18 Dec 2020 09:52:25 +0100 Subject: [PATCH 43/43] Add forgotten :meth: --- dataikuapi/dss/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 8c5e62eb..e1bd128b 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -725,7 +725,7 @@ class DSSMLAssertionParams(object): """ Object that represents parameters for one assertion Do not create this object directly, use :meth:`DSSMLAssertionsParams.get_assertion` or - `from_params` instead + :meth:`from_params` instead """ def __init__(self, data): self._internal_dict = data