diff --git a/docs/reference/util.md b/docs/reference/util.md new file mode 100644 index 00000000..9f801d05 --- /dev/null +++ b/docs/reference/util.md @@ -0,0 +1,68 @@ +## Utilities + +### Expression to Configspace + +In some cases we may have (highly) complex conditions or forbidden expressions that are already denoted as a regular expression. In that case, `ConfigSpace` can automatically convert them into a `ConfigSpace` expression using the [`parse_expression_from_string`][ConfigSpace.util.parse_expression_from_string]`parse_expression_from_string`. This function interprets the expression using the Python `Abstract Syntax Tree` parser and recursively converts it into the appropriate structure. + +!!! note + The converted expression is not added to ConfigSpace, only returned to the user. + +!!! note + If the expression contains illegal values, errors, or requires functionalities not available in `ConfigSpace`, appriopriate exceptions will be raised. + +!!! note + Expressions differentiate variables (Hyperparameter names) from constants (Categorical values) based on quotation marks; "a != b" implies hyperparameter a does not equal hyperparameter b, "a != 'b'" implies hyperparameter a does not equal categorical/ordinal value b. + +#### Adding a condition + +In this code example we show how you can add a hyperparameter condition to ConfigSpace from a string. Note that the conditional hyperparameter is specified as a seperate argument and is not part of the expression string! + +```python exec="True" result="python" source="tabbed-left" +from ConfigSpace import ConfigurationSpace +from ConfigSpace.util import parse_expression_from_string + +cs = ConfigurationSpace( + { + "a": (0, 10), # Integer from 0 to 10 + "b": ["cat", "dog"], # Categorical with choices "cat" and "dog" + "c": (0.0, 1.0), # Float from 0.0 to 1.0 + } +) +print(cs) + +# Now we add a condition and forbidden using regular expressions +condition = "b != 'cat' && c > 0.001" +condition = parse_expression_from_string(condition, cs, conditional_hyperparameter=cs["a"]) # We have to specify the conditional HP seperately here as the final argument + +print(condition) + +cs.add(condition) + +print(cs) +``` + +#### Adding a forbidden expression + +In this example we add a forbidden expression to ConfigSpace from string. Note that the conditional hyperparameter remains unspecified; this leads to ConfigSpace interpreting the expression as a forbidden expression. + +```python exec="True" result="python" source="tabbed-left" +from ConfigSpace import ConfigurationSpace +from ConfigSpace.util import parse_expression_from_string + +cs = ConfigurationSpace( + { + "a": (0, 10), # Integer from 0 to 10 + "b": ["cat", "dog"], # Categorical with choices "cat" and "dog" + "c": (0.0, 1.0), # Float from 0.0 to 1.0 + } +) +print(cs) +forbidden = "a > 5 && c >= 0.94" +forbidden = parse_expression_from_string(forbidden, cs) + +print(forbidden) + +cs.add(forbidden) + +print(cs) +``` \ No newline at end of file diff --git a/docs/reference/utils.md b/docs/reference/utils.md deleted file mode 100644 index e69de29b..00000000 diff --git a/mkdocs.yml b/mkdocs.yml index b6b453ed..d30c4f23 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -173,5 +173,5 @@ nav: - Conditions: "reference/conditions.md" - Forbidden Clauses: "reference/forbiddens.md" - Serialization: "reference/serialization.md" - - Util: "reference/utils.md" + - Util: "reference/util.md" - API: "api/" diff --git a/src/ConfigSpace/util.py b/src/ConfigSpace/util.py index 42008059..17576ecf 100644 --- a/src/ConfigSpace/util.py +++ b/src/ConfigSpace/util.py @@ -27,10 +27,12 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import annotations +import ast +import re import copy from collections import deque from collections.abc import Iterator, Sequence -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, cast, Iterable import numpy as np @@ -50,6 +52,34 @@ UniformFloatHyperparameter, UniformIntegerHyperparameter, ) + +from ConfigSpace.conditions import ( + Condition, + AndConjunction, + OrConjunction, + EqualsCondition, + GreaterThanCondition, + LessThanCondition, + NotEqualsCondition, + InCondition, +) +from ConfigSpace.forbidden import ( + ForbiddenClause, + ForbiddenAndConjunction, + ForbiddenOrConjunction, + ForbiddenEqualsClause, + ForbiddenGreaterThanClause, + ForbiddenGreaterThanEqualsClause, + ForbiddenInClause, + ForbiddenLessThanClause, + ForbiddenLessThanEqualsClause, + ForbiddenGreaterThanRelation, + ForbiddenLessThanRelation, + ForbiddenEqualsRelation, + ForbiddenGreaterThanEqualsRelation, + ForbiddenLessThanEqualsRelation, +) + from ConfigSpace.types import NotSet if TYPE_CHECKING: @@ -828,3 +858,228 @@ def _get_cartesian_product( unchecked_grid_pts.popleft() return checked_grid_pts + + +def parse_expression_from_string( + expression: str, + configspace: ConfigurationSpace, + conditional_hyperparameter: Hyperparameter | None = None, +) -> Condition | ForbiddenClause: + """Convert a logic expression to ConfigSpace expression. + + Given a logic expression, this function will return a ConfigSpace expression + that is equivalent to the logic expression. If a conditional parameter is provided, + will create a condition, otherwise a forbidden expression. + + The created expression is **NOT** automatically added to the configuration space. + + Example Condition expression parsing: + + ```python exec="true", source="material-block" result="python" + from ConfigSpace import ConfigurationSpace + from ConfigSpace.util import parse_expression_from_string + + cs = ConfigurationSpace({ "a": (0, 10), "b": (1.0, 8.0) }) + condition = parse_expression_from_string("a < 5", cs, conditional_hyperparameter=cs['b']) + print(condition) + ``` + + Example Forbidden Expression Parsing: + + ```python exec="true", source="material-block" result="python" + from ConfigSpace import ConfigurationSpace + from ConfigSpace.util import parse_expression_from_string + + cs = ConfigurationSpace({ "a": (0, 10), "b": (1.0, 8.0) }) + forbidden = parse_expression_from_string("a >= 5", cs) + print(forbidden) + ``` + + Args: + expression: The expression to convert. + configspace: The ConfigSpace to use. + conditional_hyperparameter: For conditions, will parse the expression as a condition + underwhich the provided hyperparameter will be active. + + Returns: + A ConfigSpace Condition or ForbiddenClause. + """ + # Format expression to match the ast module + # Format logical operators: + expression = re.sub(r" & ", " and ", expression) + expression = re.sub(r" && ", " and ", expression) + expression = re.sub(r" \| ", " or ", expression) + expression = re.sub(r" \|\| ", " or ", expression) + # Format (in)equality operators: + expression = re.sub(r" !== ", " != ", expression) + expression = re.sub(r" (?!=])=(? Condition | ForbiddenClause: + """Recursively parse the abstract syntax tree to a ConfigSpace expression. + + Should not be called directly, but rather through `parse_expression_from_string`. + + Args: + item: The item to parse. + configspace: The ConfigSpace to use. + conditional_hyperparameter: For conditions, will parse the expression as a condition + underwhich the hyperparameter will be active. + + Returns: + A ConfigSpace Condition or ForbiddenClause + """ + if isinstance(item, list): + if len(item) > 1: + raise ValueError(f"Can not parse list of elements: {item}.") + item = item[0] + if isinstance(item, ast.Expr): + return _recursive_conversion(item.value, configspace, conditional_hyperparameter) + if isinstance(item, ast.Name): # Convert to hyperparameter + hp = configspace.get(item.id) + if hp is None: + raise ValueError(f"Unknown hyperparameter: {item.id}") + return hp + if isinstance(item, ast.Constant): # ast.Constant are differentiated from ast.Name by integers/floats and quoted strings + return item.value + if ( + isinstance(item, ast.Tuple) + or isinstance(item, ast.Set) + or isinstance(item, ast.List) + ): + values = [] + for v in item.elts: + if isinstance(v, ast.Constant): + values.append(v.value) + elif isinstance(v, ast.Name): # Check if its a parameter + if configspace.get(v.id) is not None: + raise ValueError( + f"Only constants allowed in tuples. Found: {item.elts}" + ) + values.append(v.id) # String value was interpreted as parameter + return values + if isinstance(item, ast.BinOp): + raise NotImplementedError("Binary operations not supported by ConfigSpace.") + if isinstance(item, ast.BoolOp): + values = [ + _recursive_conversion(v, configspace, conditional_hyperparameter) for v in item.values + ] + if isinstance(item.op, ast.Or): + if conditional_hyperparameter: + return OrConjunction(*values) + return ForbiddenOrConjunction(*values) + elif isinstance(item.op, ast.And): + if conditional_hyperparameter: + return AndConjunction(*values) + return ForbiddenAndConjunction(*values) + else: + raise ValueError(f"Unknown boolean operator: {item.op}") + if isinstance(item, ast.Compare): + if len(item.ops) > 1: + raise ValueError(f"Only single comparisons allowed. Found: {item.ops}") + left = _recursive_conversion(item.left, configspace, conditional_hyperparameter) + right = _recursive_conversion(item.comparators, configspace, conditional_hyperparameter) + operator = item.ops[0] + + # CoPilot: Ensure that if there is exactly one Hyperparameter involved in the comparison, it is always on the left-hand side. This is required + # because the downstream Condition/Forbidden* constructors expect the hyperparameter to be passed as the "left" argument. + if isinstance(right, Hyperparameter) and not isinstance(left, Hyperparameter): + # Normalize expressions like "5 < hp" into "hp > 5" by swapping sides and inverting asymmetric operators. + left, right = right, left + if isinstance(operator, ast.Lt): + operator = ast.Gt() + elif isinstance(operator, ast.LtE): + operator = ast.GtE() + elif isinstance(operator, ast.Gt): + operator = ast.Lt() + elif isinstance(operator, ast.GtE): + operator = ast.LtE() + elif isinstance(operator, ast.In): + # Having a Hyperparameter only on the right-hand side of an "in" comparison (e.g. "[1, 2] in hp") is not supported. + raise ValueError( + "Invalid comparison: 'in' operator requires a hyperparameter " + "on the left-hand side." + ) + elif not isinstance(operator, (ast.Eq, ast.NotEq)): # Equality and inequality are symmetric; no operator change + # For any other unsupported operator shapes, fail. + raise ValueError( + f"Unsupported comparison between constant and hyperparameter: {ast.unparse(item)}" + ) + + if isinstance(left, Hyperparameter): # Convert to HP type + if isinstance(right, Iterable) and not isinstance(right, str): + right = [type(left.default_value)(v) for v in right] + if len(right) == 1 and not isinstance(operator, ast.In): + right = right[0] + elif isinstance(right, int): + right = type(left.default_value)(right) + elif not isinstance(right, Hyperparameter): + raise ValueError( + "Only hyperparameter comparisons allowed. Neither side is recognised as a hyperparameter in: " + f"{ast.unparse(item)}" + ) + + is_relation = isinstance(left, Hyperparameter) and isinstance(right, Hyperparameter) + if is_relation and conditional_hyperparameter: + raise ValueError("Hyperparameter relations not supported for conditions.") + + if isinstance(operator, ast.Lt): + if conditional_hyperparameter: + return LessThanCondition(conditional_hyperparameter, left, right) + if is_relation: + return ForbiddenLessThanRelation(left=left, right=right) + return ForbiddenLessThanClause(hyperparameter=left, value=right) + if isinstance(operator, ast.LtE): + if conditional_hyperparameter: + raise ValueError("LessThanEquals not supported for conditions.") + if is_relation: + return ForbiddenLessThanEqualsRelation(left=left, right=right) + return ForbiddenLessThanEqualsClause(hyperparameter=left, value=right) + if isinstance(operator, ast.Gt): + if conditional_hyperparameter: + return GreaterThanCondition(conditional_hyperparameter, left, right) + if is_relation: + return ForbiddenGreaterThanRelation(left=left, right=right) + return ForbiddenGreaterThanClause(hyperparameter=left, value=right) + if isinstance(operator, ast.GtE): + if conditional_hyperparameter: + raise ValueError("GreaterThanEquals not supported for conditions.") + if is_relation: + return ForbiddenGreaterThanEqualsRelation(left=left, right=right) + return ForbiddenGreaterThanEqualsClause(hyperparameter=left, value=right) + if isinstance(operator, ast.Eq): + if conditional_hyperparameter: + return EqualsCondition(conditional_hyperparameter, left, right) + if is_relation: + return ForbiddenEqualsRelation(left=left, right=right) + return ForbiddenEqualsClause(hyperparameter=left, value=right) + if isinstance(operator, ast.In): + if is_relation: + raise ValueError("In operator not supported for hyperparameter relations.") + if conditional_hyperparameter: + return InCondition(conditional_hyperparameter, left, right) + return ForbiddenInClause(hyperparameter=left, values=right) + if isinstance(operator, ast.NotEq): + if conditional_hyperparameter: + return NotEqualsCondition(conditional_hyperparameter, left, right) + raise ValueError("NotEq operator not supported for ForbiddenClauses.") + # The following classes do not (yet?) exist in configspace + if isinstance(operator, ast.NotIn): + raise ValueError("NotIn operator not supported for ForbiddenClauses.") + if isinstance(operator, ast.Is): + raise NotImplementedError("Is operator not supported.") + if isinstance(operator, ast.IsNot): + raise NotImplementedError("IsNot operator not supported.") + raise ValueError(f"Unsupported type: {item}") diff --git a/test/test_util.py b/test/test_util.py index f6e7f8bb..bf611239 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -28,6 +28,7 @@ from __future__ import annotations import os +import re import numpy as np import pytest @@ -43,8 +44,16 @@ EqualsCondition, ForbiddenAndConjunction, ForbiddenEqualsClause, + ForbiddenEqualsRelation, + ForbiddenGreaterThanClause, + ForbiddenGreaterThanEqualsClause, + ForbiddenGreaterThanRelation, + ForbiddenInClause, + ForbiddenLessThanClause, + ForbiddenOrConjunction, GreaterThanCondition, LessThanCondition, + NotEqualsCondition, OrConjunction, OrdinalHyperparameter, UniformFloatHyperparameter, @@ -64,6 +73,7 @@ get_one_exchange_neighbourhood, get_random_neighbor, impute_inactive_values, + parse_expression_from_string, ) @@ -658,3 +668,196 @@ def test_generate_grid(): assert dict(generated_grid[1]) == {"cat1": "F", "ord1": "2"} assert dict(generated_grid[2]) == {"cat1": "T", "ord1": "1", "int1": 0} assert dict(generated_grid[-1]) == {"cat1": "T", "ord1": "3", "int1": 1000} + + +def test_parse_expression_from_string_forbidden(): + cs = ConfigurationSpace( + { + "a": (0, 10), + "b": (0, 10), + "c": (0, 10), + "d": (0, 10), + "e": (0, 10), + "cat1": ["cat", "dog"], + "cat2": ["sun", "rain", "snow", "fog"], + "dog": ["small", "medium", "large", "cat", "dog"], + "float1": (0.0, 1.0), + "float2": (0.0, 1.0), + }, + ) + + wrong_expression = "a >!> b" + with pytest.raises(ValueError, match="Could not parse expression: 'a >!> b'"): + parse_expression_from_string(wrong_expression, cs) + + wrong_hp_name_expresion = "q <= 5" + with pytest.raises( + ValueError, + match="Unknown hyperparameter: q", + ): + cs_expression = parse_expression_from_string(wrong_hp_name_expresion, cs) + + wrong_value_expresion = "'q' <= 5" + with pytest.raises( + ValueError, + match="Only hyperparameter comparisons allowed. Neither side is recognised as a hyperparameter in: 'q' <= 5", + ): + cs_expression = parse_expression_from_string(wrong_value_expresion, cs) + + wrong_hp_value_expression = "a > 11" + with pytest.raises( + ValueError, + match=re.escape( + "Forbidden clause must be instantiated with a legal hyperparameter value for 'a, Type: UniformInteger, Range: [0, 10], Default: 5', but got '11'", + ), + ): + cs_expression = parse_expression_from_string(wrong_hp_value_expression, cs) + + wrong_hp_value_expression = "a == 'cat'" + with pytest.raises( + ValueError, + match=re.escape( + "Forbidden clause must be instantiated with a legal hyperparameter value for 'a, Type: UniformInteger, Range: [0, 10], Default: 5', but got 'cat'", + ), + ): + cs_expression = parse_expression_from_string(wrong_hp_value_expression, cs) + + wrong_forbidden_expression = "a != 5" + with pytest.raises( + ValueError, + match="NotEq operator not supported for ForbiddenClauses.", + ): + parse_expression_from_string(wrong_forbidden_expression, cs) + + wrong_forbidden_expression = "a != b" + with pytest.raises( + ValueError, + match="NotEq operator not supported for ForbiddenClauses.", + ): + parse_expression_from_string(wrong_forbidden_expression, cs) + + # In case the epxression is incorrecty ordered for ConfigSpace, the method fixes the ordering here where possible + wrong_order_expression = "5 < a" + assert parse_expression_from_string( + wrong_order_expression, + cs, + ) == ForbiddenGreaterThanClause(cs["a"], 5) + + wrong_order_expression = "5 > a" + assert parse_expression_from_string( + wrong_order_expression, + cs, + ) == ForbiddenLessThanClause(cs["a"], 5) + + wrong_order_expression = "5 == a" + assert parse_expression_from_string( + wrong_order_expression, + cs, + ) == ForbiddenEqualsClause(cs["a"], 5) + + wrong_order_expression = "[1,2,5] in a" + with pytest.raises(ValueError): + parse_expression_from_string(wrong_order_expression, cs) + + in_operator_expression = ( + "a in [1, 2, 3]" # This operator is accepted by ConfigSpace for Integer HP + ) + cs_expression = parse_expression_from_string(in_operator_expression, cs) + assert cs_expression == ForbiddenInClause(cs["a"], [1, 2, 3]) + + simple_value_expression = "a > 9" + cs_expression = parse_expression_from_string(simple_value_expression, cs) + assert cs_expression == ForbiddenGreaterThanClause(cs["a"], 9) + + simple_expression = "a > b" + cs_expression = parse_expression_from_string(simple_expression, cs) + assert cs_expression == ForbiddenGreaterThanRelation(cs["a"], cs["b"]) + + simple_expression = "a < 5" + cs_expression = parse_expression_from_string(simple_expression, cs) + assert cs_expression == ForbiddenLessThanClause(cs["a"], 5) + + complex_expression = "a > b || (c > d && e < 5 && cat1 == 'dog' && float1 >= 0.5)" + cs_expression = parse_expression_from_string(complex_expression, cs) + assert cs_expression == ForbiddenOrConjunction( + ForbiddenGreaterThanRelation(cs["a"], cs["b"]), + ForbiddenAndConjunction( + ForbiddenGreaterThanRelation(cs["c"], cs["d"]), + ForbiddenLessThanClause(cs["e"], 5), + ForbiddenEqualsClause(cs["cat1"], "dog"), + ForbiddenGreaterThanEqualsClause(cs["float1"], 0.5), + ), + ) + + complex_expression = ( + "a >= 8 and (cat1 in ['cat', 'dog'] or cat2 in ['sun', 'rain'])" + ) + cs_expression = parse_expression_from_string(complex_expression, cs) + assert cs_expression == ForbiddenAndConjunction( + ForbiddenGreaterThanEqualsClause(cs["a"], 8), + ForbiddenOrConjunction( + ForbiddenInClause(cs["cat1"], ["cat", "dog"]), + ForbiddenInClause(cs["cat2"], ["sun", "rain"]), + ), + ) + + # Check if a hyperparameter name / categorical value mixup does not occur based on the quotation marks + semi_ambigous_expression = ( + "cat1 == 'dog'" # Here we are talking about the categorical value + ) + assert parse_expression_from_string( + semi_ambigous_expression, + cs, + ) == ForbiddenEqualsClause(cs["cat1"], "dog") + semi_ambigous_expression = ( + "cat1 == dog" # Now we are referring to the Hyperparameter called dog + ) + assert parse_expression_from_string( + semi_ambigous_expression, + cs, + ) == ForbiddenEqualsRelation(cs["cat1"], cs["dog"]) + semi_ambigous_expression = ( + "dog == 'dog'" # The hyperparameter dog cannot have value 'dog' + ) + assert parse_expression_from_string( + semi_ambigous_expression, + cs, + ) == ForbiddenEqualsClause(cs["dog"], "dog") + + wrong_semi_ambigous_expression = "dog == medium" # There is no variable called medium, only a constant; quotation marks are missing + with pytest.raises(ValueError, match="Unknown hyperparameter: medium"): + parse_expression_from_string(wrong_semi_ambigous_expression, cs) + + +def test_parse_expression_from_string_condition(): + cs = ConfigurationSpace( + { + "a": (0, 10), + "b": (0, 10), + "c": (0, 10), + "d": (0, 10), + "e": (0, 10), + }, + ) + simple_expression = "a < 5" + cs_expression = parse_expression_from_string( + simple_expression, + cs, + conditional_hyperparameter=cs["e"], + ) + assert cs_expression == LessThanCondition(cs["e"], cs["a"], 5) + + simple_expression_inequality = "a != 5" + cs_expression = parse_expression_from_string( + simple_expression_inequality, + cs, + conditional_hyperparameter=cs["e"], + ) + assert cs_expression == NotEqualsCondition(cs["e"], cs["a"], 5) + + wrong_order_expression = "5 != a" + assert parse_expression_from_string( + wrong_order_expression, + cs, + conditional_hyperparameter=cs["e"], + ) == NotEqualsCondition(cs["e"], cs["a"], 5)