Transition value JSON from old to new flat format

.gitignore

@@ -1,6 +1,7 @@
 # Compiled python modules.
 *.pyc
 /.idea/
+/.spyproject/
 /docs/build/
 /docs/source/autoapi/
 /docs/source/db_mapping_schema.rst

README_dev.md

@@ -0,0 +1,24 @@
+# Developing Data Transition
+
+## Testing `alembic` Migration
+
+1. Edit `./spinedb_api/alembic.ini`, point `sqlalchemy.url` to a (copy of a) SQLite test database. ⚠️ Its data will be altered by the migration script.
+1. Edit `./spinedb_api/alembic/versions/a973ab537da2_reencode_parameter_values.py` and temporarily change
+   ```python
+   new_value = transition_data(old_value)
+   ```
+   to
+   ```python
+   new_value = b'prepend_me ' + old_value
+   ```
+1. Within the `./spinedb_api` folder, execute
+   ```bash
+   alembic upgrade head
+   ```
+1. Open your SQLite test database in a database editor and check for changed `paramater_value`s.
+
+## Developing the Data Transition Module
+
+1. Edit `./spinedb_api/compat/data_transition.py` for development.
+1. In a Python REPL, call its function `transition_data(old_json_bytes)` and check for correct output of our test cases.
+1. Once this works, revert the changes of `./spinedb_api/alembic/versions/a973ab537da2_reencode_parameter_values.py` and test the above `alembic` migration again.

pyproject.toml

@@ -25,7 +25,8 @@ dependencies = [
     "chardet >=4.0.0",
     "PyMySQL[rsa] >=1.0.2",
     "psycopg2-binary",
-    "pyarrow >= 19.0",
+    "pyarrow >= 20.0",
+    "pydantic >= 2",
     "pandas >= 2.2.3",
 ]
 

spinedb_api/alembic/versions/8b0eff478bcb_add_active_by_default_to_entity_class.py

@@ -9,7 +9,7 @@
 from alembic import op
 import sqlalchemy as sa
 import sqlalchemy.orm
-from spinedb_api.compatibility import convert_tool_feature_method_to_active_by_default
+from spinedb_api.compat.compatibility_transformations import convert_tool_feature_method_to_active_by_default
 
 # revision identifiers, used by Alembic.
 revision = "8b0eff478bcb"

spinedb_api/alembic/versions/989fccf80441_replace_values_with_reference_to_list_.py

@@ -12,7 +12,8 @@
 from sqlalchemy.sql.expression import bindparam
 from spinedb_api.exception import SpineIntegrityError
 from spinedb_api.helpers import group_concat
-from spinedb_api.parameter_value import ParameterValueFormatError, dump_db_value, from_database
+from spinedb_api.incomplete_values import dump_db_value
+from spinedb_api.parameter_value import ParameterValueFormatError, from_database
 
 # revision identifiers, used by Alembic.
 revision = "989fccf80441"

spinedb_api/alembic/versions/a973ab537da2_reencode_parameter_values.py

@@ -0,0 +1,74 @@
+"""reencode parameter_values
+
+Revision ID: a973ab537da2
+Revises: e9f2c2330cf8
+Create Date: 2025-05-21 12:49:16.861670
+
+"""
+
+from typing import Any, Optional, SupportsFloat
+from alembic import op
+import sqlalchemy as sa
+from spinedb_api.compat.converters import parse_duration
+from spinedb_api.parameter_value import DateTime, Duration, from_dict, to_database
+from spinedb_api.value_support import load_db_value
+
+# revision identifiers, used by Alembic.
+revision = "a973ab537da2"
+down_revision = "e9f2c2330cf8"
+branch_labels = None
+depends_on = None
+
+
+TYPES_TO_REENCODE = {"duration", "date_time", "time_pattern", "time_series", "array", "map"}
+
+
+def upgrade():
+    conn = op.get_bind()
+    metadata = sa.MetaData()
+    metadata.reflect(bind=conn)
+    _upgrade_table_types(metadata.tables["parameter_definition"], "default_value", "default_type", conn)
+    _upgrade_table_types(metadata.tables["parameter_value"], "value", "type", conn)
+    _upgrade_table_types(metadata.tables["list_value"], "value", "type", conn)
+
+
+def downgrade():
+    pass
+
+
+def _upgrade_table_types(table, value_label, type_label, connection):
+    value_column = getattr(table.c, value_label)
+    type_column = getattr(table.c, type_label)
+    update_statement = (
+        table.update()
+        .where(table.c.id == sa.bindparam("id"))
+        .values(
+            {
+                "id": sa.bindparam("id"),
+                value_label: sa.bindparam(value_label),
+            }
+        )
+    )
+    batch_data = []
+    for id_, type_, old_blob in connection.execute(
+        sa.select(table.c.id, type_column, value_column).where(type_column.in_(TYPES_TO_REENCODE))
+    ):
+        legacy_value = _from_database_legacy(old_blob, type_)
+        new_blob, _ = to_database(legacy_value)
+        batch_data.append({"id": id_, value_label: new_blob})
+        if len(batch_data) == 100:
+            connection.execute(update_statement, batch_data)
+            batch_data.clear()
+    if batch_data:
+        connection.execute(update_statement, batch_data)
+
+
+def _from_database_legacy(value: bytes, type_: Optional[str]) -> Optional[Any]:
+    parsed = load_db_value(value)
+    if isinstance(parsed, dict):
+        return from_dict(parsed, type_)
+    if type_ == DateTime.TYPE:
+        return DateTime(parsed)
+    if type_ == Duration.TYPE:
+        return Duration(parse_duration(parsed))
+    raise RuntimeError(f"migration for {type_} missing")

spinedb_api/alembic/versions/ca7a13da8ff6_add_type_info_for_scalars.py

@@ -9,7 +9,7 @@
 import json
 from alembic import op
 import sqlalchemy as sa
-from spinedb_api.parameter_value import type_for_scalar
+from spinedb_api.incomplete_values import type_for_scalar
 
 # revision identifiers, used by Alembic.
 revision = "ca7a13da8ff6"
@@ -40,7 +40,10 @@ def _update_scalar_type_info(table, value_label, type_label, connection):
         parsed_value = json.loads(value)
         if parsed_value is None:
             continue
-        value_type = type_for_scalar(parsed_value)
+        if isinstance(parsed_value, dict):
+            value_type = parsed_value["type"]
+        else:
+            value_type = type_for_scalar(parsed_value)
         connection.execute(update_statement.where(table.c.id == id_), {type_label: value_type})
 
 

spinedb_api/alembic/versions/ce9faa82ed59_create_entity_alternative_table.py

@@ -8,7 +8,7 @@
 
 from alembic import op
 import sqlalchemy as sa
-from spinedb_api.compatibility import convert_tool_feature_method_to_entity_alternative
+from spinedb_api.compat.compatibility_transformations import convert_tool_feature_method_to_entity_alternative
 
 # revision identifiers, used by Alembic.
 revision = "ce9faa82ed59"