Skip to content

[tooling] Add helper for database migrations, update documentation, fix latest versions #1364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BenjaminPelletier merged 1 commit into from
Feb 2, 2026
Merged

[tooling] Add helper for database migrations, update documentation, fix latest versions #1364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions build/db_schemas/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,29 @@ and named according to what the changes do. schema_versions.schema_version
should be updated as the last step of each transition; see existing .sql files
for examples.

The two new .sql files must be added to scd.libsonnet or rid.libsonnet
(for remote ID) in this folder.
You will need to create as well yugabyte migrations in the same way, in `yugabyte/` folder.

The two new .sql files must be added to scd.libsonnet, rid.libsonnet or aux_.libsonnet
in this folder.

When a new database version is created, it needs to be targeted in a number of
places:
* Both .sql files in the appropriate folder in db_schemas when setting
schema_versions.schema_version
* [DSS main.jsonnet](../../deploy/services/tanka/examples/minimum/main.jsonnet)
* [Schema manager main.jsonnet](../../deploy/services/tanka/examples/schema_manager/main.jsonnet)
* /pkg/{rid|scd}/store/datastore/store.go
* [Minikube main.jsonnet](../../deploy/services/tanka/examples/minikube/main.jsonnet)
* /pkg/{rid|scd|aux_}/store/datastore/store.go
* /deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf
* /deploy/services/helm-charts/dss/templates/schema-manager.yaml

You can use `update_latest_version.sh` to do that automatically.

## Yugabyte schema versions

Versions 1.0.0 of the schemas reflect the latest versions of the crdb schemas. If
some adaptations are required during the development phase until the first release,
changes should be done using version 1.0.1. This paragraph may be removed after the
first release.

`aux` versions are expected to be the same between Yugabyte and CockroachDB.
4 changes: 2 additions & 2 deletions build/db_schemas/aux_.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
{
data:{
"upto-v1.0.0-create_metadata_table.sql": importstr "aux_/upto-v1.0.0-create_metadata_table.sql",
"upto-v1.1.0-create_heartbeats_table.sql": importstr "aux_/upto-v1.1.0-create_heartbeats_table.sql",
"downfrom-v1.0.0-remove_metadata_table.sql": importstr "aux_/downfrom-v1.0.0-remove_metadata_table.sql",
"downfrom-v1.1.0-remove_heartbeats_table.sql": importstr "aux_/downfrom-v1.1.0-remove_heartbeats_table.sql",
"upto-v1.0.0-create_metadata_table.sql": importstr "aux_/upto-v1.0.0-create_metadata_table.sql",
"upto-v1.1.0-create_heartbeats_table.sql": importstr "aux_/upto-v1.1.0-create_heartbeats_table.sql",
},
}
12 changes: 6 additions & 6 deletions build/db_schemas/rid.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
{
data:{
"downfrom-v1.0.0-remove_isa_subs_tables.sql": importstr "rid/downfrom-v1.0.0-remove_isa_subs_tables.sql",
"downfrom-v2.0.0-remove_inverted_indices.sql": importstr "rid/downfrom-v2.0.0-remove_inverted_indices.sql",
"downfrom-v3.0.0-no_change.sql": importstr "rid/downfrom-v3.0.0-no_change.sql",
"downfrom-v3.1.0-remove_writer_column.sql": importstr "rid/downfrom-v3.1.0-remove_writer_column.sql",
"downfrom-v3.1.1-remove_index_by_time_subscriptions.sql": importstr "rid/downfrom-v3.1.1-remove_index_by_time_subscriptions.sql",
"downfrom-v4.0.0-move_rid_to_defaultdb.sql": importstr "rid/downfrom-v4.0.0-move_rid_to_defaultdb.sql",
"upto-v1.0.0-create_isa_subs_tables.sql": importstr "rid/upto-v1.0.0-create_isa_subs_tables.sql",
"upto-v2.0.0-add_inverted_indices.sql": importstr "rid/upto-v2.0.0-add_inverted_indices.sql",
"upto-v3.0.0-no_change.sql": importstr "rid/upto-v3.0.0-no_change.sql",
"upto-v3.1.0-add_writer_column.sql": importstr "rid/upto-v3.1.0-add_writer_column.sql",
"upto-v3.1.1-add_index_by_time_subscriptions.sql": importstr "rid/upto-v3.1.1-add_index_by_time_subscriptions.sql",
"upto-v4.0.0-rename_defaultdb_to_rid.sql": importstr "rid/upto-v4.0.0-rename_defaultdb_to_rid.sql",
"downfrom-v4.0.0-move_rid_to_defaultdb.sql": importstr "rid/downfrom-v4.0.0-move_rid_to_defaultdb.sql",
"downfrom-v3.1.1-remove_index_by_time_subscriptions.sql": importstr "rid/downfrom-v3.1.1-remove_index_by_time_subscriptions.sql",
"downfrom-v3.1.0-remove_writer_column.sql": importstr "rid/downfrom-v3.1.0-remove_writer_column.sql",
"downfrom-v3.0.0-no_change.sql": importstr "rid/downfrom-v3.0.0-no_change.sql",
"downfrom-v2.0.0-remove_inverted_indices.sql": importstr "rid/downfrom-v2.0.0-remove_inverted_indices.sql",
"downfrom-v1.0.0-remove_isa_subs_tables.sql": importstr "rid/downfrom-v1.0.0-remove_isa_subs_tables.sql",
},
}
10 changes: 5 additions & 5 deletions build/db_schemas/scd.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
{
data:{
"downfrom-v1.0.0-remove_initial_version.sql": importstr "scd/downfrom-v1.0.0-remove_initial_version.sql",
"downfrom-v2.0.0-remove_api_1_0_0_support.sql": importstr "scd/downfrom-v2.0.0-remove_api_1_0_0_support.sql",
"downfrom-v3.0.0-remove_inverted_indices.sql": importstr "scd/downfrom-v3.0.0-remove_inverted_indices.sql",
"downfrom-v3.1.0-remove_uss_availability.sql": importstr "scd/downfrom-v3.1.0-remove_uss_availability.sql",
"downfrom-v3.2.0-remove_ovn_columns.sql": importstr "scd/downfrom-v3.2.0-remove_ovn_columns.sql",
"upto-v1.0.0-create_initial_version.sql": importstr "scd/upto-v1.0.0-create_initial_version.sql",
"upto-v2.0.0-support_api_1_0_0.sql": importstr "scd/upto-v2.0.0-support_api_1_0_0.sql",
"upto-v3.0.0-add_inverted_indices.sql": importstr "scd/upto-v3.0.0-add_inverted_indices.sql",
"upto-v3.1.0-create_uss_availability.sql": importstr "scd/upto-v3.1.0-create_uss_availability.sql",
"upto-v3.2.0-add_ovn_columns.sql": importstr "scd/upto-v3.2.0-add_ovn_columns.sql",
"downfrom-v3.2.0-remove_ovn_columns.sql": importstr "scd/downfrom-v3.2.0-remove_ovn_columns.sql",
"downfrom-v3.1.0-remove_uss_availability.sql": importstr "scd/downfrom-v3.1.0-remove_uss_availability.sql",
"downfrom-v3.0.0-remove_inverted_indices.sql": importstr "scd/downfrom-v3.0.0-remove_inverted_indices.sql",
"downfrom-v2.0.0-remove_api_1_0_0_support.sql": importstr "scd/downfrom-v2.0.0-remove_api_1_0_0_support.sql",
"downfrom-v1.0.0-remove_initial_version.sql": importstr "scd/downfrom-v1.0.0-remove_initial_version.sql",
},
}
84 changes: 84 additions & 0 deletions build/db_schemas/update_latest_version.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
#!/usr/bin/env bash
# shellcheck disable=SC2010

set -eo pipefail

OS=$(uname)
if [[ "$OS" == "Darwin" ]]; then
# OSX uses BSD readlink
SCRIPTDIR="$(dirname "$0")"
else
SCRIPTDIR=$(readlink -e "$(dirname "$0")")
fi

BASEDIR="${SCRIPTDIR}/../.."
cd "${SCRIPTDIR}"

# Extract version
CRDB_RID=$(ls rid | grep -oE 'upto-v[0-9]+\.[0-9]+\.[0-9]+' | sed 's/upto-v//' | sort -V | tail -n 1)
CRDB_SCD=$(ls scd | grep -oE 'upto-v[0-9]+\.[0-9]+\.[0-9]+' | sed 's/upto-v//' | sort -V | tail -n 1)
YBDB_RID=$(ls yugabyte/rid | grep -oE 'upto-v[0-9]+\.[0-9]+\.[0-9]+' | sed 's/upto-v//' | sort -V | tail -n 1)
YBDB_SCD=$(ls yugabyte/scd | grep -oE 'upto-v[0-9]+\.[0-9]+\.[0-9]+' | sed 's/upto-v//' | sort -V | tail -n 1)
AUX=$(ls yugabyte/aux_ | grep -oE 'upto-v[0-9]+\.[0-9]+\.[0-9]+' | sed 's/upto-v//' | sort -V | tail -n 1)

# Replace terraform latests
sed -i -E "s/(rid_db_schema = var\.desired_rid_db_version == \"latest\" \? \(var\.datastore_type == \"cockroachdb\" \? ).*\)/\1\"$CRDB_RID\" : \"$YBDB_RID\")/" "${BASEDIR}/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf"
sed -i -E "s/(scd_db_schema = var\.desired_scd_db_version == \"latest\" \? \(var\.datastore_type == \"cockroachdb\" \? ).*\)/\1\"$CRDB_SCD\" : \"$YBDB_SCD\")/" "${BASEDIR}/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf"
sed -i -E "s/(aux_db_schema = var\.desired_aux_db_version == \"latest\" \? ).* :/\1\"$AUX\" :/" "${BASEDIR}/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf"

# Replace tanka examples / latests
sed -i -E "s/(desired_rid_db_version: ).*/\1'$YBDB_RID',/" "${BASEDIR}/deploy/services/tanka/examples/minikube/main.jsonnet"
sed -i -E "s/(desired_scd_db_version: ).*/\1'$YBDB_SCD',/" "${BASEDIR}/deploy/services/tanka/examples/minikube/main.jsonnet"
sed -i -E "s/(desired_aux_db_version: ).*/\1'$AUX',/" "${BASEDIR}/deploy/services/tanka/examples/minikube/main.jsonnet"

for file in "${BASEDIR}/deploy/services/tanka/metadata_base.libsonnet" "${BASEDIR}/deploy/services/tanka/examples/schema_manager/main.jsonnet" "${BASEDIR}/deploy/services/tanka/examples/minimum/main.jsonnet"; do
sed -i -E "s/(desired_rid_db_version: ).*/\1'$CRDB_RID',/" "${file}"
sed -i -E "s/(desired_scd_db_version: ).*/\1'$CRDB_SCD',/" "${file}"
sed -i -E "s/(desired_aux_db_version: ).*/\1'$AUX',/" "${file}"
done

# Replace helm latests
sed -i -E -e "s/(\\\$schemas := dict ).*}}/\1\"rid\" \"$CRDB_RID\" \"scd\" \"$CRDB_SCD\" \"aux_\" \"$AUX\" }}/" "${BASEDIR}/deploy/services/helm-charts/dss/templates/schema-manager.yaml"
sed -i -E -e "s/(\\\$schemas = dict ).*}}/\1\"rid\" \"$YBDB_RID\" \"scd\" \"$YBDB_SCD\" \"aux_\" \"$AUX\" }}/" "${BASEDIR}/deploy/services/helm-charts/dss/templates/schema-manager.yaml"

# Generate libsonnet files with list of migrations
cat <<EOF > rid.libsonnet
{
data:{
$(find rid -type f -print0 | sort -z | xargs -0 -I {} basename {} | awk '{print " \"" $1 "\": importstr \"rid/" $1 "\","}')
},
}
EOF

cat <<EOF > scd.libsonnet
{
data:{
$(find scd -type f -print0 | sort -z | xargs -0 -I {} basename {} | awk '{print " \"" $1 "\": importstr \"scd/" $1 "\","}')
},
}
EOF

cat <<EOF > aux_.libsonnet
{
data:{
$(find aux_ -type f -print0 | sort -z | xargs -0 -I {} basename {} | awk '{print " \"" $1 "\": importstr \"aux_/" $1 "\","}')
},
}
EOF

# Extract major versions
CRDB_RID_MAJOR=$(echo "$CRDB_RID" | cut -d. -f1)
CRDB_SCD_MAJOR=$(echo "$CRDB_SCD" | cut -d. -f1)
YBDB_RID_MAJOR=$(echo "$YBDB_RID" | cut -d. -f1)
YBDB_SCD_MAJOR=$(echo "$YBDB_SCD" | cut -d. -f1)
AUX_MAJOR=$(echo "$AUX" | cut -d. -f1)

# Replace major versions in datastore files
sed -i -E "s/(currentCrdbMajorSchemaVersion.*= )[0-9]/\1$CRDB_SCD_MAJOR/" "${BASEDIR}/pkg/scd/store/datastore/store.go"
sed -i -E "s/(currentYugabyteMajorSchemaVersion.*= )[0-9]/\1$YBDB_SCD_MAJOR/" "${BASEDIR}/pkg/scd/store/datastore/store.go"

sed -i -E "s/(currentCrdbMajorSchemaVersion.*= )[0-9]/\1$CRDB_RID_MAJOR/" "${BASEDIR}/pkg/rid/store/datastore/store.go"
sed -i -E "s/(currentYugabyteMajorSchemaVersion.*= )[0-9]/\1$YBDB_RID_MAJOR/" "${BASEDIR}/pkg/rid/store/datastore/store.go"

sed -i -E "s/(currentCrdbMajorSchemaVersion.*= )[0-9]/\1$AUX_MAJOR/" "${BASEDIR}/pkg/aux_/store/datastore/store.go"
sed -i -E "s/(currentYugabyteMajorSchemaVersion.*= )[0-9]/\1$AUX_MAJOR/" "${BASEDIR}/pkg/aux_/store/datastore/store.go"
2 changes: 1 addition & 1 deletion deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
locals {
rid_db_schema = var.desired_rid_db_version == "latest" ? (var.datastore_type == "cockroachdb" ? "4.0.0" : "1.0.1") : var.desired_rid_db_version
scd_db_schema = var.desired_scd_db_version == "latest" ? (var.datastore_type == "cockroachdb" ? "3.2.0" : "1.0.1") : var.desired_scd_db_version
aux_db_schema = var.desired_aux_db_version == "latest" ? "1.0.0" : var.desired_aux_db_version
aux_db_schema = var.desired_aux_db_version == "latest" ? "1.1.0" : var.desired_aux_db_version
}
4 changes: 2 additions & 2 deletions deploy/services/helm-charts/dss/templates/schema-manager.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@
{{- $jobVersion := .Release.Revision -}} {{/* Jobs template definition is immutable, using the revision in the name forces the job to be recreated at each helm upgrade. */}}

{{- $waitForDatastore := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $datastoreHost)) -}}
{{- $schemas := dict "rid" "4.0.0" "scd" "3.2.0" "aux_" "1.0.0" }}
{{- $schemas := dict "rid" "4.0.0" "scd" "3.2.0" "aux_" "1.1.0" }}

{{- if .Values.yugabyte.enabled }}
{{- $waitForDatastore = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $datastoreHost)) -}}
{{- $schemas = dict "rid" "1.0.1" "scd" "1.0.1" "aux_" "1.0.0" }}
{{- $schemas = dict "rid" "1.0.1" "scd" "1.0.1" "aux_" "1.1.0" }}
{{- end -}}

{{- range $service, $schemaVersion := $schemas }}
Expand Down
2 changes: 1 addition & 1 deletion deploy/services/tanka/examples/minikube/main.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ local metadata = metadataBase {
image: 'docker.io/interuss-local/dss:latest',
desired_rid_db_version: '1.0.1',
desired_scd_db_version: '1.0.1',
desired_aux_db_version: '1.0.0',
desired_aux_db_version: '1.1.0',
},
evict+: {
scd+: {
Expand Down
2 changes: 1 addition & 1 deletion deploy/services/tanka/examples/minimum/main.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ local metadata = metadataBase {
image: 'VAR_DOCKER_IMAGE_NAME',
desired_rid_db_version: '4.0.0',
desired_scd_db_version: '3.2.0',
desired_aux_db_version: '1.0.0',
desired_aux_db_version: '1.1.0',
},
prometheus+: {
storageClass: 'VAR_STORAGE_CLASS',
Expand Down
2 changes: 1 addition & 1 deletion deploy/services/tanka/examples/schema_manager/main.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ local metadata = metadataBase {
image: 'VAR_DOCKER_IMAGE_NAME',
desired_rid_db_version: '4.0.0',
desired_scd_db_version: '3.2.0',
desired_aux_db_version: '1.0.0',
desired_aux_db_version: '1.1.0',
},
};

Expand Down
2 changes: 1 addition & 1 deletion deploy/services/tanka/metadata_base.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@
image: error 'must specify image',
desired_rid_db_version: '4.0.0',
desired_scd_db_version: '3.2.0',
desired_aux_db_version: '1.0.0',
desired_aux_db_version: '1.1.0',
},
evict: {
scd: {
Expand Down
Loading