diff --git a/build/db_schemas/README.md b/build/db_schemas/README.md index e6d930ea3..8b582f17f 100644 --- a/build/db_schemas/README.md +++ b/build/db_schemas/README.md @@ -9,8 +9,10 @@ and named according to what the changes do. schema_versions.schema_version should be updated as the last step of each transition; see existing .sql files for examples. -The two new .sql files must be added to scd.libsonnet or rid.libsonnet -(for remote ID) in this folder. +You will need to create as well yugabyte migrations in the same way, in `yugabyte/` folder. + +The two new .sql files must be added to scd.libsonnet, rid.libsonnet or aux_.libsonnet + in this folder. When a new database version is created, it needs to be targeted in a number of places: @@ -18,13 +20,18 @@ places: schema_versions.schema_version * [DSS main.jsonnet](../../deploy/services/tanka/examples/minimum/main.jsonnet) * [Schema manager main.jsonnet](../../deploy/services/tanka/examples/schema_manager/main.jsonnet) -* /pkg/{rid|scd}/store/datastore/store.go +* [Minikube main.jsonnet](../../deploy/services/tanka/examples/minikube/main.jsonnet) +* /pkg/{rid|scd|aux_}/store/datastore/store.go * /deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf * /deploy/services/helm-charts/dss/templates/schema-manager.yaml +You can use `update_latest_version.sh` to do that automatically. + ## Yugabyte schema versions Versions 1.0.0 of the schemas reflect the latest versions of the crdb schemas. If some adaptations are required during the development phase until the first release, changes should be done using version 1.0.1. This paragraph may be removed after the first release. + +`aux` versions are expected to be the same between Yugabyte and CockroachDB. diff --git a/build/db_schemas/aux_.libsonnet b/build/db_schemas/aux_.libsonnet index 46e2d6bd2..0c062f902 100644 --- a/build/db_schemas/aux_.libsonnet +++ b/build/db_schemas/aux_.libsonnet @@ -1,8 +1,8 @@ { data:{ - "upto-v1.0.0-create_metadata_table.sql": importstr "aux_/upto-v1.0.0-create_metadata_table.sql", - "upto-v1.1.0-create_heartbeats_table.sql": importstr "aux_/upto-v1.1.0-create_heartbeats_table.sql", "downfrom-v1.0.0-remove_metadata_table.sql": importstr "aux_/downfrom-v1.0.0-remove_metadata_table.sql", "downfrom-v1.1.0-remove_heartbeats_table.sql": importstr "aux_/downfrom-v1.1.0-remove_heartbeats_table.sql", + "upto-v1.0.0-create_metadata_table.sql": importstr "aux_/upto-v1.0.0-create_metadata_table.sql", + "upto-v1.1.0-create_heartbeats_table.sql": importstr "aux_/upto-v1.1.0-create_heartbeats_table.sql", }, } diff --git a/build/db_schemas/rid.libsonnet b/build/db_schemas/rid.libsonnet index 010d77071..c4850b09e 100644 --- a/build/db_schemas/rid.libsonnet +++ b/build/db_schemas/rid.libsonnet @@ -1,16 +1,16 @@ { data:{ + "downfrom-v1.0.0-remove_isa_subs_tables.sql": importstr "rid/downfrom-v1.0.0-remove_isa_subs_tables.sql", + "downfrom-v2.0.0-remove_inverted_indices.sql": importstr "rid/downfrom-v2.0.0-remove_inverted_indices.sql", + "downfrom-v3.0.0-no_change.sql": importstr "rid/downfrom-v3.0.0-no_change.sql", + "downfrom-v3.1.0-remove_writer_column.sql": importstr "rid/downfrom-v3.1.0-remove_writer_column.sql", + "downfrom-v3.1.1-remove_index_by_time_subscriptions.sql": importstr "rid/downfrom-v3.1.1-remove_index_by_time_subscriptions.sql", + "downfrom-v4.0.0-move_rid_to_defaultdb.sql": importstr "rid/downfrom-v4.0.0-move_rid_to_defaultdb.sql", "upto-v1.0.0-create_isa_subs_tables.sql": importstr "rid/upto-v1.0.0-create_isa_subs_tables.sql", "upto-v2.0.0-add_inverted_indices.sql": importstr "rid/upto-v2.0.0-add_inverted_indices.sql", "upto-v3.0.0-no_change.sql": importstr "rid/upto-v3.0.0-no_change.sql", "upto-v3.1.0-add_writer_column.sql": importstr "rid/upto-v3.1.0-add_writer_column.sql", "upto-v3.1.1-add_index_by_time_subscriptions.sql": importstr "rid/upto-v3.1.1-add_index_by_time_subscriptions.sql", "upto-v4.0.0-rename_defaultdb_to_rid.sql": importstr "rid/upto-v4.0.0-rename_defaultdb_to_rid.sql", - "downfrom-v4.0.0-move_rid_to_defaultdb.sql": importstr "rid/downfrom-v4.0.0-move_rid_to_defaultdb.sql", - "downfrom-v3.1.1-remove_index_by_time_subscriptions.sql": importstr "rid/downfrom-v3.1.1-remove_index_by_time_subscriptions.sql", - "downfrom-v3.1.0-remove_writer_column.sql": importstr "rid/downfrom-v3.1.0-remove_writer_column.sql", - "downfrom-v3.0.0-no_change.sql": importstr "rid/downfrom-v3.0.0-no_change.sql", - "downfrom-v2.0.0-remove_inverted_indices.sql": importstr "rid/downfrom-v2.0.0-remove_inverted_indices.sql", - "downfrom-v1.0.0-remove_isa_subs_tables.sql": importstr "rid/downfrom-v1.0.0-remove_isa_subs_tables.sql", }, } diff --git a/build/db_schemas/scd.libsonnet b/build/db_schemas/scd.libsonnet index 99a2799fd..d5544d3fb 100644 --- a/build/db_schemas/scd.libsonnet +++ b/build/db_schemas/scd.libsonnet @@ -1,14 +1,14 @@ { data:{ + "downfrom-v1.0.0-remove_initial_version.sql": importstr "scd/downfrom-v1.0.0-remove_initial_version.sql", + "downfrom-v2.0.0-remove_api_1_0_0_support.sql": importstr "scd/downfrom-v2.0.0-remove_api_1_0_0_support.sql", + "downfrom-v3.0.0-remove_inverted_indices.sql": importstr "scd/downfrom-v3.0.0-remove_inverted_indices.sql", + "downfrom-v3.1.0-remove_uss_availability.sql": importstr "scd/downfrom-v3.1.0-remove_uss_availability.sql", + "downfrom-v3.2.0-remove_ovn_columns.sql": importstr "scd/downfrom-v3.2.0-remove_ovn_columns.sql", "upto-v1.0.0-create_initial_version.sql": importstr "scd/upto-v1.0.0-create_initial_version.sql", "upto-v2.0.0-support_api_1_0_0.sql": importstr "scd/upto-v2.0.0-support_api_1_0_0.sql", "upto-v3.0.0-add_inverted_indices.sql": importstr "scd/upto-v3.0.0-add_inverted_indices.sql", "upto-v3.1.0-create_uss_availability.sql": importstr "scd/upto-v3.1.0-create_uss_availability.sql", "upto-v3.2.0-add_ovn_columns.sql": importstr "scd/upto-v3.2.0-add_ovn_columns.sql", - "downfrom-v3.2.0-remove_ovn_columns.sql": importstr "scd/downfrom-v3.2.0-remove_ovn_columns.sql", - "downfrom-v3.1.0-remove_uss_availability.sql": importstr "scd/downfrom-v3.1.0-remove_uss_availability.sql", - "downfrom-v3.0.0-remove_inverted_indices.sql": importstr "scd/downfrom-v3.0.0-remove_inverted_indices.sql", - "downfrom-v2.0.0-remove_api_1_0_0_support.sql": importstr "scd/downfrom-v2.0.0-remove_api_1_0_0_support.sql", - "downfrom-v1.0.0-remove_initial_version.sql": importstr "scd/downfrom-v1.0.0-remove_initial_version.sql", }, } diff --git a/build/db_schemas/update_latest_version.sh b/build/db_schemas/update_latest_version.sh new file mode 100755 index 000000000..2a0dfe37d --- /dev/null +++ b/build/db_schemas/update_latest_version.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2010 + +set -eo pipefail + +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + SCRIPTDIR="$(dirname "$0")" +else + SCRIPTDIR=$(readlink -e "$(dirname "$0")") +fi + +BASEDIR="${SCRIPTDIR}/../.." +cd "${SCRIPTDIR}" + +# Extract version +CRDB_RID=$(ls rid | grep -oE 'upto-v[0-9]+\.[0-9]+\.[0-9]+' | sed 's/upto-v//' | sort -V | tail -n 1) +CRDB_SCD=$(ls scd | grep -oE 'upto-v[0-9]+\.[0-9]+\.[0-9]+' | sed 's/upto-v//' | sort -V | tail -n 1) +YBDB_RID=$(ls yugabyte/rid | grep -oE 'upto-v[0-9]+\.[0-9]+\.[0-9]+' | sed 's/upto-v//' | sort -V | tail -n 1) +YBDB_SCD=$(ls yugabyte/scd | grep -oE 'upto-v[0-9]+\.[0-9]+\.[0-9]+' | sed 's/upto-v//' | sort -V | tail -n 1) +AUX=$(ls yugabyte/aux_ | grep -oE 'upto-v[0-9]+\.[0-9]+\.[0-9]+' | sed 's/upto-v//' | sort -V | tail -n 1) + +# Replace terraform latests +sed -i -E "s/(rid_db_schema = var\.desired_rid_db_version == \"latest\" \? \(var\.datastore_type == \"cockroachdb\" \? ).*\)/\1\"$CRDB_RID\" : \"$YBDB_RID\")/" "${BASEDIR}/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf" +sed -i -E "s/(scd_db_schema = var\.desired_scd_db_version == \"latest\" \? \(var\.datastore_type == \"cockroachdb\" \? ).*\)/\1\"$CRDB_SCD\" : \"$YBDB_SCD\")/" "${BASEDIR}/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf" +sed -i -E "s/(aux_db_schema = var\.desired_aux_db_version == \"latest\" \? ).* :/\1\"$AUX\" :/" "${BASEDIR}/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf" + +# Replace tanka examples / latests +sed -i -E "s/(desired_rid_db_version: ).*/\1'$YBDB_RID',/" "${BASEDIR}/deploy/services/tanka/examples/minikube/main.jsonnet" +sed -i -E "s/(desired_scd_db_version: ).*/\1'$YBDB_SCD',/" "${BASEDIR}/deploy/services/tanka/examples/minikube/main.jsonnet" +sed -i -E "s/(desired_aux_db_version: ).*/\1'$AUX',/" "${BASEDIR}/deploy/services/tanka/examples/minikube/main.jsonnet" + +for file in "${BASEDIR}/deploy/services/tanka/metadata_base.libsonnet" "${BASEDIR}/deploy/services/tanka/examples/schema_manager/main.jsonnet" "${BASEDIR}/deploy/services/tanka/examples/minimum/main.jsonnet"; do + sed -i -E "s/(desired_rid_db_version: ).*/\1'$CRDB_RID',/" "${file}" + sed -i -E "s/(desired_scd_db_version: ).*/\1'$CRDB_SCD',/" "${file}" + sed -i -E "s/(desired_aux_db_version: ).*/\1'$AUX',/" "${file}" +done + +# Replace helm latests +sed -i -E -e "s/(\\\$schemas := dict ).*}}/\1\"rid\" \"$CRDB_RID\" \"scd\" \"$CRDB_SCD\" \"aux_\" \"$AUX\" }}/" "${BASEDIR}/deploy/services/helm-charts/dss/templates/schema-manager.yaml" +sed -i -E -e "s/(\\\$schemas = dict ).*}}/\1\"rid\" \"$YBDB_RID\" \"scd\" \"$YBDB_SCD\" \"aux_\" \"$AUX\" }}/" "${BASEDIR}/deploy/services/helm-charts/dss/templates/schema-manager.yaml" + +# Generate libsonnet files with list of migrations +cat < rid.libsonnet +{ + data:{ +$(find rid -type f -print0 | sort -z | xargs -0 -I {} basename {} | awk '{print " \"" $1 "\": importstr \"rid/" $1 "\","}') + }, +} +EOF + +cat < scd.libsonnet +{ + data:{ +$(find scd -type f -print0 | sort -z | xargs -0 -I {} basename {} | awk '{print " \"" $1 "\": importstr \"scd/" $1 "\","}') + }, +} +EOF + +cat < aux_.libsonnet +{ + data:{ +$(find aux_ -type f -print0 | sort -z | xargs -0 -I {} basename {} | awk '{print " \"" $1 "\": importstr \"aux_/" $1 "\","}') + }, +} +EOF + +# Extract major versions +CRDB_RID_MAJOR=$(echo "$CRDB_RID" | cut -d. -f1) +CRDB_SCD_MAJOR=$(echo "$CRDB_SCD" | cut -d. -f1) +YBDB_RID_MAJOR=$(echo "$YBDB_RID" | cut -d. -f1) +YBDB_SCD_MAJOR=$(echo "$YBDB_SCD" | cut -d. -f1) +AUX_MAJOR=$(echo "$AUX" | cut -d. -f1) + +# Replace major versions in datastore files +sed -i -E "s/(currentCrdbMajorSchemaVersion.*= )[0-9]/\1$CRDB_SCD_MAJOR/" "${BASEDIR}/pkg/scd/store/datastore/store.go" +sed -i -E "s/(currentYugabyteMajorSchemaVersion.*= )[0-9]/\1$YBDB_SCD_MAJOR/" "${BASEDIR}/pkg/scd/store/datastore/store.go" + +sed -i -E "s/(currentCrdbMajorSchemaVersion.*= )[0-9]/\1$CRDB_RID_MAJOR/" "${BASEDIR}/pkg/rid/store/datastore/store.go" +sed -i -E "s/(currentYugabyteMajorSchemaVersion.*= )[0-9]/\1$YBDB_RID_MAJOR/" "${BASEDIR}/pkg/rid/store/datastore/store.go" + +sed -i -E "s/(currentCrdbMajorSchemaVersion.*= )[0-9]/\1$AUX_MAJOR/" "${BASEDIR}/pkg/aux_/store/datastore/store.go" +sed -i -E "s/(currentYugabyteMajorSchemaVersion.*= )[0-9]/\1$AUX_MAJOR/" "${BASEDIR}/pkg/aux_/store/datastore/store.go" diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf index ba9be4b63..50b2af8af 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf @@ -1,5 +1,5 @@ locals { rid_db_schema = var.desired_rid_db_version == "latest" ? (var.datastore_type == "cockroachdb" ? "4.0.0" : "1.0.1") : var.desired_rid_db_version scd_db_schema = var.desired_scd_db_version == "latest" ? (var.datastore_type == "cockroachdb" ? "3.2.0" : "1.0.1") : var.desired_scd_db_version - aux_db_schema = var.desired_aux_db_version == "latest" ? "1.0.0" : var.desired_aux_db_version + aux_db_schema = var.desired_aux_db_version == "latest" ? "1.1.0" : var.desired_aux_db_version } diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml index 8df9d4265..4d5998a63 100644 --- a/deploy/services/helm-charts/dss/templates/schema-manager.yaml +++ b/deploy/services/helm-charts/dss/templates/schema-manager.yaml @@ -5,11 +5,11 @@ {{- $jobVersion := .Release.Revision -}} {{/* Jobs template definition is immutable, using the revision in the name forces the job to be recreated at each helm upgrade. */}} {{- $waitForDatastore := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $datastoreHost)) -}} -{{- $schemas := dict "rid" "4.0.0" "scd" "3.2.0" "aux_" "1.0.0" }} +{{- $schemas := dict "rid" "4.0.0" "scd" "3.2.0" "aux_" "1.1.0" }} {{- if .Values.yugabyte.enabled }} {{- $waitForDatastore = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $datastoreHost)) -}} -{{- $schemas = dict "rid" "1.0.1" "scd" "1.0.1" "aux_" "1.0.0" }} +{{- $schemas = dict "rid" "1.0.1" "scd" "1.0.1" "aux_" "1.1.0" }} {{- end -}} {{- range $service, $schemaVersion := $schemas }} diff --git a/deploy/services/tanka/examples/minikube/main.jsonnet b/deploy/services/tanka/examples/minikube/main.jsonnet index c9934d796..27c607bdf 100644 --- a/deploy/services/tanka/examples/minikube/main.jsonnet +++ b/deploy/services/tanka/examples/minikube/main.jsonnet @@ -39,7 +39,7 @@ local metadata = metadataBase { image: 'docker.io/interuss-local/dss:latest', desired_rid_db_version: '1.0.1', desired_scd_db_version: '1.0.1', - desired_aux_db_version: '1.0.0', + desired_aux_db_version: '1.1.0', }, evict+: { scd+: { diff --git a/deploy/services/tanka/examples/minimum/main.jsonnet b/deploy/services/tanka/examples/minimum/main.jsonnet index e94738ab1..ffa2f612d 100644 --- a/deploy/services/tanka/examples/minimum/main.jsonnet +++ b/deploy/services/tanka/examples/minimum/main.jsonnet @@ -59,7 +59,7 @@ local metadata = metadataBase { image: 'VAR_DOCKER_IMAGE_NAME', desired_rid_db_version: '4.0.0', desired_scd_db_version: '3.2.0', - desired_aux_db_version: '1.0.0', + desired_aux_db_version: '1.1.0', }, prometheus+: { storageClass: 'VAR_STORAGE_CLASS', diff --git a/deploy/services/tanka/examples/schema_manager/main.jsonnet b/deploy/services/tanka/examples/schema_manager/main.jsonnet index adedc99f3..b1edb2d96 100644 --- a/deploy/services/tanka/examples/schema_manager/main.jsonnet +++ b/deploy/services/tanka/examples/schema_manager/main.jsonnet @@ -19,7 +19,7 @@ local metadata = metadataBase { image: 'VAR_DOCKER_IMAGE_NAME', desired_rid_db_version: '4.0.0', desired_scd_db_version: '3.2.0', - desired_aux_db_version: '1.0.0', + desired_aux_db_version: '1.1.0', }, }; diff --git a/deploy/services/tanka/metadata_base.libsonnet b/deploy/services/tanka/metadata_base.libsonnet index 1cf23151b..cd8e1a57e 100644 --- a/deploy/services/tanka/metadata_base.libsonnet +++ b/deploy/services/tanka/metadata_base.libsonnet @@ -88,7 +88,7 @@ image: error 'must specify image', desired_rid_db_version: '4.0.0', desired_scd_db_version: '3.2.0', - desired_aux_db_version: '1.0.0', + desired_aux_db_version: '1.1.0', }, evict: { scd: {