ConversationStore: Implement migration to postgres

changelog.d/0-release-notes/conversation-migration

@@ -0,0 +1,52 @@
+Starting this release, existing deployments can migrate the conversation data to
+PostgreSQL from Cassandra. This is necessary for channel search and management
+of channels from the team-management UI. It is highly recommended to take a
+backup of the Galley Cassandra before triggering the migration.
+
+The migration needs to happen in 3 steps:
+
+1. Prepare wire-server for migration.
+
+   This step make sure that wire-server keep working as expected during the
+   migration. To do this deploy wire-server with this config change:
+
+   ```yaml
+   galley:
+     config:
+       postgresqlMigration:
+         conversation: migrate-to-postgresql
+   ```
+
+   This change should restart all the galley pods, any new conversations will
+   now be written to PostgreSQL.
+
+2. Trigger the migration and wait.
+
+   This step will actually carry out the migration. To do this deploy
+   wire-server with this config change:
+
+   ```yaml
+   background-worker:
+     config:
+       migrateConversations: true
+   ```
+
+   This change should restart the background-worker pods. It is recommended to
+   watch the logs and wait for both of these two metrics to report `1.0`:
+   `wire_local_convs_migration_finished` and `wire_user_remote_convs_migration_finished`.
+   This can take a long time depending on number of conversations in the DB.
+
+3. Configure wire-server to only use PostgreSQL for conversations.
+
+   This will be the configuration which must be used from now on for every new
+   release.
+
+   ```yaml
+   galley:
+     config:
+       postgresqlMigration:
+         conversation: postgresql
+   background-worker:
+     config:
+       migrateConversations: false
+   ```

changelog.d/2-features/conversation-migration

@@ -0,0 +1 @@
+Support migration of all conversation data to Postgresql.

charts/background-worker/templates/_helpers.tpl

@@ -23,3 +23,19 @@ created one (in case the CA is provided as PEM string.)
 {{- dict "name" "background-worker-cassandra" "key" "ca.pem" | toYaml -}}
 {{- end -}}
 {{- end -}}
+
+{{- define "useCassandraTLSGalley" -}}
+{{ or (hasKey .cassandraGalley "tlsCa") (hasKey .cassandraGalley "tlsCaSecretRef") }}
+{{- end -}}
+
+{{/* Return a Dict of TLS CA secret name and key
+This is used to switch between provided secret (e.g. by cert-manager) and
+created one (in case the CA is provided as PEM string.)
+*/}}
+{{- define "tlsSecretRefGalley" -}}
+{{- if .cassandraGalley.tlsCaSecretRef -}}
+{{ .cassandraGalley.tlsCaSecretRef | toYaml }}
+{{- else }}
+{{- dict "name" "background-worker-cassandra-galley" "key" "ca.pem" | toYaml -}}
+{{- end -}}
+{{- end -}}

charts/background-worker/templates/cassandra-galley-secret.yaml

@@ -0,0 +1,15 @@
+{{/* Secret for the provided Cassandra TLS CA. */}}
+{{- if not (empty .Values.config.cassandraGalley.tlsCa) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: background-worker-cassandra-galley
+  labels:
+    app: background-worker
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+type: Opaque
+data:
+  ca.pem: {{ .Values.config.cassandraGalley.tlsCa | b64enc | quote }}
+{{- end }}

charts/background-worker/templates/configmap.yaml

@@ -30,6 +30,24 @@ data:
       tlsCa: /etc/wire/background-worker/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }}
       {{- end }}
 
+    cassandraGalley:
+      endpoint:
+        host: {{ .cassandraGalley.host }}
+        port: 9042
+      keyspace: galley
+      {{- if hasKey .cassandraGalley "filterNodesByDatacentre" }}
+      filterNodesByDatacentre: {{ .cassandraGalley.filterNodesByDatacentre }}
+      {{- end }}
+      {{- if eq (include "useCassandraTLSGalley" .) "true" }}
+      tlsCa: /etc/wire/background-worker/cassandra-galley/{{- (include "tlsSecretRefGalley" . | fromYaml).key }}
+      {{- end }}
+
+    postgresql: {{ toYaml .postgresql | nindent 6 }}
+    postgresqlPool: {{ toYaml .postgresqlPool | nindent 6 }}
+    {{- if hasKey $.Values.secrets "pgPassword" }}
+    postgresqlPassword: /etc/wire/background-worker/secrets/pgPassword
+    {{- end }}
+
     {{- with .rabbitmq }}
     rabbitmq:
       host: {{ .host }}
@@ -46,6 +64,8 @@ data:
       {{- end }}
     {{- end }}
 
+    migrateConversations: {{ .migrateConversations }}
+
     backendNotificationPusher:
 {{toYaml .backendNotificationPusher | indent 6 }}
   {{- end }}

charts/background-worker/templates/deployment.yaml

@@ -27,6 +27,7 @@ spec:
         checksum/configmap: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum }}
         checksum/secret: {{ include (print .Template.BasePath "/secret.yaml") . | sha256sum }}
         checksum/cassandra-secret: {{ include (print .Template.BasePath "/cassandra-secret.yaml") . | sha256sum }}
+        checksum/cassandra-galley-secret: {{ include (print .Template.BasePath "/cassandra-galley-secret.yaml") . | sha256sum }}
         fluentbit.io/parser: json
     spec:
       serviceAccount: null
@@ -44,11 +45,19 @@ spec:
           secret:
             secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }}
         {{- end }}
+        {{- if eq (include "useCassandraTLSGalley" .Values.config) "true" }}
+        - name: "background-worker-cassandra-galley"
+          secret:
+            secretName: {{ (include "tlsSecretRefGalley" .Values.config | fromYaml).name }}
+        {{- end }}
         {{- if .Values.config.rabbitmq.tlsCaSecretRef }}
         - name: "rabbitmq-ca"
           secret:
             secretName: {{ .Values.config.rabbitmq.tlsCaSecretRef.name }}
         {{- end }}
+        {{- if .Values.additionalVolumes }}
+        {{ toYaml .Values.additionalVolumes | nindent 8 }}
+        {{- end }}
       containers:
         - name: background-worker
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
@@ -64,10 +73,17 @@ spec:
           - name: "background-worker-cassandra"
             mountPath: "/etc/wire/background-worker/cassandra"
           {{- end }}
+          {{- if eq (include "useCassandraTLSGalley" .Values.config) "true" }}
+          - name: "background-worker-cassandra-galley"
+            mountPath: "/etc/wire/background-worker/cassandra-galley"
+          {{- end }}
           {{- if .Values.config.rabbitmq.tlsCaSecretRef }}
           - name: "rabbitmq-ca"
             mountPath: "/etc/wire/background-worker/rabbitmq-ca/"
           {{- end }}
+          {{- if .Values.additionalVolumeMounts }}
+          {{ toYaml .Values.additionalVolumeMounts | nindent 10 }}
+          {{- end }}
           env:
           - name: RABBITMQ_USERNAME
             valueFrom:

charts/background-worker/templates/secret.yaml

@@ -15,4 +15,7 @@ data:
   {{- with .Values.secrets }}
   rabbitmqUsername: {{ .rabbitmq.username | b64enc | quote }}
   rabbitmqPassword: {{ .rabbitmq.password | b64enc | quote }}
+  {{- if .pgPassword }}
+  pgPassword: {{ .pgPassword | b64enc | quote }}
+  {{- end }}
   {{- end }}

charts/background-worker/values.yaml

@@ -31,6 +31,39 @@ config:
     #   key: <ca-attribute>
   cassandra:
     host: aws-cassandra
+  cassandraGalley:
+    host: aws-cassandra
+
+  # Postgres connection settings
+  #
+  # Values are described in https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS
+  # To set the password via a brig secret see `secrets.pgPassword`.
+  #
+  # `additionalVolumeMounts` and `additionalVolumes` can be used to mount
+  # additional files (e.g. certificates) into the brig container. This way
+  # does not work for password files (parameter `passfile`), because
+  # libpq-connect requires access rights (mask 0600) for them that we cannot
+  # provide for random uids.
+  #
+  # Below is an example configuration we're using for our CI tests.
+  postgresql:
+    host: postgresql # DNS name without protocol
+    port: "5432"
+    user: wire-server
+    dbname: wire-server
+  postgresqlPool:
+    size: 5
+    acquisitionTimeout: 10s
+    agingTimeout: 1d
+    idlenessTimeout: 10m
+
+
+  # Setting this to `true` will start conversation migration to postgresql.
+  #
+  # NOTE: It is very important that galley be configured to with
+  # `settings.postgresMigration.conversation` with `migration-to-postgresql`
+  # before setting this to `true`.
+  migrateConversations: false
 
   backendNotificationPusher:
     pushBackoffMinWait: 10000 # in microseconds, so 10ms

charts/integration/templates/integration-integration.yaml

@@ -60,10 +60,15 @@ spec:
     - name: "federator-ca"
       configMap:
         name: "federator-ca"
+
     - name: "background-worker-config"
       configMap:
         name: "background-worker"
 
+    - name: "background-worker-secrets"
+      secret:
+        secretName: "background-worker"
+
     - name: "stern-config"
       configMap:
         name: "backoffice"
@@ -250,6 +255,9 @@ spec:
     - name: background-worker-config
       mountPath: /etc/wire/background-worker/conf
 
+    - name: background-worker-secrets
+      mountPath: /etc/wire/background-worker/secrets
+
     - name: stern-config
       mountPath: /etc/wire/stern/conf
 

docs/src/developer/reference/config-options.md

@@ -1683,10 +1683,9 @@ used as `password` field.
 
 ### Using PostgreSQL for storing conversation data
 
-This is currently not the default and is experimental.
-The migration path from Cassandra is yet to be programmed.
+#### New Installations
 
-However, new installations can use this by configuring the wire-server helm
+New installations can use this by configuring the wire-server helm
 chart like this:
 
 ```yaml
@@ -1696,6 +1695,61 @@ galley:
       conversation: postgresql
 ```
 
+#### Migration for existing installations
+
+Existing installations should migrate the conversation data to PostgreSQL from
+Cassandra. This is necessary for channel search and management of channels from
+the team-management UI. It is highly recommended to take a backup of the Galley
+Cassandra before triggering the migration.
+
+The migration needs to happen in 3 steps:
+
+1. Prepare wire-server for migration.
+
+   This step make sure that wire-server keep working as expected during the
+   migration. To do this deploy wire-server with this config change:
+
+   ```yaml
+   galley:
+     config:
+       postgresqlMigration:
+         conversation: migrate-to-postgresql
+   ```
+
+   This change should restart all the galley pods, any new conversations will
+   now be written to PostgreSQL.
+
+2. Trigger the migration and wait.
+
+   This step will actually carry out the migration. To do this deploy
+   wire-server with this config change:
+
+   ```yaml
+   background-worker:
+     config:
+       migrateConversations: true
+   ```
+
+   This change should restart the background-worker pods. It is recommended to
+   watch the logs and wait for both of these two metrics to report `1.0`:
+   `wire_local_convs_migration_finished` and `wire_user_remote_convs_migration_finished`.
+   This can take a long time depending on number of conversations in the DB.
+
+3. Configure wire-server to only use PostgreSQL for conversations.
+
+   This will be the configuration which must be used from now on for every new
+   release.
+
+   ```yaml
+   galley:
+     config:
+       postgresqlMigration:
+         conversation: postgresql
+   background-worker:
+     config:
+       migrateConversations: false
+   ```
+
 ## Configure Cells
 
 If Cells integration is enabled, gundeck must be configured with the name of

hack/bin/integration-test.sh

@@ -63,7 +63,7 @@ kubectl -n wire-federation-v1 get secrets rabbitmq -ojson | jq 'del(.metadata.na
 mkdir -p ~/.parallel && touch ~/.parallel/will-cite
 printf '%s\n' "${tests[@]}" | parallel echo "Running helm tests for {}..."
 printf '%s\n' "${tests[@]}" | parallel -P "${HELM_PARALLELISM}" \
-    helm test -n "${NAMESPACE}" "${CHART}" --timeout 900s --filter name="${CHART}-{}-integration" '> logs-{};' \
+    helm test -n "${NAMESPACE}" "${CHART}" --timeout 1200s --filter name="${CHART}-{}-integration" '> logs-{};' \
     echo '$? > stat-{};' \
     echo "==== Done testing {}. ====" '};' \
     kubectl -n "${NAMESPACE}" logs "${CHART}-{}-integration" '>> logs-{};'

hack/helm_vars/postgresql/values.yaml.gotmpl

@@ -2,11 +2,11 @@ auth:
   postgresPassword: "posty-the-gres"
   username: wire-server
   password: "posty-the-gres"
-resources:
-  requests:
-    cpu: "1"
-  limits:
-    cpu: "1"
-postgresql:
-  configuration: |
-    max_connections = 500
+primary:
+  resources:
+    requests:
+      cpu: 1
+      memory: 2Gi
+    limits: {}
+  extendedConfiguration: |
+    max_connections = 1500

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -604,6 +604,11 @@ background-worker:
       pushBackoffMinWait: 1000 # 1ms
       pushBackoffMaxWait: 500000 # 0.5s
       remotesRefreshInterval: 1000000 # 1s
+    postgresql:
+      host: "postgresql"
+      port: "5432"
+      user: wire-server
+      dbname: wire-server
     cassandra:
       host: {{ .Values.cassandraHost }}
       replicaCount: 1
@@ -612,6 +617,14 @@ background-worker:
           name: "cassandra-jks-keystore"
           key: "ca.crt"
       {{- end }}
+    cassandraGalley:
+      host: {{ .Values.cassandraHost }}
+      replicaCount: 1
+      {{- if .Values.useK8ssandraSSL.enabled }}
+      tlsCaSecretRef:
+          name: "cassandra-jks-keystore"
+          key: "ca.crt"
+      {{- end }}
     rabbitmq:
       port: 5671
       adminPort: 15671
@@ -621,6 +634,7 @@ background-worker:
         name: "rabbitmq-certificate"
         key: "ca.crt"
   secrets:
+    pgPassword: "posty-the-gres"
     rabbitmq:
       username: {{ .Values.rabbitmqUsername }}
       password: {{ .Values.rabbitmqPassword }}

integration/integration.cabal

@@ -126,6 +126,7 @@ library
     Test.Client
     Test.Connection
     Test.Conversation
+    Test.Conversation.Migration
     Test.Demo
     Test.DNSMock
     Test.DomainVerification