fix: covering migrations for wrappers across all versions` (#1876)

* fix: covering migrations for wrappers across all versions`

* chore: remove comment

* fix: remove copy and deref

* Always run in-place upgrade extension in default nixos tests

We do not want to uninstall and install the extension during an upgrade test.

* chore: bump to release

---------

Co-authored-by: Jean-François Roche <jfroche@pyxel.be>

Author: samrose

ansible/vars.yml

@@ -10,9 +10,9 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.5.1.051-orioledb"
-  postgres17: "17.6.1.030"
-  postgres15: "15.14.1.030"
+  postgresorioledb-17: "17.5.1.052-orioledb"
+  postgres17: "17.6.1.031"
+  postgres15: "15.14.1.031"
 
 # Non Postgres Extensions
 pgbouncer_release: 1.19.0

nix/ext/tests/default.nix

@@ -131,7 +131,6 @@ let
             "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}],
           }
           extension_name = "${pname}"
-          support_upgrade = False
           pg17_configuration = "${pg17-configuration}"
           ext_has_background_worker = ${
             if (installedExtension "15") ? hasBackgroundWorker then "True" else "False"
@@ -146,7 +145,7 @@ let
           server.wait_for_unit("multi-user.target")
           server.wait_for_unit("postgresql.service")
 
-          test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade)
+          test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory)
 
           with subtest("Check upgrade path with postgresql 15"):
             test.check_upgrade_path("15")

nix/ext/wrappers/default.nix

@@ -171,7 +171,12 @@ let
         };
       }
     );
-  previouslyPackagedVersions = [
+  # All versions that were previously packaged (historical list)
+  allPreviouslyPackagedVersions = [
+    "0.4.3"
+    "0.4.2"
+    "0.4.1"
+    "0.3.0"
     "0.2.0"
     "0.1.19"
     "0.1.18"
@@ -191,14 +196,19 @@ let
     "0.1.1"
     "0.1.0"
   ];
-  numberOfPreviouslyPackagedVersions = builtins.length previouslyPackagedVersions;
   allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).wrappers;
   supportedVersions = lib.filterAttrs (
     _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
   ) allVersions;
   versions = lib.naturalSort (lib.attrNames supportedVersions);
   latestVersion = lib.last versions;
   numberOfVersions = builtins.length versions;
+  # Filter out previously packaged versions that are actually built for this PG version
+  # This prevents double-counting when a version appears in both lists
+  previouslyPackagedVersions = builtins.filter (
+    v: !(builtins.elem v versions)
+  ) allPreviouslyPackagedVersions;
+  numberOfPreviouslyPackagedVersions = builtins.length previouslyPackagedVersions;
   packages = builtins.attrValues (
     lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions
   );
@@ -231,26 +241,40 @@ buildEnv {
     }
 
     create_migration_sql_files() {
+
+
       PREVIOUS_VERSION=""
       while IFS= read -r i; do
         FILENAME=$(basename "$i")
-        DIRNAME=$(dirname "$i")
         VERSION="$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+' <<< $FILENAME)"
         if [[ "$PREVIOUS_VERSION" != "" ]]; then
-          echo "Processing $i"
-          MIGRATION_FILENAME="$DIRNAME/''${FILENAME/$VERSION/$PREVIOUS_VERSION--$VERSION}"
-          cp "$i" "$MIGRATION_FILENAME"
+          # Always write to $out/share/postgresql/extension, not $DIRNAME
+          # because $DIRNAME might be a symlinked read-only path from the Nix store
+          # We use -L with cp to dereference symlinks (copy the actual file content, not the symlink)
+          MIGRATION_FILENAME="$out/share/postgresql/extension/''${FILENAME/$VERSION/$PREVIOUS_VERSION--$VERSION}"
+          cp -L "$i" "$MIGRATION_FILENAME"
         fi
         PREVIOUS_VERSION="$VERSION"
       done < <(find $out -name '*.sql' | sort -V)
 
+      # Create empty SQL files for previously packaged versions that don't exist
+      # This compensates for versions that failed to produce SQL files in the past
+      for prev_version in ${lib.concatStringsSep " " previouslyPackagedVersions}; do
+        sql_file="$out/share/postgresql/extension/wrappers--$prev_version.sql"
+        if [ ! -f "$sql_file" ]; then
+          echo "-- Empty migration file for previously packaged version $prev_version" > "$sql_file"
+        fi
+      done
+
       # Create migration SQL files from previous versions to newer versions
+      # Skip if the migration file already exists (to avoid conflicts with the first loop)
       for prev_version in ${lib.concatStringsSep " " previouslyPackagedVersions}; do
         for curr_version in ${lib.concatStringsSep " " versions}; do
           if [[ "$(printf '%s\n%s' "$prev_version" "$curr_version" | sort -V | head -n1)" == "$prev_version" ]] && [[ "$prev_version" != "$curr_version" ]]; then
             main_sql_file="$out/share/postgresql/extension/wrappers--$curr_version.sql"
-            if [ -f "$main_sql_file" ]; then
-              new_file="$out/share/postgresql/extension/wrappers--$prev_version--$curr_version.sql"
+            new_file="$out/share/postgresql/extension/wrappers--$prev_version--$curr_version.sql"
+            # Only create if it doesn't already exist (first loop may have created it)
+            if [ -f "$main_sql_file" ] && [ ! -f "$new_file" ]; then
               cp "$main_sql_file" "$new_file"
               sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$new_file"
             fi
@@ -263,6 +287,7 @@ buildEnv {
     create_lib_files
     create_migration_sql_files
 
+    # Verify library count matches expected
     (test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${
       toString (numberOfVersions + numberOfPreviouslyPackagedVersions + 1)
     }")