Skip to content

Commit 0fc42a5

Browse files
jfrochesamrose
jfroche
and
samrose
authored
feat: support multiple versions of the pg_repack extension (#1688)
* feat: support multiple versions of the pg_repack extension Build multiple versions of the pg_repack extension on different PostgreSQL versions. Add test for the extensions and their upgrade on PostgreSQL 15 and 17. * chore: bump to test * tests: basic test for repack * chore: bump to release --------- Co-authored-by: Sam Rose <samuel@supabase.io>
1 parent 234da12 commit 0fc42a5

File tree

6 files changed

+400
-48
lines changed

6 files changed

+400
-48
lines changed

‎ansible/vars.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ postgres_major:
1010

1111
# Full version strings for each major version
1212
postgres_release:
13-
postgresorioledb-17: "17.5.1.047-orioledb"
14-
postgres17: "17.6.1.026"
15-
postgres15: "15.14.1.026"
13+
postgresorioledb-17: "17.5.1.048-orioledb"
14+
postgres17: "17.6.1.027"
15+
postgres15: "15.14.1.027"
1616

1717
# Non Postgres Extensions
1818
pgbouncer_release: 1.19.0

‎nix/ext/pg_repack.nix‎

Lines changed: 121 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -5,57 +5,133 @@
55
postgresql,
66
postgresqlTestHook,
77
testers,
8+
buildEnv,
89
}:
9-
10-
stdenv.mkDerivation (finalAttrs: {
10+
let
1111
pname = "pg_repack";
12-
version = "1.5.2";
1312

14-
buildInputs = postgresql.buildInputs ++ [ postgresql ];
13+
# Load version configuration from external file
14+
allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname};
1515

16-
src = fetchFromGitHub {
17-
owner = "reorg";
18-
repo = "pg_repack";
19-
rev = "ver_${finalAttrs.version}";
20-
hash = "sha256-wfjiLkx+S3zVrAynisX1GdazueVJ3EOwQEPcgUQt7eA=";
21-
};
16+
# Filter versions compatible with current PostgreSQL version
17+
supportedVersions = lib.filterAttrs (
18+
_: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
19+
) allVersions;
2220

23-
installPhase = ''
24-
install -D bin/pg_repack -t $out/bin/
25-
install -D lib/pg_repack${postgresql.dlSuffix} -t $out/lib/
26-
install -D lib/{pg_repack--${finalAttrs.version}.sql,pg_repack.control} -t $out/share/postgresql/extension
27-
'';
21+
# Derived version information
22+
versions = lib.naturalSort (lib.attrNames supportedVersions);
23+
latestVersion = lib.last versions;
24+
numberOfVersions = builtins.length versions;
25+
packages = builtins.attrValues (
26+
lib.mapAttrs (name: value: build name value.hash) supportedVersions
27+
);
28+
29+
# Build function for individual versions
30+
build =
31+
version: hash:
32+
stdenv.mkDerivation (finalAttrs: {
33+
inherit pname version;
34+
35+
buildInputs = postgresql.buildInputs ++ [ postgresql ];
36+
37+
src = fetchFromGitHub {
38+
owner = "reorg";
39+
repo = "pg_repack";
40+
rev = "ver_${finalAttrs.version}";
41+
inherit hash;
42+
};
43+
44+
installPhase = ''
45+
mkdir -p $out/{lib,share/postgresql/extension,bin}
46+
47+
mv bin/${pname} $out/bin/${pname}-${version}
48+
49+
# Install shared library with version suffix
50+
mv lib/${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix}
2851
29-
passthru.tests = {
30-
version = testers.testVersion { package = finalAttrs.finalPackage; };
31-
extension = stdenv.mkDerivation {
32-
name = "plpgsql-check-test";
33-
dontUnpack = true;
34-
doCheck = true;
35-
buildInputs = [ postgresqlTestHook ];
36-
nativeCheckInputs = [ (postgresql.withPackages (ps: [ ps.pg_repack ])) ];
37-
postgresqlTestUserOptions = "LOGIN SUPERUSER";
38-
failureHook = "postgresqlStop";
39-
checkPhase = ''
40-
runHook preCheck
41-
psql -a -v ON_ERROR_STOP=1 -c "CREATE EXTENSION pg_repack;"
42-
runHook postCheck
52+
# Create version-specific control file
53+
sed -e "/^default_version =/d" \
54+
-e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}-${version}'|" \
55+
lib/${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control
56+
57+
# Copy SQL install script
58+
cp lib/${pname}--${version}.sql $out/share/postgresql/extension
59+
60+
# For the latest version, create default control file and symlink and copy SQL upgrade scripts
61+
if [[ "${version}" == "${latestVersion}" ]]; then
62+
{
63+
echo "default_version = '${version}'"
64+
cat $out/share/postgresql/extension/${pname}--${version}.control
65+
} > $out/share/postgresql/extension/${pname}.control
66+
ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
67+
ln -sfn $out/bin/${pname}-${latestVersion} $out/bin/${pname}
68+
fi
69+
#install -D bin/pg_repack -t $out/bin/
70+
#install -D lib/pg_repack${postgresql.dlSuffix} -t $out/lib/
71+
#install -D lib/{pg_repack--${finalAttrs.version}.sql,pg_repack.control} -t $out/share/postgresql/extension
4372
'';
44-
installPhase = "touch $out";
45-
};
46-
};
4773

48-
meta = with lib; {
49-
description = "Reorganize tables in PostgreSQL databases with minimal locks";
50-
longDescription = ''
51-
pg_repack is a PostgreSQL extension which lets you remove bloat from tables and indexes, and optionally restore
52-
the physical order of clustered indexes. Unlike CLUSTER and VACUUM FULL it works online, without holding an
53-
exclusive lock on the processed tables during processing. pg_repack is efficient to boot,
54-
with performance comparable to using CLUSTER directly.
55-
'';
56-
homepage = "https://github.com/reorg/pg_repack";
57-
license = licenses.bsd3;
58-
inherit (postgresql.meta) platforms;
59-
mainProgram = "pg_repack";
74+
passthru.tests = {
75+
version = testers.testVersion { package = finalAttrs.finalPackage; };
76+
extension = stdenv.mkDerivation {
77+
name = "plpgsql-check-test";
78+
dontUnpack = true;
79+
doCheck = true;
80+
buildInputs = [ postgresqlTestHook ];
81+
nativeCheckInputs = [ (postgresql.withPackages (ps: [ ps.pg_repack ])) ];
82+
postgresqlTestUserOptions = "LOGIN SUPERUSER";
83+
failureHook = "postgresqlStop";
84+
checkPhase = ''
85+
runHook preCheck
86+
psql -a -v ON_ERROR_STOP=1 -c "CREATE EXTENSION pg_repack;"
87+
runHook postCheck
88+
'';
89+
installPhase = "touch $out";
90+
};
91+
};
92+
93+
meta = with lib; {
94+
description = "Reorganize tables in PostgreSQL databases with minimal locks";
95+
longDescription = ''
96+
pg_repack is a PostgreSQL extension which lets you remove bloat from tables and indexes, and optionally restore
97+
the physical order of clustered indexes. Unlike CLUSTER and VACUUM FULL it works online, without holding an
98+
exclusive lock on the processed tables during processing. pg_repack is efficient to boot,
99+
with performance comparable to using CLUSTER directly.
100+
'';
101+
homepage = "https://github.com/reorg/pg_repack";
102+
license = licenses.bsd3;
103+
inherit (postgresql.meta) platforms;
104+
mainProgram = "pg_repack";
105+
};
106+
});
107+
in
108+
buildEnv {
109+
name = pname;
110+
paths = packages;
111+
112+
pathsToLink = [
113+
"/bin"
114+
"/lib"
115+
"/share/postgresql/extension"
116+
];
117+
118+
postBuild = ''
119+
# Verify all expected library files are present
120+
expectedFiles=${toString (numberOfVersions + 1)}
121+
actualFiles=$(ls -l $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)
122+
123+
if [[ "$actualFiles" != "$expectedFiles" ]]; then
124+
echo "Error: Expected $expectedFiles library files, found $actualFiles"
125+
echo "Files found:"
126+
ls -la $out/lib/*${postgresql.dlSuffix} || true
127+
exit 1
128+
fi
129+
'';
130+
131+
passthru = {
132+
inherit versions numberOfVersions;
133+
pname = "${pname}-all";
134+
version =
135+
"multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
60136
};
61-
})
137+
}

‎nix/ext/tests/pg_repack.nix‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
{ self, pkgs }:
2+
let
3+
pname = "pg_repack";
4+
inherit (pkgs) lib;
5+
installedExtension =
6+
postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all";
7+
versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions;
8+
postgresqlWithExtension =
9+
postgresql:
10+
let
11+
majorVersion = lib.versions.major postgresql.version;
12+
pkg = pkgs.buildEnv {
13+
name = "postgresql-${majorVersion}-${pname}";
14+
paths = [
15+
postgresql
16+
postgresql.lib
17+
(installedExtension majorVersion)
18+
];
19+
passthru = {
20+
inherit (postgresql) version psqlSchema;
21+
lib = pkg;
22+
withPackages = _: pkg;
23+
};
24+
nativeBuildInputs = [ pkgs.makeWrapper ];
25+
pathsToLink = [
26+
"/"
27+
"/bin"
28+
"/lib"
29+
];
30+
postBuild = ''
31+
wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib
32+
wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib
33+
wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib
34+
'';
35+
};
36+
in
37+
pkg;
38+
in
39+
self.inputs.nixpkgs.lib.nixos.runTest {
40+
name = pname;
41+
hostPkgs = pkgs;
42+
nodes.server =
43+
{ config, ... }:
44+
{
45+
virtualisation = {
46+
forwardPorts = [
47+
{
48+
from = "host";
49+
host.port = 13022;
50+
guest.port = 22;
51+
}
52+
];
53+
};
54+
services.openssh = {
55+
enable = true;
56+
};
57+
58+
services.postgresql = {
59+
enable = true;
60+
package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15;
61+
enableTCPIP = true;
62+
initialScript = pkgs.writeText "init-postgres-with-password" ''
63+
CREATE USER test WITH PASSWORD 'secret';
64+
'';
65+
authentication = ''
66+
host test postgres samenet scram-sha-256
67+
'';
68+
};
69+
70+
networking.firewall.allowedTCPPorts = [ config.services.postgresql.settings.port ];
71+
72+
specialisation.postgresql17.configuration = {
73+
services.postgresql = {
74+
package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17);
75+
};
76+
77+
systemd.services.postgresql-migrate = {
78+
serviceConfig = {
79+
Type = "oneshot";
80+
RemainAfterExit = true;
81+
User = "postgres";
82+
Group = "postgres";
83+
StateDirectory = "postgresql";
84+
WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}";
85+
};
86+
script =
87+
let
88+
oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15;
89+
newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17;
90+
oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}";
91+
newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}";
92+
in
93+
''
94+
if [[ ! -d ${newDataDir} ]]; then
95+
install -d -m 0700 -o postgres -g postgres "${newDataDir}"
96+
${newPostgresql}/bin/initdb -D "${newDataDir}"
97+
${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \
98+
--old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin"
99+
else
100+
echo "${newDataDir} already exists"
101+
fi
102+
'';
103+
};
104+
105+
systemd.services.postgresql = {
106+
after = [ "postgresql-migrate.service" ];
107+
requires = [ "postgresql-migrate.service" ];
108+
};
109+
};
110+
};
111+
testScript =
112+
{ nodes, ... }:
113+
let
114+
pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17";
115+
in
116+
''
117+
from pathlib import Path
118+
versions = {
119+
"15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}],
120+
"17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}],
121+
}
122+
extension_name = "${pname}"
123+
support_upgrade = False
124+
pg17_configuration = "${pg17-configuration}"
125+
sql_test_directory = Path("${../../tests}")
126+
127+
${builtins.readFile ./lib.py}
128+
129+
start_all()
130+
131+
server.wait_for_unit("multi-user.target")
132+
server.wait_for_unit("postgresql.service")
133+
134+
test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade)
135+
136+
with subtest("Check upgrade path with postgresql 15"):
137+
test.check_upgrade_path("15")
138+
139+
last_version = None
140+
with subtest("Check the install of the last version of the extension"):
141+
last_version = test.check_install_last_version("15")
142+
143+
with subtest("switch to postgresql 17"):
144+
server.succeed(
145+
f"{pg17_configuration}/bin/switch-to-configuration test >&2"
146+
)
147+
148+
with subtest("Check last version of the extension after upgrade"):
149+
test.assert_version_matches(last_version)
150+
151+
with subtest("Check upgrade path with postgresql 17"):
152+
test.check_upgrade_path("17")
153+
'';
154+
}

‎nix/ext/versions.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -576,5 +576,35 @@
576576
"pgrx": "0.16.0",
577577
"rust": "1.87.0"
578578
}
579+
},
580+
"pg_repack": {
581+
"1.4.8": {
582+
"postgresql": [
583+
"15",
584+
"17"
585+
],
586+
"hash": "sha256-Et8aMRzG7ez0uy9wG6qsg57/kPPZdUhb+/gFxW86D08="
587+
},
588+
"1.5.0": {
589+
"postgresql": [
590+
"15",
591+
"17"
592+
],
593+
"hash": "sha256-do80phyMxwcRIkYyUt9z02z7byNQhK+pbSaCUmzG+4c="
594+
},
595+
"1.5.1": {
596+
"postgresql": [
597+
"15",
598+
"17"
599+
],
600+
"hash": "sha256-wJwy4qIt6/kgWqT6HbckUVqDayDkixqHpYiC1liLERw="
601+
},
602+
"1.5.2": {
603+
"postgresql": [
604+
"15",
605+
"17"
606+
],
607+
"hash": "sha256-wfjiLkx+S3zVrAynisX1GdazueVJ3EOwQEPcgUQt7eA="
608+
}
579609
}
580610
}

0 commit comments

Comments
 (0)