Skip to content

Commit 01f6b2e

Browse files
jfrochejfroche
committed
feat: support multiple versions of the pgmq extension
Build multiple versions of the pgmq extension on different PostgreSQL versions. Add test for the extensions and their upgrade on PostgreSQL 15 and 17.
1 parent 7d78f14 commit 01f6b2e

File tree

8 files changed

+312
-38
lines changed

8 files changed

+312
-38
lines changed

‎ansible/files/postgresql_extension_custom_scripts/pgmq/after-create.sql

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ begin
1818
physical backups everywhere
1919
*/
2020
-- Detach and delete the official function
21-
alter extension pgmq drop function pgmq.drop_queue;
22-
drop function pgmq.drop_queue;
21+
alter extension pgmq drop function pgmq.drop_queue(TEXT);
22+
drop function pgmq.drop_queue(TEXT);
2323

2424
-- Create and reattach the patched function
2525
CREATE FUNCTION pgmq.drop_queue(queue_name TEXT)
@@ -134,7 +134,7 @@ BEGIN
134134
END;
135135
$func$ LANGUAGE plpgsql;
136136

137-
alter extension pgmq add function pgmq.drop_queue;
137+
alter extension pgmq add function pgmq.drop_queue(TEXT);
138138

139139

140140
update pg_extension set extowner = 'postgres'::regrole where extname = 'pgmq';

‎flake.nix

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1384,6 +1384,8 @@
13841384
devShell = devShells.default;
13851385
} // pkgs.lib.optionalAttrs (system == "aarch64-linux") {
13861386
inherit (basePackages) postgresql_15_debug postgresql_15_src postgresql_orioledb-17_debug postgresql_orioledb-17_src postgresql_17_debug postgresql_17_src;
1387+
} // pkgs.lib.optionalAttrs (system == "x86_64-linux") {
1388+
pgmq = import ./nix/ext/tests/pgmq.nix { inherit self; inherit pkgs; };
13871389
};
13881390

13891391
# Apps is a list of names of things that can be executed with 'nix run';

‎nix/ext/pgmq.nix

Lines changed: 85 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,92 @@
1-
{ lib, stdenv, fetchFromGitHub, postgresql }:
2-
3-
stdenv.mkDerivation rec {
1+
{
2+
lib,
3+
stdenv,
4+
fetchFromGitHub,
5+
postgresql,
6+
buildEnv,
7+
}:
8+
let
49
pname = "pgmq";
5-
version = "1.4.4";
6-
buildInputs = [ postgresql ];
7-
src = fetchFromGitHub {
8-
owner = "tembo-io";
9-
repo = pname;
10-
rev = "v${version}";
11-
hash = "sha256-z+8/BqIlHwlMnuIzMz6eylmYbSmhtsNt7TJf/CxbdVw=";
12-
};
1310

14-
buildPhase = ''
15-
cd pgmq-extension
16-
'';
11+
# Load version configuration from external file
12+
allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname};
13+
14+
# Filter versions compatible with current PostgreSQL version
15+
supportedVersions = lib.filterAttrs (
16+
_: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
17+
) allVersions;
18+
19+
# Derived version information
20+
versions = lib.naturalSort (lib.attrNames supportedVersions);
21+
latestVersion = lib.last versions;
22+
numberOfVersions = builtins.length versions;
23+
packages = builtins.attrValues (
24+
lib.mapAttrs (name: value: build name value.hash) supportedVersions
25+
);
26+
27+
# Build function for individual versions
28+
build =
29+
version: hash:
30+
stdenv.mkDerivation rec {
31+
inherit pname version;
32+
buildInputs = [ postgresql ];
33+
src = fetchFromGitHub {
34+
owner = "tembo-io";
35+
repo = pname;
36+
rev = "v${version}";
37+
inherit hash;
38+
};
39+
40+
buildPhase = ''
41+
cd pgmq-extension
42+
'';
43+
44+
installPhase = ''
45+
runHook preInstall
46+
47+
mkdir -p $out/share/postgresql/extension
48+
49+
# Create versioned sql install script
50+
cp sql/${pname}.sql $out/share/postgresql/extension/${pname}--${version}.sql
51+
52+
# Create versioned control file with modified module path
53+
sed -e "/^default_version =/d" \
54+
-e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}'|" \
55+
${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control
56+
57+
# For the latest version, create default control file and symlink and copy SQL upgrade scripts
58+
if [[ "${version}" == "${latestVersion}" ]]; then
59+
{
60+
echo "default_version = '${version}'"
61+
cat $out/share/postgresql/extension/${pname}--${version}.control
62+
} > $out/share/postgresql/extension/${pname}.control
63+
cp sql/*.sql $out/share/postgresql/extension
64+
fi
65+
66+
runHook postInstall
67+
'';
1768

18-
installPhase = ''
19-
mkdir -p $out/{lib,share/postgresql/extension}
69+
meta = with lib; {
70+
description = "A lightweight message queue. Like AWS SQS and RSMQ but on Postgres.";
71+
homepage = "https://github.com/tembo-io/pgmq";
72+
maintainers = with maintainers; [ olirice ];
73+
inherit (postgresql.meta) platforms;
74+
license = licenses.postgresql;
75+
};
76+
};
77+
in
78+
buildEnv {
79+
name = pname;
80+
paths = packages;
2081

21-
mv sql/pgmq.sql $out/share/postgresql/extension/pgmq--${version}.sql
22-
cp sql/*.sql $out/share/postgresql/extension
23-
cp *.control $out/share/postgresql/extension
24-
'';
82+
pathsToLink = [
83+
"/share/postgresql/extension"
84+
];
2585

26-
meta = with lib; {
27-
description = "A lightweight message queue. Like AWS SQS and RSMQ but on Postgres.";
28-
homepage = "https://github.com/tembo-io/pgmq";
29-
maintainers = with maintainers; [ olirice ];
30-
platforms = postgresql.meta.platforms;
31-
license = licenses.postgresql;
86+
passthru = {
87+
inherit versions numberOfVersions;
88+
pname = "${pname}-all";
89+
version =
90+
"multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
3291
};
3392
}

‎nix/ext/tests/pgmq.nix

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
{ self, pkgs }:
2+
let
3+
pname = "pgmq";
4+
inherit (pkgs) lib;
5+
installedExtension =
6+
postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all";
7+
versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions;
8+
postgresqlWithExtension =
9+
postgresql:
10+
let
11+
majorVersion = lib.versions.major postgresql.version;
12+
pkg = pkgs.buildEnv {
13+
name = "postgresql-${majorVersion}-${pname}";
14+
paths = [
15+
postgresql
16+
postgresql.lib
17+
(installedExtension majorVersion)
18+
];
19+
passthru = {
20+
inherit (postgresql) version psqlSchema;
21+
lib = pkg;
22+
withPackages = _: pkg;
23+
};
24+
nativeBuildInputs = [ pkgs.makeWrapper ];
25+
pathsToLink = [
26+
"/"
27+
"/bin"
28+
"/lib"
29+
];
30+
postBuild = ''
31+
wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib
32+
wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib
33+
wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib
34+
'';
35+
};
36+
in
37+
pkg;
38+
in
39+
self.inputs.nixpkgs.lib.nixos.runTest {
40+
name = pname;
41+
hostPkgs = pkgs;
42+
nodes.server =
43+
{ config, ... }:
44+
{
45+
virtualisation = {
46+
forwardPorts = [
47+
{
48+
from = "host";
49+
host.port = 13022;
50+
guest.port = 22;
51+
}
52+
];
53+
};
54+
services.openssh = {
55+
enable = true;
56+
};
57+
58+
services.postgresql = {
59+
enable = true;
60+
package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15;
61+
};
62+
63+
specialisation.postgresql17.configuration = {
64+
services.postgresql = {
65+
package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17);
66+
};
67+
68+
systemd.services.postgresql-migrate = {
69+
serviceConfig = {
70+
Type = "oneshot";
71+
RemainAfterExit = true;
72+
User = "postgres";
73+
Group = "postgres";
74+
StateDirectory = "postgresql";
75+
WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}";
76+
};
77+
script =
78+
let
79+
oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15;
80+
newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17;
81+
oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}";
82+
newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}";
83+
in
84+
''
85+
if [[ ! -d ${newDataDir} ]]; then
86+
install -d -m 0700 -o postgres -g postgres "${newDataDir}"
87+
${newPostgresql}/bin/initdb -D "${newDataDir}"
88+
${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \
89+
--old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin"
90+
else
91+
echo "${newDataDir} already exists"
92+
fi
93+
'';
94+
};
95+
96+
systemd.services.postgresql = {
97+
after = [ "postgresql-migrate.service" ];
98+
requires = [ "postgresql-migrate.service" ];
99+
};
100+
};
101+
102+
};
103+
testScript =
104+
{ nodes, ... }:
105+
let
106+
pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17";
107+
in
108+
''
109+
versions = {
110+
"15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}],
111+
"17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}],
112+
}
113+
114+
def run_sql(query):
115+
return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip()
116+
117+
def check_upgrade_path(pg_version):
118+
with subtest("Check ${pname} upgrade path"):
119+
firstVersion = versions[pg_version][0]
120+
server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'")
121+
run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""")
122+
installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""")
123+
assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}"
124+
for version in versions[pg_version][1:]:
125+
run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""")
126+
installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""")
127+
assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}"
128+
129+
start_all()
130+
131+
server.wait_for_unit("multi-user.target")
132+
server.wait_for_unit("postgresql.service")
133+
134+
check_upgrade_path("15")
135+
136+
with subtest("Check ${pname} latest extension version"):
137+
server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'")
138+
server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'")
139+
installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""")
140+
latestVersion = versions["15"][-1]
141+
assert f"${pname},{latestVersion}" in installed_extensions
142+
143+
with subtest("switch to postgresql 17"):
144+
server.succeed(
145+
"${pg17-configuration}/bin/switch-to-configuration test >&2"
146+
)
147+
148+
with subtest("Check ${pname} latest extension version after upgrade"):
149+
installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""")
150+
latestVersion = versions["17"][-1]
151+
assert f"${pname},{latestVersion}" in installed_extensions
152+
153+
check_upgrade_path("17")
154+
'';
155+
}

‎nix/ext/versions.json

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"pgmq": {
3+
"1.4.4": {
4+
"postgresql": [
5+
"15"
6+
],
7+
"hash": "sha256-z+8/BqIlHwlMnuIzMz6eylmYbSmhtsNt7TJf/CxbdVw="
8+
},
9+
"1.5.1": {
10+
"postgresql": [
11+
"15",
12+
"17"
13+
],
14+
"hash": "sha256-IU+i6ONPwtgsFKdzya6E+222ualR66gkbb0lDr+7Rb8="
15+
}
16+
17+
}
18+
}

‎nix/tests/expected/pgmq.out

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ order by
160160
-------------+-------------------------------+----------
161161
pgmq | _belongs_to_pgmq | postgres
162162
pgmq | _ensure_pg_partman_installed | postgres
163+
pgmq | _extension_exists | postgres
163164
pgmq | _get_partition_col | postgres
164165
pgmq | _get_pg_partman_major_version | postgres
165166
pgmq | _get_pg_partman_schema | postgres
@@ -174,6 +175,7 @@ order by
174175
pgmq | delete | postgres
175176
pgmq | detach_archive | postgres
176177
pgmq | drop_queue | postgres
178+
pgmq | drop_queue | postgres
177179
pgmq | format_table_name | postgres
178180
pgmq | list_queues | postgres
179181
pgmq | metrics | postgres
@@ -183,8 +185,18 @@ order by
183185
pgmq | read | postgres
184186
pgmq | read_with_poll | postgres
185187
pgmq | send | postgres
188+
pgmq | send | postgres
189+
pgmq | send | postgres
190+
pgmq | send | postgres
191+
pgmq | send | postgres
192+
pgmq | send | postgres
193+
pgmq | send_batch | postgres
194+
pgmq | send_batch | postgres
195+
pgmq | send_batch | postgres
196+
pgmq | send_batch | postgres
197+
pgmq | send_batch | postgres
186198
pgmq | send_batch | postgres
187199
pgmq | set_vt | postgres
188200
pgmq | validate_queue_name | postgres
189-
(28 rows)
201+
(40 rows)
190202

0 commit comments

Comments
 (0)