Skip to content

Commit 29aa02a

Browse files
authored
Merge branch 'develop' into fix/postgres-storage-grants
2 parents 368343c + f5ba2a5 commit 29aa02a

File tree

8 files changed

+269
-22
lines changed

8 files changed

+269
-22
lines changed

.github/workflows/qemu-image-build.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ on:
1010
- 'qemu-arm64-nix.pkr.hcl'
1111
- 'common-nix.vars.pkr.hcl'
1212
- 'ansible/vars.yml'
13+
- 'scripts/*'
1314
workflow_dispatch:
1415

1516
permissions:
@@ -30,7 +31,7 @@ jobs:
3031
- name: Set PostgreSQL versions - only builds pg17 atm
3132
id: set-versions
3233
run: |
33-
VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[1,2]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
34+
VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[1]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
3435
echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
3536
3637
build:

Dockerfile-kubernetes

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
1-
FROM alpine:3.21
1+
FROM alpine:3.22
22

33
ADD ./output-cloudimg/packer-cloudimg /disk/focal.qcow2
44

5-
RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh-client nftables cloud-utils-localds aavmf virtiofsd
6-
# dev stuff
7-
# RUN apk add --no-cache iproute2
5+
RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh-client aavmf virtiofsd \
6+
&& truncate -s 64M /root/varstore.img \
7+
&& truncate -s 64M /root/efi.img \
8+
&& dd if=/usr/share/AAVMF/QEMU_EFI.fd of=/root/efi.img conv=notrunc \
9+
&& qemu-img create -f qcow2 /tmp/disk.qcow2 -b /disk/focal.qcow2 -F qcow2 \
10+
&& apk del --no-cache aavmf qemu-img
811

912
CMD exec /bin/sh -c "trap : TERM INT; sleep 9999999999d & wait"

ansible/files/gotrue.service.j2

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,12 @@ Description=Gotrue
44
[Service]
55
Type=simple
66
WorkingDirectory=/opt/gotrue
7+
{% if qemu_mode is defined and qemu_mode %}
8+
ExecStart=/opt/gotrue/gotrue
9+
{% else %}
710
ExecStart=/opt/gotrue/gotrue --config-dir /etc/auth.d
11+
{% endif %}
12+
813
User=gotrue
914
Restart=always
1015
RestartSec=3

ansible/tasks/setup-postgrest.yml

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,28 @@
66
url: https://www.postgresql.org/media/keys/ACCC4CF8.asc
77
state: present
88

9-
- name: PostgREST - add Postgres PPA
9+
- name: PostgREST - add Postgres PPA main
1010
apt_repository:
11-
repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg {{ postgresql_major }}"
11+
repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main"
1212
state: present
13+
filename: postgresql-pgdg
1314

14-
- name: PostgREST - update apt cache
15+
- name: PostgREST - install system dependencies
1516
apt:
16-
update_cache: yes
17-
18-
# libpq is a C library that enables user programs to communicate with
19-
# the PostgreSQL database server.
20-
- name: PostgREST - system dependencies
21-
apt:
22-
pkg:
17+
package:
2318
- libpq5
2419
- libnuma-dev
20+
update_cache: yes
21+
state: present
22+
23+
- name: PostgREST - verify libpq5 version
24+
shell: dpkg -l libpq5 | grep '^ii' | awk '{print $3}'
25+
register: libpq5_version
26+
changed_when: false
27+
28+
- name: Show installed libpq5 version
29+
debug:
30+
msg: "Installed libpq5 version: {{ libpq5_version.stdout }}"
2531

2632
- name: PostgREST - remove Postgres PPA gpg key
2733
apt_key:
@@ -30,7 +36,7 @@
3036

3137
- name: PostgREST - remove Postgres PPA
3238
apt_repository:
33-
repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg {{ postgresql_major }}"
39+
repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main"
3440
state: absent
3541

3642
- name: postgis - ensure dependencies do not get autoremoved
@@ -88,7 +94,6 @@
8894
#! /usr/bin/env bash
8995
set -euo pipefail
9096
set -x
91-
9297
cd "$(dirname "$0")"
9398
cat $@ > merged.conf
9499
dest: /etc/postgrest/merge.sh

ansible/vars.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ postgres_major:
99

1010
# Full version strings for each major version
1111
postgres_release:
12-
postgresorioledb-17: "17.0.1.094-orioledb"
13-
postgres17: "17.4.1.044"
14-
postgres15: "15.8.1.101"
12+
postgresorioledb-17: "17.0.1.096-orioledb"
13+
postgres17: "17.4.1.046"
14+
postgres15: "15.8.1.103"
1515

1616
# Non Postgres Extensions
1717
pgbouncer_release: "1.19.0"

ebssurrogate/scripts/qemu-bootstrap-nix.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,6 @@ execute_stage2_playbook
155155
# we do not want to ship an initialized DB as this is performed as needed
156156
mkdir -p /db/template
157157
mv /data/pgdata /db/template
158+
cloud-init clean --logs
158159
clean_legacy_things
159160
clean_system
160-
cloud-init clean --logs

scripts/90-cleanup-qemu.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,10 @@ elif [ -n "$(command -v apt-get)" ]; then
4343
add-apt-repository --yes --remove ppa:ansible/ansible
4444

4545
source /etc/os-release
46-
apt-get -y remove --purge linux-headers-5.11.0-1021-aws
46+
47+
apt-mark manual libevent-2.1-7 # required for pgbouncer
48+
apt-get -y remove --purge ansible-core apport appstream bash-completion bcache-tools bind9-dnsutils bind9-host bind9-libs bolt btrfs-progs byobu command-not-found console-setup distro-info eject fonts-ubuntu-console friendly-recovery ftp fwupd gawk gdisk keyboard-configuration libvolume-key1 libicu66 libssl-dev lvm2 lxd-agent-loader man-db mdadm modemmanager mtd-utils nano netcat-openbsd nfs-common ntfs-3g parted pastebinit screen strace thin-provisioning-tools tmux usb-modeswitch vim vim-runtime wget whiptail xfsprogs
49+
apt remove -y --purge libc6-dev linux-libc-dev libevent-dev libpcre3-dev libsystemd-dev
4750

4851
apt-get -y update
4952
apt-get -y upgrade

testinfra/test_ami_nix.py

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -507,3 +507,233 @@ def test_postgrest_ending_empty_key_query_parameter_is_removed(host):
507507
},
508508
)
509509
assert res.ok
510+
511+
512+
def test_postgresql_version(host):
513+
"""Print the PostgreSQL version being tested and ensure it's >= 14."""
514+
result = run_ssh_command(host['ssh'], "sudo -u postgres psql -c 'SELECT version();'")
515+
if result['succeeded']:
516+
print(f"\nPostgreSQL Version:\n{result['stdout']}")
517+
# Extract version number from the output
518+
version_line = result['stdout'].strip().split('\n')[2] # Skip header and get the actual version
519+
# Extract major version number (e.g., "15.8" -> 15)
520+
import re
521+
version_match = re.search(r'PostgreSQL (\d+)\.', version_line)
522+
if version_match:
523+
major_version = int(version_match.group(1))
524+
print(f"PostgreSQL major version: {major_version}")
525+
assert major_version >= 14, f"PostgreSQL version {major_version} is less than 14"
526+
else:
527+
assert False, "Could not parse PostgreSQL version number"
528+
else:
529+
print(f"\nFailed to get PostgreSQL version: {result['stderr']}")
530+
assert False, "Failed to get PostgreSQL version"
531+
532+
# Also get the version from the command line
533+
result = run_ssh_command(host['ssh'], "sudo -u postgres psql --version")
534+
if result['succeeded']:
535+
print(f"PostgreSQL Client Version: {result['stdout'].strip()}")
536+
else:
537+
print(f"Failed to get PostgreSQL client version: {result['stderr']}")
538+
539+
print("✓ PostgreSQL version is >= 14")
540+
541+
542+
def test_libpq5_version(host):
543+
"""Print the libpq5 version installed and ensure it's >= 14."""
544+
# Try different package managers to find libpq5
545+
result = run_ssh_command(host['ssh'], "dpkg -l | grep libpq5 || true")
546+
if result['succeeded'] and result['stdout'].strip():
547+
print(f"\nlibpq5 package info:\n{result['stdout']}")
548+
# Extract version from dpkg output (format: ii libpq5:arm64 17.5-1.pgdg20.04+1)
549+
import re
550+
version_match = re.search(r'libpq5[^ ]* +(\d+)\.', result['stdout'])
551+
if version_match:
552+
major_version = int(version_match.group(1))
553+
print(f"libpq5 major version: {major_version}")
554+
assert major_version >= 14, f"libpq5 version {major_version} is less than 14"
555+
else:
556+
print("Could not parse libpq5 version from dpkg output")
557+
else:
558+
print("\nlibpq5 not found via dpkg")
559+
560+
# Also try to find libpq.so files
561+
result = run_ssh_command(host['ssh'], "find /usr -name '*libpq*' -type f 2>/dev/null | head -10")
562+
if result['succeeded'] and result['stdout'].strip():
563+
print(f"\nlibpq files found:\n{result['stdout']}")
564+
else:
565+
print("\nNo libpq files found")
566+
567+
# Check if we can get version from a libpq file
568+
result = run_ssh_command(host['ssh'], "ldd /usr/bin/psql | grep libpq || true")
569+
if result['succeeded'] and result['stdout'].strip():
570+
print(f"\npsql libpq dependency:\n{result['stdout']}")
571+
else:
572+
print("\nCould not find libpq dependency for psql")
573+
574+
# Try to get version from libpq directly
575+
result = run_ssh_command(host['ssh'], "psql --version 2>&1 | head -1")
576+
if result['succeeded'] and result['stdout'].strip():
577+
print(f"\npsql version output: {result['stdout'].strip()}")
578+
# The psql version should match the libpq version
579+
import re
580+
version_match = re.search(r'psql \(PostgreSQL\) (\d+)\.', result['stdout'])
581+
if version_match:
582+
major_version = int(version_match.group(1))
583+
print(f"psql/libpq major version: {major_version}")
584+
assert major_version >= 14, f"psql/libpq version {major_version} is less than 14"
585+
else:
586+
print("Could not parse psql version")
587+
588+
print("✓ libpq5 version is >= 14")
589+
590+
591+
def test_postgrest_read_only_session_attrs(host):
592+
"""Test PostgREST with target_session_attrs=read-only and check for session errors."""
593+
# First, check if PostgreSQL is configured for read-only mode
594+
result = run_ssh_command(host['ssh'], "sudo -u postgres psql -c \"SHOW default_transaction_read_only;\"")
595+
if result['succeeded']:
596+
default_read_only = result['stdout'].strip()
597+
print(f"PostgreSQL default_transaction_read_only: {default_read_only}")
598+
else:
599+
print("Could not check PostgreSQL read-only setting")
600+
default_read_only = "unknown"
601+
602+
# Check if PostgreSQL is in recovery mode (standby)
603+
result = run_ssh_command(host['ssh'], "sudo -u postgres psql -c \"SELECT pg_is_in_recovery();\"")
604+
if result['succeeded']:
605+
in_recovery = result['stdout'].strip()
606+
print(f"PostgreSQL pg_is_in_recovery: {in_recovery}")
607+
else:
608+
print("Could not check PostgreSQL recovery status")
609+
in_recovery = "unknown"
610+
611+
# Find PostgreSQL configuration file
612+
result = run_ssh_command(host['ssh'], "sudo -u postgres psql -c \"SHOW config_file;\"")
613+
if result['succeeded']:
614+
config_file = result['stdout'].strip().split('\n')[2].strip() # Skip header and get the actual path
615+
print(f"PostgreSQL config file: {config_file}")
616+
else:
617+
print("Could not find PostgreSQL config file")
618+
config_file = "/etc/postgresql/15/main/postgresql.conf" # Default fallback
619+
620+
# Backup PostgreSQL config
621+
result = run_ssh_command(host['ssh'], f"sudo cp {config_file} {config_file}.backup")
622+
assert result['succeeded'], "Failed to backup PostgreSQL config"
623+
624+
# Add read-only setting to PostgreSQL config
625+
result = run_ssh_command(host['ssh'], f"echo 'default_transaction_read_only = on' | sudo tee -a {config_file}")
626+
assert result['succeeded'], "Failed to add read-only setting to PostgreSQL config"
627+
628+
# Restart PostgreSQL to apply the new configuration
629+
result = run_ssh_command(host['ssh'], "sudo systemctl restart postgresql")
630+
assert result['succeeded'], "Failed to restart PostgreSQL"
631+
632+
# Wait for PostgreSQL to start up
633+
sleep(5)
634+
635+
# Verify the change took effect
636+
result = run_ssh_command(host['ssh'], "sudo -u postgres psql -c \"SHOW default_transaction_read_only;\"")
637+
if result['succeeded']:
638+
new_default_read_only = result['stdout'].strip()
639+
print(f"PostgreSQL default_transaction_read_only after change: {new_default_read_only}")
640+
else:
641+
print("Could not verify PostgreSQL read-only setting change")
642+
643+
# First, backup the current PostgREST config
644+
result = run_ssh_command(host['ssh'], "sudo cp /etc/postgrest/base.conf /etc/postgrest/base.conf.backup")
645+
assert result['succeeded'], "Failed to backup PostgREST config"
646+
647+
try:
648+
# Read the current config to get the db-uri
649+
result = run_ssh_command(host['ssh'], "sudo cat /etc/postgrest/base.conf | grep '^db-uri'")
650+
assert result['succeeded'], "Failed to read current db-uri"
651+
652+
current_db_uri = result['stdout'].strip()
653+
print(f"Current db-uri: {current_db_uri}")
654+
655+
# Extract just the URI part (remove the db-uri = " prefix and trailing quote)
656+
uri_start = current_db_uri.find('"') + 1
657+
uri_end = current_db_uri.rfind('"')
658+
base_uri = current_db_uri[uri_start:uri_end]
659+
660+
# Modify the URI to add target_session_attrs=read-only
661+
if '?' in base_uri:
662+
# URI already has parameters, add target_session_attrs
663+
modified_uri = base_uri + "&target_session_attrs=read-only"
664+
else:
665+
# URI has no parameters, add target_session_attrs
666+
modified_uri = base_uri + "?target_session_attrs=read-only"
667+
668+
print(f"Modified URI: {modified_uri}")
669+
670+
# Use awk to replace the db-uri line more reliably
671+
result = run_ssh_command(host['ssh'], f"sudo awk '{{if ($1 == \"db-uri\") print \"db-uri = \\\"{modified_uri}\\\"\"; else print $0}}' /etc/postgrest/base.conf > /tmp/new_base.conf && sudo mv /tmp/new_base.conf /etc/postgrest/base.conf")
672+
assert result['succeeded'], "Failed to update db-uri in config"
673+
674+
# Verify the change was made correctly
675+
result = run_ssh_command(host['ssh'], "sudo cat /etc/postgrest/base.conf | grep '^db-uri'")
676+
print(f"Updated db-uri line: {result['stdout'].strip()}")
677+
678+
# Also show the full config to debug
679+
result = run_ssh_command(host['ssh'], "sudo cat /etc/postgrest/base.conf")
680+
print(f"Full config after change:\n{result['stdout']}")
681+
682+
# Restart PostgREST to apply the new configuration
683+
result = run_ssh_command(host['ssh'], "sudo systemctl restart postgrest")
684+
assert result['succeeded'], "Failed to restart PostgREST"
685+
686+
# Wait a moment for PostgREST to start up
687+
sleep(5)
688+
689+
# Check if PostgREST is running
690+
result = run_ssh_command(host['ssh'], "sudo systemctl is-active postgrest")
691+
if not (result['succeeded'] and result['stdout'].strip() == 'active'):
692+
# If PostgREST failed to start, check the logs to see why
693+
log_result = run_ssh_command(host['ssh'], "sudo journalctl -u postgrest --since '5 seconds ago' --no-pager")
694+
print(f"PostgREST failed to start. Recent logs:\n{log_result['stdout']}")
695+
assert False, "PostgREST failed to start after config change"
696+
697+
# Make a test request to trigger any potential session errors
698+
try:
699+
response = requests.get(
700+
f"http://{host['ip']}/rest/v1/",
701+
headers={"apikey": anon_key, "authorization": f"Bearer {anon_key}"},
702+
timeout=10
703+
)
704+
print(f"Test request status: {response.status_code}")
705+
except Exception as e:
706+
print(f"Test request failed: {str(e)}")
707+
708+
# Check PostgREST logs for "session is not read-only" errors
709+
result = run_ssh_command(host['ssh'], "sudo journalctl -u postgrest --since '5 seconds ago' | grep -i 'session is not read-only' || true")
710+
711+
if result['stdout'].strip():
712+
print(f"\nFound 'session is not read-only' errors in PostgREST logs:\n{result['stdout']}")
713+
assert False, "PostgREST logs contain 'session is not read-only' errors even though PostgreSQL is configured for read-only mode"
714+
else:
715+
print("\nNo 'session is not read-only' errors found in PostgREST logs")
716+
717+
finally:
718+
# Restore the original configuration
719+
result = run_ssh_command(host['ssh'], "sudo cp /etc/postgrest/base.conf.backup /etc/postgrest/base.conf")
720+
if result['succeeded']:
721+
result = run_ssh_command(host['ssh'], "sudo systemctl restart postgrest")
722+
if result['succeeded']:
723+
print("Restored original PostgREST configuration")
724+
else:
725+
print("Warning: Failed to restart PostgREST after restoring config")
726+
else:
727+
print("Warning: Failed to restore original PostgREST configuration")
728+
729+
# Restore PostgreSQL to original configuration
730+
result = run_ssh_command(host['ssh'], f"sudo cp {config_file}.backup {config_file}")
731+
if result['succeeded']:
732+
result = run_ssh_command(host['ssh'], "sudo systemctl restart postgresql")
733+
if result['succeeded']:
734+
print("Restored PostgreSQL to original configuration")
735+
else:
736+
print("Warning: Failed to restart PostgreSQL after restoring config")
737+
else:
738+
print("Warning: Failed to restore PostgreSQL configuration")
739+

0 commit comments

Comments
 (0)