diff --git a/ansible/files/pgbouncer_config/pgbouncer.service.j2 b/ansible/files/pgbouncer_config/pgbouncer.service.j2 deleted file mode 100644 index 7fcc9acbe..000000000 --- a/ansible/files/pgbouncer_config/pgbouncer.service.j2 +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=connection pooler for PostgreSQL -Documentation=man:pgbouncer(1) -Documentation=https://www.pgbouncer.org/ -After=network.target -{% if supabase_internal is defined %} -Requires=database-optimizations.service -After=database-optimizations.service -{% endif %} - -[Service] -Type=notify -User=pgbouncer -ExecStart=/usr/local/bin/pgbouncer /etc/pgbouncer/pgbouncer.ini -ExecReload=/bin/kill -HUP $MAINPID -KillSignal=SIGINT -LimitNOFILE=65536 -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 b/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 deleted file mode 100644 index d5d2cd49d..000000000 --- a/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 +++ /dev/null @@ -1,2 +0,0 @@ -# Directory for PostgreSQL sockets, lockfiles and stats tempfiles -d /run/pgbouncer 2775 pgbouncer postgres - - \ No newline at end of file diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 0991a813a..f28803683 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -13,7 +13,7 @@ dest: "00-schema.sql", } - { source: "stat_extension.sql", dest: "01-extension.sql" } - + environment: PATH: /usr/lib/postgresql/bin:{{ ansible_env.PATH }} @@ -29,13 +29,6 @@ - name: Install Postgres from source import_tasks: tasks/setup-postgres.yml - - name: Install PgBouncer - import_tasks: tasks/setup-pgbouncer.yml - tags: - - install-pgbouncer - - install-supabase-internal - when: debpkg_mode or nixpkg_mode - - name: Install WAL-G import_tasks: tasks/setup-wal-g.yml when: debpkg_mode or nixpkg_mode or stage2_nix @@ -46,7 +39,7 @@ - install-gotrue - install-supabase-internal when: debpkg_mode or nixpkg_mode - + - name: Install PostgREST import_tasks: tasks/setup-postgrest.yml tags: @@ -96,7 +89,7 @@ src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic when: debpkg_mode or nixpkg_mode - + - name: Transfer init SQL files copy: src: files/{{ item.source }} @@ -131,13 +124,13 @@ tags: - install-supabase-internal when: debpkg_mode or stage2_nix - + - name: Finalize AMI import_tasks: tasks/finalize-ami.yml tags: - install-supabase-internal when: debpkg_mode or nixpkg_mode - + - name: Enhance fail2ban import_tasks: tasks/setup-fail2ban.yml when: debpkg_mode or nixpkg_mode @@ -218,7 +211,7 @@ systemctl stop postgresql.service when: stage2_nix - - name: Remove osquery + - name: Remove osquery become: yes shell: | sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile remove osquery" diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml deleted file mode 100644 index 4381ba24d..000000000 --- a/ansible/tasks/setup-pgbouncer.yml +++ /dev/null @@ -1,135 +0,0 @@ -# PgBouncer -- name: PgBouncer - download & install dependencies - apt: - pkg: - - build-essential - - libssl-dev - - pkg-config - - libevent-dev - - libsystemd-dev - update_cache: yes - cache_valid_time: 3600 - -- name: PgBouncer - download latest release - get_url: - url: "https://www.pgbouncer.org/downloads/files/{{ pgbouncer_release }}/pgbouncer-{{ pgbouncer_release }}.tar.gz" - dest: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz - checksum: "{{ pgbouncer_release_checksum }}" - timeout: 60 - -- name: PgBouncer - unpack archive - unarchive: - remote_src: yes - src: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz - dest: /tmp - become: yes - -- name: PgBouncer - configure - shell: - cmd: "./configure --prefix=/usr/local --with-systemd" - chdir: /tmp/pgbouncer-{{ pgbouncer_release }} - become: yes - -- name: PgBouncer - build - make: - chdir: /tmp/pgbouncer-{{ pgbouncer_release }} - become: yes - -- name: PgBouncer - install - make: - chdir: /tmp/pgbouncer-{{ pgbouncer_release }} - target: install - become: yes - -- name: Create pgbouncer user - user: - name: pgbouncer - shell: /bin/false - comment: PgBouncer user - groups: postgres,ssl-cert - -- name: PgBouncer - create a directory if it does not exist - file: - path: /etc/pgbouncer - state: directory - owner: pgbouncer - group: pgbouncer - mode: '0700' - -- name: PgBouncer - create a directory if it does not exist - file: - state: directory - owner: pgbouncer - group: pgbouncer - path: '{{ item }}' - mode: '0775' - with_items: - - '/etc/pgbouncer-custom' - -- name: create placeholder config files - file: - path: '/etc/pgbouncer-custom/{{ item }}' - state: touch - owner: pgbouncer - group: pgbouncer - mode: 0664 - with_items: - - 'generated-optimizations.ini' - - 'custom-overrides.ini' - - 'ssl-config.ini' - -- name: PgBouncer - adjust pgbouncer.ini - copy: - src: files/pgbouncer_config/pgbouncer.ini.j2 - dest: /etc/pgbouncer/pgbouncer.ini - owner: pgbouncer - mode: '0700' - -- name: PgBouncer - create a directory if it does not exist - file: - path: /etc/pgbouncer/userlist.txt - state: touch - owner: pgbouncer - mode: '0700' - -- name: import /etc/tmpfiles.d/pgbouncer.conf - template: - src: files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 - dest: /etc/tmpfiles.d/pgbouncer.conf - become: yes - -- name: PgBouncer - By default allow ssl connections. - become: yes - copy: - dest: /etc/pgbouncer-custom/ssl-config.ini - content: | - client_tls_sslmode = allow - -- name: Grant pg_hba and pgbouncer grp perm for adminapi updates - shell: | - chmod g+w /etc/postgresql/pg_hba.conf - chmod g+w /etc/pgbouncer-custom/ssl-config.ini - -# Add fail2ban filter -- name: import jail.d/pgbouncer.conf - template: - src: files/fail2ban_config/jail-pgbouncer.conf.j2 - dest: /etc/fail2ban/jail.d/pgbouncer.conf - become: yes - -- name: import filter.d/pgbouncer.conf - template: - src: files/fail2ban_config/filter-pgbouncer.conf.j2 - dest: /etc/fail2ban/filter.d/pgbouncer.conf - become: yes - -# Add systemd file for PgBouncer -- name: PgBouncer - import postgresql.service - template: - src: files/pgbouncer_config/pgbouncer.service.j2 - dest: /etc/systemd/system/pgbouncer.service - become: yes - -- name: PgBouncer - reload systemd - systemd: - daemon_reload: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index 8cba94dbb..df6ad4b70 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -13,10 +13,6 @@ postgres_release: postgres17: "17.6.1.003-nixpkgs-4" postgres15: "15.14.1.003-nixpkgs-4" -# Non Postgres Extensions -pgbouncer_release: "1.19.0" -pgbouncer_release_checksum: sha256:af0b05e97d0e1fd9ad45fe00ea6d2a934c63075f67f7e2ccef2ca59e3d8ce682 - # The checksum can be found under "Assets", in the GitHub release page for each version. # The binaries used are: ubuntu-aarch64 and linux-static. # https://github.com/PostgREST/postgrest/releases diff --git a/nix/checks.nix b/nix/checks.nix index 1217a0312..1c890e407 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -315,6 +315,9 @@ inherit self; inherit pkgs; }) - ); + ) + // pkgs.lib.optionalAttrs (pkgs.stdenv.hostPlatform.isLinux) { + inherit (self'.packages) ansible-test run-testinfra docker-image-ubuntu; + }; }; } diff --git a/nix/systemConfigs.nix b/nix/systemConfigs.nix index 356e591f0..2b4ff8f21 100644 --- a/nix/systemConfigs.nix +++ b/nix/systemConfigs.nix @@ -1,12 +1,14 @@ { self, inputs, ... }: let mkModules = system: [ + self.systemModules.pgbouncer self.systemModules.postgres ( { pkgs, ... }: { services.nginx.enable = true; nixpkgs.hostPlatform = system; + supabase.services.pgbouncer.enable = true; supabase.services.postgres = { enable = true; package = self.packages.${system}."psql_17/bin"; diff --git a/nix/systemModules/default.nix b/nix/systemModules/default.nix index 4dd9eb6cd..8624b699c 100644 --- a/nix/systemModules/default.nix +++ b/nix/systemModules/default.nix @@ -6,6 +6,7 @@ flake = { systemModules = { postgres = ./postgres; + pgbouncer = ./pgbouncer.nix; }; }; } diff --git a/nix/systemModules/dummy-firewall.nix b/nix/systemModules/dummy-firewall.nix new file mode 100644 index 000000000..bf16fd00d --- /dev/null +++ b/nix/systemModules/dummy-firewall.nix @@ -0,0 +1,6 @@ +{ lib, ... }: +{ + options.networking.firewall = lib.mkOption { + type = lib.types.attrs; + }; +} diff --git a/nix/systemModules/pgbouncer.nix b/nix/systemModules/pgbouncer.nix new file mode 100644 index 000000000..2bd2b46a9 --- /dev/null +++ b/nix/systemModules/pgbouncer.nix @@ -0,0 +1,96 @@ +{ + lib, + pkgs, + nixosModulesPath, + system, + config, + ... +}: +let + cfg = config.supabase.services.pgbouncer; + + # From https://github.com/mightyiam/catppuccin-nix/blob/main/modules/lib/default.nix#L78-L89 + fromINI = + file: + let + json = pkgs.runCommand "converted.json" { } '' + ${lib.getExe pkgs.jc} --ini < ${file} > $out + ''; + in + builtins.fromJSON (builtins.readFile json); +in +{ + imports = [ + # TODO: actually open the ports it needs with ufw + ./dummy-firewall.nix + ] + ++ map (path: nixosModulesPath + path) [ + "/services/databases/pgbouncer.nix" + ]; + + options = { + supabase.services.pgbouncer = { + enable = lib.mkEnableOption "Whether to enable PostgreSQL connection pooler."; + }; + }; + + config = lib.mkIf cfg.enable { + environment.etc = { + # By default allow ssl connections. + "/etc/pgbouncer-custom/ssl-config.ini".text = '' + client_tls_sslmode = allow + ''; + }; + + # Nixpkgs pgbouncer systemd service is quite what we had set up by ansible before: + # + # [Service] + # Type=notify + # User=pgbouncer + # ExecStart=/usr/local/bin/pgbouncer /etc/pgbouncer/pgbouncer.ini + # ExecReload=/bin/kill -HUP $MAINPID + # KillSignal=SIGINT + # LimitNOFILE=65536 + # Restart=always + # RestartSec=5 + services.pgbouncer = { + enable = true; + package = + (import (fetchTarball { + # pgbouncer v1.19.0 + url = "https://github.com/NixOS/nixpkgs/archive/db7534df5fb9b7dfd3404ec26d977997ff2cc1a0.tar.gz"; + sha256 = "sha256:0lrsnz80a3jfjdyjs4njipvmq34w6wjr5ql645z1l1s9f9cyvk0g"; + }) { system = system; }).pgbouncer; + settings = + let + iniJson = fromINI ./pgbouncer/pgbouncer.ini; + in + iniJson + // { + pgbouncer = iniJson.pgbouncer // { + # jc --ini treat all values as strings, so we must manually convert + # every numeric option to its expected type for NixOS module validation ... + default_pool_size = lib.toInt iniJson.pgbouncer.default_pool_size; + listen_port = lib.toInt iniJson.pgbouncer.listen_port; + }; + }; + user = "pgbouncer"; # n.b. this is the nixpkgs default, but since everything depends on it ... + group = "pgbouncer"; # ... we might as well be explicit here! + }; + systemd.services.pgbouncer = { + wantedBy = lib.mkForce [ + "system-manager.target" + ]; + }; + + # TODO: double check if all these are really needed + systemd.tmpfiles.rules = [ + "d /run/pgbouncer 2775 pgbouncer postgres - -" + "d /etc/pgbouncer-custom 0775 pgbouncer pgbouncer - -" + "C /etc/pgbouncer/userlist.txt 0700 pgbouncer pgbouncer - -" + "C /etc/pgbouncer-custom/custom-overrides.ini 0664 pgbouncer pgbouncer - -" + "C /etc/pgbouncer-custom/generated-optimizations.ini 0664 pgbouncer pgbouncer - -" + "C /etc/pgbouncer-custom/ssl-config.ini 0664 pgbouncer pgbouncer - -" + ]; + }; +} diff --git a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 b/nix/systemModules/pgbouncer/pgbouncer.ini similarity index 96% rename from ansible/files/pgbouncer_config/pgbouncer.ini.j2 rename to nix/systemModules/pgbouncer/pgbouncer.ini index e4518c007..dfe4b1163 100644 --- a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 +++ b/nix/systemModules/pgbouncer/pgbouncer.ini @@ -359,6 +359,8 @@ default_pool_size = 15 ;; Read additional config from other file ;%include /etc/pgbouncer/pgbouncer-other.ini -%include /etc/pgbouncer-custom/generated-optimizations.ini -%include /etc/pgbouncer-custom/custom-overrides.ini -%include /etc/pgbouncer-custom/ssl-config.ini +;; TODO: I have no idea how to include these files, since there're not defined in this repo, +;; jc --ini isn't able to parse %include, and settings.pgbouncer doesn't have a way to add those either. +; %include /etc/pgbouncer-custom/generated-optimizations.ini +; %include /etc/pgbouncer-custom/custom-overrides.ini +; %include /etc/pgbouncer-custom/ssl-config.ini diff --git a/nix/systemModules/tests/test_pgbouncer.py b/nix/systemModules/tests/test_pgbouncer.py new file mode 100644 index 000000000..b17b62f70 --- /dev/null +++ b/nix/systemModules/tests/test_pgbouncer.py @@ -0,0 +1,25 @@ +# from time import sleep + + +def test_pgbouncer_service(host): + # sleep(5000) # Handy for interactive debugging (with docker exec -it $CONTAINER_ID /bin/bash) + assert host.service("pgbouncer.service").is_valid + assert host.service("pgbouncer.service").is_running, ( + "Auth service should be running but failed: {}".format( + host.run("systemctl status pgbouncer.service").stdout + ) + ) + + +# FIXME: AssertionError: Auth service should be running but failed: × pgbouncer.service - PgBouncer - PostgreSQL connection pooler +# Loaded: loaded (/etc/systemd/system/pgbouncer.service; enabled; preset: enabled) +# Active: failed (Result: exit-code) since Fri 2025-09-19 12:36:00 UTC; 12s ago +# Process: 372 ExecStart=/nix/store/bcj53gxm9i2y4hd21jr7zpi2r1hw8wlq-pgbouncer-1.24.1/bin/pgbouncer /etc/pgbouncer/pgbouncer.ini (code=exited, status=217/USER) +# Main PID: 372 (code=exited, status=217/USER) +# CPU: 4ms +# +# Sep 19 12:36:00 f803c2922bff systemd[1]: Starting pgbouncer.service - PgBouncer - PostgreSQL connection pooler... +# Sep 19 12:36:00 f803c2922bff (gbouncer)[372]: pgbouncer.service: Failed to determine user credentials: No such process +# Sep 19 12:36:00 f803c2922bff systemd[1]: pgbouncer.service: Main process exited, code=exited, status=217/USER +# Sep 19 12:36:00 f803c2922bff systemd[1]: pgbouncer.service: Failed with result 'exit-code'. +# Sep 19 12:36:00 f803c2922bff systemd[1]: Failed to start pgbouncer.service - PgBouncer - PostgreSQL connection pooler.