feat: run pg_regress during extension tests (#1812)

While testing the upgrade of our PostgreSQL extensions, we also want to
run the pg_regress tests that come with the extension. This ensures that
the extension is functioning correctly after installation and upgrades.

Author: jfroche

nix/ext/pgvector.nix

@@ -88,5 +88,6 @@ pkgs.buildEnv {
     pname = "${pname}-all";
     version =
       "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
+    pgRegressTestName = "pgvector";
   };
 }

nix/ext/tests/default.nix

@@ -125,6 +125,7 @@ let
           pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17";
         in
         ''
+          from pathlib import Path
           versions = {
             "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}],
             "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}],
@@ -135,6 +136,8 @@ let
           ext_has_background_worker = ${
             if (installedExtension "15") ? hasBackgroundWorker then "True" else "False"
           }
+          sql_test_directory = Path("${../../tests}")
+          pg_regress_test_name = "${(installedExtension "15").pgRegressTestName or pname}"
 
           ${builtins.readFile ./lib.py}
 
@@ -143,11 +146,14 @@ let
           server.wait_for_unit("multi-user.target")
           server.wait_for_unit("postgresql.service")
 
-          test = PostgresExtensionTest(server, extension_name, versions, support_upgrade)
+          test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade)
 
           with subtest("Check upgrade path with postgresql 15"):
             test.check_upgrade_path("15")
 
+          with subtest("Check pg_regress with postgresql 15"):
+            test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name)
+
           last_version = None
           with subtest("Check the install of the last version of the extension"):
             last_version = test.check_install_last_version("15")
@@ -166,6 +172,9 @@ let
 
           with subtest("Check upgrade path with postgresql 17"):
             test.check_upgrade_path("17")
+
+          with subtest("Check pg_regress with postgresql 17"):
+            test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name)
         '';
     };
 in

nix/ext/tests/lib.py

@@ -18,6 +18,7 @@ def __init__(
         vm: Machine,
         extension_name: str,
         versions: Versions,
+        sql_test_dir: Path,
         support_upgrade: bool = True,
     ):
         """Initialize the PostgreSQL extension test framework.
@@ -26,12 +27,14 @@ def __init__(
             vm: Test machine instance for executing commands
             extension_name: Name of the PostgreSQL extension to test
             versions: Mapping of PostgreSQL versions to available extension versions
+            sql_test_dir: Directory containing SQL test files for pg_regress
             support_upgrade: Whether the extension supports in-place upgrades
         """
         self.vm = vm
         self.extension_name = extension_name
         self.versions = versions
         self.support_upgrade = support_upgrade
+        self.sql_test_dir = sql_test_dir
 
     def run_sql(self, query: str) -> str:
         return self.vm.succeed(
@@ -101,9 +104,9 @@ def check_upgrade_path(self, pg_version: str):
             )
 
         # Install and verify first version
-        firstVersion = available_versions[0]
+        first_version = available_versions[0]
         self.drop_extension()
-        self.install_extension(firstVersion)
+        self.install_extension(first_version)
 
         # Test remaining versions
         for version in available_versions[1:]:
@@ -160,10 +163,41 @@ def check_switch_extension_with_background_worker(
             f"{first_version}.so"
         ), f"Expected {self.extension_name} version {first_version}, but found {ext_version}"
 
-        # Switch to the first version
+        # Switch to the last version
         self.vm.succeed(f"switch_{self.extension_name}_version {last_version}")
         # Check that we are using the last version now
         ext_version = self.vm.succeed(f"readlink -f {extension_lib_path}").strip()
         assert ext_version.endswith(
             f"{last_version}.so"
         ), f"Expected {self.extension_name} version {last_version}, but found {ext_version}"
+
+    def check_pg_regress(self, pg_regress: Path, pg_version: str, test_name: str):
+        """Run pg_regress tests for the extension on a given PostgreSQL version.
+
+        Args:
+            pg_regress: Path to the pg_regress binary
+            pg_version: PostgreSQL version to test (e.g., "14", "15")
+            test_name: SQL test file to run with pg_regress
+        """
+        sql_file = self.sql_test_dir / "sql" / f"{test_name}.sql"
+        if not sql_file.exists():
+            # check if we have a postgres version specific sql file
+            test_name = f"z_{pg_version}_{test_name}"
+            sql_file = self.sql_test_dir / "sql" / f"{test_name}.sql"
+            if not sql_file.exists():
+                print(f"Skipping pg_regress test for {pg_version}, no sql file found")
+                return
+        try:
+            print(
+                self.vm.succeed(
+                    f"""sudo -u postgres {pg_regress} --inputdir={self.sql_test_dir} --debug --use-existing --dbname=postgres --outputdir=/tmp/regression_output_{pg_version} "{test_name}" """
+                )
+            )
+        except:
+            print("Error running pg_regress, diff:")
+            print(
+                self.vm.succeed(
+                    f"cat /tmp/regression_output_{pg_version}/regression.diffs"
+                )
+            )
+            raise

nix/ext/tests/timescaledb.nix

@@ -66,8 +66,9 @@ self.inputs.nixpkgs.lib.nixos.runTest {
       }
       extension_name = "${pname}"
       support_upgrade = True
+      sql_test_directory = Path("${../../tests}")
 
-      test = PostgresExtensionTest(server, extension_name, versions, support_upgrade)
+      test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade)
 
       with subtest("Check upgrade path with postgresql 15"):
         test.check_upgrade_path("15")