Skip to content

Commit 4e1bdb8

Browse files
jfrochesamrose
jfroche
and
samrose
authored
feat: release orioledb with rewind functionality (#1880)
* feat: enable orioledb build with specific revision Allow specifying git revision hash instead of version tags for orioledb sources, enabling builds from specific commits rather than only tagged releases. * feat: update orioledb extension Expose patchset and revision attributes in postgresql to build orioledb properly. Add NixOS VM test for orioledb extension validation. * tests: migration tests needed updating * fix: test needs restrictkey handling * feat: apply changes to Dockerfile for orioledb chore: bump version * chore: switch back to using tags to specify version * tests: new testing prefix * chore: bump to release --------- Co-authored-by: Sam Rose <[email protected]>
1 parent 7c2878c commit 4e1bdb8

File tree

11 files changed

+193
-36
lines changed

11 files changed

+193
-36
lines changed

‎Dockerfile-orioledb-17‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -204,11 +204,17 @@ RUN sed -i \
204204
# Remove items from postgresql.conf
205205
RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf"
206206
#as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present
207-
RUN sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf"
208-
RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "/etc/postgresql-custom/supautils.conf"
209-
RUN sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "/etc/postgresql/postgresql.conf"
210-
RUN echo "default_table_access_method = 'orioledb'" >> "/etc/postgresql/postgresql.conf"
211-
207+
RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "/etc/postgresql-custom/supautils.conf"
208+
RUN sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "/etc/postgresql/postgresql.conf"
209+
RUN echo "default_table_access_method = 'orioledb'" >> "/etc/postgresql/postgresql.conf"
210+
211+
# OrioleDB rewind configuration
212+
# Enables time-based rewind capability for up to 20 minutes (1200 seconds)
213+
# Buffer size: 1280 buffers * 8KB = 10MB for transaction retention
214+
RUN echo "orioledb.enable_rewind = true" >> "/etc/postgresql/postgresql.conf" && \
215+
echo "orioledb.rewind_max_time = 1200" >> "/etc/postgresql/postgresql.conf" && \
216+
echo "orioledb.rewind_max_transactions = 100000" >> "/etc/postgresql/postgresql.conf" && \
217+
echo "orioledb.rewind_buffers = 1280" >> "/etc/postgresql/postgresql.conf"
212218

213219

214220
# # Include schema migrations

‎ansible/tasks/stage2-setup-postgres.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,38 @@
5151
line: "default_table_access_method = 'orioledb'"
5252
path: '/etc/postgresql/postgresql.conf'
5353
state: 'present'
54+
55+
- name: Enable OrioleDB rewind feature
56+
ansible.builtin.lineinfile:
57+
path: /etc/postgresql/postgresql.conf
58+
line: "orioledb.enable_rewind = true"
59+
state: present
60+
when: is_psql_oriole and stage2_nix
61+
become: yes
62+
63+
- name: Set OrioleDB rewind max time (20 minutes)
64+
ansible.builtin.lineinfile:
65+
path: /etc/postgresql/postgresql.conf
66+
line: "orioledb.rewind_max_time = 1200"
67+
state: present
68+
when: is_psql_oriole and stage2_nix
69+
become: yes
70+
71+
- name: Set OrioleDB rewind max transactions
72+
ansible.builtin.lineinfile:
73+
path: /etc/postgresql/postgresql.conf
74+
line: "orioledb.rewind_max_transactions = 100000"
75+
state: present
76+
when: is_psql_oriole and stage2_nix
77+
become: yes
78+
79+
- name: Set OrioleDB rewind buffers (1280 buffers = 10MB)
80+
ansible.builtin.lineinfile:
81+
path: /etc/postgresql/postgresql.conf
82+
line: "orioledb.rewind_buffers = 1280"
83+
state: present
84+
when: is_psql_oriole and stage2_nix
85+
become: yes
5486

5587
- name: Add ORIOLEDB_ENABLED environment variable
5688
ansible.builtin.lineinfile:

‎ansible/vars.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ postgres_major:
1010

1111
# Full version strings for each major version
1212
postgres_release:
13-
postgresorioledb-17: "17.5.1.073-orioledb-plv8-1"
14-
postgres17: "17.6.1.052-plv8-1"
15-
postgres15: "15.14.1.052-plv8-1"
13+
postgresorioledb-17: "17.6.0.010-orioledb"
14+
postgres17: "17.6.1.053"
15+
postgres15: "15.14.1.053"
1616

1717
# Non Postgres Extensions
1818
pgbouncer_release: 1.19.0

‎migrations/schema-orioledb-17.sql‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22
-- PostgreSQL database dump
33
--
44

5-
-- Dumped from database version 17.5
6-
-- Dumped by pg_dump version 17.5
5+
\restrict SupabaseTestDumpKey123
6+
7+
-- Dumped from database version 17.6
8+
-- Dumped by pg_dump version 17.6
79

810
SET statement_timeout = 0;
911
SET lock_timeout = 0;
@@ -1012,3 +1014,5 @@ CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop
10121014
-- PostgreSQL database dump complete
10131015
--
10141016

1017+
\unrestrict SupabaseTestDumpKey123
1018+

‎nix/config.nix‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ let
2020
options = {
2121
version = lib.mkOption { type = lib.types.str; };
2222
hash = lib.mkOption { type = lib.types.str; };
23+
revision = lib.mkOption {
24+
type = lib.types.nullOr lib.types.str;
25+
default = null;
26+
};
2327
};
2428
};
2529
supabaseSubmodule = lib.types.submodule {
@@ -52,8 +56,8 @@ in
5256
};
5357
orioledb = {
5458
"17" = {
55-
version = "17_11";
56-
hash = "sha256-RZYU955PmGZExfX2JKw1dIQMMuuswtAXpXjZ9CLbOsw=";
59+
version = "17_15";
60+
hash = "sha256-1v3FGIN0UW+E4ilLwbsV3nXvef3n9O8bVmgyjRFEJsU=";
5761
};
5862
};
5963
};

‎nix/ext/orioledb.nix‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,20 @@ stdenv.mkDerivation rec {
1515
src = fetchFromGitHub {
1616
owner = "orioledb";
1717
repo = "orioledb";
18-
rev = "beta12";
19-
sha256 = "sha256-5dyVdKD1PzW+F5OPW3TR3OKBaJxxR3OhLbzK+o9Wf+Q=";
18+
rev = "beta13";
19+
sha256 = "sha256-80RQHOgkEC7Hq3+N1VhsuKEUL+SNT/WfDN5vmXpaQG4=";
2020
};
21-
version = "beta12";
21+
version = "beta13";
2222
buildInputs = [
2323
curl
2424
libkrb5
2525
postgresql
2626
python3
2727
openssl
2828
];
29-
buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=11";
29+
buildPhase = ''
30+
make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=15
31+
'';
3032
installPhase = ''
3133
runHook preInstall
3234

‎nix/ext/tests/orioledb.nix‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
{ self, pkgs }:
2+
let
3+
pname = "orioledb";
4+
inherit (pkgs) lib;
5+
postgresqlWithExtension =
6+
postgresql:
7+
let
8+
majorVersion = lib.versions.major postgresql.version;
9+
pkg = pkgs.buildEnv {
10+
name = "postgresql-${majorVersion}-${pname}";
11+
paths = [
12+
postgresql
13+
postgresql.lib
14+
self.packages.${pkgs.system}."psql_orioledb-17/exts/orioledb"
15+
];
16+
passthru = {
17+
inherit (postgresql) version psqlSchema;
18+
lib = pkg;
19+
withPackages = _: pkg;
20+
};
21+
nativeBuildInputs = [ pkgs.makeWrapper ];
22+
pathsToLink = [
23+
"/"
24+
"/bin"
25+
"/lib"
26+
];
27+
postBuild = ''
28+
wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib
29+
wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib
30+
wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib
31+
'';
32+
};
33+
in
34+
pkg;
35+
psql_orioledb = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_orioledb-17;
36+
in
37+
self.inputs.nixpkgs.lib.nixos.runTest {
38+
name = pname;
39+
hostPkgs = pkgs;
40+
nodes.server =
41+
{ ... }:
42+
{
43+
virtualisation = {
44+
forwardPorts = [
45+
{
46+
from = "host";
47+
host.port = 13022;
48+
guest.port = 22;
49+
}
50+
];
51+
};
52+
services.openssh = {
53+
enable = true;
54+
};
55+
56+
services.postgresql = {
57+
enable = true;
58+
package = psql_orioledb;
59+
settings = {
60+
shared_preload_libraries = "orioledb";
61+
default_table_access_method = "orioledb";
62+
};
63+
initdbArgs = [
64+
"--allow-group-access"
65+
"--locale-provider=icu"
66+
"--encoding=UTF-8"
67+
"--icu-locale=en_US.UTF-8"
68+
];
69+
initialScript = pkgs.writeText "init-postgres-with-orioledb" ''
70+
CREATE EXTENSION orioledb CASCADE;
71+
'';
72+
};
73+
};
74+
testScript =
75+
{ ... }:
76+
''
77+
start_all()
78+
79+
server.wait_for_unit("multi-user.target")
80+
server.wait_for_unit("postgresql.service")
81+
'';
82+
}

‎nix/postgresql/default.nix‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ let
2020
pkgs.lib.nameValuePair "${namePrefix}${versionSuffix}" (
2121
pkgs.callPackage ./generic.nix {
2222
inherit isOrioleDB;
23-
inherit (config) version hash;
23+
inherit (config) version hash revision;
2424
jitSupport = jitSupport;
2525
self = pkgs;
2626
}

‎nix/postgresql/generic.nix‎

Lines changed: 35 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ let
4646
# source specification
4747
version,
4848
hash,
49+
revision ? null,
4950
muslPatches ? { },
5051

5152
# for tests
@@ -88,11 +89,17 @@ let
8889
pname = pname + lib.optionalString jitSupport "-jit";
8990

9091
src =
91-
if (builtins.match "[0-9][0-9]_.*" version != null) then
92-
fetchurl {
93-
url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz";
94-
inherit hash;
95-
}
92+
if isOrioleDB then
93+
if revision != null then
94+
fetchurl {
95+
url = "https://github.com/orioledb/postgres/archive/${revision}.tar.gz";
96+
inherit hash;
97+
}
98+
else
99+
fetchurl {
100+
url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz";
101+
inherit hash;
102+
}
96103
else
97104
fetchurl {
98105
url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2";
@@ -124,18 +131,15 @@ let
124131
++ lib.optionals gssSupport [ libkrb5 ]
125132
++ lib.optionals stdenv'.isLinux [ linux-pam ]
126133
++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ]
127-
++
128-
lib.optionals
129-
((builtins.match "[0-9][0-9]_.*" version != null) || (lib.versionAtLeast version "17"))
130-
[
131-
perl
132-
bison
133-
flex
134-
docbook_xsl
135-
docbook_xml_dtd_45
136-
docbook_xsl_ns
137-
libxslt
138-
];
134+
++ lib.optionals (isOrioleDB || (lib.versionAtLeast version "17")) [
135+
perl
136+
bison
137+
flex
138+
docbook_xsl
139+
docbook_xml_dtd_45
140+
docbook_xsl_ns
141+
libxslt
142+
];
139143

140144
nativeBuildInputs =
141145
[
@@ -285,13 +289,20 @@ let
285289
in
286290
{
287291
psqlSchema = lib.versions.major version;
292+
inherit revision;
288293

289294
withJIT = if jitSupport then this else jitToggle;
290295
withoutJIT = if jitSupport then jitToggle else this;
291296

292297
dlSuffix = if olderThan "16" then ".so" else stdenv.hostPlatform.extensions.sharedLibrary;
293298
inherit isOrioleDB;
294299

300+
patchset =
301+
if isOrioleDB then
302+
if revision != null then revision else builtins.elemAt (builtins.split "_" version) 2
303+
else
304+
null;
305+
295306
pkgs =
296307
let
297308
scope = {
@@ -404,7 +415,13 @@ let
404415
'';
405416

406417
passthru = {
407-
inherit (postgresql) version psqlSchema isOrioleDB;
418+
inherit (postgresql)
419+
version
420+
revision
421+
patchset
422+
psqlSchema
423+
isOrioleDB
424+
;
408425
};
409426
};
410427
in

‎nix/tools/dbmate-tool.sh.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ perform_dump() {
182182

183183
# Only use --restrict-key for standard PostgreSQL 15 and 17 versions
184184
# OrioleDB doesn't support this flag yet
185-
if [ "$PSQL_VERSION" = "15" ] || [ "$PSQL_VERSION" = "17" ]; then
185+
if [ "$PSQL_VERSION" = "15" ] || [ "$PSQL_VERSION" = "17" ] || [ "$PSQL_VERSION" = "orioledb-17" ]; then
186186
# Use a fixed restrict key for reproducible test dumps
187187
# This is safe in testing contexts but should not be used in production
188188
dump_cmd="$dump_cmd --restrict-key=SupabaseTestDumpKey123"

0 commit comments

Comments
 (0)