From 47d6c350455d129e16c4f5f83518402421e4bd75 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 17:54:44 +0300 Subject: [PATCH 01/13] chore: enable & update pljava --- .github/workflows/dockerhub-release.yml | 37 ++++++++++--------- Dockerfile | 9 ++--- .../tasks/postgres-extensions/12-pljava.yml | 30 ++++++--------- ansible/vars.yml | 4 +- common.vars.pkr.hcl | 2 +- migrations/tests/extensions/test.sql | 2 +- 6 files changed, 38 insertions(+), 46 deletions(-) diff --git a/.github/workflows/dockerhub-release.yml b/.github/workflows/dockerhub-release.yml index 1f1453c38..c014fe471 100644 --- a/.github/workflows/dockerhub-release.yml +++ b/.github/workflows/dockerhub-release.yml @@ -4,6 +4,7 @@ on: push: branches: - develop + - pcnc/fix-pljava paths: - ".github/workflows/dockerhub-release.yml" - "common.vars*" @@ -58,15 +59,15 @@ jobs: platforms: linux/${{ matrix.arch }} cache-from: type=gha cache-to: type=gha,mode=max - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: "gha-failures-notifier" - SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres ${{ matrix.arch }} image failed" - SLACK_FOOTER: "" + # - name: Slack Notification + # if: ${{ failure() }} + # uses: rtCamp/action-slack-notify@v2 + # env: + # SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + # SLACK_USERNAME: "gha-failures-notifier" + # SLACK_COLOR: "danger" + # SLACK_MESSAGE: "Building Postgres ${{ matrix.arch }} image failed" + # SLACK_FOOTER: "" merge_manifest: needs: [settings, build_image] @@ -82,15 +83,15 @@ jobs: docker buildx imagetools create -t ${{ needs.settings.outputs.image_tag }} \ ${{ needs.settings.outputs.image_tag }}_amd64 \ ${{ needs.settings.outputs.image_tag }}_arm64 - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: "gha-failures-notifier" - SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres image failed" - SLACK_FOOTER: "" + # - name: Slack Notification + # if: ${{ failure() }} + # uses: rtCamp/action-slack-notify@v2 + # env: + # SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + # SLACK_USERNAME: "gha-failures-notifier" + # SLACK_COLOR: "danger" + # SLACK_MESSAGE: "Building Postgres image failed" + # SLACK_FOOTER: "" publish: needs: [settings, merge_manifest] diff --git a/Dockerfile b/Dockerfile index 35539dbf5..030810b33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -403,11 +403,10 @@ RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --pkgver #################### FROM builder as pljava-source # Download and extract -# TODO: revert to using main repo after PG15 support is merged: https://github.com/tada/pljava/pull/413 -ARG pljava_release=master -ARG pljava_release_checksum=sha256:e99b1c52f7b57f64c8986fe6ea4a6cc09d78e779c1643db060d0ac66c93be8b6 +ARG pljava_release=1_6_5 +ARG pljava_release_checksum=sha256:e1492b237eac8aaed6792feb26b8fa06b9c6ec3bd0a61ecd7a90e175dd1f1c5a ADD --checksum=${pljava_release_checksum} \ - "https://github.com/supabase/pljava/archive/refs/heads/${pljava_release}.tar.gz" \ + "https://github.com/tada/pljava/archive/V{pljava_release}.tar.gz" \ /tmp/pljava.tar.gz RUN tar -xvf /tmp/pljava.tar.gz -C /tmp && \ rm -rf /tmp/pljava.tar.gz @@ -832,7 +831,7 @@ COPY --from=plpgsql_check-source /tmp/*.deb /tmp/ COPY --from=pg-safeupdate-source /tmp/*.deb /tmp/ COPY --from=timescaledb-source /tmp/*.deb /tmp/ COPY --from=wal2json-source /tmp/*.deb /tmp/ -# COPY --from=pljava /tmp/*.deb /tmp/ +COPY --from=pljava /tmp/*.deb /tmp/ COPY --from=plv8 /tmp/*.deb /tmp/ COPY --from=pg_plan_filter-source /tmp/*.deb /tmp/ COPY --from=pg_net-source /tmp/*.deb /tmp/ diff --git a/ansible/tasks/postgres-extensions/12-pljava.yml b/ansible/tasks/postgres-extensions/12-pljava.yml index bd16d1cec..61e052e11 100644 --- a/ansible/tasks/postgres-extensions/12-pljava.yml +++ b/ansible/tasks/postgres-extensions/12-pljava.yml @@ -9,27 +9,19 @@ update_cache: yes install_recommends: no -#TODO: revert to using main repo after PG15 support is merged: https://github.com/tada/pljava/pull/413 -# - name: pljava - download latest release -# get_url: -# url: https://github.com/tada/pljava/archive/V{{ pljava_release }}.tar.gz -# dest: /tmp/pljava-{{ pljava_release }}.tar.gz -# checksum: "{{ pljava_release_checksum }}" -# timeout: 60 - -# - name: pljava - unpack archive -# unarchive: -# remote_src: yes -# src: /tmp/pljava-{{ pljava_release }}.tar.gz -# dest: /tmp -# become: yes - - name: pljava - download latest release + get_url: + url: https://github.com/tada/pljava/archive/V{{ pljava_release }}.tar.gz + dest: /tmp/pljava-{{ pljava_release }}.tar.gz + checksum: "{{ pljava_release_checksum }}" + timeout: 60 + +- name: pljava - unpack archive + unarchive: + remote_src: yes + src: /tmp/pljava-{{ pljava_release }}.tar.gz + dest: /tmp become: yes - git: - repo: https://github.com/supabase/pljava.git - dest: /tmp/pljava-{{ pljava_release }} - version: "{{ pljava_release }}" - name: pljava - build become: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index 5380b1e69..190a7ee0a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -80,8 +80,8 @@ wal2json_release_checksum: sha256:b516653575541cf221b99cf3f8be9b6821f6dbcfc12567 supautils_release: "1.7.2" supautils_release_checksum: sha256:527e645112e1348ea4b6be6740037b444797fbbc5d3ed98103ca582419e06084 -pljava_release: master -pljava_release_checksum: sha256:e99b1c52f7b57f64c8986fe6ea4a6cc09d78e779c1643db060d0ac66c93be8b6 +pljava_release: "1_6_5" +pljava_release_checksum: sha256:e1492b237eac8aaed6792feb26b8fa06b9c6ec3bd0a61ecd7a90e175dd1f1c5a plv8_release: "3.1.5" plv8_release_checksum: sha256:1e108d5df639e4c189e1c5bdfa2432a521c126ca89e7e5a969d46899ca7bf106 diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 306c6376e..40a04d9f8 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.0.100" +postgres-version = "15.1.0.101-rc0" diff --git a/migrations/tests/extensions/test.sql b/migrations/tests/extensions/test.sql index 7e0d1f38d..3c73410eb 100644 --- a/migrations/tests/extensions/test.sql +++ b/migrations/tests/extensions/test.sql @@ -10,7 +10,7 @@ \ir 09-pg-safeupdate.sql \ir 10-timescaledb.sql \ir 11-wal2json.sql --- \ir 12-pljava.sql +\ir 12-pljava.sql \ir 13-plv8.sql \ir 14-pg_plan_filter.sql \ir 15-pg_net.sql From 7a77831b0bd0e746fd1a992f94eab0a5c3e45af5 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 19:03:09 +0300 Subject: [PATCH 02/13] chore: fix pljava --- .github/workflows/dockerhub-release.yml | 1 + Dockerfile | 16 ++++------------ ansible/tasks/postgres-extensions/12-pljava.yml | 2 +- migrations/tests/extensions/12-pljava.sql | 2 +- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/workflows/dockerhub-release.yml b/.github/workflows/dockerhub-release.yml index c014fe471..a8139be98 100644 --- a/.github/workflows/dockerhub-release.yml +++ b/.github/workflows/dockerhub-release.yml @@ -8,6 +8,7 @@ on: paths: - ".github/workflows/dockerhub-release.yml" - "common.vars*" + workflow_dispatch: jobs: settings: diff --git a/Dockerfile b/Dockerfile index 030810b33..f275bedd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ ARG plpgsql_check_release=2.2.5 ARG pg_safeupdate_release=1.4 ARG timescaledb_release=2.9.1 ARG wal2json_release=2_5 -ARG pljava_release=1.6.4 +ARG pljava_release=1_6_5 ARG plv8_release=3.1.5 ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492 ARG pg_net_release=0.7.1 @@ -406,7 +406,7 @@ FROM builder as pljava-source ARG pljava_release=1_6_5 ARG pljava_release_checksum=sha256:e1492b237eac8aaed6792feb26b8fa06b9c6ec3bd0a61ecd7a90e175dd1f1c5a ADD --checksum=${pljava_release_checksum} \ - "https://github.com/tada/pljava/archive/V{pljava_release}.tar.gz" \ + "https://github.com/tada/pljava/archive/refs/tags/V${pljava_release}.tar.gz" \ /tmp/pljava.tar.gz RUN tar -xvf /tmp/pljava.tar.gz -C /tmp && \ rm -rf /tmp/pljava.tar.gz @@ -421,15 +421,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /tmp/pljava-${pljava_release} RUN mvn -T 1C clean install -Dmaven.test.skip -DskipTests -Dmaven.javadoc.skip=true # Create debian package -RUN cp pljava-packaging/target/pljava-pg${postgresql_major}.jar /tmp/ - -FROM base as pljava -# Download pre-built packages -RUN apt-get update && apt-get install -y --no-install-recommends --download-only \ - default-jdk-headless \ - postgresql-${postgresql_major}-pljava \ - && rm -rf /var/lib/apt/lists/* -RUN mv /var/cache/apt/archives/*.deb /tmp/ +RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --pkgname=pljava --pkgversion="\${pljava_release//_/.}" --requires=libssl-dev,default-jdk --exclude /proc/ --nodoc java -jar pljava-packaging/target/pljava-pg${postgresql_major}.jar #################### # 13-plv8.yml @@ -831,7 +823,7 @@ COPY --from=plpgsql_check-source /tmp/*.deb /tmp/ COPY --from=pg-safeupdate-source /tmp/*.deb /tmp/ COPY --from=timescaledb-source /tmp/*.deb /tmp/ COPY --from=wal2json-source /tmp/*.deb /tmp/ -COPY --from=pljava /tmp/*.deb /tmp/ +COPY --from=pljava-source /tmp/*.deb /tmp/ COPY --from=plv8 /tmp/*.deb /tmp/ COPY --from=pg_plan_filter-source /tmp/*.deb /tmp/ COPY --from=pg_net-source /tmp/*.deb /tmp/ diff --git a/ansible/tasks/postgres-extensions/12-pljava.yml b/ansible/tasks/postgres-extensions/12-pljava.yml index 61e052e11..0b907abca 100644 --- a/ansible/tasks/postgres-extensions/12-pljava.yml +++ b/ansible/tasks/postgres-extensions/12-pljava.yml @@ -11,7 +11,7 @@ - name: pljava - download latest release get_url: - url: https://github.com/tada/pljava/archive/V{{ pljava_release }}.tar.gz + url: https://github.com/tada/pljava/archive/refs/tags/V{{ pljava_release }}.tar.gz dest: /tmp/pljava-{{ pljava_release }}.tar.gz checksum: "{{ pljava_release_checksum }}" timeout: 60 diff --git a/migrations/tests/extensions/12-pljava.sql b/migrations/tests/extensions/12-pljava.sql index b51c82499..81f3399ff 100644 --- a/migrations/tests/extensions/12-pljava.sql +++ b/migrations/tests/extensions/12-pljava.sql @@ -1,3 +1,3 @@ BEGIN; -create extension if not exists pljava with schema "sqlj"; +create extension if not exists pljava; ROLLBACK; From 095d81e17752ab804fc8e901b465d0bcfe71c139 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 20:02:23 +0300 Subject: [PATCH 03/13] chore: remove notifications --- .github/workflows/ami-release.yml | 32 +++++++------- docker/Dockerfile.u18 | 72 +++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 docker/Dockerfile.u18 diff --git a/.github/workflows/ami-release.yml b/.github/workflows/ami-release.yml index 209894de7..0707e9574 100644 --- a/.github/workflows/ami-release.yml +++ b/.github/workflows/ami-release.yml @@ -131,22 +131,22 @@ jobs: run: | aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz - - name: Create release - uses: softprops/action-gh-release@v1 - with: - name: ${{ steps.process_release_version.outputs.version }} - tag_name: ${{ steps.process_release_version.outputs.version }} - target_commitish: ${{github.sha}} - - - name: Slack Notification on Failure - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: 'gha-failures-notifier' - SLACK_COLOR: 'danger' - SLACK_MESSAGE: 'Building Postgres AMI failed' - SLACK_FOOTER: '' + # - name: Create release + # uses: softprops/action-gh-release@v1 + # with: + # name: ${{ steps.process_release_version.outputs.version }} + # tag_name: ${{ steps.process_release_version.outputs.version }} + # target_commitish: ${{github.sha}} + + # - name: Slack Notification on Failure + # if: ${{ failure() }} + # uses: rtCamp/action-slack-notify@v2 + # env: + # SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + # SLACK_USERNAME: 'gha-failures-notifier' + # SLACK_COLOR: 'danger' + # SLACK_MESSAGE: 'Building Postgres AMI failed' + # SLACK_FOOTER: '' - name: Cleanup resources on build cancellation if: ${{ cancelled() }} diff --git a/docker/Dockerfile.u18 b/docker/Dockerfile.u18 new file mode 100644 index 000000000..3547b63d5 --- /dev/null +++ b/docker/Dockerfile.u18 @@ -0,0 +1,72 @@ +ARG ubuntu_release=focal +FROM ubuntu:${ubuntu_release} as base + +ARG ubuntu_release +ARG ubuntu_release_no +ARG postgresql_major=15 +ARG postgresql_release=${postgresql_major}.1 + +FROM base as pg-source + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gnupg \ + dpkg-dev \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Add Postgres PPA +ARG postgresql_gpg_key=B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "${postgresql_gpg_key}" && \ + echo "deb https://apt-archive.postgresql.org/pub/repos/apt ${ubuntu_release}-pgdg-archive main" > /etc/apt/sources.list.d/pgdg.list && \ + echo "deb-src https://apt-archive.postgresql.org/pub/repos/apt ${ubuntu_release}-pgdg-archive main" > /etc/apt/sources.list.d/pgdg.list + +# Create local PPA +WORKDIR /tmp/build +RUN echo "deb [ trusted=yes ] file:///tmp/build ./" > /etc/apt/sources.list.d/temp.list && \ + dpkg-scanpackages . > Packages && \ + apt-get -o Acquire::GzipIndexes=false update + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONDONTWRITEBYTECODE=1 +ENV DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" + +# Configure processor optimised build +ARG CPPFLAGS="-mcpu=neoverse-n1" +ENV DEB_CPPFLAGS_APPEND="${CPPFLAGS} -fsigned-char" + +RUN apt-get build-dep -y postgresql-common pgdg-keyring && \ + apt-get source --compile postgresql-common pgdg-keyring && \ + dpkg-scanpackages . > Packages && \ + apt-get -o Acquire::GzipIndexes=false update + +# RUN apt-get build-dep -y "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \ +# apt-get source --compile "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \ +# dpkg-scanpackages . > Packages && \ +# apt-get -o Acquire::GzipIndexes=false update + +# Remove source directories +RUN rm -rf /tmp/build/*/ + +FROM base as pg + +# Inherit args from base stage +ARG postgresql_major +ARG postgresql_release + +COPY --from=pg-source /tmp/build /tmp/build + +ENV DEBIAN_FRONTEND=noninteractive +RUN echo "deb [ trusted=yes ] file:///tmp/build ./" > /etc/apt/sources.list.d/temp.list && \ + apt-get -o Acquire::GzipIndexes=false update && \ + apt-get install -y --no-install-recommends postgresql-common && \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf && \ + apt-get install -y --no-install-recommends "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/build /etc/apt/sources.list.d/temp.list + +ENV PATH $PATH:/usr/lib/postgresql/${postgresql_major}/bin + +FROM scratch as pg-deb + +COPY --from=pg-source /tmp/build /tmp From 655ff6f0a8764ad38b2eee5427b5f37c0e925cce Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 20:06:33 +0300 Subject: [PATCH 04/13] chore: output postgresql.conf --- ansible/tasks/test-image.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index cd92a27d6..3a6eb8f78 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -10,6 +10,14 @@ cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf when: ebssurrogate_mode +- name: Display /etc/postgresql/postgresql.conf contents + command: cat postgresql.conf chdir=/etc/postgresql.conf + register: command_output + +- name: Print to console + debug: + msg: "{{command_output.stdout}}" + - name: Start Postgres Database to load all extensions. become: yes become_user: postgres From 764cbc2a6c5499bc78b86ef88a7f76bcae919bec Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 20:25:22 +0300 Subject: [PATCH 05/13] chore: fix ubuntu 18 ami build --- .github/workflows/ami-build-ubuntu-18.yml | 1 + docker/Dockerfile | 2 + docker/Dockerfile.u18 | 72 ----------------------- 3 files changed, 3 insertions(+), 72 deletions(-) delete mode 100644 docker/Dockerfile.u18 diff --git a/.github/workflows/ami-build-ubuntu-18.yml b/.github/workflows/ami-build-ubuntu-18.yml index 81c562feb..893abf94e 100644 --- a/.github/workflows/ami-build-ubuntu-18.yml +++ b/.github/workflows/ami-build-ubuntu-18.yml @@ -81,6 +81,7 @@ jobs: postgresql_major=${{ steps.version.outputs.postgresql_major }} postgresql_release=${{ steps.version.outputs.postgresql_release }} CPPFLAGS=-mcpu=${{ matrix.mcpu }} + DEB_BUILD_PROFILES=pkg.postgresql.nozstd tags: supabase/postgres:deb-u18 platforms: linux/${{ matrix.arch }} cache-from: type=gha diff --git a/docker/Dockerfile b/docker/Dockerfile index 2a1a06521..dd64c43e2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,6 +34,8 @@ ENV DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" # Configure processor optimised build ARG CPPFLAGS="-mcpu=neoverse-n1" ENV DEB_CPPFLAGS_APPEND="${CPPFLAGS} -fsigned-char" +ARG DEB_BUILD_PROFILES="" +ENV DEB_BUILD_PROFILES="${DEB_BUILD_PROFILES}" RUN apt-get build-dep -y postgresql-common pgdg-keyring && \ apt-get source --compile postgresql-common pgdg-keyring && \ diff --git a/docker/Dockerfile.u18 b/docker/Dockerfile.u18 deleted file mode 100644 index 3547b63d5..000000000 --- a/docker/Dockerfile.u18 +++ /dev/null @@ -1,72 +0,0 @@ -ARG ubuntu_release=focal -FROM ubuntu:${ubuntu_release} as base - -ARG ubuntu_release -ARG ubuntu_release_no -ARG postgresql_major=15 -ARG postgresql_release=${postgresql_major}.1 - -FROM base as pg-source - -# Install build dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - gnupg \ - dpkg-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# Add Postgres PPA -ARG postgresql_gpg_key=B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "${postgresql_gpg_key}" && \ - echo "deb https://apt-archive.postgresql.org/pub/repos/apt ${ubuntu_release}-pgdg-archive main" > /etc/apt/sources.list.d/pgdg.list && \ - echo "deb-src https://apt-archive.postgresql.org/pub/repos/apt ${ubuntu_release}-pgdg-archive main" > /etc/apt/sources.list.d/pgdg.list - -# Create local PPA -WORKDIR /tmp/build -RUN echo "deb [ trusted=yes ] file:///tmp/build ./" > /etc/apt/sources.list.d/temp.list && \ - dpkg-scanpackages . > Packages && \ - apt-get -o Acquire::GzipIndexes=false update - -ENV DEBIAN_FRONTEND=noninteractive -ENV PYTHONDONTWRITEBYTECODE=1 -ENV DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" - -# Configure processor optimised build -ARG CPPFLAGS="-mcpu=neoverse-n1" -ENV DEB_CPPFLAGS_APPEND="${CPPFLAGS} -fsigned-char" - -RUN apt-get build-dep -y postgresql-common pgdg-keyring && \ - apt-get source --compile postgresql-common pgdg-keyring && \ - dpkg-scanpackages . > Packages && \ - apt-get -o Acquire::GzipIndexes=false update - -# RUN apt-get build-dep -y "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \ -# apt-get source --compile "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \ -# dpkg-scanpackages . > Packages && \ -# apt-get -o Acquire::GzipIndexes=false update - -# Remove source directories -RUN rm -rf /tmp/build/*/ - -FROM base as pg - -# Inherit args from base stage -ARG postgresql_major -ARG postgresql_release - -COPY --from=pg-source /tmp/build /tmp/build - -ENV DEBIAN_FRONTEND=noninteractive -RUN echo "deb [ trusted=yes ] file:///tmp/build ./" > /etc/apt/sources.list.d/temp.list && \ - apt-get -o Acquire::GzipIndexes=false update && \ - apt-get install -y --no-install-recommends postgresql-common && \ - sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf && \ - apt-get install -y --no-install-recommends "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \ - rm -rf /var/lib/apt/lists/* && \ - rm -rf /tmp/build /etc/apt/sources.list.d/temp.list - -ENV PATH $PATH:/usr/lib/postgresql/${postgresql_major}/bin - -FROM scratch as pg-deb - -COPY --from=pg-source /tmp/build /tmp From b8a8fc5c84a35f302b78e98352adf9624a71d57f Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 21:03:27 +0300 Subject: [PATCH 06/13] chore: ignore errors when pushing docker cache --- .github/workflows/ami-build-ubuntu-18.yml | 2 +- .github/workflows/ami-release.yml | 4 +++- ansible/tasks/test-image.yml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ami-build-ubuntu-18.yml b/.github/workflows/ami-build-ubuntu-18.yml index 893abf94e..c7febf197 100644 --- a/.github/workflows/ami-build-ubuntu-18.yml +++ b/.github/workflows/ami-build-ubuntu-18.yml @@ -85,7 +85,7 @@ jobs: tags: supabase/postgres:deb-u18 platforms: linux/${{ matrix.arch }} cache-from: type=gha - cache-to: type=gha,mode=max + cache-to: type=gha,mode=max,ignore-error=true - name: Extract Postgres deb run: | mkdir -p /tmp/build ansible/files/postgres diff --git a/.github/workflows/ami-release.yml b/.github/workflows/ami-release.yml index 0707e9574..0bb420831 100644 --- a/.github/workflows/ami-release.yml +++ b/.github/workflows/ami-release.yml @@ -38,6 +38,8 @@ jobs: - uses: docker/setup-buildx-action@v2 with: endpoint: builders + driver-opts: image=moby/buildkit:v0.11.6 + buildkitd-flags: --debug - uses: docker/build-push-action@v3 with: push: false @@ -75,7 +77,7 @@ jobs: tags: supabase/postgres:deb platforms: linux/${{ matrix.arch }} cache-from: type=gha - cache-to: type=gha,mode=max + cache-to: type=gha,mode=max,ignore-error=true - name: Extract Postgres deb run: | mkdir -p /tmp/build ansible/files/postgres diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 3a6eb8f78..ff221ae6c 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -11,7 +11,7 @@ when: ebssurrogate_mode - name: Display /etc/postgresql/postgresql.conf contents - command: cat postgresql.conf chdir=/etc/postgresql.conf + command: cat postgresql.conf chdir=/etc/postgresql/ register: command_output - name: Print to console From 4f8131dfd0ba51ef1315eda71e188b70e6b332f4 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 21:30:11 +0300 Subject: [PATCH 07/13] chore: debug --- docker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index dd64c43e2..2cb5abc04 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -42,10 +42,10 @@ RUN apt-get build-dep -y postgresql-common pgdg-keyring && \ dpkg-scanpackages . > Packages && \ apt-get -o Acquire::GzipIndexes=false update -RUN apt-get build-dep -y "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \ - apt-get source --compile "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \ - dpkg-scanpackages . > Packages && \ - apt-get -o Acquire::GzipIndexes=false update +RUN apt-get build-dep -y "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" +RUN apt-get source --compile "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" +RUN dpkg-scanpackages . > Packages +RUN apt-get -o Acquire::GzipIndexes=false update # Remove source directories RUN rm -rf /tmp/build/*/ From 5b4de21323995d7e06ccc21d59f49add9da92753 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 21:45:24 +0300 Subject: [PATCH 08/13] chore: more u18 fixes --- .github/workflows/ami-build-ubuntu-18.yml | 2 +- docker/Dockerfile | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ami-build-ubuntu-18.yml b/.github/workflows/ami-build-ubuntu-18.yml index c7febf197..5df39cabb 100644 --- a/.github/workflows/ami-build-ubuntu-18.yml +++ b/.github/workflows/ami-build-ubuntu-18.yml @@ -80,7 +80,7 @@ jobs: ubuntu_release_no=${{ matrix.ubuntu_version }} postgresql_major=${{ steps.version.outputs.postgresql_major }} postgresql_release=${{ steps.version.outputs.postgresql_release }} - CPPFLAGS=-mcpu=${{ matrix.mcpu }} + CPPFLAGS= DEB_BUILD_PROFILES=pkg.postgresql.nozstd tags: supabase/postgres:deb-u18 platforms: linux/${{ matrix.arch }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 2cb5abc04..dd64c43e2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -42,10 +42,10 @@ RUN apt-get build-dep -y postgresql-common pgdg-keyring && \ dpkg-scanpackages . > Packages && \ apt-get -o Acquire::GzipIndexes=false update -RUN apt-get build-dep -y "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" -RUN apt-get source --compile "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" -RUN dpkg-scanpackages . > Packages -RUN apt-get -o Acquire::GzipIndexes=false update +RUN apt-get build-dep -y "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \ + apt-get source --compile "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \ + dpkg-scanpackages . > Packages && \ + apt-get -o Acquire::GzipIndexes=false update # Remove source directories RUN rm -rf /tmp/build/*/ From 5bea56e9a79c7871c9cbb9308f6a25f140986fd4 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 21:53:54 +0300 Subject: [PATCH 09/13] chore: build without testing pljava --- .github/workflows/ami-build-ubuntu-18.yml | 4 +++- .github/workflows/ami-release.yml | 2 +- ansible/tasks/test-image.yml | 8 -------- migrations/tests/extensions/test.sql | 2 +- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ami-build-ubuntu-18.yml b/.github/workflows/ami-build-ubuntu-18.yml index 5df39cabb..08d1a7a8d 100644 --- a/.github/workflows/ami-build-ubuntu-18.yml +++ b/.github/workflows/ami-build-ubuntu-18.yml @@ -46,6 +46,8 @@ jobs: - uses: docker/setup-buildx-action@v2 with: endpoint: builders + driver-opts: image=moby/buildkit:v0.11.6 + buildkitd-flags: --debug - uses: docker/build-push-action@v3 with: push: false @@ -56,7 +58,7 @@ jobs: tags: supabase/postgres:extensions platforms: linux/${{ matrix.arch }} cache-from: type=gha - cache-to: type=gha,mode=max + cache-to: type=gha,mode=max,ignore-error=true - name: Extract built packages run: | diff --git a/.github/workflows/ami-release.yml b/.github/workflows/ami-release.yml index 0bb420831..bbb570ba4 100644 --- a/.github/workflows/ami-release.yml +++ b/.github/workflows/ami-release.yml @@ -50,7 +50,7 @@ jobs: tags: supabase/postgres:extensions platforms: linux/${{ matrix.arch }} cache-from: type=gha - cache-to: type=gha,mode=max + cache-to: type=gha,mode=max,ignore-error=true - name: Extract built packages run: | mkdir -p /tmp/extensions ansible/files/extensions diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index ff221ae6c..cd92a27d6 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -10,14 +10,6 @@ cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf when: ebssurrogate_mode -- name: Display /etc/postgresql/postgresql.conf contents - command: cat postgresql.conf chdir=/etc/postgresql/ - register: command_output - -- name: Print to console - debug: - msg: "{{command_output.stdout}}" - - name: Start Postgres Database to load all extensions. become: yes become_user: postgres diff --git a/migrations/tests/extensions/test.sql b/migrations/tests/extensions/test.sql index 3c73410eb..7e0d1f38d 100644 --- a/migrations/tests/extensions/test.sql +++ b/migrations/tests/extensions/test.sql @@ -10,7 +10,7 @@ \ir 09-pg-safeupdate.sql \ir 10-timescaledb.sql \ir 11-wal2json.sql -\ir 12-pljava.sql +-- \ir 12-pljava.sql \ir 13-plv8.sql \ir 14-pg_plan_filter.sql \ir 15-pg_net.sql From d057338c0fc4ca44e49b6c3f01b7df191eccff65 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 22:00:30 +0300 Subject: [PATCH 10/13] chore: fix pljava test --- ansible/tasks/setup-docker.yml | 7 +++++++ migrations/tests/extensions/test.sql | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/setup-docker.yml b/ansible/tasks/setup-docker.yml index c2a56cc30..94e94f552 100644 --- a/ansible/tasks/setup-docker.yml +++ b/ansible/tasks/setup-docker.yml @@ -38,6 +38,13 @@ state: present line: auto_explain.log_min_duration = 10s +- name: pljava - set pljava.libjvm_location + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-{{ platform }}/lib/server/libjvm.so' + # supautils - name: supautils - add supautils to session_preload_libraries become: yes diff --git a/migrations/tests/extensions/test.sql b/migrations/tests/extensions/test.sql index 7e0d1f38d..3c73410eb 100644 --- a/migrations/tests/extensions/test.sql +++ b/migrations/tests/extensions/test.sql @@ -10,7 +10,7 @@ \ir 09-pg-safeupdate.sql \ir 10-timescaledb.sql \ir 11-wal2json.sql --- \ir 12-pljava.sql +\ir 12-pljava.sql \ir 13-plv8.sql \ir 14-pg_plan_filter.sql \ir 15-pg_net.sql From 6a53f3fa9bc802f8d13600229c78e62a3acbd2e4 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 22:21:31 +0300 Subject: [PATCH 11/13] chore: install the correct package on ubuntu 18.04 --- ansible/tasks/setup-postgres.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 4f350c336..df84eba05 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -21,6 +21,13 @@ apt: name: postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1 install_recommends: no + when: ansible_distribution_version != "18.04" + +- name: Postgres - install server - Ubuntu 18.04 + apt: + name: postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg18.04+1 + install_recommends: no + when: ansible_distribution_version == "18.04" - name: Postgres - remove PPA apt_repository: From 0e9c321fed437422dee304623e84929bd7f28dea Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 5 Jul 2023 23:18:30 +0300 Subject: [PATCH 12/13] chore: cleanup; bump version --- .github/workflows/ami-release.yml | 32 +++++++++++----------- .github/workflows/dockerhub-release.yml | 36 ++++++++++++------------- common.vars.pkr.hcl | 2 +- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ami-release.yml b/.github/workflows/ami-release.yml index bbb570ba4..89bca3d32 100644 --- a/.github/workflows/ami-release.yml +++ b/.github/workflows/ami-release.yml @@ -133,22 +133,22 @@ jobs: run: | aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz - # - name: Create release - # uses: softprops/action-gh-release@v1 - # with: - # name: ${{ steps.process_release_version.outputs.version }} - # tag_name: ${{ steps.process_release_version.outputs.version }} - # target_commitish: ${{github.sha}} - - # - name: Slack Notification on Failure - # if: ${{ failure() }} - # uses: rtCamp/action-slack-notify@v2 - # env: - # SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - # SLACK_USERNAME: 'gha-failures-notifier' - # SLACK_COLOR: 'danger' - # SLACK_MESSAGE: 'Building Postgres AMI failed' - # SLACK_FOOTER: '' + - name: Create release + uses: softprops/action-gh-release@v1 + with: + name: ${{ steps.process_release_version.outputs.version }} + tag_name: ${{ steps.process_release_version.outputs.version }} + target_commitish: ${{github.sha}} + + - name: Slack Notification on Failure + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + SLACK_USERNAME: 'gha-failures-notifier' + SLACK_COLOR: 'danger' + SLACK_MESSAGE: 'Building Postgres AMI failed' + SLACK_FOOTER: '' - name: Cleanup resources on build cancellation if: ${{ cancelled() }} diff --git a/.github/workflows/dockerhub-release.yml b/.github/workflows/dockerhub-release.yml index a8139be98..6a6e738f4 100644 --- a/.github/workflows/dockerhub-release.yml +++ b/.github/workflows/dockerhub-release.yml @@ -60,15 +60,15 @@ jobs: platforms: linux/${{ matrix.arch }} cache-from: type=gha cache-to: type=gha,mode=max - # - name: Slack Notification - # if: ${{ failure() }} - # uses: rtCamp/action-slack-notify@v2 - # env: - # SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - # SLACK_USERNAME: "gha-failures-notifier" - # SLACK_COLOR: "danger" - # SLACK_MESSAGE: "Building Postgres ${{ matrix.arch }} image failed" - # SLACK_FOOTER: "" + - name: Slack Notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + SLACK_USERNAME: "gha-failures-notifier" + SLACK_COLOR: "danger" + SLACK_MESSAGE: "Building Postgres ${{ matrix.arch }} image failed" + SLACK_FOOTER: "" merge_manifest: needs: [settings, build_image] @@ -84,15 +84,15 @@ jobs: docker buildx imagetools create -t ${{ needs.settings.outputs.image_tag }} \ ${{ needs.settings.outputs.image_tag }}_amd64 \ ${{ needs.settings.outputs.image_tag }}_arm64 - # - name: Slack Notification - # if: ${{ failure() }} - # uses: rtCamp/action-slack-notify@v2 - # env: - # SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - # SLACK_USERNAME: "gha-failures-notifier" - # SLACK_COLOR: "danger" - # SLACK_MESSAGE: "Building Postgres image failed" - # SLACK_FOOTER: "" + - name: Slack Notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + SLACK_USERNAME: "gha-failures-notifier" + SLACK_COLOR: "danger" + SLACK_MESSAGE: "Building Postgres image failed" + SLACK_FOOTER: "" publish: needs: [settings, merge_manifest] diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 40a04d9f8..3c240fa60 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.0.101-rc0" +postgres-version = "15.1.0.101" From 36d6a7a06840d2c33fe1110264ea6569fd1d9e6f Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Thu, 6 Jul 2023 11:27:15 +0300 Subject: [PATCH 13/13] chore: remove libssl-dev, switch `default-jdk` to `default-jdk-headless` when building the .deb for pljava Co-authored-by: Han Qiao --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f275bedd5..e303e5247 100644 --- a/Dockerfile +++ b/Dockerfile @@ -421,7 +421,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /tmp/pljava-${pljava_release} RUN mvn -T 1C clean install -Dmaven.test.skip -DskipTests -Dmaven.javadoc.skip=true # Create debian package -RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --pkgname=pljava --pkgversion="\${pljava_release//_/.}" --requires=libssl-dev,default-jdk --exclude /proc/ --nodoc java -jar pljava-packaging/target/pljava-pg${postgresql_major}.jar +RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --pkgname=pljava --pkgversion="\${pljava_release//_/.}" --requires=default-jdk-headless --exclude /proc/ --nodoc java -jar pljava-packaging/target/pljava-pg${postgresql_major}.jar #################### # 13-plv8.yml