Skip to content

Commit c60f817

Browse files
encimaencima
committed
merge with develop
2 parents 00e4594 + c4d26b8 commit c60f817

File tree

248 files changed

+19438
-3162
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

248 files changed

+19438
-3162
lines changed

‎.envrc.recommended‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
watch_file nix/devShells.nix
2+
3+
use flake

‎.github/actions/shared-checkout/action.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
name: Checkout
2+
description: Checkout repository for pull requests and branches
3+
runs:
4+
using: "composite"
5+
steps:
6+
- name: Checkout repository
7+
uses: actions/checkout@v4
8+
with:
9+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
10+
ref: ${{ github.event.pull_request.head.sha || github.sha }}
11+
fetch-depth: 0
12+
fetch-tags: true

‎.github/workflows/ami-release-nix-single.yml‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
name: Release Single AMI Nix
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
postgres_version:
7+
description: 'PostgreSQL major version to build (e.g. 15)'
8+
required: true
9+
type: string
10+
branch:
11+
description: 'Branch to run the workflow from'
12+
required: true
13+
type: string
14+
default: 'main'
15+
16+
permissions:
17+
contents: write
18+
id-token: write
19+
20+
jobs:
21+
build:
22+
runs-on: large-linux-arm
23+
timeout-minutes: 150
24+
25+
steps:
26+
- name: Checkout Repo
27+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
28+
with:
29+
ref: ${{ github.event.inputs.branch }}
30+
- name: aws-creds
31+
uses: aws-actions/configure-aws-credentials@v4
32+
with:
33+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
34+
aws-region: "us-east-1"
35+
output-credentials: true
36+
role-duration-seconds: 7200
37+
38+
- name: Get current branch SHA
39+
id: get_sha
40+
run: |
41+
echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
42+
43+
- name: Install nix
44+
uses: cachix/install-nix-action@v27
45+
with:
46+
install_url: https://releases.nixos.org/nix/nix-2.29.1/install
47+
extra_nix_config: |
48+
substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
49+
trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
50+
51+
- name: Set PostgreSQL version environment variable
52+
run: |
53+
echo "POSTGRES_MAJOR_VERSION=${{ github.event.inputs.postgres_version }}" >> $GITHUB_ENV
54+
echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV
55+
56+
- name: Generate common-nix.vars.pkr.hcl
57+
run: |
58+
PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ env.POSTGRES_MAJOR_VERSION }}'"]' ansible/vars.yml)
59+
PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes
60+
echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
61+
# Ensure there's a newline at the end of the file
62+
echo "" >> common-nix.vars.pkr.hcl
63+
64+
- name: Build AMI stage 1
65+
env:
66+
POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
67+
run: |
68+
GIT_SHA=${{ steps.get_sha.outputs.sha }}
69+
nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
70+
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
71+
72+
- name: Build AMI stage 2
73+
env:
74+
POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
75+
run: |
76+
GIT_SHA=${{ steps.get_sha.outputs.sha }}
77+
nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl
78+
POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
79+
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
80+
81+
- name: Grab release version
82+
id: process_release_version
83+
run: |
84+
VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
85+
echo "version=$VERSION" >> $GITHUB_OUTPUT
86+
87+
- name: Create nix flake revision tarball
88+
run: |
89+
GIT_SHA=${{ steps.get_sha.outputs.sha }}
90+
MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
91+
92+
mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
93+
echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
94+
tar -czf "/tmp/pg_binaries.tar.gz" -C "/tmp/pg_upgrade_bin" .
95+
96+
- name: configure aws credentials - staging
97+
uses: aws-actions/configure-aws-credentials@v4
98+
with:
99+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
100+
aws-region: "us-east-1"
101+
102+
- name: Upload software manifest to s3 staging
103+
run: |
104+
cd ansible
105+
ansible-playbook -i localhost \
106+
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
107+
-e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
108+
-e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
109+
manifest-playbook.yml
110+
111+
- name: Upload nix flake revision to s3 staging
112+
run: |
113+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
114+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
115+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
116+
117+
- name: configure aws credentials - prod
118+
uses: aws-actions/configure-aws-credentials@v4
119+
with:
120+
role-to-assume: ${{ secrets.PROD_AWS_ROLE }}
121+
aws-region: "us-east-1"
122+
123+
- name: Upload software manifest to s3 prod
124+
run: |
125+
cd ansible
126+
ansible-playbook -i localhost \
127+
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
128+
-e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
129+
-e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
130+
manifest-playbook.yml
131+
132+
- name: Upload nix flake revision to s3 prod
133+
run: |
134+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
135+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
136+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
137+
138+
- name: Create release
139+
uses: softprops/action-gh-release@v2
140+
with:
141+
name: ${{ steps.process_release_version.outputs.version }}
142+
tag_name: ${{ steps.process_release_version.outputs.version }}
143+
target_commitish: ${{ steps.get_sha.outputs.sha }}
144+
145+
- name: Slack Notification on Failure
146+
if: ${{ failure() }}
147+
uses: rtCamp/action-slack-notify@v2
148+
env:
149+
SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
150+
SLACK_USERNAME: 'gha-failures-notifier'
151+
SLACK_COLOR: 'danger'
152+
SLACK_MESSAGE: 'Building Postgres AMI failed'
153+
SLACK_FOOTER: ''
154+
155+
- name: Cleanup resources after build
156+
if: ${{ always() }}
157+
run: |
158+
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
159+
160+
- name: Cleanup resources on build cancellation
161+
if: ${{ cancelled() }}
162+
run: |
163+
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
164+

‎.github/workflows/ami-release-nix.yml‎

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,20 @@ permissions:
1717

1818
jobs:
1919
prepare:
20-
runs-on: ubuntu-latest
20+
runs-on: blacksmith-4vcpu-ubuntu-2404
2121
outputs:
2222
postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }}
2323
steps:
2424
- name: Checkout Repo
25-
uses: actions/checkout@v3
26-
27-
- uses: DeterminateSystems/nix-installer-action@main
25+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
26+
27+
- name: Install nix
28+
uses: cachix/install-nix-action@v27
29+
with:
30+
install_url: https://releases.nixos.org/nix/nix-2.29.1/install
31+
extra_nix_config: |
32+
substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
33+
trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
2834
2935
- name: Set PostgreSQL versions
3036
id: set-versions
@@ -38,35 +44,46 @@ jobs:
3844
matrix:
3945
postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
4046
include:
41-
- runner: arm-runner
42-
arch: arm64
43-
ubuntu_release: focal
44-
ubuntu_version: 20.04
45-
mcpu: neoverse-n1
47+
- runner: blacksmith-2vcpu-ubuntu-2404-arm
4648
runs-on: ${{ matrix.runner }}
4749
timeout-minutes: 150
4850

4951
steps:
5052
- name: Checkout Repo
51-
uses: actions/checkout@v3
53+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
54+
- name: aws-creds
55+
uses: aws-actions/configure-aws-credentials@v4
56+
with:
57+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
58+
aws-region: "us-east-1"
59+
output-credentials: true
60+
role-duration-seconds: 7200
5261

53-
- uses: DeterminateSystems/nix-installer-action@main
62+
- name: Install nix
63+
uses: cachix/install-nix-action@v27
64+
with:
65+
install_url: https://releases.nixos.org/nix/nix-2.29.1/install
66+
extra_nix_config: |
67+
substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
68+
trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
5469
5570
- name: Run checks if triggered manually
5671
if: ${{ github.event_name == 'workflow_dispatch' }}
5772
run: |
58-
SUFFIX=$(sudo nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/')
73+
SUFFIX=$(nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/')
5974
if [[ -z "$SUFFIX" ]] ; then
6075
echo "Version must include non-numeric characters if built manually."
6176
exit 1
6277
fi
6378
6479
- name: Set PostgreSQL version environment variable
65-
run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
80+
run: |
81+
echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
82+
echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV
6683
6784
- name: Generate common-nix.vars.pkr.hcl
6885
run: |
69-
PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
86+
PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
7087
PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes
7188
echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
7289
# Ensure there's a newline at the end of the file
@@ -76,19 +93,19 @@ jobs:
7693
env:
7794
POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
7895
run: |
79-
packer init amazon-arm64-nix.pkr.hcl
8096
GIT_SHA=${{github.sha}}
97+
nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
8198
# why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined?
82-
packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
99+
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
83100
84101
- name: Build AMI stage 2
85102
env:
86103
POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
87104
run: |
88-
packer init stage2-nix-psql.pkr.hcl
89105
GIT_SHA=${{github.sha}}
106+
nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl
90107
POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
91-
packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
108+
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
92109
93110
- name: Grab release version
94111
id: process_release_version
@@ -123,6 +140,8 @@ jobs:
123140
- name: Upload nix flake revision to s3 staging
124141
run: |
125142
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
143+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
144+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
126145
127146
- name: configure aws credentials - prod
128147
uses: aws-actions/configure-aws-credentials@v4
@@ -142,6 +161,8 @@ jobs:
142161
- name: Upload nix flake revision to s3 prod
143162
run: |
144163
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
164+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
165+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
145166
146167
- name: Create release
147168
uses: softprops/action-gh-release@v2
@@ -163,9 +184,9 @@ jobs:
163184
- name: Cleanup resources after build
164185
if: ${{ always() }}
165186
run: |
166-
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
187+
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
167188
168189
- name: Cleanup resources on build cancellation
169190
if: ${{ cancelled() }}
170191
run: |
171-
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
192+
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids

‎.github/workflows/check-shellscripts.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,19 @@ on:
77
pull_request:
88
workflow_dispatch:
99

10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
12+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
13+
1014
permissions:
1115
contents: read
1216

1317
jobs:
1418
build:
1519
runs-on: ubuntu-latest
1620
steps:
17-
- uses: actions/checkout@v3
21+
- name: Checkout Repo
22+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
1823
- name: Run ShellCheck
1924
uses: ludeeus/action-shellcheck@master
2025
env:

‎.github/workflows/ci.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,17 @@ on:
66
permissions:
77
contents: read
88

9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
11+
cancel-in-progress: true
12+
913
jobs:
1014
check-release-version:
1115
timeout-minutes: 5
1216
runs-on: ubuntu-latest
1317
steps:
1418
- name: Checkout Repo
15-
uses: actions/checkout@v3
19+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
1620

1721
- name: Load postgres_release values
1822
id: load_postgres_release

0 commit comments

Comments
 (0)