Skip to content

Commit 0f428f2

Browse files
BourgoisMickaelBourgoisMickael
committed
(wip) CI
1 parent fca083b commit 0f428f2

File tree

2 files changed

+180
-0
lines changed

2 files changed

+180
-0
lines changed

‎.github/docker/docker-compose.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ services:
88
- /tmp/ssl-kmip:/ssl-kmip
99
- ${HOME}/.aws/credentials:/root/.aws/credentials
1010
- /tmp/artifacts/${JOB_NAME}:/artifacts
11+
# using artesca container, with persistent volumes for sse migration
12+
- ../../localData:/usr/src/app/localData
13+
- ../../localMetadata:/usr/src/app/localMetadata
1114
environment:
1215
- CI=true
1316
- ENABLE_LOCAL_CACHE=true
@@ -42,6 +45,16 @@ services:
4245
extra_hosts:
4346
- "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1"
4447
- "pykmip.local:127.0.0.1"
48+
cloudserver-sse-migration:
49+
extends: cloudserver
50+
profiles: [sse-migration]
51+
volumes:
52+
# using artesca container
53+
- ../../tests/functional/sse-kms-migration/config.${SSE_CONF}.json:/conf/config.json
54+
- ../../localData:/usr/src/app/localData
55+
- ../../localMetadata:/usr/src/app/localMetadata
56+
environment:
57+
- S3_CONFIG_FILE=/conf/config.json
4558
redis:
4659
image: redis:alpine
4760
network_mode: "host"

‎.github/workflows/tests.yaml‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,8 @@ jobs:
218218
source: /tmp/artifacts
219219
if: always()
220220

221+
# All tests use non federation images
222+
221223
file-ft-tests:
222224
strategy:
223225
matrix:
@@ -345,3 +347,168 @@ jobs:
345347
password: ${{ secrets.ARTIFACTS_PASSWORD }}
346348
source: /tmp/artifacts
347349
if: always()
350+
351+
sse-kms-migration-tests:
352+
strategy:
353+
matrix:
354+
include:
355+
- kmsHideScalityArn: 'showArn'
356+
job-name: sse-kms-migration-tests-show-arn
357+
# To use this one, the test needs to stop expecting an Arn in rsponse headers
358+
# But the check it should query the metadata directly to ensure the md has the arn
359+
# - kmsHideScalityArn: 'hideArn'
360+
# job-name: sse-kms-migration-tests-hide-arn
361+
name: ${{ matrix.job-name }}
362+
runs-on: ubuntu-latest
363+
needs: build
364+
env:
365+
S3BACKEND: file
366+
S3VAULT: mem
367+
CLOUDSERVER_IMAGE_BEFORE_SSE_MIGRATION: ghcr.io/${{ github.repository }}:7.70.21-11
368+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
369+
MPU_TESTING: "yes"
370+
JOB_NAME: ${{ matrix.job-name }}
371+
steps:
372+
- name: Checkout
373+
uses: actions/checkout@v4
374+
- uses: actions/setup-python@v5
375+
with:
376+
python-version: 3.9
377+
- name: Setup CI environment
378+
uses: ./.github/actions/setup-ci
379+
- name: Setup matrix job artifacts directory
380+
shell: bash
381+
run: |
382+
set -exu
383+
mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
384+
- name: Setup CI services (with old cloudserver image before sse migration)
385+
run: docker compose up -d
386+
working-directory: .github/docker
387+
env:
388+
CLOUDSERVER_IMAGE: ${{ env.CLOUDSERVER_IMAGE_BEFORE_SSE_MIGRATION }}
389+
SSE_CONF: before
390+
- name: Run SSE before migration tests (setup buckets and objects)
391+
env:
392+
# yarn run does a cd into the test folder
393+
S3_CONFIG_FILE: config.before.json
394+
run: |-
395+
set -o pipefail;
396+
bash wait_for_local_port.bash 8000 40
397+
398+
echo Ensures the expected version of cloudserver is old one:
399+
VERSION=$(docker compose -f .github/docker/docker-compose.yaml \
400+
exec cloudserver cat package.json | jq -r .version)
401+
if [[ "$VERSION" != "7.70.21-11" ]]; then
402+
echo "bad version of container. Should be 7.70.21-11. Was $VERSION" >&2
403+
exit 1
404+
else
405+
echo OK $VERSION
406+
fi
407+
408+
yarn run ft_sse_before_migration | tee /tmp/artifacts/${{ matrix.job-name }}/beforeMigration.log
409+
- name: Replace old cloudserver image with current one
410+
run: |-
411+
docker compose down cloudserver
412+
mv /tmp/artifacts/${{ matrix.job-name }}/s3.log /tmp/artifacts/${{ matrix.job-name }}/s3.old.log
413+
docker compose up -d cloudserver-sse-migration
414+
working-directory: .github/docker
415+
env:
416+
SSE_CONF: sseMigration.${{ matrix.kmsHideScalityArn }}
417+
- name: Run SSE migration tests
418+
env:
419+
# yarn run does a cd into the test folder
420+
S3_CONFIG_FILE: config.sseMigration.${{ matrix.kmsHideScalityArn }}.json
421+
run: |-
422+
set -o pipefail;
423+
bash wait_for_local_port.bash 8000 40
424+
425+
echo Ensures the expected version of cloudserver is NOT old one
426+
VERSION=$(docker compose -f .github/docker/docker-compose.yaml \
427+
exec cloudserver-sse-migration cat package.json | jq -r .version)
428+
if [[ "$VERSION" == "7.70.21-11" ]]; then
429+
echo "bad version of container. Should NOT be 7.70.21-11. Was $VERSION" >&2
430+
exit 1
431+
else
432+
echo OK $VERSION
433+
fi
434+
435+
yarn run ft_sse_migration | tee /tmp/artifacts/${{ matrix.job-name }}/migration.log
436+
- name: Run SSE arnPrefix tests
437+
env:
438+
S3_CONFIG_FILE: config.sseMigration.${{ matrix.kmsHideScalityArn }}.json
439+
run: |-
440+
set -o pipefail;
441+
yarn run ft_sse_arn | tee /tmp/artifacts/${{ matrix.job-name }}/arnPrefix.log
442+
- name: Print docker compose logs
443+
run: docker compose logs cloudserver cloudserver-sse-migration
444+
working-directory: .github/docker
445+
if: failure()
446+
- name: Upload logs to artifacts
447+
uses: scality/action-artifacts@v4
448+
with:
449+
method: upload
450+
url: https://artifacts.scality.net
451+
user: ${{ secrets.ARTIFACTS_USER }}
452+
password: ${{ secrets.ARTIFACTS_PASSWORD }}
453+
source: /tmp/artifacts
454+
if: always()
455+
456+
# Temporary until I have a good test taking into account kmsHideScalityArn
457+
tmp-kmsHideScalityArn-file-ft-tests:
458+
strategy:
459+
matrix:
460+
include:
461+
- enable-null-compat: ''
462+
job-name: tmp-kmsHideScalityArn-file-ft-tests
463+
name: ${{ matrix.job-name }}
464+
runs-on: ubuntu-latest
465+
needs: build
466+
env:
467+
S3BACKEND: file
468+
S3VAULT: mem
469+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
470+
MPU_TESTING: "yes"
471+
ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}"
472+
JOB_NAME: ${{ matrix.job-name }}
473+
steps:
474+
- name: Checkout
475+
uses: actions/checkout@v4
476+
- uses: actions/setup-python@v5
477+
with:
478+
python-version: 3.9
479+
- name: Setup CI environment
480+
uses: ./.github/actions/setup-ci
481+
- name: Setup matrix job artifacts directory
482+
shell: bash
483+
run: |
484+
set -exu
485+
mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
486+
- name: Setup python test environment
487+
run: |
488+
sudo apt-get install -y libdigest-hmac-perl
489+
pip install 's3cmd==2.3.0'
490+
- name: Setup CI services
491+
run: |-
492+
docker compose up -d
493+
docker compose down cloudserver
494+
docker compose up -d cloudserver-sse-migration
495+
working-directory: .github/docker
496+
env:
497+
SSE_CONF: sseMigration.hideArn
498+
- name: Run file ft tests
499+
env:
500+
# need absolute path as tests move into folders
501+
S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.sseMigration.hideArn.json
502+
run: |-
503+
set -o pipefail;
504+
bash wait_for_local_port.bash 8000 40
505+
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
506+
- name: Upload logs to artifacts
507+
uses: scality/action-artifacts@v4
508+
with:
509+
method: upload
510+
url: https://artifacts.scality.net
511+
user: ${{ secrets.ARTIFACTS_USER }}
512+
password: ${{ secrets.ARTIFACTS_PASSWORD }}
513+
source: /tmp/artifacts
514+
if: always()

0 commit comments

Comments
 (0)