diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..f9cd32d1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,225 @@ +name: MinIO Rust Library CI + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +env: + RUST_LOG: debug + CARGO_TERM_COLOR: always + +jobs: + # Run once - same code for both variants + check-format: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check format + run: | + cargo fmt --all -- --check + + # Run once - same code for both variants + clippy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: clippy + run: cargo clippy --all-targets --all-features --workspace -- -D warnings + + # Run once - same code for both variants + build: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - name: Build + run: | + cargo --version + cargo build --bins --examples --tests --benches --verbose + + # Test against AIStor MinIO + test-aistor-multi-thread: + runs-on: ubuntu-latest + needs: [check-format, clippy, build] + steps: + - uses: actions/checkout@v4 + - name: Validate AIStor license secret + run: | + if [ -z "${{ secrets.AISTOR_LICENSE }}" ]; then + echo "❌ ERROR: AISTOR_LICENSE secret is not set or is empty" + echo "Please configure the AISTOR_LICENSE secret in GitHub repository settings:" + echo " Settings -> Secrets and variables -> Actions -> New repository secret" + exit 1 + fi + echo "✅ AISTOR_LICENSE secret is configured" + - name: Start AIStor MinIO server + env: + MINIO_LICENSE: ${{ secrets.AISTOR_LICENSE }} + MINIO_CI_CD: true + MINIO_ROOT_USER: minioadmin # TODO: Change to ${{ secrets.AISTOR_USER }} when secret is set + MINIO_ROOT_PASSWORD: minioadmin # TODO: Change to ${{ secrets.AISTOR_PASSWORD }} when secret is set + MINIO_NOTIFY_WEBHOOK_ENABLE_miniojavatest: on + MINIO_NOTIFY_WEBHOOK_ENDPOINT_miniojavatest: http://example.org/ + run: | + wget --quiet https://dl.minio.io/aistor/minio/release/linux-amd64/minio + chmod +x minio + echo "AIStor MinIO Server Version:" + ./minio --version + mkdir -p /tmp/certs + cp ./tests/public.crt ./tests/private.key /tmp/certs/ + + # Start server and redirect output to log file + ./minio server /tmp/test-xl/{1...4}/ --certs-dir /tmp/certs/ > /tmp/minio.log 2>&1 & + MINIO_PID=$! + echo "MinIO started with PID: $MINIO_PID" + + # Wait for server to start + sleep 5 + + # Check if process is still running + if ! ps -p $MINIO_PID > /dev/null; then + echo "❌ MinIO server process died" + echo "=== MinIO Server Log ===" + cat /tmp/minio.log + exit 1 + fi + + # Check for FATAL errors in log + if grep -q "FATAL" /tmp/minio.log; then + echo "❌ MinIO server encountered FATAL error" + echo "=== MinIO Server Log ===" + cat /tmp/minio.log + exit 1 + fi + + # Additional wait for server to be fully ready + sleep 5 + + # Final check + if ! ps -p $MINIO_PID > /dev/null; then + echo "❌ MinIO server process died during startup" + echo "=== MinIO Server Log ===" + cat /tmp/minio.log + exit 1 + fi + + echo "✅ MinIO server started successfully" + echo "=== MinIO Server Log (startup) ===" + head -20 /tmp/minio.log + - name: Run tests (multi-thread) + run: | + export SERVER_ENDPOINT=localhost:9000 + export ACCESS_KEY=minioadmin # TODO: Change to ${{ secrets.AISTOR_USER }} when secret is set + export SECRET_KEY=minioadmin # TODO: Change to ${{ secrets.AISTOR_PASSWORD }} when secret is set + export ENABLE_HTTPS=1 + export MINIO_SSL_CERT_FILE=./tests/public.crt + MINIO_TEST_TOKIO_RUNTIME_FLAVOR="multi_thread" cargo test -- --nocapture + + test-aistor-current-thread: + if: false # Temporarily disabled + runs-on: ubuntu-latest + needs: [check-format, clippy, build] + steps: + - uses: actions/checkout@v4 + - name: Validate AIStor license secret + run: | + if [ -z "${{ secrets.AISTOR_LICENSE }}" ]; then + echo "❌ ERROR: AISTOR_LICENSE secret is not set or is empty" + echo "Please configure the AISTOR_LICENSE secret in GitHub repository settings:" + echo " Settings -> Secrets and variables -> Actions -> New repository secret" + exit 1 + fi + echo "✅ AISTOR_LICENSE secret is configured" + - name: Start AIStor MinIO server + env: + MINIO_LICENSE: ${{ secrets.AISTOR_LICENSE }} + MINIO_CI_CD: true + MINIO_ROOT_USER: minioadmin # TODO: Change to ${{ secrets.AISTOR_USER }} when secret is set + MINIO_ROOT_PASSWORD: minioadmin # TODO: Change to ${{ secrets.AISTOR_PASSWORD }} when secret is set + MINIO_NOTIFY_WEBHOOK_ENABLE_miniojavatest: on + MINIO_NOTIFY_WEBHOOK_ENDPOINT_miniojavatest: http://example.org/ + run: | + wget --quiet https://dl.minio.io/aistor/minio/release/linux-amd64/minio + chmod +x minio + echo "AIStor MinIO Server Version:" + ./minio --version + mkdir -p /tmp/certs + cp ./tests/public.crt ./tests/private.key /tmp/certs/ + + # Start server and redirect output to log file + ./minio server /tmp/test-xl/{1...4}/ --certs-dir /tmp/certs/ > /tmp/minio.log 2>&1 & + MINIO_PID=$! + echo "MinIO started with PID: $MINIO_PID" + + # Wait for server to start + sleep 5 + + # Check if process is still running + if ! ps -p $MINIO_PID > /dev/null; then + echo "❌ MinIO server process died" + echo "=== MinIO Server Log ===" + cat /tmp/minio.log + exit 1 + fi + + # Check for FATAL errors in log + if grep -q "FATAL" /tmp/minio.log; then + echo "❌ MinIO server encountered FATAL error" + echo "=== MinIO Server Log ===" + cat /tmp/minio.log + exit 1 + fi + + # Additional wait for server to be fully ready + sleep 5 + + # Final check + if ! ps -p $MINIO_PID > /dev/null; then + echo "❌ MinIO server process died during startup" + echo "=== MinIO Server Log ===" + cat /tmp/minio.log + exit 1 + fi + + echo "✅ MinIO server started successfully" + echo "=== MinIO Server Log (startup) ===" + head -20 /tmp/minio.log + - name: Run tests (current-thread) + run: | + export SERVER_ENDPOINT=localhost:9000 + export ACCESS_KEY=minioadmin # TODO: Change to ${{ secrets.AISTOR_USER }} when secret is set + export SECRET_KEY=minioadmin # TODO: Change to ${{ secrets.AISTOR_PASSWORD }} when secret is set + export ENABLE_HTTPS=1 + export MINIO_SSL_CERT_FILE=./tests/public.crt + MINIO_TEST_TOKIO_RUNTIME_FLAVOR="current_thread" cargo test -- --nocapture + + # Test against OSS MinIO + test-oss-multi-thread: + if: false # Temporarily disabled + runs-on: ubuntu-latest + needs: [check-format, clippy, build] + steps: + - uses: actions/checkout@v4 + - name: Start OSS MinIO server + run: | + wget --quiet https://dl.min.io/server/minio/release/linux-amd64/minio + chmod +x minio + echo "OSS MinIO Server Version:" + ./minio --version + mkdir -p /tmp/certs + cp ./tests/public.crt ./tests/private.key /tmp/certs/ + MINIO_CI_CD=true \ + MINIO_NOTIFY_WEBHOOK_ENABLE_miniojavatest=on \ + MINIO_NOTIFY_WEBHOOK_ENDPOINT_miniojavatest=http://example.org/ \ + ./minio server /tmp/test-xl/{1...4}/ --certs-dir /tmp/certs/ & + sleep 10 + - name: Run tests (multi-thread) + run: | + export SERVER_ENDPOINT=localhost:9000 + export ACCESS_KEY=minioadmin + export SECRET_KEY=minioadmin + export ENABLE_HTTPS=1 + export MINIO_SSL_CERT_FILE=./tests/public.crt + MINIO_TEST_TOKIO_RUNTIME_FLAVOR="multi_thread" cargo test -- --nocapture diff --git a/.github/workflows/deployer.yml b/.github/workflows/deployer.yml new file mode 100644 index 00000000..9ac9c2d3 --- /dev/null +++ b/.github/workflows/deployer.yml @@ -0,0 +1,142 @@ +name: Build Container + +on: + push: + branches: + - "master" +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +env: + REGISTRY_IMAGE: registry.min.dev/aistor/minio-rs + +jobs: + prep: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Prepare + id: prep + run: | + TIMESTAMP=$(date '+%Y-%m-%dT%H-%M-%SZ') + SHORT_SHA=${GITHUB_SHA:0:7} + echo "tag=RELEASE.${TIMESTAMP}" >> $GITHUB_OUTPUT + echo "tagged_image=RELEASE.${TIMESTAMP}" >> $GITHUB_OUTPUT + echo "latest_image=edge" >> $GITHUB_OUTPUT + echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + outputs: + tag: ${{ steps.prep.outputs.tag }} + tagged_image: ${{ steps.prep.outputs.tagged_image }} + latest_image: ${{ steps.prep.outputs.latest_image }} + short_sha: ${{ steps.prep.outputs.short_sha }} + + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + needs: prep + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_IMAGE }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Registry + uses: docker/login-action@v3 + with: + registry: ${{ secrets.REGISTRY_URL }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + build-args: | + RELEASE=${{ needs.prep.outputs.tag }} + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ strategy.job-index }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-latest + needs: + - prep + - build + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: digests-* + merge-multiple: true + path: /tmp/digests + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_IMAGE }} + tags: | + type=raw,value=${{ needs.prep.outputs.tagged_image }} + type=raw,value=${{ needs.prep.outputs.short_sha }} + type=raw,value=edge + type=raw,value=latest + + - name: Login to Registry + uses: docker/login-action@v3 + with: + registry: ${{ secrets.REGISTRY_URL }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ needs.prep.outputs.tagged_image }} + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ needs.prep.outputs.short_sha }} + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:edge + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:latest diff --git a/.github/workflows/license-header-check.yml b/.github/workflows/license-header-check.yml new file mode 100644 index 00000000..6d09f713 --- /dev/null +++ b/.github/workflows/license-header-check.yml @@ -0,0 +1,146 @@ +name: License Header Check +on: + pull_request: + branches: + - master +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true +permissions: + contents: read + pull-requests: write +jobs: + license-header-check: + name: Validate License Headers + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check License Headers + run: | + # Create the license header validation script + cat > check_license_headers.sh << 'EOF' + #!/bin/bash + + # Expected license header for Rust files (all 14 comment lines) + EXPECTED_HEADER='// MinIO Rust Library for Amazon S3 Compatible Cloud Storage + // Copyright 2023 MinIO, Inc. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. + // You may obtain a copy of the License at + // + // http://www.apache.org/licenses/LICENSE-2.0 + // + // Unless required by applicable law or agreed to in writing, software + // distributed under the License is distributed on an "AS IS" BASIS, + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + // See the License for the specific language governing permissions and + // limitations under the License.' + + # Get list of new/added Rust files in this PR - excluding generated files + # that have "// This file is @generated" or similar markers + NEW_FILES=$(git diff --name-only --diff-filter=A origin/${{ github.base_ref }}..HEAD | \ + grep '\.rs$' || true) + + if [ -z "$NEW_FILES" ]; then + echo "No new Rust files found in this PR." + exit 0 + fi + + echo "Checking license headers in new Rust files:" + echo "$NEW_FILES" + echo "" + + EXIT_CODE=0 + + for file in $NEW_FILES; do + if [ ! -f "$file" ]; then + continue + fi + + # Check if file is generated (skip license check for generated files) + if head -5 "$file" | grep -q "@generated\|automatically generated"; then + echo "⏭️ Skipping generated file: $file" + continue + fi + + echo "Checking $file..." + + # Extract the first 14 lines (license header) + ACTUAL_HEADER=$(head -14 "$file" | sed 's/[[:space:]]*$//') + + # Compare headers + if [ "$ACTUAL_HEADER" != "$EXPECTED_HEADER" ]; then + echo "❌ License header missing or incorrect in: $file" + EXIT_CODE=1 + + echo "Expected header:" + echo "$EXPECTED_HEADER" + echo "" + echo "Actual header (first 14 lines):" + head -14 "$file" + echo "" + + # Generate git diff suggestion + TEMP_FILE=$(mktemp) + cat > "$TEMP_FILE" << 'HEADER_EOF' + // MinIO Rust Library for Amazon S3 Compatible Cloud Storage + // Copyright 2023 MinIO, Inc. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. + // You may obtain a copy of the License at + // + // http://www.apache.org/licenses/LICENSE-2.0 + // + // Unless required by applicable law or agreed to in writing, software + // distributed under the License is distributed on an "AS IS" BASIS, + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + // See the License for the specific language governing permissions and + // limitations under the License. + + HEADER_EOF + + # Add existing content after header (skip existing partial header if any) + if head -1 "$file" | grep -q "// MinIO\|//.*Copyright\|//.*Licensed"; then + # File has some form of header, skip lines until we find non-comment/non-empty + tail -n +1 "$file" | awk ' + BEGIN { skip = 1; found_content = 0 } + /^\/\// { if (skip) next } + /^[[:space:]]*$/ { if (skip) next } + { skip = 0; found_content = 1; print } + END { if (!found_content) print "" } + ' >> "$TEMP_FILE" + else + # No header, just append entire file + cat "$file" >> "$TEMP_FILE" + fi + + echo "Suggested diff for $file:" + git diff --no-index --no-prefix "$file" "$TEMP_FILE" | head -50 || true + echo "" + + rm -f "$TEMP_FILE" + else + echo "✅ License header correct in: $file" + fi + done + + if [ $EXIT_CODE -ne 0 ]; then + echo "" + echo "❌ License header validation failed!" + echo "Please add the required Apache 2.0 license header to the files listed above." + echo "The header should be placed at the top of each new Rust source file." + exit 1 + else + echo "" + echo "✅ All new Rust files have correct license headers!" + fi + EOF + + chmod +x check_license_headers.sh + ./check_license_headers.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index 4ee3af48..00000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: MinIO Rust Library - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - -env: - RUST_LOG: debug - CARGO_TERM_COLOR: always - -jobs: - check-format: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Check format - run: | - cargo fmt --all -- --check - - clippy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: clippy - run: cargo clippy --all-targets --all-features --workspace -- -D warnings - test-multi-thread: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run tests - run: | - ./tests/start-server.sh - export SERVER_ENDPOINT=localhost:9000 - export ACCESS_KEY=minioadmin - export SECRET_KEY=minioadmin - export ENABLE_HTTPS=1 - export MINIO_SSL_CERT_FILE=./tests/public.crt - MINIO_TEST_TOKIO_RUNTIME_FLAVOR="multi_thread" cargo test -- --nocapture - test-current-thread: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run tests - run: | - ./tests/start-server.sh - export SERVER_ENDPOINT=localhost:9000 - export ACCESS_KEY=minioadmin - export SECRET_KEY=minioadmin - export ENABLE_HTTPS=1 - export MINIO_SSL_CERT_FILE=./tests/public.crt - MINIO_TEST_TOKIO_RUNTIME_FLAVOR="current_thread" cargo test -- --nocapture - - build: - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v4 - - name: Build - run: | - cargo --version - cargo build --bins --examples --tests --benches --verbose diff --git a/tests/start-server.sh b/tests/start-server.sh deleted file mode 100755 index b0c8fd62..00000000 --- a/tests/start-server.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -x -set -e - -wget --quiet https://dl.min.io/server/minio/release/linux-amd64/minio -chmod +x minio - -echo "MinIO Server Version:" -./minio --version - -mkdir -p /tmp/certs -cp ./tests/public.crt ./tests/private.key /tmp/certs/ - -(MINIO_CI_CD=true \ - MINIO_NOTIFY_WEBHOOK_ENABLE_miniojavatest=on \ - MINIO_NOTIFY_WEBHOOK_ENDPOINT_miniojavatest=http://example.org/ \ - ./minio server /tmp/test-xl/{1...4}/ --certs-dir /tmp/certs/ &) - -sleep 10 - - -