diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000000..a47e40edbc3 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,27 @@ +self-hosted-runner: + # Define custom labels for Blacksmith runners + # Ubuntu 24.04 x64 runners + labels: + - blacksmith-2vcpu-ubuntu-2404 + - blacksmith-4vcpu-ubuntu-2404 + - blacksmith-8vcpu-ubuntu-2404 + - blacksmith-16vcpu-ubuntu-2404 + - blacksmith-32vcpu-ubuntu-2404 + # Ubuntu 24.04 ARM runners + - blacksmith-2vcpu-ubuntu-2404-arm + - blacksmith-4vcpu-ubuntu-2404-arm + - blacksmith-8vcpu-ubuntu-2404-arm + - blacksmith-16vcpu-ubuntu-2404-arm + - blacksmith-32vcpu-ubuntu-2404-arm + # Ubuntu 22.04 x64 runners + - blacksmith-2vcpu-ubuntu-2204 + - blacksmith-4vcpu-ubuntu-2204 + - blacksmith-8vcpu-ubuntu-2204 + - blacksmith-16vcpu-ubuntu-2204 + - blacksmith-32vcpu-ubuntu-2204 + # Ubuntu 22.04 ARM runners + - blacksmith-2vcpu-ubuntu-2204-arm + - blacksmith-4vcpu-ubuntu-2204-arm + - blacksmith-8vcpu-ubuntu-2204-arm + - blacksmith-16vcpu-ubuntu-2204-arm + - blacksmith-32vcpu-ubuntu-2204-arm diff --git a/.github/workflows/cd-dgraph-nightly.yml b/.github/workflows/cd-dgraph-nightly.yml index f5c9fb6216c..8d5319d21af 100644 --- a/.github/workflows/cd-dgraph-nightly.yml +++ b/.github/workflows/cd-dgraph-nightly.yml @@ -10,9 +10,10 @@ permissions: jobs: build: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - - uses: docker/setup-buildx-action@v3.11.1 + - name: Setup Blacksmith Builder + uses: useblacksmith/setup-docker-builder@v1 - name: Log in to GitHub Container Registry uses: docker/login-action@v3.5.0 @@ -22,7 +23,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Docker image - uses: docker/build-push-action@v6.18.0 + uses: useblacksmith/build-push-action@v2 with: push: true platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/cd-dgraph.yml b/.github/workflows/cd-dgraph.yml index 36e527efe94..9f27c90cd3a 100644 --- a/.github/workflows/cd-dgraph.yml +++ b/.github/workflows/cd-dgraph.yml @@ -21,7 +21,7 @@ permissions: jobs: dgraph-build-amd64: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 steps: - uses: actions/checkout@v5 @@ -222,7 +222,7 @@ jobs: graph-docker-image-and-manifests-push: needs: [dgraph-build-amd64, dgraph-build-arm64] - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-core-tests.yml b/.github/workflows/ci-dgraph-core-tests.yml index 7fedfc46ba5..c9c0225b868 100644 --- a/.github/workflows/ci-dgraph-core-tests.yml +++ b/.github/workflows/ci-dgraph-core-tests.yml @@ -26,7 +26,7 @@ permissions: jobs: dgraph-core-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-core-upgrade-tests.yml b/.github/workflows/ci-dgraph-core-upgrade-tests.yml index 9446016f12d..f88eb9b8845 100644 --- a/.github/workflows/ci-dgraph-core-upgrade-tests.yml +++ b/.github/workflows/ci-dgraph-core-upgrade-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-upgrade-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-fuzz.yml b/.github/workflows/ci-dgraph-fuzz.yml index a1f2f0f61db..4e0f9015743 100644 --- a/.github/workflows/ci-dgraph-fuzz.yml +++ b/.github/workflows/ci-dgraph-fuzz.yml @@ -23,7 +23,7 @@ permissions: jobs: fuzz-test: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-integration2-tests.yml b/.github/workflows/ci-dgraph-integration2-tests.yml index cb94826215d..40c3ef28c9d 100644 --- a/.github/workflows/ci-dgraph-integration2-tests.yml +++ b/.github/workflows/ci-dgraph-integration2-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-integration2-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-jepsen-tests.yml b/.github/workflows/ci-dgraph-jepsen-tests.yml index fa5466e9f07..7709af4264d 100644 --- a/.github/workflows/ci-dgraph-jepsen-tests.yml +++ b/.github/workflows/ci-dgraph-jepsen-tests.yml @@ -12,7 +12,7 @@ permissions: jobs: dgraph-jepsen-tests: - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - name: Checkout dgraph repo diff --git a/.github/workflows/ci-dgraph-ldbc-tests.yml b/.github/workflows/ci-dgraph-ldbc-tests.yml index 5e193f7eb95..6b56855ea79 100644 --- a/.github/workflows/ci-dgraph-ldbc-tests.yml +++ b/.github/workflows/ci-dgraph-ldbc-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-ldbc-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: - name: Checkout Dgraph diff --git a/.github/workflows/ci-dgraph-load-tests.yml b/.github/workflows/ci-dgraph-load-tests.yml index 688fdb6770b..0aee7076e43 100644 --- a/.github/workflows/ci-dgraph-load-tests.yml +++ b/.github/workflows/ci-dgraph-load-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-load-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml similarity index 86% rename from .github/workflows/ci-dgraph-system-upgrade-tests.yml rename to .github/workflows/ci-dgraph-system-upgrade-tests-mt.yml index bd6a64a3985..f30f6cba304 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml @@ -1,4 +1,4 @@ -name: ci-dgraph-system-upgrade-tests +name: ci-dgraph-system-upgrade-tests-multi-tenancy on: pull_request: @@ -24,7 +24,7 @@ permissions: jobs: dgraph-upgrade-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 steps: - uses: actions/checkout@v5 @@ -50,8 +50,6 @@ jobs: cp dgraph/dgraph ~/go/bin/dgraph # run the sytem upgrade tests go test -v -timeout=120m -failfast -tags=upgrade \ - github.com/hypermodeinc/dgraph/v25/systest/mutations-and-queries \ - github.com/hypermodeinc/dgraph/v25/systest/plugin \ github.com/hypermodeinc/dgraph/v25/systest/multi-tenancy # clean up docker containers after test execution go clean -testcache diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml new file mode 100644 index 00000000000..6e8a043842b --- /dev/null +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml @@ -0,0 +1,57 @@ +name: ci-dgraph-system-upgrade-tests-mutations-and-queries + +on: + pull_request: + paths: + - "!**/*.md" + - "**/*.go" + - "**/go.mod" + - "**/*.yml" + - "**/Dockerfile" + - "**/Makefile" + types: + - opened + - reopened + - synchronize + - ready_for_review + branches: + - main + - release/** + +permissions: + contents: read + +jobs: + dgraph-upgrade-tests: + if: github.event.pull_request.draft == false + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + - name: Make Linux Build and Docker Image + run: make docker-image + - name: Clean Up Environment + run: | + #!/bin/bash + # clean cache + go clean -testcache + - name: Run System Upgrade Tests + run: | + #!/bin/bash + # go env settings + export GOPATH=~/go + # move the binary + cp dgraph/dgraph ~/go/bin/dgraph + # run the sytem upgrade tests + go test -v -timeout=120m -failfast -tags=upgrade \ + github.com/hypermodeinc/dgraph/v25/systest/mutations-and-queries + # clean up docker containers after test execution + go clean -testcache + # sleep + sleep 5 diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml new file mode 100644 index 00000000000..10dfe7767a3 --- /dev/null +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml @@ -0,0 +1,57 @@ +name: ci-dgraph-system-upgrade-tests-plugin + +on: + pull_request: + paths: + - "!**/*.md" + - "**/*.go" + - "**/go.mod" + - "**/*.yml" + - "**/Dockerfile" + - "**/Makefile" + types: + - opened + - reopened + - synchronize + - ready_for_review + branches: + - main + - release/** + +permissions: + contents: read + +jobs: + dgraph-upgrade-tests: + if: github.event.pull_request.draft == false + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + - name: Make Linux Build and Docker Image + run: make docker-image + - name: Clean Up Environment + run: | + #!/bin/bash + # clean cache + go clean -testcache + - name: Run System Upgrade Tests + run: | + #!/bin/bash + # go env settings + export GOPATH=~/go + # move the binary + cp dgraph/dgraph ~/go/bin/dgraph + # run the sytem upgrade tests + go test -v -timeout=120m -failfast -tags=upgrade \ + github.com/hypermodeinc/dgraph/v25/systest/plugin + # clean up docker containers after test execution + go clean -testcache + # sleep + sleep 5 diff --git a/.github/workflows/ci-dgraph-systest-tests.yml b/.github/workflows/ci-dgraph-systest-tests.yml index a199880f46d..90b2b0bf3dc 100644 --- a/.github/workflows/ci-dgraph-systest-tests.yml +++ b/.github/workflows/ci-dgraph-systest-tests.yml @@ -26,7 +26,7 @@ permissions: jobs: dgraph-systest-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 @@ -63,6 +63,7 @@ jobs: #!/bin/bash # go env settings export GOPATH=~/go + export DEBUG_SHOW_ERROR=1 # move the binary cp dgraph/dgraph ~/go/bin/dgraph # run the unit and systests diff --git a/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml b/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml index 15e0a447325..13b5280fabe 100644 --- a/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml +++ b/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml @@ -9,7 +9,7 @@ permissions: jobs: dgraph-upgrade-fixed-versions-tests: - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-vector-tests.yml b/.github/workflows/ci-dgraph-vector-tests.yml index a4682c7007c..d0b6ca88201 100644 --- a/.github/workflows/ci-dgraph-vector-tests.yml +++ b/.github/workflows/ci-dgraph-vector-tests.yml @@ -26,7 +26,7 @@ permissions: jobs: dgraph-vector-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ab7db66d9f1..3b84ba57608 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -18,7 +18,7 @@ permissions: jobs: analyze: name: Analyze (${{ matrix.language }}) - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 360 permissions: security-events: write diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index e25d544aef3..7c1d9e41dcb 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -20,7 +20,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - uses: actions/checkout@v5 - uses: actions/labeler@v6 diff --git a/SECURITY.md b/SECURITY.md index 90c2a08dd32..e27e3140bb2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,6 +5,6 @@ vulnerability in Dgraph, we encourage you to let us know right away. We will investigate all legitimate reports and do our best to quickly fix the problem. Please report any issues or vulnerabilities via GitHub Security Advisories instead of posting a public issue in -GitHub. You can also send security communications to security@hypermode.com. +GitHub. You can also send security communications to dgraph-admin@istaridigital.com. Please include the version identifier and details on how the vulnerability can be exploited. diff --git a/dgraphtest/image.go b/dgraphtest/image.go index a91eb4e1875..a472486e551 100644 --- a/dgraphtest/image.go +++ b/dgraphtest/image.go @@ -21,6 +21,7 @@ import ( var ( cloneOnce sync.Once + gitMutex sync.Mutex // Protects git operations on shared repoDir ) func (c *LocalCluster) dgraphImage() string { @@ -49,7 +50,23 @@ func (c *LocalCluster) setupBinary() error { return copyBinary(fromDir, c.tempBinDir, c.conf.version) } - isFileThere, err := fileExists(filepath.Join(binariesPath, fmt.Sprintf(binaryNameFmt, c.conf.version))) + binaryPath := filepath.Join(binariesPath, fmt.Sprintf(binaryNameFmt, c.conf.version)) + + // First check without lock (fast path) + isFileThere, err := fileExists(binaryPath) + if err != nil { + return err + } + if isFileThere { + return copyBinary(binariesPath, c.tempBinDir, c.conf.version) + } + + // Lock git operations to prevent parallel tests from conflicting + gitMutex.Lock() + defer gitMutex.Unlock() + + // Double-check after acquiring lock - another parallel test may have built it + isFileThere, err = fileExists(binaryPath) if err != nil { return err } @@ -199,6 +216,16 @@ func copy(src, dst string) error { return errors.Errorf("%s is not a regular file", src) } + // Check if destination already exists and matches source size + if destStat, err := os.Stat(dst); err == nil { + if destStat.Size() == sourceFileStat.Size() { + log.Printf("[INFO] destination file %s already exists with matching size, skipping copy", dst) + return nil + } + log.Printf("[WARNING] destination file %s exists but size mismatch (source=%d, dest=%d), will overwrite", + dst, sourceFileStat.Size(), destStat.Size()) + } + // Open source file source, err := os.Open(src) if err != nil { diff --git a/systest/backup/common/utils.go b/systest/backup/common/utils.go index adb2774821e..efafaf2a7b8 100644 --- a/systest/backup/common/utils.go +++ b/systest/backup/common/utils.go @@ -278,5 +278,8 @@ func CopyToLocalFsFromNFS(t *testing.T, backupDst string, copyBackupDirectory st // "docker cp" to create a copy that is not owned by the root user. require.NoError(t, os.RemoveAll(copyBackupDirectory)) srcPath := testutil.DockerPrefix + "_nfs_1:/data" + backupDst - require.NoError(t, testutil.DockerCp(srcPath, copyBackupDirectory)) + if err := testutil.DockerCp(srcPath, copyBackupDirectory); err != nil { + t.Logf("DockerCp failed: src=%s dst=%s error=%v", srcPath, copyBackupDirectory, err) + require.NoError(t, err) + } } diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index 1c18257302c..aa9f609428f 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -18,10 +18,9 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/stretchr/testify/require" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" "github.com/dgraph-io/dgo/v250" "github.com/dgraph-io/dgo/v250/protos/api" @@ -39,24 +38,59 @@ var ( ) func TestBackupHAClust(t *testing.T) { - - backupRestoreTest(t, testutil.GetSockAddr(), testutil.GetSockAddrAlpha4Http(), - testutil.GetSockAddrZeroHttp(), backupDstHA, testutil.GetSockAddrHttp()) + t.Skip("Skipping HA backup test via NFS") + backupRestoreTest(t, "alpha1_backup_clust_ha", "zero1_backup_clust_ha", + "alpha4_restore_clust_ha", backupDstHA) } func TestBackupNonHAClust(t *testing.T) { - - backupRestoreTest(t, testutil.GetSockAddrAlpha7(), testutil.GetSockAddrAlpha8Http(), - testutil.GetSockAddrZero7Http(), backupDstNonHA, testutil.GetSockAddrAlpha7Http()) + t.Skip("Skipping Non-HA backup test via NFS") + backupRestoreTest(t, "alpha7_backup_clust_non_ha", "zero7_backup_clust_non_ha", + "alpha8_restore_clust_non_ha", backupDstNonHA) } -func backupRestoreTest(t *testing.T, backupAlphaSocketAddr string, restoreAlphaAddr string, - backupZeroAddr string, backupDst string, backupAlphaSocketAddrHttp string) { +func backupRestoreTest(t *testing.T, backupAlphaName string, backupZeroName string, + restoreAlphaName string, backupDst string) { - conn, err := grpc.NewClient(backupAlphaSocketAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) - require.NoError(t, err) - dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) + // Wait for containers to be healthy before proceeding + t.Logf("Waiting for %s to be healthy...", backupAlphaName) + backupAlpha := testutil.ContainerInstance{Name: backupAlphaName, Prefix: testutil.DockerPrefix} + require.NoError(t, backupAlpha.BestEffortWaitForHealthy(8080)) + + t.Logf("Waiting for %s to be healthy...", backupZeroName) + backupZero := testutil.ContainerInstance{Name: backupZeroName, Prefix: testutil.DockerPrefix} + require.NoError(t, backupZero.BestEffortWaitForHealthy(6080)) + + // Resolve addresses after containers are healthy + backupAlphaSocketAddr := testutil.ContainerAddr(backupAlphaName, 9080) + backupAlphaSocketAddrHttp := testutil.ContainerAddr(backupAlphaName, 8080) + restoreAlphaAddr := testutil.ContainerAddr(restoreAlphaName, 8080) + backupZeroAddr := testutil.ContainerAddr(backupZeroName, 6080) + + var dg *dgo.Dgraph + var err error ctx := context.Background() + + // Wait for gRPC connection to be ready with retries + t.Log("Waiting for gRPC connection to be ready...") + for i := 0; i < 30; i++ { + var connErr error + dg, connErr = dgo.Open(fmt.Sprintf("dgraph://%s?sslmode=disable", backupAlphaSocketAddr)) + if connErr != nil { + err = connErr + t.Logf("Failed to create Dgraph client (attempt %d/30): %v", i+1, connErr) + time.Sleep(time.Second) + continue + } + _, err = testutil.RetryQuery(dg, `schema {}`) + if err == nil { + break + } + t.Logf("Health query failed (attempt %d/30): %v", i+1, err) + time.Sleep(time.Second) + } + require.NoError(t, err, "Failed to connect to gRPC after 30 attempts") + require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) // Add schema and types. require.NoError(t, dg.Alter(ctx, &api.Operation{Schema: `movie: string . diff --git a/systest/backup/nfs-backup/docker-compose.yml b/systest/backup/nfs-backup/docker-compose.yml index 4a06a691dba..8046fe69262 100644 --- a/systest/backup/nfs-backup/docker-compose.yml +++ b/systest/backup/nfs-backup/docker-compose.yml @@ -25,6 +25,16 @@ services: - -c - | mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + for i in $(seq 1 30); do + if mountpoint -q /mnt 2>/dev/null; then + break + fi + sleep 1 + done + if ! mountpoint -q /mnt 2>/dev/null; then + echo "ERROR: Failed to mount NFS after 30 seconds" + exit 1 + fi /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha1_backup_clust_ha:7080 --zero=zero1_backup_clust_ha:5080,zero2_backup_clust_ha:5080,zero3_backup_clust_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" alpha2_backup_clust_ha: @@ -158,6 +168,16 @@ services: - -c - | mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + for i in $(seq 1 30); do + if mountpoint -q /mnt 2>/dev/null; then + break + fi + sleep 1 + done + if ! mountpoint -q /mnt 2>/dev/null; then + echo "ERROR: Failed to mount NFS after 30 seconds" + exit 1 + fi /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha4_restore_clust_ha:7080 --zero=zero4_restore_clust_ha:5080,zero5_restore_clust_ha:5080,zero6_restore_clust_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" alpha5_restore_clust_ha: @@ -312,6 +332,16 @@ services: - -c - | mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + for i in $(seq 1 30); do + if mountpoint -q /mnt 2>/dev/null; then + break + fi + sleep 1 + done + if ! mountpoint -q /mnt 2>/dev/null; then + echo "ERROR: Failed to mount NFS after 30 seconds" + exit 1 + fi /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha7_backup_clust_non_ha:7080 --zero=zero7_backup_clust_non_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" #non HA restore cluster @@ -359,6 +389,16 @@ services: - -c - | mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + for i in $(seq 1 30); do + if mountpoint -q /mnt 2>/dev/null; then + break + fi + sleep 1 + done + if ! mountpoint -q /mnt 2>/dev/null; then + echo "ERROR: Failed to mount NFS after 30 seconds" + exit 1 + fi /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha8_restore_clust_non_ha:7080 --zero=zero8_restore_clust_non_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" nfs: diff --git a/testutil/testaudit/audit.go b/testutil/testaudit/audit.go index 48d03c3d994..4e843f6d329 100644 --- a/testutil/testaudit/audit.go +++ b/testutil/testaudit/audit.go @@ -20,30 +20,55 @@ import ( ) func VerifyLogs(t *testing.T, path string, cmds []string) { - // to make sure that the audit log is flushed - time.Sleep(time.Second * 5) abs, err := filepath.Abs(path) require.NoError(t, err) - f, err := os.Open(abs) - require.NoError(t, err) type log struct { Msg string `json:"endpoint"` } - logMap := make(map[string]bool) - fileScanner := bufio.NewScanner(f) - for fileScanner.Scan() { - bytes := fileScanner.Bytes() - l := new(log) - require.NoError(t, json.Unmarshal(bytes, l)) - logMap[l.Msg] = true - } - for _, m := range cmds { - if !logMap[m] { - t.Fatalf("audit logs not present for command %s", m) + maxRetries := 10 + for attempt := 1; attempt <= maxRetries; attempt++ { + f, err := os.Open(abs) + if err != nil { + if attempt == maxRetries { + require.NoError(t, err, "failed to open audit log after %d attempts", maxRetries) + } + time.Sleep(time.Second) + continue + } + + logMap := make(map[string]bool) + fileScanner := bufio.NewScanner(f) + for fileScanner.Scan() { + bytes := fileScanner.Bytes() + l := new(log) + if err := json.Unmarshal(bytes, l); err != nil { + f.Close() + if attempt == maxRetries { + require.NoError(t, err, "failed to unmarshal audit log after %d attempts", maxRetries) + } + time.Sleep(time.Second) + continue + } + logMap[l.Msg] = true + } + f.Close() + + missingCmds := []string{} + for _, m := range cmds { + if !logMap[m] { + missingCmds = append(missingCmds, m) + } + } + if len(missingCmds) == 0 { + return + } + if attempt == maxRetries { + t.Fatalf("audit logs not present after %d attempts. Missing commands: %v", maxRetries, missingCmds) } + time.Sleep(time.Second) } }