diff --git a/.github/workflows/tests-outerloop.yml b/.github/workflows/tests-outerloop.yml index 8aa81ffa506..b414a07ee90 100644 --- a/.github/workflows/tests-outerloop.yml +++ b/.github/workflows/tests-outerloop.yml @@ -12,88 +12,60 @@ concurrency: jobs: - test: - name: ${{ matrix.os.title }} - runs-on: ${{ matrix.os.name }} - strategy: - fail-fast: false - matrix: - os: - - name: ubuntu-latest - title: Linux - - name: windows-latest - title: Windows + generate_tests_matrix: + name: Generate test runsheet + runs-on: windows-latest + if: ${{ github.repository_owner == 'dotnet' }} + outputs: + runsheet: ${{ steps.generate_tests_matrix.outputs.runsheet }} steps: - - name: Setup vars (Linux) - if: ${{ matrix.os.name == 'ubuntu-latest' }} - run: | - echo "DOTNET_SCRIPT=./dotnet.sh" >> $GITHUB_ENV - echo "BUILD_SCRIPT=./build.sh" >> $GITHUB_ENV + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Setup vars (Windows) - if: ${{ matrix.os.name == 'windows-latest' }} + # We need to build the whole solution, so that we can interrogate each test project + # and find out whether it contains any quarantined tests. + - name: Build the solution run: | - echo "DOTNET_SCRIPT=.\dotnet.cmd" >> $env:GITHUB_ENV - echo "BUILD_SCRIPT=.\build.cmd" >> $env:GITHUB_ENV + ./build.cmd -restore -build -c Release -ci /p:CI=false /p:GeneratePackageOnBuild=false /p:InstallBrowsersForPlaywright=false - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Trust HTTPS development certificate (Linux) - if: matrix.os.name == 'ubuntu-latest' - run: ${{ env.DOTNET_SCRIPT }} dev-certs https --trust - - - name: Run quarantined tests - env: - CI: false + - name: Generate test runsheet + id: generate_tests_matrix run: | - ${{ env.BUILD_SCRIPT }} -projects ${{ github.workspace }}/tests/Shared/SolutionTests.proj -ci -restore -build -test -c Release /p:RunQuarantinedTests=true /bl:${{ github.workspace }}/artifacts/log/Release/test-quarantined.binlog + ./build.cmd -test /p:TestRunnerName=QuarantinedTestRunsheetBuilder /p:RunQuarantinedTests=true -c Release -ci /p:CI=false /p:Restore=false /p:Build=false /bl:./artifacts/log/Release/runsheet.binlog - - name: Keep only relevant test logs - if: always() - shell: pwsh - run: | - # Define the directory to search for log files - $logDirectory = "${{ github.workspace }}/artifacts/log/**/TestLogs" + - name: Upload logs, and test results + if: ${{ always() }} + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: logs-runsheet + path: | + ${{ github.workspace }}/artifacts/log/*/*.binlog + ${{ github.workspace }}/artifacts/log/*/TestLogs/** + ${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json + retention-days: 5 - # Define the text to search for in the log files - $searchText = "No test matches the given testcase filter" - $resultsFilePattern = "Results File: (.+)" + run_tests: + name: Test + needs: generate_tests_matrix + strategy: + fail-fast: false + matrix: + tests: ${{ fromJson(needs.generate_tests_matrix.outputs.runsheet) }} - # Get all .log files in the specified directory and its subdirectories - $logFiles = Get-ChildItem -Path $logDirectory -Filter *.log -Recurse + runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix + if: ${{ github.repository_owner == 'dotnet' }} - foreach ($logFile in $logFiles) { - # Read the content of the log file - $content = Get-Content -Path $logFile.FullName + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - # Check if the content contains the specified text - if ($content -match $searchText) { - # Remove the log file if it contains the specified text - Remove-Item -Path $logFile.FullName -Force - Write-Host "Removed file: $($logFile.FullName)" - } - else { - # Extract paths from lines containing "Results File: " - foreach ($line in $content) { - if ($line -match $resultsFilePattern) { - $resultsFilePath = $matches[1] - Write-Host "Found results file: $resultsFilePath" - - # Copy the results file to the TestLogs folder - $destinationPath = (Split-Path -Path $logFile.FullName -Parent) - Copy-Item -Path $resultsFilePath -Destination $destinationPath -Force - Write-Host "Copied $resultsFilePath to $destinationPath" - } - } - } - } + - name: Test ${{ matrix.tests.project }} + run: | + ${{ matrix.tests.command }} - name: Process logs and post results if: always() shell: pwsh run: | - $logDirectory = "${{ github.workspace }}/artifacts/log/**/TestLogs" + $logDirectory = "${{ github.workspace }}/artifacts/TestResults" $trxFiles = Get-ChildItem -Path $logDirectory -Filter *.trx -Recurse $testResults = @() # Initialize an array to store test results @@ -157,19 +129,13 @@ jobs: $table | Out-File -FilePath $outputPath -Encoding utf8 Write-Host "Test results saved to $outputPath" - # Windows-specific: Check for failed tests and set the exit code accordingly - # This is a workaround for the issue with the `exit` command in PowerShell - if ($failedTests -gt 0) { - Write-Host "::error::Build failed. Check errors above." - exit 1 - } - - name: Upload logs, and test results - if: always() + if: failure() uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: - name: logs-${{ matrix.os.name }} + name: logs-${{ matrix.tests.os }}-${{ matrix.tests.project }} path: | ${{ github.workspace }}/artifacts/log/*/*.binlog ${{ github.workspace }}/artifacts/log/*/TestLogs/** + ${{ github.workspace }}/artifacts/TestResults/*/*.trx retention-days: 5 diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets new file mode 100644 index 00000000000..cadc5b7fbf3 --- /dev/null +++ b/eng/AfterSolutionBuild.targets @@ -0,0 +1,90 @@ + + + + + + + + <_CombinedRunsheetFile>$(ArtifactsTmpDir)/combined_runsheet.json + <_Command> + $combined = @() + Get-ChildItem -Path '$(ArtifactsTmpDir)' -Filter '*.runsheet.json' | + ForEach-Object { + $content = Get-Content -Raw $_.FullName | ConvertFrom-Json + if ($content -is [Array]) { + $combined += $content + } + else { + $combined += @($content) + } + } + $jsonString = ($combined | ConvertTo-Json -Depth 10 -Compress) + $jsonString | Set-Content '$(_CombinedRunsheetFile)'; + + # determine if the script is running in a GitHub Actions environment + if ($env:CI -and $env:GITHUB_ACTIONS) { + "runsheet=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + } + else { + Write-Host "runsheet=$jsonString" + } + + <_Script>$([MSBuild]::NormalizePath($(ArtifactsTmpDir), 'create-runsheet.ps1')) + + + + + + + + + diff --git a/eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets b/eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets new file mode 100644 index 00000000000..d464a433c2d --- /dev/null +++ b/eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets @@ -0,0 +1,171 @@ + + + + + + + + <_TestEnvironment>%(TestToRun.EnvironmentDisplay) + <_TestAssembly>%(TestToRun.Identity) + <_TestAssembly Condition="'$(OS)'=='Windows_NT'">$([System.IO.Path]::ChangeExtension($(_TestAssembly), '.exe')) + <_TestAssembly Condition="'$(OS)'!='Windows_NT'">$(_TestAssembly.TrimEnd('.dll')) + <_TestTimeout>%(TestToRun.TestTimeout) + + <_TargetDir>$([System.IO.Path]::GetDirectoryName('$(_TestAssembly)'))\ + + <_TestRunnerCommand>$(_TestAssembly) --list-tests $(_QuarantinedTestRunAdditionalArgs) + + + <_TestRunnerCommand Condition="'$(TestCaptureOutput)' != 'false'">$(_TestRunnerCommand) >> "%(TestToRun.ResultsStdOutPath)" 2>&1 + + + + <_OutputFiles Include="%(TestToRun.ResultsStdOutPath)" /> + + + + + + + + + + + + + + + <_ResultsFileToDisplay>%(TestToRun.ResultsStdOutPath) + + + + + + + + + <_HasQuarantinedTests>true + <_HasQuarantinedTests Condition=" '$(_TestErrorCode)' == '8' ">false + + + + + + <_TestRunsheet>$(MSBuildProjectName) + <_TestBinLog>$([MSBuild]::NormalizePath($(ArtifactsLogDir), '$(_TestRunsheet).binlog')) + + <_RelativeTestProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '%24(pwd)/')) + <_RelativeTestBinLog>$([System.String]::Copy('$(_TestBinLog)').Replace('$(RepoRoot)', '%24(pwd)/')) + + <_TestRunnerWindows>./eng/build.ps1 + <_TestRunnerLinux>./eng/build.sh + <_TestCommand>-restore -build -test -projects "$(_RelativeTestProjectPath)" /bl:"$(_RelativeTestBinLog)" -c $(Configuration) -ci /p:RunQuarantinedTests=true /p:CI=false + + + <_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('"', '\"')) + + <_TestRunsheetWindows>{ "project": "$(_TestRunsheet)", "os": "windows-latest", "command": "./eng/build.ps1 $(_TestCommand)" } + <_TestRunsheetLinux>{ "project": "$(_TestRunsheet)", "os": "ubuntu-latest", "command": "./eng/build.sh $(_TestCommand)" } + + + + + + + + + + + + + + diff --git a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj index 78faf4d9376..9fdc76892f8 100644 --- a/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj +++ b/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj @@ -24,6 +24,9 @@ gets built independently on helix. --> $(NoWarn);xUnit1051 + + + true diff --git a/tests/Shared/SolutionTests.proj b/tests/Shared/SolutionTests.proj deleted file mode 100644 index 15d49df2396..00000000000 --- a/tests/Shared/SolutionTests.proj +++ /dev/null @@ -1,30 +0,0 @@ - - - - $(DefaultTargetFramework) - - - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Shared\**\*Tests.csproj" /> - <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" /> - <_TestProjectsToExclude Include="$(RepoRoot)tests\TestingAppHost1\**\*Tests.csproj" /> - - - <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.Templates.Tests\**\*Tests.csproj" /> - - <_TestProjectsToInclude Include="$(RepoRoot)tests\**\*Tests.csproj" /> - - - - - -