From 65a9f1ef5e060ca396e0c3a5521a43e1b966ba67 Mon Sep 17 00:00:00 2001 From: Jakub Majocha <1760221+majocha@users.noreply.github.com> Date: Mon, 16 Dec 2024 19:23:39 +0100 Subject: [PATCH] Parallelize tests (#17872) * rebase * orchestrate instead of Async.Sleep * xunit customizations behind conditional compilation * make console capture explicit * diff * try fix test * lazy pattern * reenable extras * test temptath * cleaup test dir * remove unneeded file * fix merge * another shot at fixing gc test * another attempt at improving Async.Parallel test * smaller diff * restore test * restore tests * diff * unblock service tests * rename attributes * fix * revert - should now work without * add comment * diff * diff Co-authored-by: Petr * just exclude gc test * diff * exclude test case * try to trigger clr error in ci * try unflake test * unflake asyncmemo * ilverify * fix merge * some cleanup * Update tests/fsharp/Compiler/CodeGen/EmittedIL/DeterministicTests.fs Co-authored-by: Petr * remove timeout * Make FsiTests sequential * restore ci throttling * min diff * add comment * bring down the parallelizm a notch on mac and linux --------- Co-authored-by: Petr Co-authored-by: Vlad Zarytovskii Co-authored-by: Kevin Ransom (msft) --- eng/Build.ps1 | 50 +++--- eng/build.sh | 3 +- .../FSharp.Build.UnitTests.fsproj | 8 +- .../FSharp.Build.UnitTests/xunit.runner.json | 5 + .../CompilerOptions/fsc/misc/utf8output.fs | 1 + .../CompilerOptions/fsc/times/times.fs | 2 + .../CompilerService/AsyncMemoize.fs | 2 +- .../AttributeUsage/AssemblyVersion04.fs | 2 +- .../Events/Basic/Basic.fs | 1 + .../FSharpChecker/TransparentCompiler.fs | 5 +- ...rnTypeDirectedPartialActivePatternTests.fs | 2 + .../SequenceExpressionTests.fs | 2 + .../Miscellaneous/FsharpSuiteMigrated.fs | 8 +- .../Miscellaneous/MigratedCoreTests.fs | 40 +++-- .../TypeChecks/Graph/Utils.fs | 4 +- .../xunit.runner.json | 3 +- .../DependencyManagerInteractiveTests.fs | 1 + ...ompiler.Private.Scripting.UnitTests.fsproj | 2 +- .../xunit.runner.json | 7 +- .../AssemblyContentProviderTests.fs | 2 +- .../AssemblyInfo.fs | 7 - tests/FSharp.Compiler.Service.Tests/Common.fs | 2 +- .../FSharp.Compiler.Service.Tests.fsproj | 1 - .../FileSystemTests.fs | 2 + .../FsiHelpTests.fs | 1 - .../FSharp.Compiler.Service.Tests/FsiTests.fs | 10 +- .../ModuleReaderCancellationTests.fs | 2 + .../PerfTests.fs | 3 +- .../ProjectAnalysisTests.fs | 69 ++++---- .../ScriptOptionsTests.fs | 2 + .../xunit.runner.json | 3 +- .../FSharp.Core.UnitTests.fsproj | 2 +- .../FSharp.Core/ComparersRegression.fs | 1 + .../Microsoft.FSharp.Control/AsyncType.fs | 5 +- .../Microsoft.FSharp.Control/Cancellation.fs | 2 +- .../MailboxProcessorType.fs | 1 + .../Microsoft.FSharp.Control/Tasks.fs | 3 +- .../Microsoft.FSharp.Control/TasksDynamic.fs | 5 +- tests/FSharp.Core.UnitTests/xunit.runner.json | 3 +- tests/FSharp.Test.Utilities/CompilerAssert.fs | 3 +- .../FSharp.Test.Utilities.fsproj | 5 +- tests/FSharp.Test.Utilities/ILChecker.fs | 3 +- tests/FSharp.Test.Utilities/Peverifier.fs | 3 +- .../ProjectGeneration.fs | 5 +- tests/FSharp.Test.Utilities/ScriptHelpers.fs | 2 +- tests/FSharp.Test.Utilities/TestFramework.fs | 17 +- tests/FSharp.Test.Utilities/Utilities.fs | 6 +- tests/FSharp.Test.Utilities/XunitHelpers.fs | 162 +++++++++++++++++- tests/FSharp.Test.Utilities/XunitSetup.fs | 5 + .../CodeGen/EmittedIL/DeterministicTests.fs | 2 +- tests/fsharp/FSharpSuite.Tests.fsproj | 1 - tests/fsharp/TypeProviderTests.fs | 2 +- tests/fsharp/XunitHelpers.fs | 8 - tests/fsharp/tests.fs | 2 +- tests/fsharp/xunit.runner.json | 3 +- tests/scripts/scriptlib.fsx | 13 +- 56 files changed, 351 insertions(+), 165 deletions(-) create mode 100644 tests/FSharp.Build.UnitTests/xunit.runner.json delete mode 100644 tests/FSharp.Compiler.Service.Tests/AssemblyInfo.fs delete mode 100644 tests/fsharp/XunitHelpers.fs diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 9d137104b9c..feaf768f4f2 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -362,14 +362,16 @@ function VerifyAssemblyVersionsAndSymbols() { } } -function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [boolean] $asBackgroundJob = $false) { +function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [boolean] $asBackgroundJob = $false, [string] $settings = "") { $dotnetPath = InitializeDotNetCli $dotnetExe = Join-Path $dotnetPath "dotnet.exe" $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject) - $testLogPath = "$ArtifactsDir\TestResults\$configuration\${projectName}_$targetFramework.xml" + # {assembly} and {framework} will expand respectively. See https://github.com/spekt/testlogger/wiki/Logger-Configuration#logfilepath + # This is useful to deconflict log filenames when there are many test assemblies, e.g. when testing a whole solution. + $testLogPath = "$ArtifactsDir\TestResults\$configuration\{assembly}_{framework}.xml" $testBinLogPath = "$LogDir\${projectName}_$targetFramework.binlog" $args = "test $testProject -c $configuration -f $targetFramework -v n --test-adapter-path $testadapterpath --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath" - $args += " --blame --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=false" + $args += " --blame --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=true" if (-not $noVisualStudio -or $norestore) { $args += " --no-restore" @@ -379,17 +381,20 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str $args += " --no-build" } + $args += " $settings" + if ($asBackgroundJob) { + Write-Host Write-Host("Starting on the background: $args") Write-Host("------------------------------------") - $bgJob = Start-Job -ScriptBlock { - & $using:dotnetExe test $using:testProject -c $using:configuration -f $using:targetFramework -v n --test-adapter-path $using:testadapterpath --logger "xunit;LogFilePath=$using:testLogPath" /bl:$using:testBinLogPath --blame --results-directory $using:ArtifactsDir\TestResults\$using:configuration + Start-Job -ScriptBlock { + $argArray = $using:args -Split " " + & $using:dotnetExe $argArray if ($LASTEXITCODE -ne 0) { throw "Command failed to execute with exit code $($LASTEXITCODE): $using:dotnetExe $using:args" } } - return $bgJob - } else{ + } else { Write-Host("$args") Exec-Console $dotnetExe $args } @@ -595,21 +600,20 @@ try { $script:BuildCategory = "Test" $script:BuildMessage = "Failure running tests" - if ($testCoreClr) { - $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true - - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\" + function Receive($job) { + while($job.HasMoreData) { + Receive-Job $job | Write-Host + Start-Sleep -Seconds 1 + } + Receive-Job $job -Wait -ErrorAction Stop + } - # Collect output from background jobs - Wait-job $bgJob | out-null - Receive-Job $bgJob -ErrorAction Stop + if ($testCoreClr) { + $cpuLimit = if ($ci) { "-m:2 -- xUnit.MaxParallelThreads=0.25x" } else { "" } + TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -settings $cpuLimit } - if ($testDesktop) { + if ($testDesktop -and $ci) { $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" @@ -618,9 +622,11 @@ try { TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\" TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\" - # Collect output from background jobs - Wait-job $bgJob | out-null - Receive-Job $bgJob -ErrorAction Stop + Receive -job $bgJob + } + + if ($testDesktop -and -not $ci ) { + TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" } if ($testFSharpQA) { diff --git a/eng/build.sh b/eng/build.sh index e6e27732c8f..b377904e2a6 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -219,7 +219,8 @@ function Test() { projectname=$(basename -- "$testproject") projectname="${projectname%.*}" testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml" - args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false" + args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false" + args+=" -- xUnit.MaxParallelThreads=3" "$DOTNET_INSTALL_DIR/dotnet" $args || exit $? } diff --git a/tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj b/tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj index e689cf1bf39..853737b3ca8 100644 --- a/tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj +++ b/tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj @@ -4,7 +4,7 @@ net472;$(FSharpNetCoreProductTargetFramework) - $(FSharpNetCoreProductTargetFramework) + $(FSharpNetCoreProductTargetFramework) Library true xunit @@ -18,6 +18,12 @@ + + + PreserveNewest + + + diff --git a/tests/FSharp.Build.UnitTests/xunit.runner.json b/tests/FSharp.Build.UnitTests/xunit.runner.json new file mode 100644 index 00000000000..b01c50a3cb5 --- /dev/null +++ b/tests/FSharp.Build.UnitTests/xunit.runner.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "appDomain": "denied", + "parallelizeAssembly": true +} diff --git a/tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/misc/utf8output.fs b/tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/misc/utf8output.fs index 6c6216a8377..c17a5c01f07 100644 --- a/tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/misc/utf8output.fs +++ b/tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/misc/utf8output.fs @@ -7,6 +7,7 @@ open FSharp.Test open FSharp.Test.Compiler open System +[] module utf8output = [] diff --git a/tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/times/times.fs b/tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/times/times.fs index b18e5472ec0..7b137c24c15 100644 --- a/tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/times/times.fs +++ b/tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/times/times.fs @@ -8,6 +8,8 @@ open FSharp.Test.Compiler open System open System.IO +// reportTime uses global state. +[] module times = // This test was automatically generated (moved from FSharpQA suite - CompilerOptions/fsc/times) diff --git a/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs b/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs index cdc7bdb1400..3e940c51ea6 100644 --- a/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs +++ b/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs @@ -28,7 +28,7 @@ let internal observe (cache: AsyncMemoize<_,_,_>) = cache.Event.Add arrivals.Post - let next () = collected.Receive(10_000) + let next () = collected.Receive() next diff --git a/tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/CustomAttributes/AttributeUsage/AssemblyVersion04.fs b/tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/CustomAttributes/AttributeUsage/AssemblyVersion04.fs index 2a67d906999..fd876e72abb 100644 --- a/tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/CustomAttributes/AttributeUsage/AssemblyVersion04.fs +++ b/tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/CustomAttributes/AttributeUsage/AssemblyVersion04.fs @@ -14,5 +14,5 @@ let success = asm.Version.Major = 1 && asm.Version.Minor = 2 && asm.Version.Build = 3 && - (abs (asm.Version.Revision - (int defaultRevision))) < 10 // default value is seconds in the current day / 2. Check if within 10 sec of that. + (abs (asm.Version.Revision - (int defaultRevision))) < 60 // default value is seconds in the current day / 2. Check if within 60 sec of that. if success then () else failwith "Failed: 1" \ No newline at end of file diff --git a/tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/Events/Basic/Basic.fs b/tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/Events/Basic/Basic.fs index 87d12baea7c..caede79cd6b 100644 --- a/tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/Events/Basic/Basic.fs +++ b/tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/Events/Basic/Basic.fs @@ -6,6 +6,7 @@ open Xunit open FSharp.Test open FSharp.Test.Compiler +[] module Events = let verifyCompile compilation = diff --git a/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs b/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs index 4b5987f65e5..b523b814b72 100644 --- a/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs +++ b/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs @@ -1,4 +1,5 @@ -module FSharpChecker.TransparentCompiler +[] +module FSharpChecker.TransparentCompiler open System.Collections.Concurrent open System.Diagnostics @@ -991,6 +992,8 @@ type private LoadClosureTestShim(currentFileSystem: IFileSystem) = ?shouldShadowCopy = shouldShadowCopy ) +// Because it is mutating FileSystem! +[] module TestsMutatingFileSystem = [] diff --git a/tests/FSharp.Compiler.ComponentTests/Language/BooleanReturningAndReturnTypeDirectedPartialActivePatternTests.fs b/tests/FSharp.Compiler.ComponentTests/Language/BooleanReturningAndReturnTypeDirectedPartialActivePatternTests.fs index 6562e347bbd..8ae8d85db6e 100644 --- a/tests/FSharp.Compiler.ComponentTests/Language/BooleanReturningAndReturnTypeDirectedPartialActivePatternTests.fs +++ b/tests/FSharp.Compiler.ComponentTests/Language/BooleanReturningAndReturnTypeDirectedPartialActivePatternTests.fs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// Because of shared fsi session. +[] module Language.BooleanReturningAndReturnTypeDirectedPartialActivePatternTests open Xunit diff --git a/tests/FSharp.Compiler.ComponentTests/Language/SequenceExpressions/SequenceExpressionTests.fs b/tests/FSharp.Compiler.ComponentTests/Language/SequenceExpressions/SequenceExpressionTests.fs index 6d46a3718f0..9d98106bcdb 100644 --- a/tests/FSharp.Compiler.ComponentTests/Language/SequenceExpressions/SequenceExpressionTests.fs +++ b/tests/FSharp.Compiler.ComponentTests/Language/SequenceExpressions/SequenceExpressionTests.fs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// Run sequentially because of shared fsiSession. +[] module Language.SequenceExpression.SequenceExpressionTests open FSharp.Test diff --git a/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs b/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs index 5ab961176f5..8ab6a4ac584 100644 --- a/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs +++ b/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs @@ -36,13 +36,15 @@ module ScriptRunner = let res = evalScriptFromDiskInSharedSession engine cu match res with | CompilationResult.Failure _ -> res - | CompilationResult.Success s -> + | CompilationResult.Success _ -> if capture.OutText |> TestFramework.outputPassed then res else - failwith $"Results looked correct, but 'TEST PASSED OK' was not printed. Result: %A{s}" + failwith $"Results looked correct, but 'TEST PASSED OK' was not printed." - | _ -> failwith $"Compilation unit other than fsharp is not supported, cannot process %A{cu}" + | _ -> + printfn $"Cannot process %A{cu}" + failwith $"Compilation unit other than fsharp is not supported." /// This test file was created by porting over (slower) FsharpSuite.Tests /// In order to minimize human error, the test definitions have been copy-pasted and this adapter provides implementations of the test functions diff --git a/tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs b/tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs index 2d9c6657901..d0cd80472f1 100644 --- a/tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs +++ b/tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs @@ -93,14 +93,17 @@ let ``comprehensions-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehens [] let ``comprehensions-FSI`` () = singleTestBuildAndRun "core/comprehensions" FSI -[] -let ``comprehensionshw-FSC_DEBUG`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_DEBUG +// Cancels default token. +[] +module Comprehensionshw = + [] + let ``comprehensionshw-FSC_DEBUG`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_DEBUG -[] -let ``comprehensionshw-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_OPTIMIZED + [] + let ``comprehensionshw-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_OPTIMIZED -[] -let ``comprehensionshw-FSI`` () = singleTestBuildAndRun "core/comprehensions-hw" FSI + [] + let ``comprehensionshw-FSI`` () = singleTestBuildAndRun "core/comprehensions-hw" FSI [] let ``genericmeasures-FSC_DEBUG`` () = singleTestBuildAndRun "core/genericmeasures" FSC_DEBUG @@ -375,18 +378,21 @@ let ``recordResolution-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/recordRe [] let ``recordResolution-FSI`` () = singleTestBuildAndRun "core/recordResolution" FSI -// This test has hardcoded expectations about current synchronization context -// Will be moved out of FsharpSuite.Tests in a later phase for desktop framework -[] -let ``control-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/control" FSC_OPTIMIZED +// Cancels default token. +[] +module CoreControl = + // This test has hardcoded expectations about current synchronization context + // Will be moved out of FsharpSuite.Tests in a later phase for desktop framework + [] + let ``control-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/control" FSC_OPTIMIZED -[] -let ``control-FSI`` () = singleTestBuildAndRun "core/control" FSI + [] + let ``control-FSI`` () = singleTestBuildAndRun "core/control" FSI -[] -let ``control --tailcalls`` () = - let cfg = "core/control" - singleTestBuildAndRunAux cfg ["--tailcalls"] FSC_OPTIMIZED + [] + let ``control --tailcalls`` () = + let cfg = "core/control" + singleTestBuildAndRunAux cfg ["--tailcalls"] FSC_OPTIMIZED [] let ``controlChamenos-FSC_OPTIMIZED`` () = @@ -401,7 +407,7 @@ let ``controlChamenos-FSI`` () = [] let ``controlMailbox-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/controlMailbox" FSC_OPTIMIZED -[] +[] let ``controlMailbox-FSI`` () = singleTestBuildAndRun "core/controlMailbox" FSI [] diff --git a/tests/FSharp.Compiler.ComponentTests/TypeChecks/Graph/Utils.fs b/tests/FSharp.Compiler.ComponentTests/TypeChecks/Graph/Utils.fs index d14ddb44de1..02dd073625d 100644 --- a/tests/FSharp.Compiler.ComponentTests/TypeChecks/Graph/Utils.fs +++ b/tests/FSharp.Compiler.ComponentTests/TypeChecks/Graph/Utils.fs @@ -5,7 +5,7 @@ open FSharp.Compiler.GraphChecking open FSharp.Compiler.Text open FSharp.Compiler.Syntax -let private checker = FSharpChecker.Create() +open FSharp.Test let parseSourceCode (name: string, code: string) = let sourceText = SourceText.ofString code @@ -16,7 +16,7 @@ let parseSourceCode (name: string, code: string) = } let result = - checker.ParseFile(name, sourceText, parsingOptions) |> Async.RunSynchronously + CompilerAssert.Checker.ParseFile(name, sourceText, parsingOptions) |> Async.RunSynchronously result.ParseTree diff --git a/tests/FSharp.Compiler.ComponentTests/xunit.runner.json b/tests/FSharp.Compiler.ComponentTests/xunit.runner.json index d1866b85c61..b01c50a3cb5 100644 --- a/tests/FSharp.Compiler.ComponentTests/xunit.runner.json +++ b/tests/FSharp.Compiler.ComponentTests/xunit.runner.json @@ -1,6 +1,5 @@ { "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", "appDomain": "denied", - "parallelizeTestCollections": false, - "maxParallelThreads": 1 + "parallelizeAssembly": true } diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs b/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs index 20c5919e00e..7f320fea23f 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs @@ -26,6 +26,7 @@ module Native = type scriptHost (?langVersion: LangVersion) = inherit FSharpScript(langVersion=defaultArg langVersion LangVersion.Preview) +[] type DependencyManagerInteractiveTests() = let getValue ((value: Result), (errors: FSharpDiagnostic[])) = diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj index a3ac4bd6d1a..e0d064e12f9 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj @@ -3,7 +3,7 @@ net472;$(FSharpNetCoreProductTargetFramework) - $(FSharpNetCoreProductTargetFramework) + $(FSharpNetCoreProductTargetFramework) Library true xunit diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json b/tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json index b180883cc0f..b01c50a3cb5 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json @@ -1,6 +1,5 @@ { - "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", - "appDomain": "denied", - "parallelizeTestCollections": false, - "maxParallelThreads": 1 + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "appDomain": "denied", + "parallelizeAssembly": true } diff --git a/tests/FSharp.Compiler.Service.Tests/AssemblyContentProviderTests.fs b/tests/FSharp.Compiler.Service.Tests/AssemblyContentProviderTests.fs index cb4521b7af4..eabf028393c 100644 --- a/tests/FSharp.Compiler.Service.Tests/AssemblyContentProviderTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/AssemblyContentProviderTests.fs @@ -22,7 +22,7 @@ let private projectOptions : FSharpProjectOptions = UnresolvedReferences = None Stamp = None } -let private checker = FSharpChecker.Create(useTransparentCompiler=CompilerAssertHelpers.UseTransparentCompiler) +let private checker = FSharpChecker.Create(useTransparentCompiler = CompilerAssertHelpers.UseTransparentCompiler) let private assertAreEqual (expected, actual) = if actual <> expected then diff --git a/tests/FSharp.Compiler.Service.Tests/AssemblyInfo.fs b/tests/FSharp.Compiler.Service.Tests/AssemblyInfo.fs deleted file mode 100644 index 3433f928ba4..00000000000 --- a/tests/FSharp.Compiler.Service.Tests/AssemblyInfo.fs +++ /dev/null @@ -1,7 +0,0 @@ -module FSharp.Compiler.Service.Tests.AssemblyInfo - -open Xunit - -[] - -do() diff --git a/tests/FSharp.Compiler.Service.Tests/Common.fs b/tests/FSharp.Compiler.Service.Tests/Common.fs index 18276ae39a9..8ccb19b0f85 100644 --- a/tests/FSharp.Compiler.Service.Tests/Common.fs +++ b/tests/FSharp.Compiler.Service.Tests/Common.fs @@ -31,7 +31,7 @@ type Async with task.Result // Create one global interactive checker instance -let checker = FSharpChecker.Create(useTransparentCompiler=FSharp.Compiler.CompilerConfig.FSharpExperimentalFeaturesEnabledAutomatically) +let checker = FSharpChecker.Create(useTransparentCompiler = FSharp.Test.CompilerAssertHelpers.UseTransparentCompiler) type TempFile(ext, contents: string) = let tmpFile = Path.ChangeExtension(getTemporaryFileName (), ext) diff --git a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj index fc64abb4138..57ce88d8f02 100644 --- a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj +++ b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj @@ -19,7 +19,6 @@ Never - XunitSetup.fs diff --git a/tests/FSharp.Compiler.Service.Tests/FileSystemTests.fs b/tests/FSharp.Compiler.Service.Tests/FileSystemTests.fs index e11d97096f8..4d191dcc7cb 100644 --- a/tests/FSharp.Compiler.Service.Tests/FileSystemTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/FileSystemTests.fs @@ -25,6 +25,8 @@ let file2 = """ module File2 let B = File1.A + File1.A""" +// FileSystem is a global shared resource. +[] type internal MyFileSystem() = inherit DefaultFileSystem() static member FilesCache = dict [(fileName1, file1); (fileName2, file2)] diff --git a/tests/FSharp.Compiler.Service.Tests/FsiHelpTests.fs b/tests/FSharp.Compiler.Service.Tests/FsiHelpTests.fs index d17b3421ed9..110cc1a09e7 100644 --- a/tests/FSharp.Compiler.Service.Tests/FsiHelpTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/FsiHelpTests.fs @@ -3,7 +3,6 @@ open FSharp.Test.Assert open Xunit -[] module FsiHelpTests = [] diff --git a/tests/FSharp.Compiler.Service.Tests/FsiTests.fs b/tests/FSharp.Compiler.Service.Tests/FsiTests.fs index f6a785af4f1..bb3e8407d6f 100644 --- a/tests/FSharp.Compiler.Service.Tests/FsiTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/FsiTests.fs @@ -14,21 +14,17 @@ type Sentinel () = module MyModule = let test(x: int) = () -[] +// Running in parallel is unstable with occasional System.IO.FileLoadException: Could not load file or assembly 'FSI-ASSEMBLY... +[] module FsiTests = let createFsiSession (useOneDynamicAssembly: bool) = - // Initialize output and input streams - let inStream = new StringReader("") - let outStream = new CompilerOutputStream() - let errStream = new CompilerOutputStream() - // Build command line arguments & start FSI session let argv = [| "C:\\fsi.exe" |] let allArgs = Array.append argv [|"--noninteractive"; if useOneDynamicAssembly then "--multiemit-" else "--multiemit+" |] let fsiConfig = FsiEvaluationSession.GetDefaultConfiguration() - FsiEvaluationSession.Create(fsiConfig, allArgs, inStream, new StreamWriter(outStream), new StreamWriter(errStream), collectible = true) + FsiEvaluationSession.Create(fsiConfig, allArgs, stdin, stdout, stderr, collectible = true) [] let ``No bound values at the start of FSI session`` () = diff --git a/tests/FSharp.Compiler.Service.Tests/ModuleReaderCancellationTests.fs b/tests/FSharp.Compiler.Service.Tests/ModuleReaderCancellationTests.fs index 995d9dabfb2..62e36de58b9 100644 --- a/tests/FSharp.Compiler.Service.Tests/ModuleReaderCancellationTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/ModuleReaderCancellationTests.fs @@ -1,3 +1,5 @@ +// Sequential execution because of shared mutable state. +[] module FSharp.Compiler.Service.Tests.ModuleReaderCancellationTests open System diff --git a/tests/FSharp.Compiler.Service.Tests/PerfTests.fs b/tests/FSharp.Compiler.Service.Tests/PerfTests.fs index aa270182895..8a9ac73740a 100644 --- a/tests/FSharp.Compiler.Service.Tests/PerfTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/PerfTests.fs @@ -1,4 +1,5 @@ -module FSharp.Compiler.Service.Tests.PerfTests +[] +module FSharp.Compiler.Service.Tests.PerfTests open Xunit open FSharp.Test.Assert diff --git a/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs b/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs index dfce65ef186..1570cd34d0c 100644 --- a/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs @@ -105,28 +105,31 @@ let ``Test project1 whole project errors`` () = wholeProjectResults.Diagnostics[0].Range.StartColumn |> shouldEqual 43 wholeProjectResults.Diagnostics[0].Range.EndColumn |> shouldEqual 44 -[] -let ``Test project1 and make sure TcImports gets cleaned up`` () = +[] +module ClearLanguageServiceRootCachesTest = + [] + let ``Test project1 and make sure TcImports gets cleaned up`` () = - // A private checker for this test. - let checker = FSharpChecker.Create() + // A private checker for this test. + let checker = FSharpChecker.Create() - let test () = - let _, checkFileAnswer = checker.ParseAndCheckFileInProject(Project1.fileName1, 0, Project1.fileSource1, Project1.options) |> Async.RunImmediate - match checkFileAnswer with - | FSharpCheckFileAnswer.Aborted -> failwith "should not be aborted" - | FSharpCheckFileAnswer.Succeeded checkFileResults -> - let tcImportsOpt = checkFileResults.TryGetCurrentTcImports () - Assert.True tcImportsOpt.IsSome - let weakTcImports = WeakReference tcImportsOpt.Value - Assert.True weakTcImports.IsAlive - weakTcImports - - // Here we are only keeping a handle to weakTcImports and nothing else - let weakTcImports = test () - checker.InvalidateConfiguration Project1.options - checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() - System.Threading.SpinWait.SpinUntil(fun () -> not weakTcImports.IsAlive) + let test () = + let _, checkFileAnswer = checker.ParseAndCheckFileInProject(Project1.fileName1, 0, Project1.fileSource1, Project1.options) |> Async.RunImmediate + match checkFileAnswer with + | FSharpCheckFileAnswer.Aborted -> failwith "should not be aborted" + | FSharpCheckFileAnswer.Succeeded checkFileResults -> + let tcImportsOpt = checkFileResults.TryGetCurrentTcImports () + Assert.True tcImportsOpt.IsSome + let weakTcImports = WeakReference tcImportsOpt.Value + Assert.True weakTcImports.IsAlive + weakTcImports + + // Here we are only keeping a handle to weakTcImports and nothing else + let weakTcImports = test () + checker.InvalidateConfiguration Project1.options + checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() + GC.Collect() + System.Threading.SpinWait.SpinUntil(fun () -> not weakTcImports.IsAlive) [] let ``Test Project1 should have protected FullName and TryFullName return same results`` () = @@ -5788,18 +5791,20 @@ let checkContentAsScript content = | FSharpCheckFileAnswer.Aborted -> failwith "no check results" | FSharpCheckFileAnswer.Succeeded r -> r -[] -let ``References from #r nuget are included in script project options`` () = - let checkResults = checkContentAsScript """ -#i "nuget:https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" -#r "nuget: Dapper" -""" - let assemblyNames = - checkResults.ProjectContext.GetReferencedAssemblies() - |> Seq.choose (fun f -> f.FileName |> Option.map Path.GetFileName) - |> Seq.distinct - printfn "%s" (assemblyNames |> String.concat "\n") - Assert.Contains("Dapper.dll", assemblyNames) +[] +module ScriptClosureCacheUse = + [] + let ``References from #r nuget are included in script project options`` () = + let checkResults = checkContentAsScript """ + #i "nuget:https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" + #r "nuget: Dapper" + """ + let assemblyNames = + checkResults.ProjectContext.GetReferencedAssemblies() + |> Seq.choose (fun f -> f.FileName |> Option.map Path.GetFileName) + |> Seq.distinct + printfn "%s" (assemblyNames |> String.concat "\n") + Assert.Contains("Dapper.dll", assemblyNames) module internal EmptyProject = let base2 = getTemporaryFileName () diff --git a/tests/FSharp.Compiler.Service.Tests/ScriptOptionsTests.fs b/tests/FSharp.Compiler.Service.Tests/ScriptOptionsTests.fs index 328272defb8..43dfe12a0c0 100644 --- a/tests/FSharp.Compiler.Service.Tests/ScriptOptionsTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/ScriptOptionsTests.fs @@ -1,3 +1,5 @@ +// Because of script closure cache. +[] module FSharp.Compiler.Service.Tests.ScriptOptionsTests open Xunit diff --git a/tests/FSharp.Compiler.Service.Tests/xunit.runner.json b/tests/FSharp.Compiler.Service.Tests/xunit.runner.json index 2b65d19ea3e..b01c50a3cb5 100644 --- a/tests/FSharp.Compiler.Service.Tests/xunit.runner.json +++ b/tests/FSharp.Compiler.Service.Tests/xunit.runner.json @@ -1,4 +1,5 @@ { "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", - "appDomain": "denied" + "appDomain": "denied", + "parallelizeAssembly": true } diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj b/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj index 305d234a837..3e6bbbd282d 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj +++ b/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj @@ -4,7 +4,7 @@ $(FSharpNetCoreProductTargetFramework);net472 - $(FSharpNetCoreProductTargetFramework) + $(FSharpNetCoreProductTargetFramework) Library FSharp.Core.UnitTests diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/ComparersRegression.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/ComparersRegression.fs index a8e42790090..59497654b5b 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/ComparersRegression.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/ComparersRegression.fs @@ -1747,6 +1747,7 @@ module ComparersRegression = open ComparersRegression open Xunit +[] type GeneratedTests () = let _ = () // ------------------------------------------------------------------------------ diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/AsyncType.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/AsyncType.fs index 97ea8fca1a8..ef75dff9e00 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/AsyncType.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/AsyncType.fs @@ -11,6 +11,8 @@ open Xunit open System.Threading open System.Threading.Tasks +// Cancels default token. +[] module AsyncType = type ExpectedContinuation = Success | Exception | Cancellation @@ -38,8 +40,7 @@ module AsyncType = async { return () } |> expect Success - - +[] type AsyncType() = let ignoreSynchCtx f = diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/Cancellation.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/Cancellation.fs index b4a75843369..3fc449546e1 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/Cancellation.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/Cancellation.fs @@ -9,7 +9,7 @@ open FSharp.Test open System.Threading open System.Threading.Tasks - +[] type CancellationType() = let ordered() = diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/MailboxProcessorType.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/MailboxProcessorType.fs index 1e2fcd58545..4dd7797ae9f 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/MailboxProcessorType.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/MailboxProcessorType.fs @@ -24,6 +24,7 @@ type StartImmediateThreadInfo = type StartImmediateMessage = | GetThreadInfo of AsyncReplyChannel +[] type MailboxProcessorType() = let getSimpleMailbox() = diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/Tasks.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/Tasks.fs index 8097c2d10f5..cce271d58bb 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/Tasks.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/Tasks.fs @@ -192,6 +192,7 @@ module Helpers = let require x msg = if not x then failwith msg let failtest str = raise (TestException str) +[] type Basics() = [] member _.testShortCircuitResult() = @@ -1201,8 +1202,6 @@ type Basics() = } |> ignore -[] -type BasicsNotInParallel() = [] member _.testTaskUsesSyncContext() = diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/TasksDynamic.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/TasksDynamic.fs index 7f844e99d96..430b1b526e4 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/TasksDynamic.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/TasksDynamic.fs @@ -313,6 +313,7 @@ module Helpers = let require x msg = if not x then failwith msg let failtest str = raise (TestException str) +[] type Basics() = [] member _.testShortCircuitResult() = @@ -1259,10 +1260,6 @@ type Basics() = } |> ignore - -[] -type BasicsNotInParallel() = - [] member _.testTaskUsesSyncContext() = printfn "Running testBackgroundTask..." diff --git a/tests/FSharp.Core.UnitTests/xunit.runner.json b/tests/FSharp.Core.UnitTests/xunit.runner.json index d1866b85c61..b01c50a3cb5 100644 --- a/tests/FSharp.Core.UnitTests/xunit.runner.json +++ b/tests/FSharp.Core.UnitTests/xunit.runner.json @@ -1,6 +1,5 @@ { "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", "appDomain": "denied", - "parallelizeTestCollections": false, - "maxParallelThreads": 1 + "parallelizeAssembly": true } diff --git a/tests/FSharp.Test.Utilities/CompilerAssert.fs b/tests/FSharp.Test.Utilities/CompilerAssert.fs index 5253ef16934..0f7baf597fb 100644 --- a/tests/FSharp.Test.Utilities/CompilerAssert.fs +++ b/tests/FSharp.Test.Utilities/CompilerAssert.fs @@ -622,8 +622,7 @@ module CompilerAssertHelpers = let runtimeconfigPath = Path.ChangeExtension(outputFilePath, ".runtimeconfig.json") File.WriteAllText(runtimeconfigPath, runtimeconfig) #endif - let timeout = 30000 - let rc, output, errors = Commands.executeProcess fileName arguments (Path.GetDirectoryName(outputFilePath)) timeout + let rc, output, errors = Commands.executeProcess fileName arguments (Path.GetDirectoryName(outputFilePath)) let output = String.Join(Environment.NewLine, output) let errors = String.Join(Environment.NewLine, errors) ExitCode rc, output, errors diff --git a/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj b/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj index 52c1893073c..e16ac79fc6a 100644 --- a/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj +++ b/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj @@ -10,8 +10,11 @@ true false false + false $(OtherFlags) --warnon:1182 true + + XUNIT_EXTRAS @@ -105,7 +108,7 @@ - + diff --git a/tests/FSharp.Test.Utilities/ILChecker.fs b/tests/FSharp.Test.Utilities/ILChecker.fs index de0fbd8050b..b47ae4409f0 100644 --- a/tests/FSharp.Test.Utilities/ILChecker.fs +++ b/tests/FSharp.Test.Utilities/ILChecker.fs @@ -15,8 +15,7 @@ module ILChecker = let private exec exe args = let arguments = args |> String.concat " " - let timeout = 30000 - let exitCode, _output, errors = Commands.executeProcess exe arguments "" timeout + let exitCode, _output, errors = Commands.executeProcess exe arguments "" let errors = errors |> String.concat Environment.NewLine errors, exitCode diff --git a/tests/FSharp.Test.Utilities/Peverifier.fs b/tests/FSharp.Test.Utilities/Peverifier.fs index f3ccc7de2b1..0591a435484 100644 --- a/tests/FSharp.Test.Utilities/Peverifier.fs +++ b/tests/FSharp.Test.Utilities/Peverifier.fs @@ -24,8 +24,7 @@ module PEVerifier = let private exec exe args = let arguments = args |> String.concat " " - let timeout = 30000 - let exitCode, _output, errors = Commands.executeProcess exe arguments "" timeout + let exitCode, _output, errors = Commands.executeProcess exe arguments "" let errors = errors |> String.concat Environment.NewLine errors, exitCode diff --git a/tests/FSharp.Test.Utilities/ProjectGeneration.fs b/tests/FSharp.Test.Utilities/ProjectGeneration.fs index 197788b0e2e..fc110ded6b6 100644 --- a/tests/FSharp.Test.Utilities/ProjectGeneration.fs +++ b/tests/FSharp.Test.Utilities/ProjectGeneration.fs @@ -32,6 +32,7 @@ open FSharp.Compiler.Text open Xunit open FSharp.Test.Utilities + open OpenTelemetry open OpenTelemetry.Resources open OpenTelemetry.Trace @@ -349,11 +350,11 @@ type SyntheticProject = UnresolvedReferences = None OriginalLoadReferences = [] Stamp = None } - - + OptionsCache.GetOrAdd(key, factory).Value + member this.GetAllProjects() = [ this for p in this.DependsOn do diff --git a/tests/FSharp.Test.Utilities/ScriptHelpers.fs b/tests/FSharp.Test.Utilities/ScriptHelpers.fs index f060cb91398..aa0593fa090 100644 --- a/tests/FSharp.Test.Utilities/ScriptHelpers.fs +++ b/tests/FSharp.Test.Utilities/ScriptHelpers.fs @@ -100,4 +100,4 @@ module TestHelpers = | Ok(value) -> value | Error ex -> raise ex - let ignoreValue x = getValue x |> ignore + let ignoreValue = getValue >> ignore diff --git a/tests/FSharp.Test.Utilities/TestFramework.fs b/tests/FSharp.Test.Utilities/TestFramework.fs index f50452eff9e..38976846a4a 100644 --- a/tests/FSharp.Test.Utilities/TestFramework.fs +++ b/tests/FSharp.Test.Utilities/TestFramework.fs @@ -4,12 +4,11 @@ module TestFramework open System open System.IO -open System.Reflection open System.Diagnostics +open System.Reflection open Scripting open Xunit open FSharp.Compiler.IO -open Xunit.Sdk let getShortId() = Guid.NewGuid().ToString().[..7] @@ -64,7 +63,7 @@ module Commands = // Execute the process pathToExe passing the arguments: arguments with the working directory: workingDir timeout after timeout milliseconds -1 = wait forever // returns exit code, stdio and stderr as string arrays - let executeProcess pathToExe arguments workingDir (timeout:int) = + let executeProcess pathToExe arguments workingDir = let commandLine = ResizeArray() let errorsList = ResizeArray() let outputList = ResizeArray() @@ -103,11 +102,7 @@ module Commands = if p.Start() then p.BeginOutputReadLine() p.BeginErrorReadLine() - if not(p.WaitForExit(timeout)) then - // Timed out resolving throw a diagnostic. - raise (new TimeoutException(sprintf "Timeout executing command '%s' '%s'" (psi.FileName) (psi.Arguments))) - else - p.WaitForExit() + p.WaitForExit() #if DEBUG let workingDir' = if workingDir = "" @@ -564,8 +559,7 @@ module Command = | Output r -> use writer = openWrite r use outFile = redirectTo writer - use toLog = redirectToLog () - fCont { cmdArgs with RedirectOutput = Some (outFile.Post); RedirectError = Some (toLog.Post) } + fCont { cmdArgs with RedirectOutput = Some (outFile.Post); RedirectError = Some ignore } | OutputAndError (r1,r2) -> use writer1 = openWrite r1 use writer2 = openWrite r2 @@ -579,8 +573,7 @@ module Command = | Error r -> use writer = openWrite r use outFile = redirectTo writer - use toLog = redirectToLog () - fCont { cmdArgs with RedirectOutput = Some (toLog.Post); RedirectError = Some (outFile.Post) } + fCont { cmdArgs with RedirectOutput = Some ignore; RedirectError = Some (outFile.Post) } let exec cmdArgs = log "%s" (logExec dir path args redirect) diff --git a/tests/FSharp.Test.Utilities/Utilities.fs b/tests/FSharp.Test.Utilities/Utilities.fs index 484ee6049e2..f48c0eac880 100644 --- a/tests/FSharp.Test.Utilities/Utilities.fs +++ b/tests/FSharp.Test.Utilities/Utilities.fs @@ -18,6 +18,7 @@ open System.Collections.Generic open FSharp.Compiler.CodeAnalysis open Newtonsoft.Json open Newtonsoft.Json.Linq +open Xunit.Sdk type TheoryForNETCOREAPPAttribute() = @@ -42,7 +43,7 @@ type FactForDESKTOPAttribute() = module Utilities = type Async with - static member RunImmediate (computation: Async<'T>, ?cancellationToken ) = + static member RunImmediate (computation: Async<'T>, ?cancellationToken) = let cancellationToken = defaultArg cancellationToken Async.DefaultCancellationToken let ts = TaskCompletionSource<'T>() let task = ts.Task @@ -215,8 +216,7 @@ let main argv = 0""" File.WriteAllText(directoryBuildPropsFileName, directoryBuildProps) File.WriteAllText(directoryBuildTargetsFileName, directoryBuildTargets) - let timeout = 120000 - let exitCode, dotnetoutput, dotneterrors = Commands.executeProcess config.DotNetExe "build" projectDirectory timeout + let exitCode, dotnetoutput, dotneterrors = Commands.executeProcess config.DotNetExe "build" projectDirectory if exitCode <> 0 || errors.Length > 0 then errors <- dotneterrors diff --git a/tests/FSharp.Test.Utilities/XunitHelpers.fs b/tests/FSharp.Test.Utilities/XunitHelpers.fs index cda74f7867c..7a41338200a 100644 --- a/tests/FSharp.Test.Utilities/XunitHelpers.fs +++ b/tests/FSharp.Test.Utilities/XunitHelpers.fs @@ -1,3 +1,7 @@ +#if XUNIT_EXTRAS +#nowarn "0044" +#endif + namespace FSharp.Test open System @@ -6,7 +10,13 @@ open Xunit.Abstractions open TestFramework -/// Installs console support for parallel test runs. +/// Disables custom internal parallelization added with XUNIT_EXTRAS. +/// Execute test cases in a class or a module one by one instead of all at once. Allow other collections to run simultaneously. +[] +type RunTestCasesInSequenceAttribute() = inherit Attribute() + +#if !XUNIT_EXTRAS +/// Installs console support for parallel test runs and conditionally enables optional xUnit customizations. type FSharpXunitFramework(sink: IMessageSink) = inherit XunitTestFramework(sink) do @@ -21,3 +31,153 @@ type FSharpXunitFramework(sink: IMessageSink) = cleanUpTemporaryDirectoryOfThisTestRun () base.Dispose() +#else + +// To use xUnit means to customize it. The following abomination adds 2 features: +// - Capturing full console output individually for each test case, viewable in Test Explorer as test stdout. +// - Internally parallelize test classes and theories. Test cases and theory cases included in a single class or F# module can execute simultaneously + +/// Passes captured console output to xUnit. +type ConsoleCapturingTestRunner(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource) = + inherit XunitTestRunner(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource) + + member _.BaseInvokeTestMethodAsync aggregator = base.InvokeTestMethodAsync aggregator + override this.InvokeTestAsync (aggregator: ExceptionAggregator) = + task { + use capture = new TestConsole.ExecutionCapture() + let! executionTime = this.BaseInvokeTestMethodAsync aggregator + let output = + seq { + capture.OutText + if not (String.IsNullOrEmpty capture.ErrorText) then + "" + "=========== Standard Error ===========" + "" + capture.ErrorText + } |> String.concat Environment.NewLine + return executionTime, output + } + +module TestCaseCustomizations = + // Internally parallelize test classes and theories. + // Based on https://www.meziantou.net/parallelize-test-cases-execution-in-xunit.htm + // The trick is to assign a unique test collection to each case. + // Since test collection is xUnit's unit of parallelization, it will execute everything in parallel including theory cases. + let rewriteTestMethod (testCase: ITestCase) : ITestMethod = + let canFullyParallelize = + // does not belong to a defined collection + isNull testCase.TestMethod.TestClass.TestCollection.CollectionDefinition + && testCase.TestMethod.TestClass.Class.GetCustomAttributes(typeof) |> Seq.isEmpty + // is not marked with `[]` attribute + && testCase.TestMethod.Method.GetCustomAttributes(typeof) |> Seq.isEmpty + && testCase.TestMethod.TestClass.Class.GetCustomAttributes(typeof) |> Seq.isEmpty + + if canFullyParallelize then + let oldTestMethod = testCase.TestMethod + let oldTestClass = oldTestMethod.TestClass + let oldTestCollection = oldTestMethod.TestClass.TestCollection + + // Create a new collection with a unique id for the test case. + let newTestCollection = + new TestCollection( + oldTestCollection.TestAssembly, + oldTestCollection.CollectionDefinition, + oldTestCollection.DisplayName, + Guid.NewGuid() + ) + + let newTestClass = new TestClass(newTestCollection, oldTestClass.Class) + TestMethod(newTestClass, oldTestMethod.Method) + else + testCase.TestMethod + +type CustomTestCase = + inherit XunitTestCase + // xUinit demands this constructor for deserialization. + new() = { inherit XunitTestCase() } + + new(sink: IMessageSink, md, mdo, testMethod, testMethodArgs) = { inherit XunitTestCase(sink, md, mdo, testMethod, testMethodArgs) } + + override testCase.RunAsync (_, bus, args, aggregator, cts) = + let runner : XunitTestCaseRunner = + { new XunitTestCaseRunner(testCase, testCase.DisplayName, testCase.SkipReason, args, testCase.TestMethodArguments, bus, aggregator, cts) with + override this.CreateTestRunner(test, bus, testCase, args, testMethod, methodArgs, skipReason, attrs, aggregator, cts) = + ConsoleCapturingTestRunner(test, bus, testCase, args, testMethod, methodArgs, skipReason, attrs, aggregator, cts) + } + runner.RunAsync() + + // Initialize is ensured by xUnit to run once before any property access. + override testCase.Initialize () = + base.Initialize() + testCase.TestMethod <- TestCaseCustomizations.rewriteTestMethod testCase + +type CustomTheoryTestCase = + inherit XunitTheoryTestCase + new() = { inherit XunitTheoryTestCase() } + + new(sink: IMessageSink, md, mdo, testMethod) = { inherit XunitTheoryTestCase(sink, md, mdo, testMethod) } + + override testCase.RunAsync (sink, bus, args, aggregator, cts) = + let runner : XunitTestCaseRunner = + { new XunitTheoryTestCaseRunner(testCase, testCase.DisplayName, testCase.SkipReason, args, sink, bus, aggregator, cts) with + override this.CreateTestRunner(test, bus, testCase, args, testMethod, methodArgs, skipReason, attrs, aggregator, cts) = + ConsoleCapturingTestRunner(test, bus, testCase, args, testMethod, methodArgs, skipReason, attrs, aggregator, cts) + } + runner.RunAsync() + + override testCase.Initialize () = + base.Initialize() + testCase.TestMethod <- TestCaseCustomizations.rewriteTestMethod testCase + +/// `XunitTestFramework` providing parallel console support and conditionally enabling optional xUnit customizations. +type FSharpXunitFramework(sink: IMessageSink) = + inherit XunitTestFramework(sink) + do + // Because xUnit v2 lacks assembly fixture, the next best place to ensure things get called + // right at the start of the test run is here in the constructor. + // This gets executed once per test assembly. + log "FSharpXunitFramework with XUNIT_EXTRAS installing TestConsole redirection" + TestConsole.install() + + interface IDisposable with + member _.Dispose() = + cleanUpTemporaryDirectoryOfThisTestRun () + base.Dispose() + + override this.CreateDiscoverer (assemblyInfo) = + { new XunitTestFrameworkDiscoverer(assemblyInfo, this.SourceInformationProvider, this.DiagnosticMessageSink) with + override _.FindTestsForType (testClass, includeSourceInformation, messageBus, options) = + // Intercepts test discovery messages to augment test cases with additional capabilities. + let customizingBus = + { new IMessageBus with + member _.QueueMessage (message: IMessageSinkMessage) = + match message with + | :? ITestCaseDiscoveryMessage as discoveryMessage -> + let customized: ITestCase = + match discoveryMessage.TestCase with + | :? XunitTheoryTestCase -> + new CustomTheoryTestCase( + sink, + options.MethodDisplayOrDefault(), + options.MethodDisplayOptionsOrDefault(), + discoveryMessage.TestCase.TestMethod, + SourceInformation = discoveryMessage.TestCase.SourceInformation + ) + | :? XunitTestCase -> + new CustomTestCase( + sink, + options.MethodDisplayOrDefault(), + options.MethodDisplayOptionsOrDefault(), + discoveryMessage.TestCase.TestMethod, + discoveryMessage.TestCase.TestMethodArguments, + SourceInformation = discoveryMessage.TestCase.SourceInformation + ) + | testCase -> testCase + messageBus.QueueMessage(TestCaseDiscoveryMessage customized) + | _ -> + messageBus.QueueMessage message + member _.Dispose () = messageBus.Dispose() } + base.FindTestsForType(testClass, includeSourceInformation, customizingBus, options) + } + +#endif diff --git a/tests/FSharp.Test.Utilities/XunitSetup.fs b/tests/FSharp.Test.Utilities/XunitSetup.fs index fb43569b1d0..97b4adbba01 100644 --- a/tests/FSharp.Test.Utilities/XunitSetup.fs +++ b/tests/FSharp.Test.Utilities/XunitSetup.fs @@ -2,6 +2,11 @@ namespace FSharp.Test open Xunit +/// Exclude from parallelization. Execute test cases in sequence and do not run any other collections at the same time. +/// see https://github.com/xunit/xunit/issues/1999#issuecomment-522635397 +[] +type NotThreadSafeResourceCollection = class end + module XUnitSetup = [] diff --git a/tests/fsharp/Compiler/CodeGen/EmittedIL/DeterministicTests.fs b/tests/fsharp/Compiler/CodeGen/EmittedIL/DeterministicTests.fs index 057004ff01f..c29c0ecb8a5 100644 --- a/tests/fsharp/Compiler/CodeGen/EmittedIL/DeterministicTests.fs +++ b/tests/fsharp/Compiler/CodeGen/EmittedIL/DeterministicTests.fs @@ -8,7 +8,7 @@ open FSharp.Test.Compiler open Xunit - +[] module DeterministicTests = let commonOptions = ["--refonly";"--deterministic";"--nooptimizationdata"] diff --git a/tests/fsharp/FSharpSuite.Tests.fsproj b/tests/fsharp/FSharpSuite.Tests.fsproj index dc8fb5a164e..753af0ce4aa 100644 --- a/tests/fsharp/FSharpSuite.Tests.fsproj +++ b/tests/fsharp/FSharpSuite.Tests.fsproj @@ -20,7 +20,6 @@ XunitSetup.fs - diff --git a/tests/fsharp/TypeProviderTests.fs b/tests/fsharp/TypeProviderTests.fs index ab9f2e7e5ef..8ce84d59338 100644 --- a/tests/fsharp/TypeProviderTests.fs +++ b/tests/fsharp/TypeProviderTests.fs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. #if INTERACTIVE //#r @"../../release/net40/bin/FSharp.Compiler.dll" diff --git a/tests/fsharp/XunitHelpers.fs b/tests/fsharp/XunitHelpers.fs deleted file mode 100644 index 6bfb5031f13..00000000000 --- a/tests/fsharp/XunitHelpers.fs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Xunit - -open Xunit - -module Assert = - - [] - do() diff --git a/tests/fsharp/tests.fs b/tests/fsharp/tests.fs index 8b13fc003a3..0dc078e506c 100644 --- a/tests/fsharp/tests.fs +++ b/tests/fsharp/tests.fs @@ -763,7 +763,7 @@ module CoreTests = #endif -#if !NETCOREAPP +#if !NETCOREAPP [] let quotes () = let cfg = testConfig "core/quotes" diff --git a/tests/fsharp/xunit.runner.json b/tests/fsharp/xunit.runner.json index c222a4167a8..f47fec5d745 100644 --- a/tests/fsharp/xunit.runner.json +++ b/tests/fsharp/xunit.runner.json @@ -1,4 +1,5 @@ { "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", - "appDomain": "denied" + "appDomain": "denied", + "parallelizeAssembly": true } \ No newline at end of file diff --git a/tests/scripts/scriptlib.fsx b/tests/scripts/scriptlib.fsx index 04b23a93fa8..ea1a5a63737 100644 --- a/tests/scripts/scriptlib.fsx +++ b/tests/scripts/scriptlib.fsx @@ -31,7 +31,7 @@ module Scripting = let info = ProcessStartInfo(Arguments=arguments, UseShellExecute=false, RedirectStandardOutput=true, RedirectStandardError=true, CreateNoWindow=true, FileName=fileName) - let p = new Process(StartInfo=info) + use p = new Process(StartInfo=info) p.OutputDataReceived.Add(fun x -> processWriteMessage stdout x.Data) p.ErrorDataReceived.Add(fun x -> processWriteMessage stderr x.Data) if p.Start() then @@ -121,7 +121,7 @@ module Scripting = ignore envs // work out what to do about this - let p = new Process() + use p = new Process() p.EnableRaisingEvents <- true p.StartInfo <- processInfo let out = StringBuilder() @@ -163,11 +163,14 @@ module Scripting = p.WaitForExit() + printf $"{string out}" + eprintf $"{string err}" + match p.ExitCode with | 0 -> Success(string out) | errCode -> - let msg = sprintf "Error running command '%s' with args '%s' in directory '%s'.\n---- stdout below --- \n%s\n---- stderr below --- \n%s " exePath arguments workDir (out.ToString()) (err.ToString()) + let msg = sprintf "Error running command '%s' with args '%s' in directory '%s'" exePath arguments workDir ErrorLevel (msg, errCode) type OutPipe (writer: TextWriter) = @@ -177,8 +180,6 @@ module Scripting = let redirectTo (writer: TextWriter) = new OutPipe (writer) - let redirectToLog () = redirectTo System.Console.Out - #if !NETCOREAPP let defaultPlatform = match Environment.OSVersion.Platform, Environment.Is64BitOperatingSystem with @@ -192,7 +193,7 @@ module Scripting = let info = ProcessStartInfo(Arguments=arguments, UseShellExecute=false, RedirectStandardOutput=true, RedirectStandardError=true,RedirectStandardInput=true, CreateNoWindow=true, FileName=fileName) - let p = new Process(StartInfo=info) + use p = new Process(StartInfo=info) if p.Start() then async { try