diff --git a/README.md b/README.md index 730d41f..f525fd3 100644 --- a/README.md +++ b/README.md @@ -161,5 +161,22 @@ and define the following in your `csproj`: ``` These attributes will ask for the runtime test engine to replace the ones defined by the `Uno.UI.RuntimeTests.Engine` package. +## Test runner (UnitTestsControl) filtering syntax +- Search terms are separated by space. Multiple consecutive spaces are treated same as one. +- Multiple search terms are chained with AND logic. +- Search terms are case insensitive. +- `-` can be used before any term for exclusion, effectively inverting the results. +- Special tags can be used to match certain part of the test: // syntax: tag:term + - `class` or `c` matches the class name + - `method` or `m` matches the method name + - `displayname` or `d` matches the display name in [DataRow] +- Search term without a prefixing tag will match either of method name or class name. + +Examples: +- `listview` +- `listview measure` +- `listview measure -recycle` +- `c:listview m:measure -m:recycle` + ## Running the tests automatically during CI _TBD_ diff --git a/src/TestApp/shared/EngineFeatureTests.cs b/src/TestApp/shared/EngineFeatureTests.cs new file mode 100644 index 0000000..3dc96e1 --- /dev/null +++ b/src/TestApp/shared/EngineFeatureTests.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +#if HAS_UNO_WINUI || WINDOWS_WINUI +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; +#else +using Windows.UI.Xaml; +using Windows.UI.Xaml.Controls; +#endif + +namespace Uno.UI.RuntimeTests.Engine +{ + /// + /// Contains tests relevant to the RTT engine features. + /// + [TestClass] + public class MetaTests + { + [TestMethod] + [RunsOnUIThread] + public async Task When_Test_ContentHelper() + { + var SUT = new TextBlock() { Text = "Hello" }; + UnitTestsUIContentHelper.Content = SUT; + + await UnitTestsUIContentHelper.WaitForIdle(); + await UnitTestsUIContentHelper.WaitForLoaded(SUT); + } + + [TestMethod] + [DataRow("hello", DisplayName = "hello test")] + [DataRow("goodbye", DisplayName = "goodbye test")] + public void When_DisplayName(string text) + { + } + } +} diff --git a/src/TestApp/shared/SanityTests.cs b/src/TestApp/shared/SanityTests.cs index c2b0039..4a00eca 100644 --- a/src/TestApp/shared/SanityTests.cs +++ b/src/TestApp/shared/SanityTests.cs @@ -14,6 +14,9 @@ namespace Uno.UI.RuntimeTests.Engine { + /// + /// Contains sanity/smoke tests used to assert basic scenarios. + /// [TestClass] public class SanityTests { @@ -28,24 +31,6 @@ public async Task Is_Still_Sane() await Task.Delay(2000); } - [TestMethod] - [RunsOnUIThread] - public async Task When_Test_ContentHelper() - { - var SUT = new TextBlock() { Text = "Hello" }; - UnitTestsUIContentHelper.Content = SUT; - - await UnitTestsUIContentHelper.WaitForIdle(); - await UnitTestsUIContentHelper.WaitForLoaded(SUT); - } - - [TestMethod] - [DataRow("hello", DisplayName = "hello test")] - [DataRow("goodbye", DisplayName = "goodbye test")] - public void Is_Sane_With_Cases(string text) - { - } - #if DEBUG [TestMethod] public async Task No_Longer_Sane() // expected to fail diff --git a/src/TestApp/shared/uno.ui.runtimetests.engine.Shared.projitems b/src/TestApp/shared/uno.ui.runtimetests.engine.Shared.projitems index 0083c1d..7632538 100644 --- a/src/TestApp/shared/uno.ui.runtimetests.engine.Shared.projitems +++ b/src/TestApp/shared/uno.ui.runtimetests.engine.Shared.projitems @@ -22,6 +22,7 @@ MainPage.xaml + diff --git a/src/Uno.UI.RuntimeTests.Engine.Library/Extensions/StringExtensions.cs b/src/Uno.UI.RuntimeTests.Engine.Library/Extensions/StringExtensions.cs new file mode 100644 index 0000000..7c7aaf5 --- /dev/null +++ b/src/Uno.UI.RuntimeTests.Engine.Library/Extensions/StringExtensions.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace Uno.UI.RuntimeTests.Extensions; + +internal static partial class StringExtensions +{ + /// + /// Like , but allows exception to be made with a Regex pattern. + /// + /// + /// + /// segments matched by the regex will not be splited. + /// + /// + public static string[] SplitWithIgnore(this string input, char separator, string ignoredPattern, bool skipEmptyEntries) + { + var ignores = Regex.Matches(input, ignoredPattern); + + var shards = new List(); + for (int i = 0; i < input.Length; i++) + { + var nextSpaceDelimiter = input.IndexOf(separator, i); + + // find the next space, if inside a quote + while (nextSpaceDelimiter != -1 && ignores.FirstOrDefault(x => InRange(x, nextSpaceDelimiter)) is { } enclosingIgnore) + { + nextSpaceDelimiter = enclosingIgnore.Index + enclosingIgnore.Length + 1 is { } afterIgnore && afterIgnore < input.Length + ? input.IndexOf(separator, afterIgnore) + : -1; + } + + if (nextSpaceDelimiter != -1) + { + shards.Add(input.Substring(i, nextSpaceDelimiter - i)); + i = nextSpaceDelimiter; + + // skip multiple continous spaces + while (skipEmptyEntries && i + 1 < input.Length && input[i + 1] == separator) i++; + } + else + { + shards.Add(input.Substring(i)); + break; + } + } + + return shards.ToArray(); + + bool InRange(Match x, int index) => x.Index <= index && index < (x.Index + x.Length); + } +} diff --git a/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestEngineConfig.cs b/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestEngineConfig.cs index c66d02f..a19f6f3 100644 --- a/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestEngineConfig.cs +++ b/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestEngineConfig.cs @@ -10,7 +10,7 @@ public class UnitTestEngineConfig public static UnitTestEngineConfig Default { get; } = new UnitTestEngineConfig(); - public string[]? Filters { get; set; } + public string? Query { get; set; } public int Attempts { get; set; } = DefaultRepeatCount; diff --git a/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestsControl.Filtering.cs b/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestsControl.Filtering.cs new file mode 100644 index 0000000..ad9fe75 --- /dev/null +++ b/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestsControl.Filtering.cs @@ -0,0 +1,171 @@ +#if !UNO_RUNTIMETESTS_DISABLE_UI + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Reflection; +using System.Text; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Uno.UI.RuntimeTests.Extensions; +using static System.StringComparison; + +namespace Uno.UI.RuntimeTests; + +partial class UnitTestsControl +{ + private static IEnumerable FilterTests(UnitTestClassInfo testClassInfo, string? query) + { + var tests = testClassInfo.Tests?.AsEnumerable() ?? Array.Empty(); + foreach (var filter in SearchPredicate.ParseQuery(query)) + { + // chain filters with AND logic + tests = tests.Where(x => + filter.Exclusion ^ // use xor to flip the result based on Exclusion + filter.Tag?.ToLowerInvariant() switch + { + "class" => MatchClassName(x, filter.Text), + "displayname" => MatchDisplayName(x, filter.Text), + "method" => MatchMethodName(x, filter.Text), + + _ => MatchClassName(x, filter.Text) || MatchMethodName(x, filter.Text), + } + ); + } + + bool MatchClassName(MethodInfo x, string value) => x.DeclaringType?.Name.Contains(value, InvariantCultureIgnoreCase) ?? false; + bool MatchMethodName(MethodInfo x, string value) => x.Name.Contains(value, InvariantCultureIgnoreCase); + bool MatchDisplayName(MethodInfo x, string value) => + // fixme: since we are returning MethodInfo for match, there is no way to specify + // which of the [DataRow] or which row within [DynamicData] without refactoring. + // fixme: support [DynamicData] + x.GetCustomAttributes().Any(y => y.DisplayName.Contains(value, InvariantCultureIgnoreCase)); + + return tests; + } + + + public record SearchPredicate(string Raw, string Text, bool Exclusion = false, string? Tag = null, params SearchPredicatePart[] Parts) + { + public static SearchPredicateComparer DefaultComparer = new(); + public static SearchQueryComparer DefaultQueryComparer = new(); + + // fixme: domain specific configuration should be injectable + private static readonly IReadOnlyDictionary NamespaceAliases = + new Dictionary(StringComparer.InvariantCultureIgnoreCase) + { + ["c"] = "class", + ["m"] = "method", + ["d"] = "display_name", + }; + + public static SearchPredicate[] ParseQuery(string? query) + { + if (string.IsNullOrWhiteSpace(query)) return Array.Empty(); + + return query!.SplitWithIgnore(' ', @""".*?(?() // trim null + .Where(x => x.Text.Length > 0) // ignore empty tag query eg: "c:" + .ToArray(); + } + + public static SearchPredicate? ParseFragment(string criteria) + { + if (string.IsNullOrWhiteSpace(criteria)) return null; + + var raw = criteria.Trim(); + var text = raw; + if (text.StartsWith('-') is var exclusion && exclusion) + { + text = text.Substring(1); + } + var tag = default(string?); + if (text.Split(':', 2) is { Length: 2 } tagParts) + { + tag = NamespaceAliases.TryGetValue(tagParts[0], out var value) ? value : tagParts[0]; + text = tagParts[1]; + } + var parts = text.SplitWithIgnore(',', @""".*?(? Parts + .Any(x => (x.MatchStart, x.MatchEnd) switch + { + (true, false) => input.StartsWith(x.Text), + (false, true) => input.EndsWith(x.Text), + + _ => input.Contains(x.Text), + }); + + public class SearchPredicateComparer : IEqualityComparer + { + public int GetHashCode(SearchPredicate? obj) => obj?.GetHashCode() ?? -1; + public bool Equals(SearchPredicate? x, SearchPredicate? y) + { + return (x, y) switch + { + (null, null) => true, + (null, _) => false, + (_, null) => false, + _ => + x.Raw == y.Raw && + x.Text == y.Text && + x.Exclusion == y.Exclusion && + x.Tag == y.Tag && + x.Parts.SequenceEqual(y.Parts), + }; + } + } + + public class SearchQueryComparer : IEqualityComparer + { + public int GetHashCode(SearchPredicate[]? obj) => obj?.GetHashCode() ?? -1; + public bool Equals(SearchPredicate[]? x, SearchPredicate[]? y) + { + return (x, y) switch + { + (null, null) => true, + (null, _) => false, + (_, null) => false, + _ => x.SequenceEqual(y, DefaultComparer), + }; + } + } + } + public record SearchPredicatePart(string Raw, string Text, bool MatchStart = false, bool MatchEnd = false) + { + public static SearchPredicatePart Parse(string part) + { + var raw = part; + var text = raw; + + if (text.Length >= 2 && text.StartsWith('"') && text.EndsWith('"')) + { + // within quoted string, unquote and unescape \" to " + text = text + .Substring(1, text.Length - 2) + .Replace("\\\"", "\""); + } + if (text.StartsWith("^") is { } matchStart && matchStart) + { + text = text.Substring(1); + } + if (text.EndsWith("$") is { } matchEnd && matchEnd) + { + text = text.Substring(0, text.Length - 1); + } + + return new(raw, text, matchStart, matchEnd); + } + } +} + +#endif \ No newline at end of file diff --git a/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestsControl.cs b/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestsControl.cs index 4a8c27b..c810020 100644 --- a/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestsControl.cs +++ b/src/Uno.UI.RuntimeTests.Engine.Library/UI/UnitTestsControl.cs @@ -65,7 +65,6 @@ public sealed partial class UnitTestsControl : UserControl #endif #pragma warning restore CS0109 - private const StringComparison StrComp = StringComparison.InvariantCultureIgnoreCase; private Task? _runner; private CancellationTokenSource? _cts = new CancellationTokenSource(); #if DEBUG @@ -499,7 +498,7 @@ private void EnableConfigPersistence() consoleOutput.IsChecked = config.IsConsoleOutputEnabled; runIgnored.IsChecked = config.IsRunningIgnored; retry.IsChecked = config.Attempts > 1; - testFilter.Text = string.Join(";", config.Filters ?? Array.Empty()); + testFilter.Text = config.Query ?? string.Empty; } } catch (Exception) @@ -534,15 +533,11 @@ private UnitTestEngineConfig BuildConfig() var isConsoleOutput = consoleOutput.IsChecked ?? false; var isRunningIgnored = runIgnored.IsChecked ?? false; var attempts = (retry.IsChecked ?? true) ? UnitTestEngineConfig.DefaultRepeatCount : 1; - var filter = testFilter.Text.Trim(); - if (string.IsNullOrEmpty(filter)) - { - filter = null; - } + var query = testFilter.Text.Trim() is { Length: >0 } text ? text: null; return new UnitTestEngineConfig { - Filters = filter?.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries) ?? Array.Empty(), + Query = query, IsConsoleOutputEnabled = isConsoleOutput, IsRunningIgnored = isRunningIgnored, Attempts = attempts, @@ -675,17 +670,6 @@ await _dispatcher.RunAsync(() => await GenerateTestResults(); } - private static IEnumerable FilterTests(UnitTestClassInfo testClassInfo, string[]? filters) - { - var testClassNameContainsFilters = filters?.Any(f => testClassInfo.Type?.FullName?.Contains(f, StrComp) ?? false) ?? false; - return testClassInfo.Tests?. - Where(t => ((!filters?.Any()) ?? true) - || testClassNameContainsFilters - || (filters?.Any(f => t.DeclaringType?.FullName?.Contains(f, StrComp) ?? false) ?? false) - || (filters?.Any(f => t.Name.Contains(f, StrComp)) ?? false)) - ?? Array.Empty(); - } - private async Task ExecuteTestsForInstance( CancellationToken ct, object instance, @@ -696,7 +680,7 @@ private async Task ExecuteTestsForInstance( ? ConsoleOutputRecorder.Start() : default; - var tests = UnitTestsControl.FilterTests(testClassInfo, config.Filters) + var tests = FilterTests(testClassInfo, config.Query) .Select(method => new UnitTestMethodInfo(instance, method)) .ToArray(); @@ -705,7 +689,7 @@ private async Task ExecuteTestsForInstance( return; } - ReportTestClass(testClassInfo.Type.GetTypeInfo()); + ReportTestClass(testClassInfo.Type!.GetTypeInfo()); _ = ReportMessage($"Running {tests.Length} test methods"); foreach (var test in tests) diff --git a/src/Uno.UI.RuntimeTests.Engine.Library/Uno.UI.RuntimeTests.Engine.Library.projitems b/src/Uno.UI.RuntimeTests.Engine.Library/Uno.UI.RuntimeTests.Engine.Library.projitems index 3c7c839..80d22ea 100644 --- a/src/Uno.UI.RuntimeTests.Engine.Library/Uno.UI.RuntimeTests.Engine.Library.projitems +++ b/src/Uno.UI.RuntimeTests.Engine.Library/Uno.UI.RuntimeTests.Engine.Library.projitems @@ -9,20 +9,22 @@ Uno.UI.RuntimeTests.Engine.Library + + - + diff --git a/src/Uno.UI.RuntimeTests.Engine.sln b/src/Uno.UI.RuntimeTests.Engine.sln index 9a986aa..0983621 100644 --- a/src/Uno.UI.RuntimeTests.Engine.sln +++ b/src/Uno.UI.RuntimeTests.Engine.sln @@ -32,7 +32,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Uno.UI.RuntimeTests.Engine. EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "UWP", "UWP", "{D31D49DA-BFBD-420C-9F8E-12D77DDF3070}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Uno.UI.RuntimeTests.Engine.Mobile", "TestApp\uwp\Uno.UI.RuntimeTests.Engine.Mobile\Uno.UI.RuntimeTests.Engine.Mobile.csproj", "{C72C5858-9953-4DEB-A551-E25BC5E7AD51}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Uno.UI.RuntimeTests.Engine.Mobile", "TestApp\uwp\Uno.UI.RuntimeTests.Engine.Mobile\Uno.UI.RuntimeTests.Engine.Mobile.csproj", "{C72C5858-9953-4DEB-A551-E25BC5E7AD51}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Uno.UI.RuntimeTests.Engine.Uwp", "TestApp\uwp\Uno.UI.RuntimeTests.Engine.UWP\Uno.UI.RuntimeTests.Engine.Uwp.csproj", "{9804C1B9-A958-4A09-B975-AAECF08CFEE1}" EndProject @@ -192,6 +192,22 @@ Global {7BD72040-D89A-4DE8-8713-5A08068ADBCE}.Release|x64.Build.0 = Release|Any CPU {7BD72040-D89A-4DE8-8713-5A08068ADBCE}.Release|x86.ActiveCfg = Release|Any CPU {7BD72040-D89A-4DE8-8713-5A08068ADBCE}.Release|x86.Build.0 = Release|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Debug|arm64.ActiveCfg = Debug|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Debug|arm64.Build.0 = Debug|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Debug|x64.ActiveCfg = Debug|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Debug|x64.Build.0 = Debug|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Debug|x86.ActiveCfg = Debug|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Debug|x86.Build.0 = Debug|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Release|Any CPU.Build.0 = Release|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Release|arm64.ActiveCfg = Release|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Release|arm64.Build.0 = Release|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Release|x64.ActiveCfg = Release|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Release|x64.Build.0 = Release|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Release|x86.ActiveCfg = Release|Any CPU + {C72C5858-9953-4DEB-A551-E25BC5E7AD51}.Release|x86.Build.0 = Release|Any CPU {9804C1B9-A958-4A09-B975-AAECF08CFEE1}.Debug|Any CPU.ActiveCfg = Debug|x64 {9804C1B9-A958-4A09-B975-AAECF08CFEE1}.Debug|Any CPU.Build.0 = Debug|x64 {9804C1B9-A958-4A09-B975-AAECF08CFEE1}.Debug|Any CPU.Deploy.0 = Debug|x64 @@ -318,6 +334,8 @@ Global Uno.UI.RuntimeTests.Engine.Library\Uno.UI.RuntimeTests.Engine.Library.projitems*{9804c1b9-a958-4a09-b975-aaecf08cfee1}*SharedItemsImports = 4 TestApp\shared\Uno.UI.RuntimeTests.Engine.Shared.projitems*{a5b8155a-118f-4794-b551-c6f3cf7e5411}*SharedItemsImports = 5 Uno.UI.RuntimeTests.Engine.Library\Uno.UI.RuntimeTests.Engine.Library.projitems*{a5b8155a-118f-4794-b551-c6f3cf7e5411}*SharedItemsImports = 5 + TestApp\shared\Uno.UI.RuntimeTests.Engine.Shared.projitems*{c72c5858-9953-4deb-a551-e25bc5e7ad51}*SharedItemsImports = 5 + Uno.UI.RuntimeTests.Engine.Library\Uno.UI.RuntimeTests.Engine.Library.projitems*{c72c5858-9953-4deb-a551-e25bc5e7ad51}*SharedItemsImports = 5 TestApp\shared\Uno.UI.RuntimeTests.Engine.Shared.projitems*{dec0dfbd-2926-492b-be22-2158438556e3}*SharedItemsImports = 5 Uno.UI.RuntimeTests.Engine.Library\Uno.UI.RuntimeTests.Engine.Library.projitems*{dec0dfbd-2926-492b-be22-2158438556e3}*SharedItemsImports = 5 Uno.UI.RuntimeTests.Engine.Library\Uno.UI.RuntimeTests.Engine.Library.projitems*{e3f4af60-8456-4ed6-9992-303cdad44e0d}*SharedItemsImports = 13