Add TestExpectation system, remove continue-on-error

.github/workflows/dotnet.yml

@@ -97,15 +97,13 @@ jobs:
       run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
     - name: Test (Linux headless)
       if: ${{ matrix.os == 'ubuntu-latest' && matrix.mode == 'headless' }}
-      continue-on-error: true
       env:
         PRODUCT: ${{ matrix.browser }}
         HEADLESS: true
       run: |
         dotnet test ./src/PlaywrightSharp.Tests/PlaywrightSharp.Tests.csproj --no-build -f net10.0 -s src/PlaywrightSharp.Tests/test.runsettings
     - name: Test (Linux headful)
       if: ${{ matrix.os == 'ubuntu-latest' && matrix.mode == 'headful' }}
-      continue-on-error: true
       env:
         PRODUCT: ${{ matrix.browser }}
         HEADLESS: false
@@ -114,7 +112,6 @@ jobs:
           dotnet test ./src/PlaywrightSharp.Tests/PlaywrightSharp.Tests.csproj --no-build -f net10.0 -s src/PlaywrightSharp.Tests/test.runsettings
     - name: Test (Windows)
       if: matrix.os == 'windows-latest'
-      continue-on-error: true
       env:
         PRODUCT: ${{ matrix.browser }}
         HEADLESS: ${{ matrix.mode == 'headless' && 'true' || 'false' }}

src/PlaywrightSharp.Nunit/PlaywrightSharp.Nunit.csproj

@@ -7,4 +7,7 @@
 	<ItemGroup>
 		<PackageReference Include="NUnit" Version="4.1.0" />
 	</ItemGroup>
+	<ItemGroup>
+		<EmbeddedResource Include="TestExpectations\TestExpectations.local.json" />
+	</ItemGroup>
 </Project>

src/PlaywrightSharp.Nunit/PlaywrightTestAttribute.cs

@@ -1,7 +1,13 @@
 using System;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using System.Text.Json;
 using NUnit.Framework;
 using NUnit.Framework.Interfaces;
 using NUnit.Framework.Internal;
+using PlaywrightSharp.Nunit.TestExpectations;
 
 namespace PlaywrightSharp.Nunit
 {
@@ -11,6 +17,8 @@ namespace PlaywrightSharp.Nunit
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
     public class PlaywrightTestAttribute : NUnitAttribute, IApplyToTest
     {
+        private static TestExpectation[] _localExpectations;
+
         /// <summary>
         /// Gets whether the current product is Chromium.
         /// </summary>
@@ -71,9 +79,111 @@ public PlaywrightTestAttribute(string fileName, string describe, string nameOfTe
         /// </summary>
         public string Describe { get; }
 
+        /// <inheritdoc/>
+        public override string ToString()
+            => Describe == null ? $"[{FileName}] {TestName}" : $"[{FileName}] {Describe} {TestName}";
+
         /// <inheritdoc/>
         public void ApplyToTest(Test test)
         {
+            if (test == null)
+            {
+                return;
+            }
+
+            if (ShouldSkipByExpectation(test, out TestExpectation expectation))
+            {
+                test.RunState = RunState.Ignored;
+                test.Properties.Set(PropertyNames.SkipReason, $"Skipped by expectation {expectation.TestIdPattern}");
+            }
+        }
+
+        private bool ShouldSkipByExpectation(Test test, out TestExpectation output)
+        {
+            TestExpectation.TestExpectationPlatform currentPlatform = GetCurrentExpectationPlatform();
+            TestExpectation.TestExpectationsParameter browserParam = IsChromium
+                ? TestExpectation.TestExpectationsParameter.Chromium
+                : IsFirefox
+                    ? TestExpectation.TestExpectationsParameter.Firefox
+                    : TestExpectation.TestExpectationsParameter.Webkit;
+
+            string headlessEnv = Environment.GetEnvironmentVariable("HEADLESS");
+            TestExpectation.TestExpectationsParameter modeParam =
+                string.Equals(headlessEnv, "false", StringComparison.OrdinalIgnoreCase)
+                    ? TestExpectation.TestExpectationsParameter.Headful
+                    : TestExpectation.TestExpectationsParameter.Headless;
+
+            TestExpectation.TestExpectationsParameter[] parameters = new[] { browserParam, modeParam };
+
+            TestExpectation[] localExpectations = GetLocalExpectations();
+            string testIdStr = ToString();
+
+            foreach (TestExpectation expectation in localExpectations)
+            {
+                if (expectation.TestIdRegex.IsMatch(testIdStr))
+                {
+                    bool platformMatch = expectation.Platforms.Contains(currentPlatform);
+                    bool paramsMatch = expectation.Parameters.Length == 0 ||
+                        expectation.Parameters.All(p => parameters.Contains(p));
+
+                    if (platformMatch && paramsMatch)
+                    {
+                        bool shouldSkip =
+                            expectation.Expectations.Contains(TestExpectation.TestExpectationResult.Skip) ||
+                            expectation.Expectations.Contains(TestExpectation.TestExpectationResult.Fail) ||
+                            expectation.Expectations.Contains(TestExpectation.TestExpectationResult.Timeout);
+
+                        if (shouldSkip)
+                        {
+                            output = expectation;
+                            return true;
+                        }
+
+                        if (expectation.Expectations.Contains(TestExpectation.TestExpectationResult.Pass))
+                        {
+                            output = null;
+                            return false;
+                        }
+                    }
+                }
+            }
+
+            output = null;
+            return false;
+        }
+
+        private static TestExpectation.TestExpectationPlatform GetCurrentExpectationPlatform()
+        {
+            if (RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows))
+            {
+                return TestExpectation.TestExpectationPlatform.Win32;
+            }
+
+            if (RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX))
+            {
+                return TestExpectation.TestExpectationPlatform.Darwin;
+            }
+
+            return TestExpectation.TestExpectationPlatform.Linux;
+        }
+
+        private static TestExpectation[] GetLocalExpectations() =>
+            _localExpectations ??= LoadExpectationsFromResource("PlaywrightSharp.Nunit.TestExpectations.TestExpectations.local.json");
+
+        private static readonly JsonSerializerOptions DefaultJsonSerializerOptions =
+            new()
+            {
+                PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+            };
+
+        private static TestExpectation[] LoadExpectationsFromResource(string resourceName)
+        {
+            Assembly assembly = Assembly.GetExecutingAssembly();
+
+            using Stream stream = assembly.GetManifestResourceStream(resourceName);
+            using StreamReader reader = new StreamReader(stream);
+            string fileContent = reader.ReadToEnd();
+            return JsonSerializer.Deserialize<TestExpectation[]>(fileContent, DefaultJsonSerializerOptions);
         }
     }
 }

src/PlaywrightSharp.Nunit/TestExpectations/TestExpectation.cs

@@ -0,0 +1,90 @@
+// * MIT License
+//  *
+//  * Copyright (c) Dario Kondratiuk
+//  *
+//  * Permission is hereby granted, free of charge, to any person obtaining a copy
+//  * of this software and associated documentation files (the "Software"), to deal
+//  * in the Software without restriction, including without limitation the rights
+//  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+//  * copies of the Software, and to permit persons to whom the Software is
+//  * furnished to do so, subject to the following conditions:
+//  *
+//  * The above copyright notice and this permission notice shall be included in all
+//  * copies or substantial portions of the Software.
+//  *
+//  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+//  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+//  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+//  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+//  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+//  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+//  * SOFTWARE.
+
+using System;
+using System.Runtime.Serialization;
+using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;
+
+namespace PlaywrightSharp.Nunit.TestExpectations;
+
+public class TestExpectation
+{
+    private Lazy<Regex> _testIdRegex;
+
+    public string TestIdPattern
+    {
+        get => _testIdPattern;
+        set
+        {
+            _testIdPattern = value;
+            _testIdRegex = new Lazy<Regex>(() =>
+            {
+                // Replace `*` with a placeholder before escaping special characters.
+                string patternRegExString = Regex.Escape(_testIdPattern.Replace("*", "--STAR--"));
+
+                // Replace placeholder with greedy match
+                patternRegExString = patternRegExString.Replace("--STAR--", "(.*)?");
+
+                // Match beginning and end explicitly
+                return new Regex($"^{patternRegExString}$");
+            });
+        }
+    }
+
+    private string _testIdPattern;
+
+    public Regex TestIdRegex => _testIdRegex?.Value;
+
+    public TestExpectationPlatform[] Platforms { get; set; }
+
+    public TestExpectationsParameter[] Parameters { get; set; }
+
+    public TestExpectationResult[] Expectations { get; set; }
+
+    [JsonConverter(typeof(JsonStringEnumConverter<TestExpectationResult>))]
+    public enum TestExpectationResult
+    {
+        [EnumMember(Value = "FAIL")] Fail,
+        [EnumMember(Value = "PASS")] Pass,
+        [EnumMember(Value = "SKIP")] Skip,
+        [EnumMember(Value = "TIMEOUT")] Timeout,
+    }
+
+    [JsonConverter(typeof(JsonStringEnumConverter<TestExpectationsParameter>))]
+    public enum TestExpectationsParameter
+    {
+        [EnumMember(Value = "chromium")] Chromium,
+        [EnumMember(Value = "firefox")] Firefox,
+        [EnumMember(Value = "webkit")] Webkit,
+        [EnumMember(Value = "headless")] Headless,
+        [EnumMember(Value = "headful")] Headful,
+    }
+
+    [JsonConverter(typeof(JsonStringEnumConverter<TestExpectationPlatform>))]
+    public enum TestExpectationPlatform
+    {
+        [EnumMember(Value = "darwin")] Darwin,
+        [EnumMember(Value = "linux")] Linux,
+        [EnumMember(Value = "win32")] Win32,
+    }
+}