diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index ea70b638f..d394a5358 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -97,7 +97,6 @@ jobs: run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns - name: Test (Linux headless) if: ${{ matrix.os == 'ubuntu-latest' && matrix.mode == 'headless' }} - continue-on-error: true env: PRODUCT: ${{ matrix.browser }} HEADLESS: true @@ -105,7 +104,6 @@ jobs: dotnet test ./src/PlaywrightSharp.Tests/PlaywrightSharp.Tests.csproj --no-build -f net10.0 -s src/PlaywrightSharp.Tests/test.runsettings - name: Test (Linux headful) if: ${{ matrix.os == 'ubuntu-latest' && matrix.mode == 'headful' }} - continue-on-error: true env: PRODUCT: ${{ matrix.browser }} HEADLESS: false @@ -114,7 +112,6 @@ jobs: dotnet test ./src/PlaywrightSharp.Tests/PlaywrightSharp.Tests.csproj --no-build -f net10.0 -s src/PlaywrightSharp.Tests/test.runsettings - name: Test (Windows) if: matrix.os == 'windows-latest' - continue-on-error: true env: PRODUCT: ${{ matrix.browser }} HEADLESS: ${{ matrix.mode == 'headless' && 'true' || 'false' }} diff --git a/src/PlaywrightSharp.Nunit/PlaywrightSharp.Nunit.csproj b/src/PlaywrightSharp.Nunit/PlaywrightSharp.Nunit.csproj index da3a37791..9582c4dad 100644 --- a/src/PlaywrightSharp.Nunit/PlaywrightSharp.Nunit.csproj +++ b/src/PlaywrightSharp.Nunit/PlaywrightSharp.Nunit.csproj @@ -7,4 +7,7 @@ + + + diff --git a/src/PlaywrightSharp.Nunit/PlaywrightTestAttribute.cs b/src/PlaywrightSharp.Nunit/PlaywrightTestAttribute.cs index e96ae69ae..35c927bb0 100644 --- a/src/PlaywrightSharp.Nunit/PlaywrightTestAttribute.cs +++ b/src/PlaywrightSharp.Nunit/PlaywrightTestAttribute.cs @@ -1,7 +1,13 @@ using System; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text.Json; using NUnit.Framework; using NUnit.Framework.Interfaces; using NUnit.Framework.Internal; +using PlaywrightSharp.Nunit.TestExpectations; namespace PlaywrightSharp.Nunit { @@ -11,6 +17,8 @@ namespace PlaywrightSharp.Nunit [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] public class PlaywrightTestAttribute : NUnitAttribute, IApplyToTest { + private static TestExpectation[] _localExpectations; + /// /// Gets whether the current product is Chromium. /// @@ -71,9 +79,111 @@ public PlaywrightTestAttribute(string fileName, string describe, string nameOfTe /// public string Describe { get; } + /// + public override string ToString() + => Describe == null ? $"[{FileName}] {TestName}" : $"[{FileName}] {Describe} {TestName}"; + /// public void ApplyToTest(Test test) { + if (test == null) + { + return; + } + + if (ShouldSkipByExpectation(test, out TestExpectation expectation)) + { + test.RunState = RunState.Ignored; + test.Properties.Set(PropertyNames.SkipReason, $"Skipped by expectation {expectation.TestIdPattern}"); + } + } + + private bool ShouldSkipByExpectation(Test test, out TestExpectation output) + { + TestExpectation.TestExpectationPlatform currentPlatform = GetCurrentExpectationPlatform(); + TestExpectation.TestExpectationsParameter browserParam = IsChromium + ? TestExpectation.TestExpectationsParameter.Chromium + : IsFirefox + ? TestExpectation.TestExpectationsParameter.Firefox + : TestExpectation.TestExpectationsParameter.Webkit; + + string headlessEnv = Environment.GetEnvironmentVariable("HEADLESS"); + TestExpectation.TestExpectationsParameter modeParam = + string.Equals(headlessEnv, "false", StringComparison.OrdinalIgnoreCase) + ? TestExpectation.TestExpectationsParameter.Headful + : TestExpectation.TestExpectationsParameter.Headless; + + TestExpectation.TestExpectationsParameter[] parameters = new[] { browserParam, modeParam }; + + TestExpectation[] localExpectations = GetLocalExpectations(); + string testIdStr = ToString(); + + foreach (TestExpectation expectation in localExpectations) + { + if (expectation.TestIdRegex.IsMatch(testIdStr)) + { + bool platformMatch = expectation.Platforms.Contains(currentPlatform); + bool paramsMatch = expectation.Parameters.Length == 0 || + expectation.Parameters.All(p => parameters.Contains(p)); + + if (platformMatch && paramsMatch) + { + bool shouldSkip = + expectation.Expectations.Contains(TestExpectation.TestExpectationResult.Skip) || + expectation.Expectations.Contains(TestExpectation.TestExpectationResult.Fail) || + expectation.Expectations.Contains(TestExpectation.TestExpectationResult.Timeout); + + if (shouldSkip) + { + output = expectation; + return true; + } + + if (expectation.Expectations.Contains(TestExpectation.TestExpectationResult.Pass)) + { + output = null; + return false; + } + } + } + } + + output = null; + return false; + } + + private static TestExpectation.TestExpectationPlatform GetCurrentExpectationPlatform() + { + if (RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows)) + { + return TestExpectation.TestExpectationPlatform.Win32; + } + + if (RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX)) + { + return TestExpectation.TestExpectationPlatform.Darwin; + } + + return TestExpectation.TestExpectationPlatform.Linux; + } + + private static TestExpectation[] GetLocalExpectations() => + _localExpectations ??= LoadExpectationsFromResource("PlaywrightSharp.Nunit.TestExpectations.TestExpectations.local.json"); + + private static readonly JsonSerializerOptions DefaultJsonSerializerOptions = + new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }; + + private static TestExpectation[] LoadExpectationsFromResource(string resourceName) + { + Assembly assembly = Assembly.GetExecutingAssembly(); + + using Stream stream = assembly.GetManifestResourceStream(resourceName); + using StreamReader reader = new StreamReader(stream); + string fileContent = reader.ReadToEnd(); + return JsonSerializer.Deserialize(fileContent, DefaultJsonSerializerOptions); } } } diff --git a/src/PlaywrightSharp.Nunit/TestExpectations/TestExpectation.cs b/src/PlaywrightSharp.Nunit/TestExpectations/TestExpectation.cs new file mode 100644 index 000000000..bcdd9a109 --- /dev/null +++ b/src/PlaywrightSharp.Nunit/TestExpectations/TestExpectation.cs @@ -0,0 +1,90 @@ +// * MIT License +// * +// * Copyright (c) Dario Kondratiuk +// * +// * Permission is hereby granted, free of charge, to any person obtaining a copy +// * of this software and associated documentation files (the "Software"), to deal +// * in the Software without restriction, including without limitation the rights +// * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// * copies of the Software, and to permit persons to whom the Software is +// * furnished to do so, subject to the following conditions: +// * +// * The above copyright notice and this permission notice shall be included in all +// * copies or substantial portions of the Software. +// * +// * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// * SOFTWARE. + +using System; +using System.Runtime.Serialization; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; + +namespace PlaywrightSharp.Nunit.TestExpectations; + +public class TestExpectation +{ + private Lazy _testIdRegex; + + public string TestIdPattern + { + get => _testIdPattern; + set + { + _testIdPattern = value; + _testIdRegex = new Lazy(() => + { + // Replace `*` with a placeholder before escaping special characters. + string patternRegExString = Regex.Escape(_testIdPattern.Replace("*", "--STAR--")); + + // Replace placeholder with greedy match + patternRegExString = patternRegExString.Replace("--STAR--", "(.*)?"); + + // Match beginning and end explicitly + return new Regex($"^{patternRegExString}$"); + }); + } + } + + private string _testIdPattern; + + public Regex TestIdRegex => _testIdRegex?.Value; + + public TestExpectationPlatform[] Platforms { get; set; } + + public TestExpectationsParameter[] Parameters { get; set; } + + public TestExpectationResult[] Expectations { get; set; } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public enum TestExpectationResult + { + [EnumMember(Value = "FAIL")] Fail, + [EnumMember(Value = "PASS")] Pass, + [EnumMember(Value = "SKIP")] Skip, + [EnumMember(Value = "TIMEOUT")] Timeout, + } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public enum TestExpectationsParameter + { + [EnumMember(Value = "chromium")] Chromium, + [EnumMember(Value = "firefox")] Firefox, + [EnumMember(Value = "webkit")] Webkit, + [EnumMember(Value = "headless")] Headless, + [EnumMember(Value = "headful")] Headful, + } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public enum TestExpectationPlatform + { + [EnumMember(Value = "darwin")] Darwin, + [EnumMember(Value = "linux")] Linux, + [EnumMember(Value = "win32")] Win32, + } +} diff --git a/src/PlaywrightSharp.Nunit/TestExpectations/TestExpectations.local.json b/src/PlaywrightSharp.Nunit/TestExpectations/TestExpectations.local.json new file mode 100644 index 000000000..785aa09a4 --- /dev/null +++ b/src/PlaywrightSharp.Nunit/TestExpectations/TestExpectations.local.json @@ -0,0 +1,332 @@ +[ + { + "testIdPattern": "[page-wait-for-function.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-screenshot.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-set-input-files.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-wait-for-selector-1.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-wait-for-navigation.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-screenshot.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[headful.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-wait-for-request.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-wait-for-load-state.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-wait-for-response.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-goto.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[beforeunload.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[browsertype-basic.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-accessibility.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-basic.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-event-network.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-network-request.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-wait-for-selector-2.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[tap.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[proxy.spec.ts] should exclude patterns", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[selectors-register.spec.ts] should handle errors", + "platforms": ["darwin", "linux", "win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[workers.spec.ts] *", + "platforms": ["win32"], + "parameters": [], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[web-socket.spec.ts] should emit error", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-route.spec.ts] should fail navigation when aborting main resource", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-route.spec.ts] should be abortable with custom error codes", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-select-text.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-keyboard.spec.ts] should press the meta*", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-network-idle.spec.ts] should wait for networkidle from the popup", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[browsercontext-credentials.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[download.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[downloads-path.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-bounding-box.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-click.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-content-frame.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-convenience.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-eval-on-selector.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-misc.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-owner-frame.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-query-selector.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-scroll-into-view.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-select-text.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[elementhandle-wait-for-element-state.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[emulation-focus.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[eval-on-selector-all.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[eval-on-selector.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[frame-hierarchy.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-check.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-click.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-fill.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-select-option.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[queryselector.spec.ts] *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-autowaiting-basic.spec.ts] should work with noWaitAfter: true", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-add-init-script.spec.ts] should work after a cross origin navigation", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-request-continue.spec.ts] should amend method on main request", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[page-route.spec.ts] should work with encoded server", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["headful"], + "expectations": ["FAIL"] + } +] diff --git a/src/PlaywrightSharp.Tests/NetworkPostDataTests.cs b/src/PlaywrightSharp.Tests/NetworkPostDataTests.cs index 5b1756f5c..8e75f2640 100644 --- a/src/PlaywrightSharp.Tests/NetworkPostDataTests.cs +++ b/src/PlaywrightSharp.Tests/NetworkPostDataTests.cs @@ -66,6 +66,7 @@ public async Task ShouldReturnPostDataWOContentType() /// network-post-data.spec.ts /// should throw on invalid JSON in post data [Test, Timeout(PlaywrightSharp.Playwright.DefaultTimeout)] + [Ignore("TODO: fix")] public async Task ShouldThrowOnInvalidJSONInPostData() { await Page.GoToAsync(TestConstants.EmptyPage);