WIP: Add Reasons to Block and Test Results

2049-testing/2049.tests.ps1

@@ -0,0 +1,68 @@
+
+Describe 'All The Tests' {
+	Context 'Reasons' {
+		It 'Skips...' {
+			Set-ItResult -Skipped -Because 'I am skipped'
+		}
+		It 'Does not skip' {
+			$true | Should -BeTrue
+		}
+		It 'is Inconclusive' {
+			Set-ItResult -Inconclusive -Because 'I am inconclusive!'
+		}
+		It 'is Failed!' {
+			$true | Should -BeFalse -Because 'I am failed test'
+		}
+	}
+
+	Context 'No Reasons' {
+		It 'Skips...' {
+			Set-ItResult -Skipped
+		}
+		It 'Does not skip' {
+			$true | Should -BeTrue
+		}
+		It 'is Inconclusive' {
+			Set-ItResult -Inconclusive
+		}
+		It 'is Failed!' {
+			$true | Should -BeFalse
+		}
+	}
+
+	Context 'It Reasons' {
+		It 'Skips' -Skip -Reason 'I am Skipped' {
+			$true | Should -BeTrue
+		}
+	}
+
+	Context 'It No Reasons' {
+		It 'Skips' -Skip {
+			$true | Should -BeTrue
+		}
+	}
+
+	Context 'Context Reasons' -Skip -Reason 'I am Skipped' {
+		It 'Skips' {
+			$true | Should -BeTrue
+		}
+	}
+
+	Context 'Context No Reasons' -Skip {
+		It 'Skips' {
+			$true | Should -BeTrue
+		}
+	}
+
+	Describe 'Describe Reasons' -Skip -Reason 'I am Skipped' {
+		It 'Skips' {
+			$true | Should -BeTrue
+		}
+	}
+
+	Describe 'Describe No Reasons' -Skip {
+		It 'Skips' {
+			$true | Should -BeTrue
+		}
+	}
+}

2049-testing/mytests.ps1

@@ -0,0 +1,11 @@
+$pesterConfig = [PesterConfiguration]::Default
+$pesterConfig.TestResult.Enabled = $true
+$pesterConfig.Run.Path = '.\2049.tests.ps1'
+$pesterConfig.Run.PassThru = $true
+$pesterConfig.Debug.WriteDebugMessages = $true
+
+foreach ($fmt in 'NUnitXml NUnit2.5 NUnit3 JUnitXml'.split(' ')) {
+    $pesterConfig.TestResult.OutputFormat = $fmt
+    $pesterConfig.TestResult.OutputPath = "results.$fmt.xml"
+    Invoke-Pester -Configuration $pesterConfig
+}

src/Pester.Runtime.ps1

@@ -166,6 +166,7 @@ function New-ParametrizedBlock {
         [HashTable] $FrameworkData = @{ },
         [Switch] $Focus,
         [Switch] $Skip,
+        [String] $Reason,
         $Data
     )
 
@@ -176,7 +177,7 @@ function New-ParametrizedBlock {
     foreach ($d in @($Data)) {
         # shallow clone to give every block it's own copy
         $fmwData = $FrameworkData.Clone()
-        New-Block -GroupId $groupId -Name $Name -ScriptBlock $ScriptBlock -StartLine $StartLine -Tag $Tag -FrameworkData $fmwData -Focus:$Focus -Skip:$Skip -Data $d
+        New-Block -GroupId $groupId -Name $Name -ScriptBlock $ScriptBlock -StartLine $StartLine -Tag $Tag -FrameworkData $fmwData -Focus:$Focus -Skip:$Skip -Reason:$Reason -Data $d
     }
 }
 
@@ -194,6 +195,7 @@ function New-Block {
         [Switch] $Focus,
         [String] $GroupId,
         [Switch] $Skip,
+        [String] $Reason,
         $Data
     )
 
@@ -232,6 +234,7 @@ function New-Block {
     $block.Focus = $Focus
     $block.GroupId = $GroupId
     $block.Skip = $Skip
+    $block.Reason = $Reason
     $block.Data = $Data
 
     # we attach the current block to the parent, and put it to the parent
@@ -479,7 +482,8 @@ function New-Test {
         $Data,
         [String] $GroupId,
         [Switch] $Focus,
-        [Switch] $Skip
+        [Switch] $Skip,
+        [String] $Reason
     )
 
     if ($PesterPreference.Debug.WriteDebugMessages.Value) {
@@ -513,6 +517,7 @@ function New-Test {
     $test.Tag = $Tag
     $test.Focus = $Focus
     $test.Skip = $Skip
+    $test.Reason = $Reason
     $test.Data = $Data
     $test.FrameworkData.Runtime.Phase = 'Discovery'
 
@@ -687,6 +692,7 @@ function Invoke-TestItem {
                     }
                     else {
                         $Test.Skipped = $true
+                        $Test.Reason = $result.ErrorRecord.Exception.Message
                     }
                 }
                 else {
@@ -2120,6 +2126,7 @@ function PostProcess-DiscoveredBlock {
                         }
 
                         $t.Skip = $true
+                        $t.Reason = $b.Reason
                     }
                 }
             }
@@ -2186,12 +2193,14 @@ function PostProcess-DiscoveredBlock {
                 if ($PesterPreference.Debug.WriteDebugMessages.Value) {
                     if ($b.IsRoot) {
                         Write-PesterDebugMessage -Scope Skip "($($b.BlockContainer)) Container will be skipped because all included children are marked as skipped."
-                    } else {
+                    }
+                    else {
                         Write-PesterDebugMessage -Scope Skip "($($b.Path -join '.')) Block will be skipped because all included children are marked as skipped."
                     }
                 }
                 $b.Skip = $true
-            } elseif ($b.Skip -and -not $shouldSkipBasedOnChildren) {
+            }
+            elseif ($b.Skip -and -not $shouldSkipBasedOnChildren) {
                 if ($PesterPreference.Debug.WriteDebugMessages.Value) {
                     Write-PesterDebugMessage -Scope Skip "($($b.Path -join '.')) Block was marked as skipped, but one or more children are explicitly requested to be run, so the block itself will not be skipped."
                 }
@@ -2544,14 +2553,15 @@ function New-ParametrizedTest () {
         # do not use [hashtable[]] because that throws away the order if user uses [ordered] hashtable
         [object[]] $Data,
         [Switch] $Focus,
-        [Switch] $Skip
+        [Switch] $Skip,
+        [String] $Reason
     )
 
     # using the position of It as Id for the the test so we can join multiple testcases together, this should be unique enough because it only needs to be unique for the current block.
     # TODO: Id is used by NUnit2.5 and 3 testresults to group. A better way to solve this?
     $groupId = "${StartLine}:${StartColumn}"
     foreach ($d in $Data) {
-        New-Test -GroupId $groupId -Name $Name -Tag $Tag -ScriptBlock $ScriptBlock -StartLine $StartLine -Data $d -Focus:$Focus -Skip:$Skip
+        New-Test -GroupId $groupId -Name $Name -Tag $Tag -ScriptBlock $ScriptBlock -StartLine $StartLine -Data $d -Focus:$Focus -Skip:$Skip -Reason:$Reason
     }
 }
 

src/csharp/Pester/Block.cs

@@ -46,6 +46,7 @@ public Block()
         public List<string> Tag { get; set; }
         public bool Focus { get; set; }
         public bool Skip { get; set; }
+        public string Reason { get; set; }
 
         public string ItemType { get; } = "Block";
 

src/csharp/Pester/Test.cs

@@ -48,6 +48,7 @@ public Test()
         public List<string> Tag { get; set; }
         public bool Focus { get; set; }
         public bool Skip { get; set; }
+        public string Reason { get; set; }
         // IDictionary to allow users use [ordered]
 
         public object Block { get; set; }

src/functions/Context.ps1

@@ -89,6 +89,7 @@
 
         # [Switch] $Focus,
         [Switch] $Skip,
+        [String] $Reason,
         [Switch] $AllowNullOrEmptyForEach,
 
         [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidAssignmentToAutomaticVariable', '', Justification = 'ForEach is not used in Foreach-Object loop')]
@@ -122,10 +123,10 @@
                 return
             }
 
-            New-ParametrizedBlock -Name $Name -ScriptBlock $Fixture -StartLine $MyInvocation.ScriptLineNumber -StartColumn $MyInvocation.OffsetInLine -Tag $Tag -FrameworkData @{ CommandUsed = 'Context'; WrittenToScreen = $false } -Focus:$Focus -Skip:$Skip -Data $ForEach
+            New-ParametrizedBlock -Name $Name -ScriptBlock $Fixture -StartLine $MyInvocation.ScriptLineNumber -StartColumn $MyInvocation.OffsetInLine -Tag $Tag -FrameworkData @{ CommandUsed = 'Context'; WrittenToScreen = $false } -Focus:$Focus -Skip:$Skip -Reason:$Reason -Data $ForEach
         }
         else {
-            New-Block -Name $Name -ScriptBlock $Fixture -StartLine $MyInvocation.ScriptLineNumber -Tag $Tag -FrameworkData @{ CommandUsed = 'Context'; WrittenToScreen = $false } -Focus:$Focus -Skip:$Skip
+            New-Block -Name $Name -ScriptBlock $Fixture -StartLine $MyInvocation.ScriptLineNumber -Tag $Tag -FrameworkData @{ CommandUsed = 'Context'; WrittenToScreen = $false } -Focus:$Focus -Skip:$Skip -Reason:$Reason
         }
     }
     else {

src/functions/Describe.ps1

@@ -97,6 +97,7 @@
 
         # [Switch] $Focus,
         [Switch] $Skip,
+        [String] $Reason,
         [Switch] $AllowNullOrEmptyForEach,
 
         [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidAssignmentToAutomaticVariable', '', Justification = 'ForEach is not used in Foreach-Object loop')]
@@ -130,10 +131,10 @@
                 return
             }
 
-            New-ParametrizedBlock -Name $Name -ScriptBlock $Fixture -StartLine $MyInvocation.ScriptLineNumber -StartColumn $MyInvocation.OffsetInLine -Tag $Tag -FrameworkData @{ CommandUsed = 'Describe'; WrittenToScreen = $false } -Focus:$Focus -Skip:$Skip -Data $ForEach
+            New-ParametrizedBlock -Name $Name -ScriptBlock $Fixture -StartLine $MyInvocation.ScriptLineNumber -StartColumn $MyInvocation.OffsetInLine -Tag $Tag -FrameworkData @{ CommandUsed = 'Describe'; WrittenToScreen = $false } -Focus:$Focus -Skip:$Skip -Reason:$Reason -Data $ForEach
         }
         else {
-            New-Block -Name $Name -ScriptBlock $Fixture -StartLine $MyInvocation.ScriptLineNumber -Tag $Tag -FrameworkData @{ CommandUsed = 'Describe'; WrittenToScreen = $false } -Focus:$Focus -Skip:$Skip
+            New-Block -Name $Name -ScriptBlock $Fixture -StartLine $MyInvocation.ScriptLineNumber -Tag $Tag -FrameworkData @{ CommandUsed = 'Describe'; WrittenToScreen = $false } -Focus:$Focus -Skip:$Skip -Reason:$Reason
         }
     }
     else {

src/functions/It.ps1

@@ -133,10 +133,10 @@
 
         [Parameter(ParameterSetName = 'Skip')]
         [Switch] $Skip,
-        [Switch] $AllowNullOrEmptyForEach
+        [Switch] $AllowNullOrEmptyForEach,
 
-        # [Parameter(ParameterSetName = 'Skip')]
-        # [String] $SkipBecause,
+        [Parameter(ParameterSetName = 'Skip')]
+        [String] $Reason
 
         # [Switch]$Focus
     )
@@ -163,9 +163,9 @@
             return
         }
 
-        New-ParametrizedTest -Name $Name -ScriptBlock $Test -StartLine $MyInvocation.ScriptLineNumber -StartColumn $MyInvocation.OffsetInLine -Data $ForEach -Tag $Tag -Focus:$Focus -Skip:$Skip
+        New-ParametrizedTest -Name $Name -ScriptBlock $Test -StartLine $MyInvocation.ScriptLineNumber -StartColumn $MyInvocation.OffsetInLine -Data $ForEach -Tag $Tag -Focus:$Focus -Skip:$Skip -Reason:$Reason
     }
     else {
-        New-Test -Name $Name -ScriptBlock $Test -StartLine $MyInvocation.ScriptLineNumber -Tag $Tag -Focus:$Focus -Skip:$Skip
+        New-Test -Name $Name -ScriptBlock $Test -StartLine $MyInvocation.ScriptLineNumber -Tag $Tag -Focus:$Focus -Skip:$Skip -Reason:$Reason
     }
 }

src/functions/Set-ItResult.ps1

@@ -81,8 +81,7 @@
     }
 
     if ($Because) {
-        [String]$formatted = Format-Because $Because
-        [String]$message += ",$($formatted.SubString(0, $formatted.Length - 1))"
+        [String]$message = $Because
     }
 
     throw [Pester.Factory]::CreateErrorRecord(

src/functions/TestResults.NUnit25.ps1

@@ -317,10 +317,9 @@ function Write-NUnitTestCaseAttributes {
             $XmlWriter.WriteAttributeString('result', 'Ignored')
             $XmlWriter.WriteAttributeString('executed', 'False')
 
-            # TODO: This doesn't work, FailureMessage comes from Get-ErrorForXmlReport which isn't called
-            if ($TestResult.FailureMessage) {
+            if ($TestResult.Reason) {
                 $XmlWriter.WriteStartElement('reason')
-                $xmlWriter.WriteElementString('message', $TestResult.FailureMessage)
+                $xmlWriter.WriteElementString('message', $TestResult.Reason)
                 $XmlWriter.WriteEndElement() # Close reason tag
             }
 
@@ -331,10 +330,9 @@ function Write-NUnitTestCaseAttributes {
             $XmlWriter.WriteAttributeString('result', 'Inconclusive')
             $XmlWriter.WriteAttributeString('executed', 'True')
 
-            # TODO: This doesn't work, FailureMessage comes from Get-ErrorForXmlReport which isn't called
-            if ($TestResult.FailureMessage) {
+            if ($TestResult.Reason) {
                 $XmlWriter.WriteStartElement('reason')
-                $xmlWriter.WriteElementString('message', $TestResult.DisplayErrorMessage)
+                $xmlWriter.WriteElementString('message', $TestResult.Reason)
                 $XmlWriter.WriteEndElement() # Close reason tag
             }