Implement working automated tests for instrumentation

Fixed-port controller server is tested.

Author: Mohammad Dehghan

SG.CodeCoverage.TestConsole/Program.cs

@@ -15,7 +15,8 @@ static void Main()
         {
             var tester = new InstrumenterTester();
             tester.InstrumentSampleProject();
-            tester.RunSomeCode();
+            tester.RunApp();
+            tester.RunIsPrimeInApp(7);
             var result = tester.GetCoverageResult();
             Console.WriteLine($"Visited {result.GetVisitedSources().Count()} source files" +
                 $" and {result.GetVisitedMethods().Count()} methods.");

SG.CodeCoverage.Tests.NetFx/InstrumenterTester.cs

@@ -6,9 +6,12 @@
 using SG.CodeCoverage.Metadata;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Text;
+using System.Threading;
 
 namespace SG.CodeCoverage.Tests
 {
@@ -21,6 +24,8 @@ public class InstrumenterTester
         public IRecordingController RecordingController { get; private set; }
         public string InstrumentedAssemblyPath { get; private set; }
         private readonly ILogger _logger;
+        private Process _process;
+        private List<string> _processOutput;
 
         public InstrumenterTester(ILogger logger = null)
         {
@@ -68,14 +73,63 @@ private void CleanDirectory(string dirPath)
                 Directory.Delete(dir, true);
         }
 
-        public void RunSomeCode()
+        public void RunApp(string args = null)
         {
             if (InstrumentedAssemblyPath == null)
                 throw new InvalidOperationException("Sample assembly is not instrumented.");
-            Assembly.LoadFrom(Path.Combine(OutputPath, "SG.CodeCoverage.Recorder.dll"));
-            var assembly = Assembly.LoadFrom(InstrumentedAssemblyPath);
-            var calc = assembly.DefinedTypes.Where(x => x.Name == nameof(PrimeCalculator)).FirstOrDefault();
-            var res = calc.GetMethod("IsPrime").Invoke(calc.DeclaredConstructors.First().Invoke(null), new object[] { 7 });
+            _process = Process.Start(new ProcessStartInfo()
+            {
+                FileName = InstrumentedAssemblyPath,
+                Arguments = args,
+                UseShellExecute = false,
+                WorkingDirectory = OutputPath,
+                RedirectStandardInput = true,
+                RedirectStandardOutput = true
+            });
+            _processOutput = new List<string>();
+            _process.OutputDataReceived += _process_OutputDataReceived;
+            _process.BeginOutputReadLine();
+            WaitForAppIdle();
+        }
+
+        private void _process_OutputDataReceived(object sender, DataReceivedEventArgs e)
+        {
+            if (string.IsNullOrEmpty(e.Data))
+                return;
+            _processOutput.Add(e.Data);
+        }
+
+        private void WaitForAppIdle()
+        {
+            CheckAppStarted();
+            while (_processOutput.Count == 0 || _processOutput.Last() != "Enter command:")
+                Thread.Sleep(1);
+        }
+
+        public bool AppIsRunning
+            => _process != null && !_process.HasExited;
+
+        public void ExitApp()
+        {
+            CheckAppStarted();
+            _process.StandardInput.WriteLine("exit");
+            if (!_process.WaitForExit(10000))
+                throw new Exception("App did not exit withing 10 seconds.");
+            _process.WaitForExit();
+            _process = null;
+        }
+
+        public void RunIsPrimeInApp(int number)
+        {
+            CheckAppStarted();
+            _process.StandardInput.WriteLine("IsPrime " + number);
+            WaitForAppIdle();
+        }
+
+        private void CheckAppStarted()
+        {
+            if (_process == null)
+                throw new InvalidOperationException("Sample application is not started.");
         }
 
         public CoverageResult GetCoverageResult()

SG.CodeCoverage.Tests.NetFx/TestInstrumentation.cs

@@ -12,12 +12,141 @@ namespace SG.CodeCoverage.Tests.NetFx
     [TestClass]
     public class TestInstrumentation
     {
+        InstrumenterTester _tester;
+
+        [TestInitialize]
+        public void Instrument()
+        {
+            _tester = new InstrumenterTester();
+            _tester.InstrumentSampleProject();
+        }
+
+        [TestMethod]
+        public void Coverage_Startup()
+        {
+            _tester.RunApp();
+            AssertVisitedFilesAndMethods(Files.Startup, Methods.Startup);
+        }
+
         [TestMethod]
-        public void TestSampleProjectInstrumented()
+        public void Coverage_IsPrime1()
+        {
+            _tester.RunApp("IsPrime 1");
+            AssertVisitedFilesAndMethods(
+                Files.Startup
+                    .Concat(Files.PrimeCalculator),
+                Methods.Startup
+                    .Concat(Methods.RunCommand)
+                    .Concat(Methods.PrimeCalculator.IsPrimeAndIsLessThan2));
+        }
+
+        [TestMethod]
+        public void Coverage_IsPrime4()
+        {
+            _tester.RunApp("IsPrime 4");
+            AssertVisitedFilesAndMethods(
+                Files.Startup
+                    .Concat(Files.PrimeCalculator),
+                Methods.Startup
+                    .Concat(Methods.RunCommand)
+                    .Concat(Methods.PrimeCalculator.IsPrimeAndIsLessThan2)
+                    .Concat(Methods.PrimeCalculator.GetUpperBound));
+        }
+
+
+        [TestCleanup]
+        public void ExitApp()
         {
-            var tester = new InstrumenterTester();
-            tester.InstrumentSampleProject();
-            var map = InstrumentationMap.Parse(tester.MapFilePath);
+            if (_tester.AppIsRunning)
+                _tester.ExitApp();
+        }
+
+        private void AssertVisitedFilesAndMethods(
+            IEnumerable<string> expectedVisitedFiles,
+            IEnumerable<string> expectedVisitedMethods)
+        {
+            var result = _tester.GetCoverageResult();
+            var files = result.GetVisitedSources().ToList();
+            var methods = result.GetVisitedMethodNames().ToList();
+            ShouldVisit(expectedVisitedFiles.ToList(), files, "file");
+            ShouldVisit(expectedVisitedMethods.ToList(), methods, "method");
+        }
+
+        private static void ShouldVisit(IReadOnlyList<string> expectedNames, IReadOnlyList<string> actualNames, string what)
+        {
+            Assert.AreEqual(expectedNames.Count, actualNames.Count, $"visited {what}s");
+            foreach(var expected in expectedNames)
+            {
+                bool found = false;
+                foreach(var actual in actualNames)
+                {
+                    if(actual.EndsWith(expected))
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+                if (!found)
+                    Assert.Fail($"{what} {expected} is not visited.");
+            }
+        }
+
+        private static class Files
+        {
+            public static readonly IEnumerable<string> Startup = new string[]
+            {
+                "App.cs",
+                "Program.cs"
+            };
+
+            public static readonly IEnumerable<string> PrimeCalculator = new string[]
+            {
+                "PrimeCalculator.cs"
+            };
+
+            public static readonly IEnumerable<string> SampleStruct = new string[]
+            {
+                "SampleStruct.cs"
+            };
+        }
+
+        private static class Methods
+        {
+            public static readonly IEnumerable<string> Startup = new string[]
+            {
+                "Program::Main(System.String[])",
+                "App::.ctor()",
+                "App::get_Commands()",
+                "App::Run(System.String[])",
+                "Command::.ctor(System.String,System.String,System.Action`1<System.String>)",
+                "Command::get_Help()",
+            };
+
+            public static readonly IEnumerable<string> RunCommand = new string[]
+            {
+                "App::RunCommand(System.String[])",
+                "Command::get_CommandText()",
+                "Command::get_Operation()",
+            };
+
+            public static class PrimeCalculator
+            {
+                public static readonly IEnumerable<string> IsPrime = new string[]
+                {
+                    "App::IsPrime(System.String)",
+                    "PrimeCalculator::IsPrime(System.Int64)",
+                };
+                public static readonly IEnumerable<string> IsLessThan2 = new string[]
+                {
+                    "PrimeCalculator::IsLessThan2(System.Int64)"
+                };
+                public static readonly IEnumerable<string> GetUpperBound = new string[]
+                {
+                    "PrimeCalculator::GetUpperBound(System.Int64)"
+                };
+                public static IEnumerable<string> IsPrimeAndIsLessThan2
+                    => IsPrime.Concat(IsLessThan2);
+            }
         }
     }
 }

SampleProjectForTest/App.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace SampleProjectForTest
+{
+    public class App
+    {
+        public IReadOnlyList<Command> Commands { get; }
+        private bool _shouldExit = false;
+
+        public App()
+        {
+            Commands = new List<Command>()
+            {
+                new Command( "exit", "exit", Exit),
+                new Command("isprime", "IsPrime", IsPrime)
+            };
+        }
+
+        public void Run(string[] args)
+        {
+            if (args.Length > 0)
+                RunCommand(args);
+
+            Console.WriteLine("Commands:");
+            foreach (var cmd in Commands)
+                Console.WriteLine("  " + cmd.Help);
+
+            while (!_shouldExit)
+            {
+                Console.WriteLine("Enter command:");
+                var line = Console.ReadLine();
+                var parts = line.Split(Array.Empty<char>(), StringSplitOptions.RemoveEmptyEntries);
+                RunCommand(parts);
+            }
+        }
+
+        private void RunCommand(string[] parts)
+        {
+            bool found = false;
+            foreach (var cmd in Commands)
+            {
+                if (parts[0].Equals(cmd.CommandText, StringComparison.OrdinalIgnoreCase))
+                {
+                    cmd.Operation(parts.Length > 0 ? parts[1] : null);
+                    found = true;
+                    break;
+                }
+            }
+
+            if (!found)
+                throw new Exception($"Command '{parts[0]}' not found.");
+        }
+
+        private void Exit(string _)
+        {
+            _shouldExit = true;
+        }
+
+        private void IsPrime(string input)
+        {
+            var number = int.Parse(input);
+            var isPrime = new PrimeCalculator().IsPrime(number);
+            Console.WriteLine($"{number} is {(isPrime ? "" : "not ")}prime.");
+        }
+    }
+
+    public class Command
+    {
+        public Command(string commandText, string help, Action<string> operation)
+        {
+            CommandText = commandText;
+            Help = help;
+            Operation = operation;
+        }
+
+        public string CommandText { get; }
+        public string Help { get; }
+        public Action<string> Operation { get; }
+    }
+}

SampleProjectForTest/ClassWithStaticInitializer.cs

@@ -16,7 +16,7 @@ public ClassWithStaticInitializer()
             Console.WriteLine("constructor");
         }
 
-        public void PringAssemblyLocation()
+        public void PrintAssemblyLocation()
         {
             Console.WriteLine(AssemblyLocation.Value);
         }

SampleProjectForTest/GlobalSuppressions.cs

@@ -3,13 +3,9 @@
 // Project-level suppressions either have no target or are given
 // a specific target and scoped to a namespace, type, member, etc.
 
-[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1810:Initialize reference type static fields inline", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.SampleClassWithStaticConstructor.#cctor")]
-[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.SampleClassWithStaticInitializer.PringAssemblyLocation")]
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.Program.Main(System.String[])")]
-[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.SampleClassWithStaticInitializer.PrintAnotherValueAndWait(System.String)")]
-[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.SampleClassWithStaticInitializer.EmptyMethod")]
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1810:Initialize reference type static fields inline", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.StaticClassWithStaticConstructor.#cctor")]
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.PrimeCalculator.IsPrime(System.Int64)~System.Boolean")]
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.ClassWithStaticInitializer.EmptyMethod")]
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.ClassWithStaticInitializer.PrintAnotherValueAndWait(System.String)")]
-[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.ClassWithStaticInitializer.PringAssemblyLocation")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:SampleProjectForTest.ClassWithStaticInitializer.PrintAssemblyLocation")]

SampleProjectForTest/PrimeCalculator.cs

@@ -10,15 +10,29 @@ public class PrimeCalculator
     {
         public bool IsPrime(long number)
         {
-            var sqrt = Math.Sqrt(number);
-            if (number < 2)
+            if (IsLessThan2(number))
                 return false;
             if (number == 2)
                 return true;
-            for (int i = 3; i <= sqrt; i++)
+            var up = GetUpperBound(number);
+            for (int i = 2; i <= up; i++)
                 if (number % i == 0)
                     return false;
+
+            var value = new SampleStruct(4);
+            value.Multiply(3);
+
             return true;
         }
+
+        private static bool IsLessThan2(long number)
+        {
+            return number < 2;
+        }
+
+        private long GetUpperBound(long number)
+        {
+            return (long)Math.Ceiling(Math.Sqrt(number));
+        }
     }
 }