Add simple example deduplication logic for Merge-MarkdownModel (#365)

Author: vors

.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "build",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "build",
+                "${workspaceFolder}/test/Markdown.MAML.Test/Markdown.MAML.Test.csproj"
+            ],
+            "problemMatcher": "$msCompile"
+        }
+    ]
+}

src/Markdown.MAML/Model/MAML/MamlCodeBlock.cs

@@ -21,5 +21,17 @@ public MamlCodeBlock(string text, string languageMoniker = null)
         /// The text of the code block.
         /// </summary>
         public string Text { get; private set; }
+
+        /// <summary>
+        /// Serves as a hash function for a <see cref="T:Markdown.MAML.Model.MAML.MamlCodeBlock"/> object.
+        /// </summary>
+        /// <returns>A hash code for this instance that is suitable for use in hashing algorithms and data structures such as a
+        /// hash table.</returns>
+        public override int GetHashCode()
+        {
+            return
+                (Text == null ? 679 : Text.GetHashCode()) ^
+                (LanguageMoniker == null ? 765679 : LanguageMoniker.GetHashCode());
+        }
     }
 }

src/Markdown.MAML/Model/MAML/MamlExample.cs

@@ -13,5 +13,27 @@ public class MamlExample
         /// Additional options that determine how the section will be formated when rendering markdown.
         /// </summary>
         public SectionFormatOption FormatOption { get; set; }
+
+        /// <summary>
+        /// Serves as a hash function for a <see cref="T:Markdown.MAML.Model.MAML.MamlExample"/> object.
+        /// Ignores the FormatOption field.
+        /// </summary>
+        /// <returns>A hash code for this instance that is suitable for use in hashing algorithms and data structures such as a
+        /// hash table.</returns>
+        public override int GetHashCode()
+        {
+            int hash = 1;
+            if (Code != null)
+            {
+                foreach (var codeBlock in Code) {
+                    hash ^= codeBlock.GetHashCode();
+                }
+            }
+            return
+                hash ^
+                (Title == null ? 123 : Title.GetHashCode()) ^
+                (Remarks == null ? 12345 : Remarks.GetHashCode()) ^
+                (Introduction == null ? 123457 : Introduction.GetHashCode());
+        }
     }
 }

src/Markdown.MAML/Transformer/MamlMultiModelMerger.cs

@@ -46,9 +46,9 @@ public MamlCommand Merge(Dictionary<string, MamlCommand> applicableTag2Model)
             result = new MamlCommand()
             {
                 Name = referenceModel.Name,
-                Synopsis = new SectionBody(MergeText(tagsModel.ToDictionary(pair => pair.Key, pair => pair.Value.Synopsis.Text))),
-                Description = new SectionBody(MergeText(tagsModel.ToDictionary(pair => pair.Key, pair => pair.Value.Description.Text))),
-                Notes = new SectionBody(MergeText(tagsModel.ToDictionary(pair => pair.Key, pair => pair.Value.Notes.Text))),
+                Synopsis = new SectionBody(MergeText(tagsModel.ToDictionary(pair => pair.Key, pair => pair.Value.Synopsis?.Text))),
+                Description = new SectionBody(MergeText(tagsModel.ToDictionary(pair => pair.Key, pair => pair.Value.Description?.Text))),
+                Notes = new SectionBody(MergeText(tagsModel.ToDictionary(pair => pair.Key, pair => pair.Value.Notes?.Text))),
                 Extent = referenceModel.Extent
             };
 
@@ -157,13 +157,39 @@ private List<MamlParameter> MergeParameterList(IEnumerable<List<MamlParameter>>
         }
 
         /// <summary>
-        /// Combine examples together. Don't perform any reduction, but group the examples after each other and add tag name to the title.
-        /// TODO can we combine examples better perhaps?
+        /// Combine examples together.
+        /// If examples are identical, we deduplicate them.
+        /// If example is not found in all the models, add the applicable tags in parentheses.
         /// </summary>
         /// <param name="result"></param>
         /// <param name="applicableTag2Model"></param>
         private void MergeExamples(MamlCommand result, Dictionary<string, MamlCommand> applicableTag2Model)
         {
+            // We want to more or less keep the order of the original examples.
+            // To do it, we use the following algorithm:
+            // We are adding examples sequentially:
+            // 1st from every model, 2nd from every model, etc.
+            // At the same time we are doing deduplication by hashcode.
+
+            // this dictonary gives the mapping between example hashes and applicable tags
+            var hash2tag = new Dictionary<int, List<String>>();
+            foreach (var pair in applicableTag2Model)
+            {
+                foreach (var example in pair.Value.Examples) {
+                    List<String> tagList;
+                    int hash = example.GetHashCode();
+                    if (!hash2tag.TryGetValue(hash, out tagList))
+                    {
+                        tagList = new List<string>();
+                        hash2tag[hash] = tagList;
+                    }
+                    tagList.Add(pair.Key);
+                }
+            }
+
+            // this hashset contains the example hashes for already used examples
+            var usedHashes = new HashSet<int>();
+
             int max = applicableTag2Model.Select(pair => pair.Value.Examples.Count).Max();
             for (int i = 0; i < max; i++)
             {
@@ -172,7 +198,19 @@ private void MergeExamples(MamlCommand result, Dictionary<string, MamlCommand> a
                     if (pair.Value.Examples.Count > i)
                     {
                         var example = pair.Value.Examples.ElementAt(i);
-                        example.Title += string.Format(" ({0})", pair.Key);
+                        int hash = example.GetHashCode();
+                        if (usedHashes.Contains(hash)) {
+                            continue;
+                        }
+                        // if all tags are covered, don't add anything
+                        if (hash2tag[hash].Count < applicableTag2Model.Count)
+                        {
+                            // we will sort the same list few times, but that's fine
+                            hash2tag[hash].Sort();
+                            string listString = string.Join(", ", hash2tag[hash]);
+                            example.Title += string.Format(" ({0})", listString);
+                        }
+                        usedHashes.Add(hash);
                         result.Examples.Add(example);
                     }
                 }

test/Markdown.MAML.Test/Transformer/MamlMultiModelMergerTests.cs

@@ -301,7 +301,7 @@ public class MamlMultiModelMergerSyntaxTests
         [Fact]
         public void MergeDefaultSyntaxAndCreateMarkdown()
         {
-            // First merge two models with default syntax names 
+            // First merge two models with default syntax names
 
             var merger = new MamlMultiModelMerger(null, false, "! ");
             var input = new Dictionary<string, MamlCommand>();
@@ -400,4 +400,142 @@ private MamlCommand GetModel2()
             return command;
         }
     }
+
+    public class MamlMultiModelMergerExampleTest
+    {
+        [Fact]
+        public void Merge2ExamplesInOne()
+        {
+            // First merge two models with default syntax names
+
+            var merger = new MamlMultiModelMerger(null, false, "! ");
+            var input = new Dictionary<string, MamlCommand>();
+
+            input["First"] = GetModel(
+                title: "Example 1",
+                code: "PS:> 1+1",
+                remarks: "Hello"
+            );
+            input["Second"] = GetModel(
+                title: "Example 1",
+                code: "PS:> 1+1",
+                remarks: "Hello"
+            );
+
+            var result = merger.Merge(input);
+
+             // Examples
+            Assert.Single(result.Examples);
+            Assert.Equal("Example 1", result.Examples.ElementAt(0).Title);
+
+            // next render it as markdown and make sure that we don't crash
+            var renderer = new MarkdownV2Renderer(MAML.Parser.ParserMode.FormattingPreserve);
+            string markdown = renderer.MamlModelToString(result, true);
+        }
+
+        [Fact]
+        public void MergeExamplesOrder()
+        {
+            // First merge two models with default syntax names
+
+            var merger = new MamlMultiModelMerger(null, false, "! ");
+            var input = new Dictionary<string, MamlCommand>();
+
+            input["First"] = GetModel(
+                new []
+                {
+                    new MamlExample() { Title = "E1" },
+                    new MamlExample() { Title = "E2" },
+                }
+            );
+            input["Second"] = GetModel(
+                new []
+                {
+                    new MamlExample() { Title = "E2" },
+                    new MamlExample() { Title = "E1" },
+                    new MamlExample() { Title = "E3" },
+                }
+            );
+
+            var result = merger.Merge(input);
+
+             // Examples
+            Assert.Equal(3, result.Examples.Count);
+            Assert.Equal("E1", result.Examples.ElementAt(0).Title);
+            Assert.Equal("E2", result.Examples.ElementAt(1).Title);
+            Assert.Equal("E3 (Second)", result.Examples.ElementAt(2).Title);
+
+            // next render it as markdown and make sure that we don't crash
+            var renderer = new MarkdownV2Renderer(MAML.Parser.ParserMode.FormattingPreserve);
+            string markdown = renderer.MamlModelToString(result, true);
+        }
+
+        [Fact]
+        public void Merge3ExamplesIn2()
+        {
+            // First merge two models with default syntax names
+
+            var merger = new MamlMultiModelMerger(null, false, "! ");
+            var input = new Dictionary<string, MamlCommand>();
+
+            input["First"] = GetModel(
+                title: "Example 1",
+                code: "PS:> 1+1",
+                remarks: "Hello"
+            );
+            input["Second"] = GetModel(
+                title: "Example 1",
+                code: "PS:> 1+1",
+                remarks: "Hello"
+            );
+
+            input["Third"] = GetModel(
+                title: "Example 1",
+                code: "PS:> 1+1",
+                remarks: "Hello world"
+            );
+
+            var result = merger.Merge(input);
+
+             // Examples
+            Assert.Equal(2, result.Examples.Count);
+            Assert.Equal("Example 1 (First, Second)", result.Examples.ElementAt(0).Title);
+            Assert.Equal("Hello", result.Examples.ElementAt(0).Remarks);
+            Assert.Equal("Example 1 (Third)", result.Examples.ElementAt(1).Title);
+            Assert.Equal("Hello world", result.Examples.ElementAt(1).Remarks);
+
+            // next render it as markdown and make sure that we don't crash
+            var renderer = new MarkdownV2Renderer(MAML.Parser.ParserMode.FormattingPreserve);
+            string markdown = renderer.MamlModelToString(result, true);
+        }
+
+        private MamlCommand GetModel(string title, string code, string remarks)
+        {
+            MamlCommand command = new MamlCommand()
+            {
+                Name = "Get-Foo",
+            };
+
+            command.Examples.Add(new MamlExample()
+            {
+                Title = title,
+                Code = new[] { new MamlCodeBlock(code) },
+                Remarks = remarks,
+            }
+            );
+
+            return command;
+        }
+
+        private MamlCommand GetModel(MamlExample[] examples)
+        {
+            MamlCommand command = new MamlCommand()
+            {
+                Name = "Get-Foo",
+            };
+
+            command.Examples.AddRange(examples);
+            return command;
+        }
+    }
 }