From a966ae6a86d28f83c496d064afb022124ea0faef Mon Sep 17 00:00:00 2001 From: Sindhusha Yadavalli Date: Fri, 12 May 2023 23:04:11 +0530 Subject: [PATCH 1/6] Adding PackageJson LocationPath In YarnDetector --- .../yarn/YarnEntry.cs | 7 +- .../yarn/YarnLockComponentDetector.cs | 67 ++++++++++++++----- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnEntry.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnEntry.cs index 75cca5be0..ecf1c17e6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnEntry.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnEntry.cs @@ -1,4 +1,4 @@ -namespace Microsoft.ComponentDetection.Detectors.Yarn; +namespace Microsoft.ComponentDetection.Detectors.Yarn; using System.Collections.Generic; public class YarnEntry @@ -39,4 +39,9 @@ public class YarnEntry /// Gets or sets a value indicating whether or not the component is a dev dependency. /// public bool DevDependency { get; set; } + + /// + /// Gets or Sets the location for this yarnentry. Often a file path if not in test circumstances. + /// + public string Location { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index d52c2bf16..6003e2887 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -98,6 +98,11 @@ private void DetectComponents(YarnLockFile file, string location, ISingleFileCom foreach (var dependency in yarnRoots) { var root = new DetectedComponent(new NpmComponent(dependency.Name, dependency.Version)); + if (dependency.Location != null && dependency.Location.Length > 0) + { + root.AddComponentFilePath(dependency.Location); + } + this.AddDetectedComponentToGraph(root, null, singleFileComponentRecorder, isRootComponent: true); } @@ -207,9 +212,10 @@ private bool TryReadPeerPackageJsonRequestsAsYarnEntries(ISingleFileComponentRec return false; } + var workspaceDependencyVsLocationMap = new Dictionary(); if (yarnWorkspaces.Count > 0) { - this.GetWorkspaceDependencies(yarnWorkspaces, new FileInfo(location).Directory, combinedDependencies); + this.GetWorkspaceDependencies(yarnWorkspaces, new FileInfo(location).Directory, combinedDependencies, workspaceDependencyVsLocationMap); } // Convert all of the dependencies we retrieved from package.json @@ -232,13 +238,19 @@ private bool TryReadPeerPackageJsonRequestsAsYarnEntries(ISingleFileComponentRec entry.DevDependency = version.Value; yarnRoots.Add(entry); + + var locationMapDictonaryKey = name + "-" + version.Key; + if (workspaceDependencyVsLocationMap.ContainsKey(locationMapDictonaryKey)) + { + entry.Location = workspaceDependencyVsLocationMap[locationMapDictonaryKey]; + } } } return true; } - private void GetWorkspaceDependencies(IList yarnWorkspaces, DirectoryInfo root, IDictionary> dependencies) + private void GetWorkspaceDependencies(IList yarnWorkspaces, DirectoryInfo root, IDictionary> dependencies, IDictionary workspaceDependencyVsLocationMap) { var ignoreCase = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); @@ -263,31 +275,56 @@ private void GetWorkspaceDependencies(IList yarnWorkspaces, DirectoryInf foreach (var dependency in combinedDependencies) { - this.ProcessWorkspaceDependency(dependencies, dependency); + this.ProcessWorkspaceDependency(dependencies, dependency, workspaceDependencyVsLocationMap, stream.Location); } } } } - private void ProcessWorkspaceDependency(IDictionary> dependencies, KeyValuePair> newDependency) + private void ProcessWorkspaceDependency(IDictionary> dependencies, KeyValuePair> newDependency, IDictionary workspaceDependencyVsLocationMap, string streamLocation) { - if (!dependencies.TryGetValue(newDependency.Key, out var existingDependency)) - { - dependencies.Add(newDependency.Key, newDependency.Value); - return; - } - - foreach (var item in newDependency.Value) + try { - if (existingDependency.TryGetValue(item.Key, out var wasDev)) + if (!dependencies.TryGetValue(newDependency.Key, out var existingDependency)) { - existingDependency[item.Key] = wasDev && item.Value; + dependencies.Add(newDependency.Key, newDependency.Value); + foreach (var item in newDependency.Value) + { + // Adding 'Package.json stream's location'(in which workspacedependency of Yarn.lock file was found) as location of respective WorkSpaceDependency. + this.AddLocationInfoToWorkspaceDependency(workspaceDependencyVsLocationMap, streamLocation, newDependency.Key, item.Key); + } + + return; } - else + + foreach (var item in newDependency.Value) { - existingDependency[item.Key] = item.Value; + if (existingDependency.TryGetValue(item.Key, out var wasDev)) + { + existingDependency[item.Key] = wasDev && item.Value; + } + else + { + existingDependency[item.Key] = item.Value; + } + + // Adding 'Package.json stream's location'(in which workspacedependency of Yarn.lock file was found) as location of respective WorkSpaceDependency. + this.AddLocationInfoToWorkspaceDependency(workspaceDependencyVsLocationMap, streamLocation, newDependency.Key, item.Key); } } + catch (Exception ex) + { + this.Logger.LogError(ex, "Could not process workspace dependency from file {PackageJsonStreamLocation}.", streamLocation); + } + } + + private void AddLocationInfoToWorkspaceDependency(IDictionary workspaceDependencyVsLocationMap, string streamLocation, string dependencyName, string dependencyVersion) + { + var locationMapDictionaryKey = dependencyName + "-" + dependencyVersion; + if (!workspaceDependencyVsLocationMap.ContainsKey(locationMapDictionaryKey)) + { + workspaceDependencyVsLocationMap[locationMapDictionaryKey] = streamLocation; + } } private void AddDetectedComponentToGraph(DetectedComponent componentToAdd, DetectedComponent parentComponent, ISingleFileComponentRecorder singleFileComponentRecorder, bool isRootComponent = false, bool? isDevDependency = null) From aa74d7203c7df74387ccafa5fddfff2e1170f9ce Mon Sep 17 00:00:00 2001 From: Sindhusha Yadavalli Date: Thu, 18 May 2023 16:04:52 +0530 Subject: [PATCH 2/6] Adding new tests for YarnDetector to check workspace filepath addition --- .../DependencyGraph/ComponentRecorder.cs | 5 --- .../DetectedComponent.cs | 5 ++- .../YarnLockDetectorTests.cs | 44 +++++++++++++++++++ .../DetectorTestUtilityBuilder.cs | 21 +++++++++ 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs index 71b1270df..c913166d9 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs @@ -173,11 +173,6 @@ public void RegisterUsage( } #if DEBUG - if (detectedComponent.FilePaths?.Any() ?? false) - { - this.logger.LogWarning("Detector should not populate DetectedComponent.FilePaths!"); - } - if (detectedComponent.DependencyRoots?.Any() ?? false) { this.logger.LogWarning("Detector should not populate DetectedComponent.DependencyRoots!"); diff --git a/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs b/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs index 9bee98b7e..005ebf5f7 100644 --- a/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs @@ -61,7 +61,10 @@ public DetectedComponent(TypedComponent.TypedComponent component, IComponentDete private string DebuggerDisplay => $"{this.Component.DebuggerDisplay}"; /// Adds a filepath to the FilePaths hashset for this detected component. - /// Note: Dependency Graph automatically captures the location where a component is found, no need to call it at all inside package manager detectors. + /// Note: + /// (1) Dependency Graph automatically captures the location where a component is found, no need to call it at all inside package manager detectors. + /// (2) Only usecase where Detectors allowed to call this API is, in scenarios where "Detectors(eg:Yarn) further process other config files to get workspace dependencies recursively" + /// and detectors need to add WorkspaceDependency found path too. /// The file path to add to the hashset. public void AddComponentFilePath(string filePath) { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs index 70553ec77..f48fab6a4 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs @@ -240,6 +240,50 @@ public async Task WellFormedYarnLockV1WithWorkspace_FindsComponentAsync() parentComponent => parentComponent.Name == componentA.Name && parentComponent.Version == version0); } + [TestMethod] + public async Task WellFormedYarnLockV1WithWorkspace_CheckFilePathsAsync() + { + var directory = new DirectoryInfo(Path.GetTempPath()); + + var version0 = NewRandomVersion(); + var componentA = new YarnTestComponentDefinition + { + ActualVersion = version0, + RequestedVersion = $"^{version0}", + ResolvedVersion = "https://resolved0/a/resolved", + Name = Guid.NewGuid().ToString("N"), + }; + + var componentStream = YarnTestUtilities.GetMockedYarnLockStream("yarn.lock", this.CreateYarnLockV1FileContent(new List { componentA })); + + var workspaceJson = new + { + name = "testworkspace", + version = "1.0.0", + @private = true, + workspaces = new[] { "workspace" }, + }; + var str = JsonConvert.SerializeObject(workspaceJson); + var workspaceJsonComponentStream = new ComponentStream { Location = directory.ToString(), Pattern = "package.json", Stream = str.ToStream() }; + + var packageStream = NpmTestUtilities.GetPackageJsonOneRootComponentStream(componentA.Name, componentA.RequestedVersion); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("yarn.lock", componentStream.Stream) + .WithFile("package.json", workspaceJsonComponentStream.Stream, new[] { "package.json" }, Path.Combine(Path.GetTempPath(), "package.json")) + .WithFile("package.json", packageStream.Stream, new[] { "package.json" }, Path.Combine(Path.GetTempPath(), "workspace", "package.json")) + .ExecuteDetectorAsync(); + + Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + // checking workspace's package.json entry gets added as filepath. + var filePathsCount = detectedComponents.First().FilePaths.Count; + Assert.AreEqual(1, filePathsCount); + Assert.AreEqual(Path.Combine(Path.GetTempPath(), "workspace", "package.json"), detectedComponents.First().FilePaths.First()); + } + [TestMethod] public async Task WellFormedYarnLockV2WithWorkspace_FindsComponentAsync() { diff --git a/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtilityBuilder.cs b/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtilityBuilder.cs index 9c3c353db..ea8711d6d 100644 --- a/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtilityBuilder.cs +++ b/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtilityBuilder.cs @@ -1,5 +1,6 @@ namespace Microsoft.ComponentDetection.TestsUtilities; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -200,6 +201,26 @@ private void InitializeFileMocks() fileToSend.Location, fileToSend.Contents)).Select(pr => pr.ComponentStream); }); + + this.mockComponentStreamEnumerableFactory.Setup(x => + x.GetComponentStreams( + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Returns, ExcludeDirectoryPredicate, bool>( + (directoryInfo, fileMatchingPredicate, _, recurse) => + { + return filesToSend + .Where(fileToSend => fileMatchingPredicate(new FileInfo(fileToSend.Location))) + .Select(fileToSend => + this.CreateProcessRequest( + FindMatchingPattern( + fileToSend.Name, + searchPatterns), + fileToSend.Location, + fileToSend.Contents)).Select(pr => pr.ComponentStream); + }); } } } From 8cda930e1b85f28797986185336b441fac7aed75 Mon Sep 17 00:00:00 2001 From: Sindhusha Yadavalli Date: Wed, 7 Jun 2023 13:35:24 +0530 Subject: [PATCH 3/6] Update YarnLockDetectorTests.cs Addressing iteration-2 comments, and using FluentAssertions --- .../YarnLockDetectorTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs index f48fab6a4..3ffd4d405 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs @@ -274,14 +274,15 @@ public async Task WellFormedYarnLockV1WithWorkspace_CheckFilePathsAsync() .WithFile("package.json", packageStream.Stream, new[] { "package.json" }, Path.Combine(Path.GetTempPath(), "workspace", "package.json")) .ExecuteDetectorAsync(); - Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); var detectedComponents = componentRecorder.GetDetectedComponents(); // checking workspace's package.json entry gets added as filepath. var filePathsCount = detectedComponents.First().FilePaths.Count; - Assert.AreEqual(1, filePathsCount); - Assert.AreEqual(Path.Combine(Path.GetTempPath(), "workspace", "package.json"), detectedComponents.First().FilePaths.First()); + filePathsCount.Should().Be(1); + var expectedWorkSpacePath = Path.Combine(Path.GetTempPath(), "workspace", "package.json"); + detectedComponents.First().FilePaths.Contains(expectedWorkSpacePath).Should().Be(true); } [TestMethod] From 1c75d249a304d3cbdfb97dd3132832946dece85d Mon Sep 17 00:00:00 2001 From: Sindhusha Yadavalli Date: Wed, 14 Jun 2023 22:23:24 +0530 Subject: [PATCH 4/6] Added 'GetLocationMapKey' util & updated YarnLockDetectorTests's FilePath logic to check Package.json entry existence alone. --- .../yarn/YarnLockComponentDetector.cs | 14 ++++++++++---- .../YarnLockDetectorTests.cs | 9 ++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index 6003e2887..7d3bb30a1 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -1,4 +1,4 @@ -namespace Microsoft.ComponentDetection.Detectors.Yarn; +namespace Microsoft.ComponentDetection.Detectors.Yarn; using System; using System.Collections.Generic; @@ -98,7 +98,8 @@ private void DetectComponents(YarnLockFile file, string location, ISingleFileCom foreach (var dependency in yarnRoots) { var root = new DetectedComponent(new NpmComponent(dependency.Name, dependency.Version)); - if (dependency.Location != null && dependency.Location.Length > 0) + + if (!string.IsNullOrWhiteSpace(dependency.Location)) { root.AddComponentFilePath(dependency.Location); } @@ -239,7 +240,7 @@ private bool TryReadPeerPackageJsonRequestsAsYarnEntries(ISingleFileComponentRec yarnRoots.Add(entry); - var locationMapDictonaryKey = name + "-" + version.Key; + var locationMapDictonaryKey = this.GetLocationMapKey(name, version.Key); if (workspaceDependencyVsLocationMap.ContainsKey(locationMapDictonaryKey)) { entry.Location = workspaceDependencyVsLocationMap[locationMapDictonaryKey]; @@ -320,13 +321,18 @@ private void ProcessWorkspaceDependency(IDictionary workspaceDependencyVsLocationMap, string streamLocation, string dependencyName, string dependencyVersion) { - var locationMapDictionaryKey = dependencyName + "-" + dependencyVersion; + var locationMapDictionaryKey = this.GetLocationMapKey(dependencyName, dependencyVersion); if (!workspaceDependencyVsLocationMap.ContainsKey(locationMapDictionaryKey)) { workspaceDependencyVsLocationMap[locationMapDictionaryKey] = streamLocation; } } + private string GetLocationMapKey(string dependencyName, string dependencyVersion) + { + return dependencyName + "-" + dependencyVersion; + } + private void AddDetectedComponentToGraph(DetectedComponent componentToAdd, DetectedComponent parentComponent, ISingleFileComponentRecorder singleFileComponentRecorder, bool isRootComponent = false, bool? isDevDependency = null) { if (parentComponent == null) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs index 3ffd4d405..347cdacc3 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs @@ -278,11 +278,10 @@ public async Task WellFormedYarnLockV1WithWorkspace_CheckFilePathsAsync() var detectedComponents = componentRecorder.GetDetectedComponents(); - // checking workspace's package.json entry gets added as filepath. - var filePathsCount = detectedComponents.First().FilePaths.Count; - filePathsCount.Should().Be(1); - var expectedWorkSpacePath = Path.Combine(Path.GetTempPath(), "workspace", "package.json"); - detectedComponents.First().FilePaths.Contains(expectedWorkSpacePath).Should().Be(true); + // checking if workspace's "package.json FilePath entry" is added or not. + var detectedFilePaths = detectedComponents.First().FilePaths; + var expectedWorkSpacePackageJsonPath = Path.Combine(Path.GetTempPath(), "workspace", "package.json"); + detectedComponents.First().FilePaths.Contains(expectedWorkSpacePackageJsonPath).Should().Be(true); } [TestMethod] From b9acf396ed7c6a9da331609edc6a87d83ff339fd Mon Sep 17 00:00:00 2001 From: Sindhusha Yadavalli Date: Wed, 14 Jun 2023 23:12:49 +0530 Subject: [PATCH 5/6] Updating YarnDetectorTests to check filepaths count --- .../yarn/YarnLockComponentDetector.cs | 2 +- .../YarnLockDetectorTests.cs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index 7d3bb30a1..afd1576c8 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -330,7 +330,7 @@ private void AddLocationInfoToWorkspaceDependency(IDictionary wo private string GetLocationMapKey(string dependencyName, string dependencyVersion) { - return dependencyName + "-" + dependencyVersion; + return $"{dependencyName}-{dependencyVersion}"; } private void AddDetectedComponentToGraph(DetectedComponent componentToAdd, DetectedComponent parentComponent, ISingleFileComponentRecorder singleFileComponentRecorder, bool isRootComponent = false, bool? isDevDependency = null) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs index 347cdacc3..fd7f3c0f7 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs @@ -277,9 +277,11 @@ public async Task WellFormedYarnLockV1WithWorkspace_CheckFilePathsAsync() scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(1); // checking if workspace's "package.json FilePath entry" is added or not. var detectedFilePaths = detectedComponents.First().FilePaths; + detectedFilePaths.Should().HaveCount(1); var expectedWorkSpacePackageJsonPath = Path.Combine(Path.GetTempPath(), "workspace", "package.json"); detectedComponents.First().FilePaths.Contains(expectedWorkSpacePackageJsonPath).Should().Be(true); } From f051e9ec366d52c89b7abc73acd94a4f5d9d8990 Mon Sep 17 00:00:00 2001 From: Sindhusha Yadavalli Date: Fri, 16 Jun 2023 22:57:27 +0530 Subject: [PATCH 6/6] Updating YarnLockcomponentDetector version --- .../yarn/YarnLockComponentDetector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index afd1576c8..1a464fdb6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -35,7 +35,7 @@ public YarnLockComponentDetector( public override IEnumerable SupportedComponentTypes { get; } = new[] { ComponentType.Npm }; - public override int Version => 6; + public override int Version => 7; public override IEnumerable Categories => new[] { Enum.GetName(typeof(DetectorClass), DetectorClass.Npm) };