Skip to content

Commit c2f96db

Browse files
AMaini503Aayush Maini
AMaini503
and
Aayush Maini
authored
Ignore local package references when parsing go.mod at Go117 (#1421)
* Ignore local go package references * Add UT to verify that local references are ignored * Bump Go117 detector version * Add other replace directives in UT go.mod file * CR: Use unix path for test file --------- Co-authored-by: Aayush Maini <aamaini@microsoft.com>
1 parent 98619eb commit c2f96db

File tree

5 files changed

+168
-4
lines changed

5 files changed

+168
-4
lines changed

‎src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public Go117ComponentDetector(
4848

4949
public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = [ComponentType.Go];
5050

51-
public override int Version => 2;
51+
public override int Version => 3;
5252

5353
protected override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
5454
IObservable<ProcessRequest> processRequests,

‎src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs‎

Lines changed: 96 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
namespace Microsoft.ComponentDetection.Detectors.Go;
22

3+
using System;
4+
using System.Collections.Generic;
35
using System.IO;
46
using System.Text.RegularExpressions;
57
using System.Threading.Tasks;
@@ -15,11 +17,50 @@ public class GoModParser : IGoParser
1517

1618
public GoModParser(ILogger logger) => this.logger = logger;
1719

20+
/// <summary>
21+
/// Checks whether the input path is a potential local file system path
22+
/// 1. '.' checks whether the path is relative to current directory.
23+
/// 2. '..' checks whether the path is relative to some ancestor directory.
24+
/// 3. IsRootedPath checks whether it is an absolute path.
25+
/// </summary>
26+
/// <param name="path">Candidate path.</param>
27+
/// <returns>true if potential local file system path.</returns>
28+
private static bool IsLocalPath(string path)
29+
{
30+
return path.StartsWith('.') || path.StartsWith("..") || Path.IsPathRooted(path);
31+
}
32+
33+
/// <summary>
34+
/// Tries to extract source token from replace directive.
35+
/// </summary>
36+
/// <param name="directiveLine">String containing a directive after replace token.</param>
37+
/// <param name="replaceDirectives">HashSet where the token is placed if replace directive substitutes a local path.</param>
38+
private static void TryExtractReplaceDirective(string directiveLine, HashSet<string> replaceDirectives)
39+
{
40+
var parts = directiveLine.Split("=>", StringSplitOptions.RemoveEmptyEntries);
41+
if (parts.Length == 2)
42+
{
43+
var source = parts[0].Trim().Split(' ')[0];
44+
var target = parts[1].Trim();
45+
46+
if (IsLocalPath(target))
47+
{
48+
replaceDirectives.Add(source);
49+
}
50+
}
51+
}
52+
1853
public async Task<bool> ParseAsync(
1954
ISingleFileComponentRecorder singleFileComponentRecorder,
2055
IComponentStream file,
2156
GoGraphTelemetryRecord record)
2257
{
58+
// Collect replace directives that point to a local path
59+
var replaceDirectives = await this.GetAllReplacePathDirectivesAsync(file);
60+
61+
// Rewind stream after reading replace directives
62+
file.Stream.Seek(0, SeekOrigin.Begin);
63+
2364
using var reader = new StreamReader(file.Stream);
2465

2566
// There can be multiple require( ) sections in go 1.17+. loop over all of them.
@@ -38,7 +79,7 @@ public async Task<bool> ParseAsync(
3879
// are listed in the require () section
3980
if (line.StartsWith(StartString))
4081
{
41-
this.TryRegisterDependencyFromModLine(line[StartString.Length..], singleFileComponentRecorder);
82+
this.TryRegisterDependencyFromModLine(file, line[StartString.Length..], singleFileComponentRecorder, replaceDirectives);
4283
}
4384

4485
line = await reader.ReadLineAsync();
@@ -47,14 +88,14 @@ public async Task<bool> ParseAsync(
4788
// Stopping at the first ) restrict the detection to only the require section.
4889
while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(')'))
4990
{
50-
this.TryRegisterDependencyFromModLine(line, singleFileComponentRecorder);
91+
this.TryRegisterDependencyFromModLine(file, line, singleFileComponentRecorder, replaceDirectives);
5192
}
5293
}
5394

5495
return true;
5596
}
5697

57-
private void TryRegisterDependencyFromModLine(string line, ISingleFileComponentRecorder singleFileComponentRecorder)
98+
private void TryRegisterDependencyFromModLine(IComponentStream file, string line, ISingleFileComponentRecorder singleFileComponentRecorder, HashSet<string> replaceDirectives)
5899
{
59100
if (line.Trim().StartsWith("//"))
60101
{
@@ -64,6 +105,15 @@ private void TryRegisterDependencyFromModLine(string line, ISingleFileComponentR
64105

65106
if (this.TryToCreateGoComponentFromModLine(line, out var goComponent))
66107
{
108+
if (replaceDirectives.Contains(goComponent.Name))
109+
{
110+
// Skip registering this dependency since it's replaced by a local path
111+
// we will be reading this dependency somewhere else
112+
this.logger.LogInformation("Skipping {GoComponentId} from {Location} because it's a local reference.", goComponent.Id, file.Location);
113+
return;
114+
}
115+
116+
this.logger.LogError("Registering {GoComponent} from {Location}", goComponent.Name, file.Location);
67117
singleFileComponentRecorder.RegisterUsage(new DetectedComponent(goComponent));
68118
}
69119
else
@@ -90,4 +140,47 @@ private bool TryToCreateGoComponentFromModLine(string line, out GoComponent goCo
90140

91141
return true;
92142
}
143+
144+
private async Task<HashSet<string>> GetAllReplacePathDirectivesAsync(IComponentStream file)
145+
{
146+
var replacedDirectives = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
147+
const string singleReplaceDirectiveBegin = "replace ";
148+
const string multiReplaceDirectiveBegin = "replace (";
149+
using (var reader = new StreamReader(file.Stream, leaveOpen: true))
150+
{
151+
while (!reader.EndOfStream)
152+
{
153+
var line = await reader.ReadLineAsync();
154+
if (line == null)
155+
{
156+
continue;
157+
}
158+
159+
line = line.Trim();
160+
161+
// Multiline block: replace (
162+
if (line.StartsWith(multiReplaceDirectiveBegin))
163+
{
164+
while ((line = await reader.ReadLineAsync()) != null)
165+
{
166+
line = line.Trim();
167+
if (line == ")")
168+
{
169+
break;
170+
}
171+
172+
TryExtractReplaceDirective(line, replacedDirectives);
173+
}
174+
}
175+
else if (line.StartsWith(singleReplaceDirectiveBegin))
176+
{
177+
// single line block: replace
178+
var directiveContent = line[singleReplaceDirectiveBegin.Length..].Trim();
179+
TryExtractReplaceDirective(directiveContent, replacedDirectives);
180+
}
181+
}
182+
}
183+
184+
return replacedDirectives;
185+
}
93186
}

‎test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests;
88
using FluentAssertions;
99
using Microsoft.ComponentDetection.Common.Telemetry.Records;
1010
using Microsoft.ComponentDetection.Contracts;
11+
using Microsoft.ComponentDetection.Contracts.BcdeModels;
1112
using Microsoft.ComponentDetection.Detectors.Go;
1213
using Microsoft.ComponentDetection.TestsUtilities;
1314
using Microsoft.Extensions.Logging;
@@ -212,4 +213,45 @@ public async Task Go117ModDetector_ExecutingGoVersionFails_DetectorDoesNotFail()
212213

213214
goModParserMock.Verify(parser => parser.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()), Times.Once);
214215
}
216+
217+
[TestMethod]
218+
public async Task Go117ModDetector_VerifyLocalReferencesIgnored()
219+
{
220+
var goModFilePath = "./TestFiles/go_WithLocalReferences.mod"; // Replace with your actual file path
221+
var fileStream = new FileStream(goModFilePath, FileMode.Open, FileAccess.Read);
222+
223+
var goModParser = new GoModParser(this.mockLogger.Object);
224+
var mockSingleFileComponentRecorder = new Mock<ISingleFileComponentRecorder>();
225+
226+
var capturedComponents = new List<DetectedComponent>();
227+
var expectedComponentIds = new List<string>()
228+
{
229+
"github.com/grafana/grafana-app-sdk v0.23.1 - Go",
230+
"k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f - Go",
231+
};
232+
233+
mockSingleFileComponentRecorder
234+
.Setup(m => m.RegisterUsage(
235+
It.IsAny<DetectedComponent>(),
236+
It.IsAny<bool>(),
237+
It.IsAny<string>(),
238+
It.IsAny<bool?>(),
239+
It.IsAny<DependencyScope?>(),
240+
It.IsAny<string>()))
241+
.Callback<DetectedComponent, bool, string, bool?, DependencyScope?, string>((comp, _, _, _, _, _) =>
242+
{
243+
capturedComponents.Add(comp);
244+
});
245+
246+
var mockComponentStream = new Mock<IComponentStream>();
247+
mockComponentStream.Setup(mcs => mcs.Stream).Returns(fileStream);
248+
mockComponentStream.Setup(mcs => mcs.Location).Returns("Location");
249+
250+
var result = await goModParser.ParseAsync(mockSingleFileComponentRecorder.Object, mockComponentStream.Object, new GoGraphTelemetryRecord());
251+
result.Should().BeTrue();
252+
capturedComponents
253+
.Select(c => c.Component.Id)
254+
.Should()
255+
.BeEquivalentTo(expectedComponentIds);
256+
}
215257
}

‎test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,13 @@
5656
<None Update="Mocks\test.component-detection-pip-report.json">
5757
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
5858
</None>
59+
<None Update="TestFiles\go_WithLocalReferences.mod">
60+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
61+
</None>
62+
</ItemGroup>
63+
64+
<ItemGroup>
65+
<Folder Include="TestFiles\" />
5966
</ItemGroup>
6067

6168
</Project>

‎test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
module github.com/Go117Tests
2+
3+
go 1.23.5
4+
5+
replace github.com/grafana/grafana => ../../..
6+
replace k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f => k8s.io/kube-openapi v1.1.1
7+
8+
9+
require (
10+
github.com/grafana/grafana v0.0.0-00010101000000-000000000000
11+
github.com/grafana/grafana-app-sdk v0.23.1
12+
k8s.io/apimachinery v0.32.0
13+
k8s.io/apiserver v0.32.0
14+
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f
15+
)
16+
17+
replace (
18+
k8s.io/apimachinery => ../
19+
k8s.io/apiserver => ./a/b/c
20+
)
21+
22+
replace github.com/grafana/grafana-app-sdk => github.com/grafana/grafana-app-sdk v0.22.1

0 commit comments

Comments
 (0)