From 8ad2664313ceb0b22fd9195d60cd3d438d697c83 Mon Sep 17 00:00:00 2001 From: matiasdaloia <66310421+matiasdaloia@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:20:36 +0100 Subject: [PATCH] fix(dependencies): read pyproject.toml as TOML and mark dev declarations The pyproject.toml reader matched lines: any line holding "dependencies" and "[" opened a section, so a one-line dependencies = ["requests>=2.31"] was skipped, later strings such as readme = "README.md" became dependencies, Poetry tables were misread and [dependency-groups] was never read. No declaration said whether it was only for development. Read the file with the go-toml parser and record the table each dependency comes from as its Scope. Add LocalPurl.Dev, set from each manifest's own dev marker: devDependencies, Maven's test scope, Gradle test configurations, PEP 735 groups, Poetry dev groups, and extras named for development (package internal/pyscope). The poetry.lock and uv.lock resolvers use the same extra rule, so a dependency reads the same with or without a lockfile. --- CHANGELOG.md | 19 ++ pkg/dependencies/README.md | 31 ++- pkg/dependencies/internal/pyscope/pyscope.go | 63 +++++ .../internal/pyscope/pyscope_test.go | 49 ++++ pkg/dependencies/parsers/dev_test.go | 110 ++++++++ pkg/dependencies/parsers/gradle.go | 8 + pkg/dependencies/parsers/maven.go | 1 + pkg/dependencies/parsers/npm.go | 2 + pkg/dependencies/parsers/python.go | 248 ++++++++++-------- pkg/dependencies/parsers/python_test.go | 156 +++++++---- pkg/dependencies/parsers/types.go | 6 + pkg/dependencies/resolve/lockfile/poetry.go | 8 +- .../testdata/poetry/dev-scope/pyproject.toml | 3 +- .../testdata/uv/extras/graph.golden.json | 10 + .../lockfile/testdata/uv/extras/uv.lock | 8 + pkg/dependencies/resolve/lockfile/uv.go | 12 +- 16 files changed, 565 insertions(+), 169 deletions(-) create mode 100644 pkg/dependencies/internal/pyscope/pyscope.go create mode 100644 pkg/dependencies/internal/pyscope/pyscope_test.go create mode 100644 pkg/dependencies/parsers/dev_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a76821..2fd634f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`LocalPurl.Dev`** says the manifest declares a dependency only to develop the project: + `devDependencies` in `package.json` and `package-lock.json`, Maven's `test` scope, Gradle test + configurations, and the `pyproject.toml` groups and development extras. `dependencies` output + gains `"dev": true` on those entries. + +### Fixed + +- **`pyproject.toml` is read as TOML.** The line-based reader dropped a one-line + `dependencies = ["requests>=2.31"]`, invented dependencies from other strings such as + `readme = "README.md"`, misread Poetry tables, and never read `[dependency-groups]`. It now + reads `[project]` dependencies, each extra in `[project.optional-dependencies]`, each PEP 735 + group and Poetry's dependencies, dev-dependencies and groups, with the table as `Scope`, such as + `project.optional-dependencies.dev`. A malformed file is now an error instead of a partial list. +- **Development extras are dev in `poetry.lock` and `uv.lock` graphs.** An extra named for + development (`dev`, `test`, `lint`, `docs` and the like; see `pkg/dependencies/README.md`) marks + its dependencies dev, as the manifest parser does. Other extras stay runtime. + ## [0.16.0] - 2026-10-30 ### Added diff --git a/pkg/dependencies/README.md b/pkg/dependencies/README.md index 20451e8..4bda992 100644 --- a/pkg/dependencies/README.md +++ b/pkg/dependencies/README.md @@ -126,6 +126,7 @@ type LocalPurl struct { Purl string // Package URL (e.g., "pkg:npm/react@18.0.0") Requirement string // Version requirement (e.g., ">=1.0.0") Scope string // Dependency scope (e.g., "devDependencies") + Dev bool // The manifest declares it only to develop the project } ``` @@ -208,7 +209,10 @@ pkg:nuget/Newtonsoft.Json@13.0.1 - Filters out URLs, paths, and recursive dependencies - Handles operators: `==`, `>=`, `~`, `!=`, etc. - Exact versions (`==`) included in PURL, others in `Requirement` -- **pyproject.toml**: Parses dependencies from `[tool.poetry.dependencies]` section +- **pyproject.toml**: Read as TOML: `[project]` dependencies, each extra in + `[project.optional-dependencies]`, each PEP 735 `[dependency-groups]` group, and Poetry's + `dependencies`, `dev-dependencies` and groups. `Scope` is the table, such as + `project.optional-dependencies.dev`. See [Dev dependencies](#dev-dependencies) for which are dev. ### Maven Parser @@ -246,6 +250,27 @@ pkg:nuget/Newtonsoft.Json@13.0.1 - **packages.config**: Parses legacy package elements - Both extract package ID and version +### Dev dependencies + +`LocalPurl.Dev` is true when the manifest declares a dependency only to develop the project +(tests, linting, type checking, documentation, build tooling), so a consumer can tell it apart +without a lockfile: + +| Manifest | Dev | +|---|---| +| `package.json` | `devDependencies` | +| `package-lock.json` | entries marked `"dev": true` | +| `pom.xml` | `test` scope (`provided` is runtime: the platform supplies it) | +| `build.gradle` | configurations that start with `test` or contain `Test`, such as `testImplementation` and `androidTestImplementation` | +| `pyproject.toml` | every `[dependency-groups]` group, Poetry's `dev-dependencies` and every Poetry group but `main`, and every development extra | + +An extra in `[project.optional-dependencies]` is installed by any user who asks for it, so it is +dev only when its name says so: its PEP 685 normalized name, or the part before its first dash, +is one of `dev`, `devel`, `develop`, `development`, `test`, `tests`, `testing`, `lint`, +`linting`, `typing`, `mypy`, `doc`, `docs`, `coverage` or `qa`. `dev`, `Test_Integration` and +`docs-build` are dev; `socks` and `devices` are not. The `poetry.lock` and `uv.lock` resolvers +apply the same rule, so a dependency reads the same with or without a lockfile. + ## Dependency Graphs The parsers above return a flat list of PURLs per file. To know which package @@ -298,8 +323,8 @@ and edges kept, and a bad lockfile is reported on its root, never as a panic. | `pnpm-lock.yaml` (5.x, 6.x, 9.x) | lockfile only | every importer | each importer's `devDependencies` | | `Cargo.lock` | lockfile, `Cargo.toml` and workspace member manifests | `Cargo.toml` and each workspace member; a path crate that something depends on is a local node | `[dev-dependencies]`, including per target, that are not also normal or build dependencies; unknown for a root without its manifest | | `Gemfile.lock` | lockfile and `Gemfile` | `Gemfile`; `PATH` gems are local nodes | gems declared only in the `development` and `test` groups; unknown without a `Gemfile` | -| `poetry.lock` | lockfile and `pyproject.toml` | project; path dependencies are local nodes | `dev-dependencies`, every Poetry group but `main`, and `[dependency-groups]`; unknown without `pyproject.toml` | -| `uv.lock` (version 1) | lockfile only | the project and each workspace member, at its `pyproject.toml`; editable, directory and path packages are local nodes | each root's `dev-dependencies` groups; extras are runtime | +| `poetry.lock` | lockfile and `pyproject.toml` | project; path dependencies are local nodes | `dev-dependencies`, every Poetry group but `main`, every `[dependency-groups]` group, and development extras; unknown without `pyproject.toml` | +| `uv.lock` (version 1) | lockfile only | the project and each workspace member, at its `pyproject.toml`; editable, directory and path packages are local nodes | each root's `dev-dependencies` groups and development extras | | `packages.lock.json` (NuGet v1, v2) | lockfile, and the name of the project file beside it | the `.csproj`, `.fsproj` or `.vbproj` beside it, or the lockfile when there is not exactly one; every target framework is merged, and `Project` entries are local nodes | unknown: the lockfile does not record them | | `composer.lock` | lockfile and `composer.json` | project; path repository packages are local nodes | `require-dev`; unknown without `composer.json` | | `gradle.lockfile` | lockfile, and whether `build.gradle.kts` is beside it | `build.gradle`, or `build.gradle.kts` when it is the only build file; no direct members and no edges (`edges_unknown`) | unknown | diff --git a/pkg/dependencies/internal/pyscope/pyscope.go b/pkg/dependencies/internal/pyscope/pyscope.go new file mode 100644 index 0000000..5155885 --- /dev/null +++ b/pkg/dependencies/internal/pyscope/pyscope.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2026, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +// Package pyscope decides which pyproject.toml dependencies are only needed to +// develop the project. The manifest parser and the poetry.lock and uv.lock +// resolvers share it, so a dependency reads as dev with or without a lockfile. +// +// Every PEP 735 [dependency-groups] group, Poetry's dev-dependencies and every +// Poetry group but main are for development: none of them is installed with +// the package. An extra is installed by any user who asks for it, so it is +// dev only when its name says it is (DevExtra). +package pyscope + +import ( + "regexp" + "strings" +) + +// devExtraNames are the extra names the Python ecosystem uses for tests, +// linting, type checking, documentation and other development tooling. +var devExtraNames = map[string]bool{ + "dev": true, "devel": true, "develop": true, "development": true, + "test": true, "tests": true, "testing": true, + "lint": true, "linting": true, "typing": true, "mypy": true, + "doc": true, "docs": true, "coverage": true, "qa": true, +} + +var separators = regexp.MustCompile(`[-_.]+`) + +// DevExtra reports whether an extra is for development: its normalized name +// (PEP 685), or the part of it before the first dash, is a development name, +// so dev, Test, docs and test-integration are dev, and socks and devices are +// not. +func DevExtra(name string) bool { + n := separators.ReplaceAllString(strings.ToLower(strings.TrimSpace(name)), "-") + head, _, _ := strings.Cut(n, "-") + return devExtraNames[n] || devExtraNames[head] +} + +// DevPoetryGroup reports whether a Poetry dependency group is for development. +func DevPoetryGroup(name string) bool { + return name != "main" +} diff --git a/pkg/dependencies/internal/pyscope/pyscope_test.go b/pkg/dependencies/internal/pyscope/pyscope_test.go new file mode 100644 index 0000000..84259eb --- /dev/null +++ b/pkg/dependencies/internal/pyscope/pyscope_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2026, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package pyscope + +import "testing" + +func TestDevExtra(t *testing.T) { + t.Parallel() + for name, want := range map[string]bool{ + "dev": true, "Dev": true, "tests": true, "test-integration": true, "Test_Integration": true, + "docs.build": true, "lint": true, "typing": true, + "socks": false, "devices": false, "testbed": false, "all": false, "postgres": false, "": false, + } { + if got := DevExtra(name); got != want { + t.Errorf("DevExtra(%q) = %v, want %v", name, got, want) + } + } +} + +func TestDevPoetryGroup(t *testing.T) { + t.Parallel() + if DevPoetryGroup("main") { + t.Error("main is the runtime group") + } + if !DevPoetryGroup("test") { + t.Error("every group but main is for development") + } +} diff --git a/pkg/dependencies/parsers/dev_test.go b/pkg/dependencies/parsers/dev_test.go new file mode 100644 index 0000000..a46f79a --- /dev/null +++ b/pkg/dependencies/parsers/dev_test.go @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2026, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package parsers + +import ( + "maps" + "testing" +) + +// TestDevDeclarations checks that each manifest's own way of declaring a +// development-only dependency sets Dev, and that a runtime one does not. +func TestDevDeclarations(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + parse ParserFunc + file string + content string + want map[string]bool // purl -> Dev + }{ + { + name: "package.json devDependencies", + parse: ParsePackageJSON, + file: "package.json", + content: `{"dependencies": {"express": "^4.19.0"}, "devDependencies": {"jest": "^29.7.0"}}`, + want: map[string]bool{"pkg:npm/express@^4.19.0": false, "pkg:npm/jest@^29.7.0": true}, + }, + { + name: "package-lock.json dev entries", + parse: ParsePackageLock, + file: "package-lock.json", + content: `{"lockfileVersion": 3, "packages": { + "": {"name": "app"}, + "node_modules/express": {"version": "4.19.2"}, + "node_modules/jest": {"version": "29.7.0", "dev": true} +}}`, + want: map[string]bool{"pkg:npm/express@4.19.2": false, "pkg:npm/jest@29.7.0": true}, + }, + { + name: "pom.xml test scope", + parse: ParsePomXML, + file: "pom.xml", + content: ` + org.slf4jslf4j-api2.0.9 + junitjunit4.13.2test + javax.servletservlet-api2.5provided +`, + want: map[string]bool{ + "pkg:maven/org.slf4j/slf4j-api@2.0.9": false, + "pkg:maven/junit/junit@4.13.2": true, + "pkg:maven/javax.servlet/servlet-api@2.5": false, + }, + }, + { + name: "build.gradle test configurations", + parse: ParseBuildGradle, + file: "build.gradle", + content: `dependencies { + implementation 'com.google.guava:guava:32.1.2-jre' + testImplementation 'junit:junit:4.13.2' + androidTestImplementation group: 'org.mockito', name: 'mockito-core', version: '5.5.0' +} +`, + want: map[string]bool{ + "pkg:maven/com.google.guava/guava@32.1.2-jre": false, + "pkg:maven/junit/junit@4.13.2": true, + "pkg:maven/org.mockito/mockito-core@5.5.0": true, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + result, err := tc.parse([]byte(tc.content), tc.file) + if err != nil { + t.Fatalf("parse: %v", err) + } + got := map[string]bool{} + for _, p := range result.Purls { + got[p.Purl] = p.Dev + } + if !maps.Equal(got, tc.want) { + t.Errorf("got purl -> Dev %v, want %v", got, tc.want) + } + }) + } +} diff --git a/pkg/dependencies/parsers/gradle.go b/pkg/dependencies/parsers/gradle.go index d074b13..f6ec369 100644 --- a/pkg/dependencies/parsers/gradle.go +++ b/pkg/dependencies/parsers/gradle.go @@ -134,6 +134,7 @@ func ParseBuildGradle(fileContent []byte, filePath string) (*LocalDependency, er Purl: purl, Requirement: version, Scope: scope, + Dev: gradleTestConfiguration(scope), Line: lineNo, DeclaredText: strings.TrimSpace(originalLine), }) @@ -169,6 +170,7 @@ func ParseBuildGradle(fileContent []byte, filePath string) (*LocalDependency, er Purl: purl, Requirement: version, Scope: currentScope, + Dev: gradleTestConfiguration(currentScope), Line: depLine, DeclaredText: strings.TrimSpace(line), }) @@ -197,3 +199,9 @@ func ParseGradle(fileContent []byte, filePath string) (*LocalDependency, error) Purls: []LocalPurl{}, }, nil } + +// gradleTestConfiguration reports whether a dependency configuration only +// builds or runs tests, such as testImplementation or androidTestImplementation. +func gradleTestConfiguration(configuration string) bool { + return strings.HasPrefix(configuration, "test") || strings.Contains(configuration, "Test") +} diff --git a/pkg/dependencies/parsers/maven.go b/pkg/dependencies/parsers/maven.go index eaa290e..c878902 100644 --- a/pkg/dependencies/parsers/maven.go +++ b/pkg/dependencies/parsers/maven.go @@ -291,6 +291,7 @@ func ParsePomXML(fileContent []byte, filePath string) (*LocalDependency, error) Purl: purl, Requirement: version, Scope: scope, + Dev: scope == "test", Line: offsetToLine(fileContent, cd.startOff), DeclaredText: strings.TrimSpace(string(fileContent[cd.startOff:cd.endOff])), }) diff --git a/pkg/dependencies/parsers/npm.go b/pkg/dependencies/parsers/npm.go index a4a7929..a6e54af 100644 --- a/pkg/dependencies/parsers/npm.go +++ b/pkg/dependencies/parsers/npm.go @@ -224,6 +224,7 @@ func ParsePackageJSON(fileContent []byte, filePath string) (*LocalDependency, er Purl: purl, Requirement: e.value, Scope: scope, + Dev: scope == "devDependencies", Line: offsetToLine(fileContent, e.off), DeclaredText: lineTextAt(fileContent, e.off), }) @@ -351,6 +352,7 @@ func parsePackageLockV2TokenWalk(fileContent []byte, result *LocalDependency) (* Purl: purl, Requirement: pkg.Version, Scope: scope, + Dev: pkg.Dev, Line: line, DeclaredText: declText, }) diff --git a/pkg/dependencies/parsers/python.go b/pkg/dependencies/parsers/python.go index 1bf7759..2f01b5d 100644 --- a/pkg/dependencies/parsers/python.go +++ b/pkg/dependencies/parsers/python.go @@ -28,19 +28,18 @@ import ( "bytes" "path/filepath" "regexp" + "slices" "strings" -) -var ( - // requirementRegex matches Python requirement lines - // Example: django>=4.2.0 or requests==2.28.0 or flask - requirementRegex = regexp.MustCompile(`^(?P[-\w]+)\s*(?P[>=~!]*)\s*(?P[\d\.]*)`) + "github.com/pelletier/go-toml/v2/unstable" - // pyprojectDepsRegex matches dependency lines in pyproject.toml - // Example: "django>=4.2.0" or 'requests==2.28.0' - pyprojectDepsRegex = regexp.MustCompile(`["']([^"']+)["']`) + "github.com/scanoss/scanoss.go/pkg/dependencies/internal/pyscope" ) +// requirementRegex matches Python requirement lines +// Example: django>=4.2.0 or requests==2.28.0 or flask +var requirementRegex = regexp.MustCompile(`^(?P[-\w]+)\s*(?P[>=~!]*)\s*(?P[\d\.]*)`) + // ParseRequirementsTxt parses a requirements.txt file and extracts dependencies func ParseRequirementsTxt(fileContent []byte, filePath string) (*LocalDependency, error) { result := &LocalDependency{ @@ -87,39 +86,10 @@ func ParseRequirementsTxt(fileContent []byte, filePath string) (*LocalDependency continue } - // Parse the requirement - matches := requirementRegex.FindStringSubmatch(line) - if len(matches) >= 4 { - name := matches[1] - symbol := matches[2] - version := matches[3] - - var purl string - var requirement string - - // Build PURL based on version specifier - if symbol == "" { - // No version specified - purl = BuildPURL("pypi", "", name, "", nil) - } else if symbol == "==" && version != "" { - // Exact version - purl = BuildPURL("pypi", "", name, version, nil) - requirement = version - } else if version != "" { - // Other operators (>=, ~, !=, etc.) - purl = BuildPURL("pypi", "", name, "", nil) - requirement = symbol + version - } else { - // Just the symbol without version - purl = BuildPURL("pypi", "", name, "", nil) - } - - result.Purls = append(result.Purls, LocalPurl{ - Purl: purl, - Requirement: requirement, - Line: lineNo, - DeclaredText: strings.TrimSpace(raw), - }) + if lp, ok := pypiRequirement(line); ok { + lp.Line = lineNo + lp.DeclaredText = line + result.Purls = append(result.Purls, lp) } } @@ -130,7 +100,12 @@ func ParseRequirementsTxt(fileContent []byte, filePath string) (*LocalDependency return result, nil } -// ParsePyprojectToml parses a pyproject.toml file and extracts dependencies +// ParsePyprojectToml parses a pyproject.toml file and extracts the +// dependencies it declares: [project] dependencies, each extra in +// [project.optional-dependencies], each PEP 735 [dependency-groups] group, and +// Poetry's dependencies, dev-dependencies and groups. Scope is the TOML table +// the dependency is declared in, such as "project.optional-dependencies.dev", +// and Dev follows the rule in package pyscope. A malformed file is an error. func ParsePyprojectToml(fileContent []byte, filePath string) (*LocalDependency, error) { result := &LocalDependency{ File: filePath, @@ -138,79 +113,144 @@ func ParsePyprojectToml(fileContent []byte, filePath string) (*LocalDependency, Purls: []LocalPurl{}, } - scanner := bufio.NewScanner(bytes.NewReader(fileContent)) - inDependenciesSection := false - lineNo := 0 - - for scanner.Scan() { - lineNo++ - raw := scanner.Text() - line := strings.TrimSpace(raw) - - // Skip empty lines and comments - if line == "" || strings.HasPrefix(line, "#") { - continue + var p unstable.Parser + p.Reset(fileContent) + table := []string{} + for p.NextExpression() { + e := p.Expression() + switch e.Kind { + case unstable.Table: + table = tomlKey(e.Key()) + case unstable.ArrayTable: + table = nil // no dependency table is an array of tables + case unstable.KeyValue: + if table != nil { + collectPyproject(&p, result, append(slices.Clone(table), tomlKey(e.Key())...), e) + } } + } + if err := p.Error(); err != nil { + return nil, err + } + return result, nil +} - // Check for dependencies section - if strings.Contains(line, "dependencies") && strings.Contains(line, "[") { - inDependenciesSection = true - continue +// collectPyproject adds the dependencies value declares at the TOML key path, +// descending into inline tables such as optional-dependencies = {dev = [...]}. +func collectPyproject(p *unstable.Parser, result *LocalDependency, path []string, kv *unstable.Node) { + value := kv.Value() + add := func(lp LocalPurl, ok bool, scope string, dev bool, at *unstable.Node) { + if !ok { + return } - - // Check for end of section (next section starts with [) - if inDependenciesSection && strings.HasPrefix(line, "[") { - inDependenciesSection = false - continue + lp.Scope = scope + lp.Dev = dev + lp.Line = p.Shape(at.Raw).Start.Line + lp.DeclaredText = sourceLine(p.Data(), lp.Line) + result.Purls = append(result.Purls, lp) + } + addAll := func(scope string, dev bool) { + if value.Kind != unstable.Array { + return } + for it := value.Children(); it.Next(); { + // A table such as {include-group = "test"} names another group, + // which declares its dependencies itself. + if n := it.Node(); n.Kind == unstable.String { + lp, ok := pypiRequirement(string(n.Data)) + add(lp, ok, scope, dev, n) + } + } + } + addPoetry := func(name string, scope []string, dev bool) { + lp, ok := poetryRequirement(name, value) + add(lp, ok, strings.Join(scope, "."), dev, kv) + } + switch { + case slices.Equal(path, []string{"project", "dependencies"}): + addAll(strings.Join(path, "."), false) + case len(path) == 3 && path[0] == "project" && path[1] == "optional-dependencies": + addAll(strings.Join(path, "."), pyscope.DevExtra(path[2])) + case len(path) == 2 && path[0] == "dependency-groups": + addAll(strings.Join(path, "."), true) + case len(path) == 4 && slices.Equal(path[:3], []string{"tool", "poetry", "dependencies"}): + addPoetry(path[3], path[:3], false) + case len(path) == 4 && slices.Equal(path[:3], []string{"tool", "poetry", "dev-dependencies"}): + addPoetry(path[3], path[:3], true) + case len(path) == 6 && slices.Equal(path[:3], []string{"tool", "poetry", "group"}) && path[4] == "dependencies": + addPoetry(path[5], path[:5], pyscope.DevPoetryGroup(path[3])) + case value.Kind == unstable.InlineTable: + for it := value.Children(); it.Next(); { + member := it.Node() + collectPyproject(p, result, append(slices.Clone(path), tomlKey(member.Key())...), member) + } + } +} - // Parse dependencies within the section - if inDependenciesSection { - // Extract quoted strings; all deps on this physical line share lineNo and raw. - matches := pyprojectDepsRegex.FindAllStringSubmatch(line, -1) - for _, match := range matches { - if len(match) >= 2 { - dep := match[1] - - // Parse the dependency (similar to requirements.txt) - reqMatches := requirementRegex.FindStringSubmatch(dep) - if len(reqMatches) >= 4 { - name := reqMatches[1] - symbol := reqMatches[2] - version := reqMatches[3] - - var purl string - var requirement string - - if symbol == "" { - purl = BuildPURL("pypi", "", name, "", nil) - } else if symbol == "==" && version != "" { - purl = BuildPURL("pypi", "", name, version, nil) - requirement = version - } else if version != "" { - purl = BuildPURL("pypi", "", name, "", nil) - requirement = symbol + version - } else { - purl = BuildPURL("pypi", "", name, "", nil) - } - - result.Purls = append(result.Purls, LocalPurl{ - Purl: purl, - Requirement: requirement, - Line: lineNo, - DeclaredText: strings.TrimSpace(raw), - }) - } - } +// poetryRequirement reads a Poetry dependency, declared as name = "^1.2" or +// as a table with a version, into a purl, pinned when the version is exact. +// python is the interpreter, not a dependency. +func poetryRequirement(name string, value *unstable.Node) (LocalPurl, bool) { + if strings.EqualFold(name, "python") { + return LocalPurl{}, false + } + version := "" + switch value.Kind { + case unstable.String: + version = string(value.Data) + case unstable.InlineTable: + for it := value.Children(); it.Next(); { + field := it.Node() + if slices.Equal(tomlKey(field.Key()), []string{"version"}) && field.Value().Kind == unstable.String { + version = string(field.Value().Data) } } } + version = strings.TrimSpace(version) + if exact := strings.TrimPrefix(version, "=="); exact != "" && !strings.ContainsAny(exact, "<>=!~^*, |") { + return LocalPurl{Purl: BuildPURL("pypi", "", name, exact, nil), Requirement: exact}, true + } + if version == "*" { + version = "" + } + return LocalPurl{Purl: BuildPURL("pypi", "", name, "", nil), Requirement: version}, true +} + +// pypiRequirement reads a requirement such as requests>=2.31 into a purl, +// pinned when the requirement is an exact version. +func pypiRequirement(spec string) (LocalPurl, bool) { + m := requirementRegex.FindStringSubmatch(spec) + if len(m) < 4 { + return LocalPurl{}, false + } + name, symbol, version := m[1], m[2], m[3] + switch { + case symbol == "==" && version != "": + return LocalPurl{Purl: BuildPURL("pypi", "", name, version, nil), Requirement: version}, true + case symbol != "" && version != "": + return LocalPurl{Purl: BuildPURL("pypi", "", name, "", nil), Requirement: symbol + version}, true + default: + return LocalPurl{Purl: BuildPURL("pypi", "", name, "", nil)}, true + } +} - if err := scanner.Err(); err != nil { - return nil, err +// tomlKey is the parts of a dotted TOML key. +func tomlKey(it unstable.Iterator) []string { + var parts []string + for it.Next() { + parts = append(parts, string(it.Node().Data)) } + return parts +} - return result, nil +// sourceLine is the trimmed text of the 1-based line in data. +func sourceLine(data []byte, line int) string { + for i, l := range bytes.Split(data, []byte("\n")) { + if i+1 == line { + return strings.TrimSpace(string(l)) + } + } + return "" } // ParsePython is the main entry point for parsing Python dependency files diff --git a/pkg/dependencies/parsers/python_test.go b/pkg/dependencies/parsers/python_test.go index b90bf5e..f0c9fc8 100644 --- a/pkg/dependencies/parsers/python_test.go +++ b/pkg/dependencies/parsers/python_test.go @@ -97,52 +97,104 @@ flask==2.3.0 } } -// TestParsePyprojectTomlSourceLocation verifies Line and DeclaredText for pyproject.toml. -// For pyproject.toml, multiple quoted deps on a single line share the same Line and DeclaredText. -func TestParsePyprojectTomlSourceLocation(t *testing.T) { +func TestParsePyprojectToml(t *testing.T) { t.Parallel() tests := []struct { - name string - content string - wantCount int - wantLine int - wantLines []int // per-purl exact lines; checked when len > 0 - wantPurls []string + name string + content string + want []LocalPurl }{ { - name: "single dep per line under dependencies section (PEP 508 style)", - // line 1: [tool.poetry] - // line 2: name = "myapp" - // line 3: (blank) - // line 4: [tool.poetry.dependencies] - // line 5: dep1 = "requests>=2.28.0" - // line 6: dep2 = "flask>=2.3.0" - content: `[tool.poetry] -name = "myapp" - -[tool.poetry.dependencies] -dep1 = "requests>=2.28.0" -dep2 = "flask>=2.3.0" + name: "project dependencies and a dev extra", + content: `[project] +name = "earnie-staging-probe" +version = "0.1.0" +dependencies = ["requests>=2.31"] +[project.optional-dependencies] +dev = [ + "pytest>=7.4", + "pylint>=2.17", +] `, - wantCount: 2, - wantLines: []int{5, 6}, + want: []LocalPurl{ + {Purl: "pkg:pypi/requests", Requirement: ">=2.31", Scope: "project.dependencies", Line: 4, DeclaredText: `dependencies = ["requests>=2.31"]`}, + {Purl: "pkg:pypi/pytest", Requirement: ">=7.4", Scope: "project.optional-dependencies.dev", Dev: true, Line: 7, DeclaredText: `"pytest>=7.4",`}, + {Purl: "pkg:pypi/pylint", Requirement: ">=2.17", Scope: "project.optional-dependencies.dev", Dev: true, Line: 8, DeclaredText: `"pylint>=2.17",`}, + }, }, { - name: "multiple quoted deps on single line share same Line", - // line 1: [build-system] - // line 2: requires = ["setuptools>=40.8.0", "wheel"] - // line 3: (blank) - // line 4: [tool.poetry.dependencies] - // line 5: extras = ["requests>=2.28.0", "flask>=2.3.0"] + name: "other project strings and build requirements are not dependencies", content: `[build-system] -requires = ["setuptools>=40.8.0", "wheel"] +requires = ["setuptools>=61", "wheel"] + +[project] +name = "app" +description = "A tool" +requires-python = ">=3.9" +dependencies = [ + "django==4.2.0", "flask", +] +readme = "README.md" +`, + want: []LocalPurl{ + {Purl: "pkg:pypi/django@4.2.0", Requirement: "4.2.0", Scope: "project.dependencies", Line: 9, DeclaredText: `"django==4.2.0", "flask",`}, + {Purl: "pkg:pypi/flask", Scope: "project.dependencies", Line: 9, DeclaredText: `"django==4.2.0", "flask",`}, + }, + }, + { + name: "an extra is dev only when its name says so", + content: `[project] +name = "app" +optional-dependencies = { socks = ["PySocks"], Test_Integration = ["tox"], docs = ["sphinx"] } +`, + want: []LocalPurl{ + {Purl: "pkg:pypi/PySocks", Scope: "project.optional-dependencies.socks", Line: 3}, + {Purl: "pkg:pypi/tox", Scope: "project.optional-dependencies.Test_Integration", Dev: true, Line: 3}, + {Purl: "pkg:pypi/sphinx", Scope: "project.optional-dependencies.docs", Dev: true, Line: 3}, + }, + }, + { + name: "every PEP 735 group is dev, and an include-group adds nothing", + content: `[dependency-groups] +test = ["pytest>=8", {include-group = "typing"}] +typing = ["mypy"] +release = ["twine"] +`, + want: []LocalPurl{ + {Purl: "pkg:pypi/pytest", Requirement: ">=8", Scope: "dependency-groups.test", Dev: true, Line: 2}, + {Purl: "pkg:pypi/mypy", Scope: "dependency-groups.typing", Dev: true, Line: 3}, + {Purl: "pkg:pypi/twine", Scope: "dependency-groups.release", Dev: true, Line: 4}, + }, + }, + { + name: "poetry dependencies, dev-dependencies and groups", + content: `[tool.poetry] +name = "app" [tool.poetry.dependencies] -extras = ["requests>=2.28.0", "flask>=2.3.0"] +python = "^3.9" +requests = "^2.31" +flask = { version = "2.3.0", optional = true } +click = "*" + +[tool.poetry.dev-dependencies] +black = "23.1" + +[tool.poetry.group.main.dependencies] +attrs = ">=23" + +[tool.poetry.group.test.dependencies] +pytest = { path = "../pytest" } `, - wantCount: 2, - wantLine: 5, + want: []LocalPurl{ + {Purl: "pkg:pypi/requests", Requirement: "^2.31", Scope: "tool.poetry.dependencies", Line: 6}, + {Purl: "pkg:pypi/flask@2.3.0", Requirement: "2.3.0", Scope: "tool.poetry.dependencies", Line: 7}, + {Purl: "pkg:pypi/click", Scope: "tool.poetry.dependencies", Line: 8}, + {Purl: "pkg:pypi/black@23.1", Requirement: "23.1", Scope: "tool.poetry.dev-dependencies", Dev: true, Line: 11}, + {Purl: "pkg:pypi/attrs", Requirement: ">=23", Scope: "tool.poetry.group.main.dependencies", Line: 14}, + {Purl: "pkg:pypi/pytest", Scope: "tool.poetry.group.test.dependencies", Dev: true, Line: 17}, + }, }, } @@ -153,31 +205,25 @@ extras = ["requests>=2.28.0", "flask>=2.3.0"] if err != nil { t.Fatalf("ParsePyprojectToml error: %v", err) } - if len(result.Purls) != tc.wantCount { - t.Fatalf("got %d purls, want %d; purls: %+v", len(result.Purls), tc.wantCount, result.Purls) + if len(result.Purls) != len(tc.want) { + t.Fatalf("got %d purls, want %d; purls: %+v", len(result.Purls), len(tc.want), result.Purls) } - // All purls must have non-zero Line and non-empty DeclaredText - for i, p := range result.Purls { - if p.Line == 0 { - t.Errorf("[%d] Line is 0, want non-zero", i) - } - if p.DeclaredText == "" { - t.Errorf("[%d] DeclaredText is empty", i) - } - if tc.wantLine != 0 && p.Line != tc.wantLine { - t.Errorf("[%d] Line: got %d, want %d", i, p.Line, tc.wantLine) - } - if i < len(tc.wantLines) && p.Line != tc.wantLines[i] { - t.Errorf("[%d] Line: got %d, want %d", i, p.Line, tc.wantLines[i]) + for i, got := range result.Purls { + w := tc.want[i] + if w.DeclaredText == "" { + w.DeclaredText = got.DeclaredText } - } - // For multi-dep line case: all deps on same line share same DeclaredText - if tc.wantLine != 0 && len(result.Purls) == 2 { - if result.Purls[0].DeclaredText != result.Purls[1].DeclaredText { - t.Errorf("multi-dep: expected same DeclaredText for both; got %q vs %q", - result.Purls[0].DeclaredText, result.Purls[1].DeclaredText) + if got != w { + t.Errorf("[%d] got %+v, want %+v", i, got, w) } } }) } } + +func TestParsePyprojectTomlMalformed(t *testing.T) { + t.Parallel() + if _, err := ParsePyprojectToml([]byte("[project\ndependencies = [\"requests\"]\n"), "pyproject.toml"); err == nil { + t.Fatal("want an error for a malformed pyproject.toml") + } +} diff --git a/pkg/dependencies/parsers/types.go b/pkg/dependencies/parsers/types.go index 76f1608..75f5a74 100644 --- a/pkg/dependencies/parsers/types.go +++ b/pkg/dependencies/parsers/types.go @@ -38,6 +38,12 @@ type LocalPurl struct { // Scope is the optional dependency scope (e.g., "dependencies", "devDependencies", "test") Scope string `json:"scope,omitempty"` + // Dev is true when the manifest declares the dependency only to develop the + // project (tests, linting, documentation, build tooling): devDependencies in + // package.json, Maven's test scope, Gradle test configurations, and the + // pyproject.toml groups and extras package pyscope names. + Dev bool `json:"dev,omitempty"` + // Line is the 1-based line number of the dependency declaration in the manifest file. // Zero means the location could not be determined. Line int `json:"line,omitempty"` diff --git a/pkg/dependencies/resolve/lockfile/poetry.go b/pkg/dependencies/resolve/lockfile/poetry.go index 4e76f9f..d707082 100644 --- a/pkg/dependencies/resolve/lockfile/poetry.go +++ b/pkg/dependencies/resolve/lockfile/poetry.go @@ -31,6 +31,7 @@ import ( "github.com/pelletier/go-toml/v2" + "github.com/scanoss/scanoss.go/pkg/dependencies/internal/pyscope" "github.com/scanoss/scanoss.go/pkg/dependencies/resolve" "github.com/scanoss/scanoss.go/pkg/dependencies/resolve/internal/build" ) @@ -120,8 +121,7 @@ func pyprojectDirects(r *build.Root, manifest []byte, byName map[string][]resolv r.Declare(id, dev) } } - // Every Poetry group but main, and every PEP 735 group, is for - // development; extras are installed by users. + // Package pyscope decides which groups and extras are for development. poetry := py.Tool.Poetry type table struct { deps map[string]any @@ -129,7 +129,7 @@ func pyprojectDirects(r *build.Root, manifest []byte, byName map[string][]resolv } tables := []table{{poetry.Dependencies, false}, {poetry.DevDependencies, true}} for _, g := range slices.Sorted(maps.Keys(poetry.Group)) { - tables = append(tables, table{poetry.Group[g].Dependencies, g != "main"}) + tables = append(tables, table{poetry.Group[g].Dependencies, pyscope.DevPoetryGroup(g)}) } for _, t := range tables { for _, dep := range slices.Sorted(maps.Keys(t.deps)) { @@ -143,7 +143,7 @@ func pyprojectDirects(r *build.Root, manifest []byte, byName map[string][]resolv } for _, extra := range slices.Sorted(maps.Keys(py.Project.OptionalDependencies)) { for _, dep := range py.Project.OptionalDependencies[extra] { - declare(dep, true, false) + declare(dep, true, pyscope.DevExtra(extra)) } } for _, g := range slices.Sorted(maps.Keys(py.DependencyGroups)) { diff --git a/pkg/dependencies/resolve/lockfile/testdata/poetry/dev-scope/pyproject.toml b/pkg/dependencies/resolve/lockfile/testdata/poetry/dev-scope/pyproject.toml index 79eb465..bf78a75 100644 --- a/pkg/dependencies/resolve/lockfile/testdata/poetry/dev-scope/pyproject.toml +++ b/pkg/dependencies/resolve/lockfile/testdata/poetry/dev-scope/pyproject.toml @@ -5,6 +5,7 @@ dependencies = ["both>=1"] [project.optional-dependencies] fast = ["ujson>=5"] +lint = ["ruff"] [tool.poetry.dependencies] python = "^3.10" @@ -21,4 +22,4 @@ pytest = "^8.0" [dependency-groups] docs = ["Sphinx>=7"] -lint = ["ruff", { include-group = "docs" }] +lint = [{ include-group = "docs" }] diff --git a/pkg/dependencies/resolve/lockfile/testdata/uv/extras/graph.golden.json b/pkg/dependencies/resolve/lockfile/testdata/uv/extras/graph.golden.json index 4528d37..66907bd 100644 --- a/pkg/dependencies/resolve/lockfile/testdata/uv/extras/graph.golden.json +++ b/pkg/dependencies/resolve/lockfile/testdata/uv/extras/graph.golden.json @@ -19,6 +19,11 @@ "id": "pkg:pypi/httpx@0.28.1", "purl": "pkg:pypi/httpx", "version": "0.28.1" + }, + { + "id": "pkg:pypi/iniconfig@2.1.0", + "purl": "pkg:pypi/iniconfig", + "version": "2.1.0" } ], "roots": [ @@ -44,6 +49,11 @@ { "id": "pkg:pypi/httpx@0.28.1", "direct": true + }, + { + "id": "pkg:pypi/iniconfig@2.1.0", + "direct": true, + "dev": true } ], "edges": [ diff --git a/pkg/dependencies/resolve/lockfile/testdata/uv/extras/uv.lock b/pkg/dependencies/resolve/lockfile/testdata/uv/extras/uv.lock index 695c52c..19032ab 100644 --- a/pkg/dependencies/resolve/lockfile/testdata/uv/extras/uv.lock +++ b/pkg/dependencies/resolve/lockfile/testdata/uv/extras/uv.lock @@ -11,6 +11,9 @@ dependencies = [ ] [package.optional-dependencies] +test = [ + { name = "iniconfig" }, +] cli = [ { name = "click" }, ] @@ -63,3 +66,8 @@ wheels = [ http2 = [ { name = "h2" }, ] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } diff --git a/pkg/dependencies/resolve/lockfile/uv.go b/pkg/dependencies/resolve/lockfile/uv.go index 5446ed6..8c63c0b 100644 --- a/pkg/dependencies/resolve/lockfile/uv.go +++ b/pkg/dependencies/resolve/lockfile/uv.go @@ -31,6 +31,7 @@ import ( "github.com/pelletier/go-toml/v2" + "github.com/scanoss/scanoss.go/pkg/dependencies/internal/pyscope" "github.com/scanoss/scanoss.go/pkg/dependencies/resolve" "github.com/scanoss/scanoss.go/pkg/dependencies/resolve/internal/build" ) @@ -87,7 +88,8 @@ func (p uvPackage) dev() []uvDep { // parseUvLock reads uv.lock. The project and each workspace member is a root // at its pyproject.toml, found from its editable or virtual source, and its -// dev-dependencies groups are its dev dependencies. Editable, virtual, +// dev-dependencies groups and development extras (package pyscope) are its +// dev dependencies. Editable, virtual, // directory and path packages are local, and become nodes only when another // package depends on them. A dependency on a name locked at several versions // names the version; one that does not has an edge to each. @@ -172,9 +174,15 @@ func parseUvLock(b *build.Builder, data, _ []byte, _ map[string][]byte) { r := b.Root(p.project()) r.DevKnown = true unresolved := func(detail string) { r.Flag(resolve.ReasonUnresolvedDependency, detail) } - for _, d := range p.runtime() { + for _, d := range p.Dependencies { link(i, d, func(id resolve.NodeID) { r.Declare(id, false) }, unresolved) } + for _, extra := range slices.Sorted(maps.Keys(p.OptionalDependencies)) { + dev := pyscope.DevExtra(extra) + for _, d := range p.OptionalDependencies[extra] { + link(i, d, func(id resolve.NodeID) { r.Declare(id, dev) }, unresolved) + } + } for _, d := range p.dev() { link(i, d, func(id resolve.NodeID) { r.Declare(id, true) }, unresolved) }