From 4f5cee6fa194ad17c51f99fd3d0d676f19d0ff88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Thu, 30 Oct 2025 17:55:35 +0100 Subject: [PATCH 01/15] feat: add initial fileupload api client --- pkg/apiclients/fileupload/batch.go | 115 +++ pkg/apiclients/fileupload/client.go | 347 +++++++++ .../fileupload/client_integration_test.go | 85 +++ pkg/apiclients/fileupload/client_test.go | 684 ++++++++++++++++++ pkg/apiclients/fileupload/errors.go | 36 + pkg/apiclients/fileupload/fake_client.go | 99 +++ .../fileupload/files/list_sources.go | 21 + .../fileupload/files/list_sources_test.go | 51 ++ .../files/testdata/simplest/package.json | 1 + .../files/testdata/simplest/src/index.js | 0 .../files/testdata/with-ignores/.gitignore | 1 + .../files/testdata/with-ignores/package.json | 1 + .../testdata/with-ignores/src/with-ignores.js | 0 pkg/apiclients/fileupload/filter.go | 17 + pkg/apiclients/fileupload/filters/client.go | 78 ++ .../fileupload/filters/client_test.go | 77 ++ .../fileupload/filters/fake_client.go | 25 + pkg/apiclients/fileupload/filters/opts.go | 13 + pkg/apiclients/fileupload/filters/utils.go | 17 + .../fileupload/filters/utils_test.go | 20 + pkg/apiclients/fileupload/opts.go | 32 + pkg/apiclients/fileupload/types.go | 35 + .../fileupload/uploadrevision/client.go | 300 ++++++++ .../fileupload/uploadrevision/client_test.go | 565 +++++++++++++++ .../fileupload/uploadrevision/compression.go | 67 ++ .../uploadrevision/compression_test.go | 120 +++ .../fileupload/uploadrevision/errors.go | 174 +++++ .../fileupload/uploadrevision/fake_client.go | 146 ++++ .../fileupload/uploadrevision/opts.go | 13 + .../fileupload/uploadrevision/opts_test.go | 46 ++ .../fileupload/uploadrevision/types.go | 138 ++++ 31 files changed, 3324 insertions(+) create mode 100644 pkg/apiclients/fileupload/batch.go create mode 100644 pkg/apiclients/fileupload/client.go create mode 100644 pkg/apiclients/fileupload/client_integration_test.go create mode 100644 pkg/apiclients/fileupload/client_test.go create mode 100644 pkg/apiclients/fileupload/errors.go create mode 100644 pkg/apiclients/fileupload/fake_client.go create mode 100644 pkg/apiclients/fileupload/files/list_sources.go create mode 100644 pkg/apiclients/fileupload/files/list_sources_test.go create mode 100644 pkg/apiclients/fileupload/files/testdata/simplest/package.json create mode 100644 pkg/apiclients/fileupload/files/testdata/simplest/src/index.js create mode 100644 pkg/apiclients/fileupload/files/testdata/with-ignores/.gitignore create mode 100644 pkg/apiclients/fileupload/files/testdata/with-ignores/package.json create mode 100644 pkg/apiclients/fileupload/files/testdata/with-ignores/src/with-ignores.js create mode 100644 pkg/apiclients/fileupload/filter.go create mode 100644 pkg/apiclients/fileupload/filters/client.go create mode 100644 pkg/apiclients/fileupload/filters/client_test.go create mode 100644 pkg/apiclients/fileupload/filters/fake_client.go create mode 100644 pkg/apiclients/fileupload/filters/opts.go create mode 100644 pkg/apiclients/fileupload/filters/utils.go create mode 100644 pkg/apiclients/fileupload/filters/utils_test.go create mode 100644 pkg/apiclients/fileupload/opts.go create mode 100644 pkg/apiclients/fileupload/types.go create mode 100644 pkg/apiclients/fileupload/uploadrevision/client.go create mode 100644 pkg/apiclients/fileupload/uploadrevision/client_test.go create mode 100644 pkg/apiclients/fileupload/uploadrevision/compression.go create mode 100644 pkg/apiclients/fileupload/uploadrevision/compression_test.go create mode 100644 pkg/apiclients/fileupload/uploadrevision/errors.go create mode 100644 pkg/apiclients/fileupload/uploadrevision/fake_client.go create mode 100644 pkg/apiclients/fileupload/uploadrevision/opts.go create mode 100644 pkg/apiclients/fileupload/uploadrevision/opts_test.go create mode 100644 pkg/apiclients/fileupload/uploadrevision/types.go diff --git a/pkg/apiclients/fileupload/batch.go b/pkg/apiclients/fileupload/batch.go new file mode 100644 index 000000000..7f0f55fac --- /dev/null +++ b/pkg/apiclients/fileupload/batch.go @@ -0,0 +1,115 @@ +package fileupload + +import ( + "fmt" + "iter" + "os" + "path/filepath" + + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" +) + +// uploadBatch manages a batch of files for upload. +type uploadBatch struct { + files []uploadrevision.UploadFile + currentSize int64 + limits uploadrevision.Limits +} + +func newUploadBatch(limits uploadrevision.Limits) *uploadBatch { + return &uploadBatch{ + files: make([]uploadrevision.UploadFile, 0, limits.FileCountLimit), + limits: limits, + } +} + +func (b *uploadBatch) addFile(file uploadrevision.UploadFile, fileSize int64) { + b.files = append(b.files, file) + b.currentSize += fileSize +} + +func (b *uploadBatch) wouldExceedLimits(fileSize int64) bool { + wouldExceedCount := len(b.files) >= b.limits.FileCountLimit + wouldExceedSize := b.currentSize+fileSize > b.limits.TotalPayloadSizeLimit + return wouldExceedCount || wouldExceedSize +} + +func (b *uploadBatch) isEmpty() bool { + return len(b.files) == 0 +} + +func (b *uploadBatch) closeRemainingFiles() { + for _, file := range b.files { + file.File.Close() + } +} + +type batchingResult struct { + batch *uploadBatch + filteredFiles []FilteredFile +} + +func batchPaths(rootPath string, paths <-chan string, limits uploadrevision.Limits, filters ...filter) iter.Seq2[*batchingResult, error] { + return func(yield func(*batchingResult, error) bool) { + batch := newUploadBatch(limits) + filtered := []FilteredFile{} + for path := range paths { + relPath, err := filepath.Rel(rootPath, path) + if err != nil { + if !yield(nil, fmt.Errorf("failed to get relative path of file %s: %w", path, err)) { + return + } + } + + f, err := os.Open(path) + if err != nil { + f.Close() + if !yield(nil, fmt.Errorf("failed to open file %s: %w", path, err)) { + return + } + } + + fstat, err := f.Stat() + if err != nil { + f.Close() + if !yield(nil, fmt.Errorf("failed to stat file %s: %w", path, err)) { + return + } + } + + ff := applyFilters(fileToFilter{Path: relPath, Stat: fstat}, filters...) + if ff != nil { + f.Close() + filtered = append(filtered, *ff) + continue + } + + if batch.wouldExceedLimits(fstat.Size()) { + if !yield(&batchingResult{batch: batch, filteredFiles: filtered}, nil) { + return + } + batch = newUploadBatch(limits) + filtered = []FilteredFile{} + } + + batch.addFile(uploadrevision.UploadFile{ + Path: relPath, + File: f, + }, fstat.Size()) + } + + if !batch.isEmpty() || len(filtered) > 0 { + yield(&batchingResult{batch: batch, filteredFiles: filtered}, nil) + } + } +} + +func applyFilters(ff fileToFilter, filters ...filter) *FilteredFile { + for _, filter := range filters { + if ff := filter(ff); ff != nil { + return ff + } + } + + return nil +} diff --git a/pkg/apiclients/fileupload/client.go b/pkg/apiclients/fileupload/client.go new file mode 100644 index 000000000..be978239c --- /dev/null +++ b/pkg/apiclients/fileupload/client.go @@ -0,0 +1,347 @@ +package fileupload + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "runtime" + + "github.com/google/uuid" + "github.com/puzpuzpuz/xsync" + "github.com/rs/zerolog" + "github.com/snyk/go-application-framework/pkg/configuration" + "github.com/snyk/go-application-framework/pkg/utils" + "github.com/snyk/go-application-framework/pkg/workflow" + + listsources "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/files" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/filters" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" +) + +// Config contains configuration for the file upload client. +type Config struct { + BaseURL string + OrgID OrgID + IsFedRamp bool +} + +// HTTPClient provides high-level file upload functionality. +type HTTPClient struct { + uploadRevisionSealableClient uploadrevision.SealableClient + filtersClient filters.Client + cfg Config + filters Filters + logger *zerolog.Logger +} + +// Client defines the interface for the high level file upload client. +type Client interface { + CreateRevisionFromPaths(ctx context.Context, paths []string, opts UploadOptions) (UploadResult, error) + CreateRevisionFromDir(ctx context.Context, dirPath string, opts UploadOptions) (UploadResult, error) + CreateRevisionFromFile(ctx context.Context, filePath string, opts UploadOptions) (UploadResult, error) +} + +var _ Client = (*HTTPClient)(nil) + +// NewClient creates a new high-level file upload client. +func NewClient(httpClient *http.Client, cfg Config, opts ...Option) *HTTPClient { + client := &HTTPClient{ + cfg: cfg, + filters: Filters{ + supportedExtensions: xsync.NewMapOf[bool](), + supportedConfigFiles: xsync.NewMapOf[bool](), + }, + } + + for _, opt := range opts { + opt(client) + } + + if client.logger == nil { + client.logger = utils.Ptr(zerolog.Nop()) + } + + if client.uploadRevisionSealableClient == nil { + client.uploadRevisionSealableClient = uploadrevision.NewClient(uploadrevision.Config{ + BaseURL: cfg.BaseURL, + }, uploadrevision.WithHTTPClient(httpClient)) + } + + if client.filtersClient == nil { + client.filtersClient = filters.NewDeeproxyClient(filters.Config{ + BaseURL: cfg.BaseURL, + IsFedRamp: cfg.IsFedRamp, + }, filters.WithHTTPClient(httpClient)) + } + + return client +} + +// NewClientFromInvocationContext creates a new file upload client from a workflow.InvocationContext. +// This is a convenience function that extracts the necessary configuration and HTTP client +// from the invocation context. +func NewClientFromInvocationContext(ictx workflow.InvocationContext, orgID uuid.UUID) Client { + cfg := ictx.GetConfiguration() + return NewClient( + ictx.GetNetworkAccess().GetHttpClient(), + Config{ + BaseURL: cfg.GetString(configuration.API_URL), + OrgID: orgID, + IsFedRamp: cfg.GetBool(configuration.IS_FEDRAMP), + }, + WithLogger(ictx.GetEnhancedLogger()), + ) +} + +func (c *HTTPClient) loadFilters(ctx context.Context) error { + c.filters.once.Do(func() { + filtersResp, err := c.filtersClient.GetFilters(ctx, c.cfg.OrgID) + if err != nil { + c.filters.initErr = err + return + } + + for _, ext := range filtersResp.Extensions { + c.filters.supportedExtensions.Store(ext, true) + } + for _, configFile := range filtersResp.ConfigFiles { + // .gitignore and .dcignore should not be uploaded + // (https://github.com/snyk/code-client/blob/d6f6a2ce4c14cb4b05aa03fb9f03533d8cf6ca4a/src/files.ts#L138) + if configFile == ".gitignore" || configFile == ".dcignore" { + continue + } + c.filters.supportedConfigFiles.Store(configFile, true) + } + }) + return c.filters.initErr +} + +// createDeeproxyFilter creates a filter function based on the current deeproxy filtering configuration. +func (c *HTTPClient) createDeeproxyFilter(ctx context.Context) (filter, error) { + if err := c.loadFilters(ctx); err != nil { + return nil, fmt.Errorf("failed to load deeproxy filters: %w", err) + } + + return func(ff fileToFilter) *FilteredFile { + fileExt := filepath.Ext(ff.Stat.Name()) + fileName := filepath.Base(ff.Stat.Name()) + _, isSupportedExtension := c.filters.supportedExtensions.Load(fileExt) + _, isSupportedConfigFile := c.filters.supportedConfigFiles.Load(fileName) + + if !isSupportedExtension && !isSupportedConfigFile { + var reason error + if !isSupportedConfigFile { + reason = errors.Join(reason, fmt.Errorf("file name is not a part of the supported config files: %s", fileName)) + } + if !isSupportedExtension { + reason = errors.Join(reason, fmt.Errorf("file extension is not supported: %s", fileExt)) + } + return &FilteredFile{ + Path: ff.Path, + Reason: reason, + } + } + + return nil + }, nil +} + +func (c *HTTPClient) uploadBatch(ctx context.Context, revID RevisionID, batch *uploadBatch) error { + defer batch.closeRemainingFiles() + + if batch.isEmpty() { + return nil + } + + err := c.uploadRevisionSealableClient.UploadFiles(ctx, c.cfg.OrgID, revID, batch.files) + if err != nil { + return fmt.Errorf("failed to upload files: %w", err) + } + + return nil +} + +// addPathsToRevision adds multiple file paths to an existing revision. +func (c *HTTPClient) addPathsToRevision( + ctx context.Context, + revisionID RevisionID, + rootPath string, + pathsChan <-chan string, + opts UploadOptions, +) (UploadResult, error) { + res := UploadResult{ + RevisionID: revisionID, + FilteredFiles: make([]FilteredFile, 0), + } + + fileSizeFilter := func(ff fileToFilter) *FilteredFile { + fileSizeLimit := c.uploadRevisionSealableClient.GetLimits().FileSizeLimit + if ff.Stat.Size() > fileSizeLimit { + return &FilteredFile{ + Path: ff.Path, + Reason: uploadrevision.NewFileSizeLimitError(ff.Stat.Name(), ff.Stat.Size(), fileSizeLimit), + } + } + + return nil + } + + filePathLengthFilter := func(ff fileToFilter) *FilteredFile { + filePathLengthLimit := c.uploadRevisionSealableClient.GetLimits().FilePathLengthLimit + if len(ff.Path) > filePathLengthLimit { + return &FilteredFile{ + Path: ff.Path, + Reason: uploadrevision.NewFilePathLengthLimitError(ff.Path, len(ff.Path), filePathLengthLimit), + } + } + + return nil + } + + filters := []filter{ + fileSizeFilter, + filePathLengthFilter, + } + if !opts.SkipDeeproxyFiltering { + deeproxyFilter, err := c.createDeeproxyFilter(ctx) + if err != nil { + return res, err + } + + filters = append(filters, deeproxyFilter) + } + + for batchResult, err := range batchPaths(rootPath, pathsChan, c.uploadRevisionSealableClient.GetLimits(), filters...) { + if err != nil { + return res, fmt.Errorf("failed to batch files: %w", err) + } + + res.FilteredFiles = append(res.FilteredFiles, batchResult.filteredFiles...) + + err = c.uploadBatch(ctx, revisionID, batchResult.batch) + if err != nil { + return res, err + } + + res.UploadedFilesCount += len(batchResult.batch.files) + } + + return res, nil +} + +// createRevision creates a new revision and returns its ID. +func (c *HTTPClient) createRevision(ctx context.Context) (RevisionID, error) { + revision, err := c.uploadRevisionSealableClient.CreateRevision(ctx, c.cfg.OrgID) + if err != nil { + return uuid.Nil, fmt.Errorf("failed to create revision: %w", err) + } + return revision.Data.ID, nil +} + +// addFileToRevision adds a single file to an existing revision. +func (c *HTTPClient) addFileToRevision(ctx context.Context, revisionID RevisionID, filePath string, opts UploadOptions) (UploadResult, error) { + writableChan := make(chan string, 1) + writableChan <- filePath + close(writableChan) + + return c.addPathsToRevision(ctx, revisionID, filepath.Dir(filePath), writableChan, opts) +} + +// addDirToRevision adds a directory and all its contents to an existing revision. +func (c *HTTPClient) addDirToRevision(ctx context.Context, revisionID RevisionID, dirPath string, opts UploadOptions) (UploadResult, error) { + sources, err := listsources.ForPath(dirPath, c.logger, runtime.NumCPU()) + if err != nil { + return UploadResult{}, fmt.Errorf("failed to list files in directory %s: %w", dirPath, err) + } + + return c.addPathsToRevision(ctx, revisionID, dirPath, sources, opts) +} + +// sealRevision seals a revision, making it immutable. +func (c *HTTPClient) sealRevision(ctx context.Context, revisionID RevisionID) error { + _, err := c.uploadRevisionSealableClient.SealRevision(ctx, c.cfg.OrgID, revisionID) + if err != nil { + return fmt.Errorf("failed to seal revision: %w", err) + } + return nil +} + +// CreateRevisionFromPaths uploads multiple paths (files or directories), returning a revision ID. +// This is a convenience method that creates, uploads, and seals a revision. +func (c *HTTPClient) CreateRevisionFromPaths(ctx context.Context, paths []string, opts UploadOptions) (UploadResult, error) { + res := UploadResult{ + FilteredFiles: make([]FilteredFile, 0), + } + + revisionID, err := c.createRevision(ctx) + if err != nil { + return res, err + } + res.RevisionID = revisionID + + for _, pth := range paths { + info, err := os.Stat(pth) + if err != nil { + return UploadResult{}, uploadrevision.NewFileAccessError(pth, err) + } + + if info.IsDir() { + dirUploadRes, err := c.addDirToRevision(ctx, revisionID, pth, opts) + if err != nil { + return res, fmt.Errorf("failed to add directory %s: %w", pth, err) + } + res.FilteredFiles = append(res.FilteredFiles, dirUploadRes.FilteredFiles...) + res.UploadedFilesCount += dirUploadRes.UploadedFilesCount + } else { + fileUploadRes, err := c.addFileToRevision(ctx, revisionID, pth, opts) + if err != nil { + return res, fmt.Errorf("failed to add file %s: %w", pth, err) + } + res.FilteredFiles = append(res.FilteredFiles, fileUploadRes.FilteredFiles...) + res.UploadedFilesCount += fileUploadRes.UploadedFilesCount + } + } + + if res.UploadedFilesCount == 0 && len(res.FilteredFiles) == 0 { + return res, ErrNoFilesProvided + } + + if err := c.sealRevision(ctx, revisionID); err != nil { + return res, err + } + + return res, nil +} + +// CreateRevisionFromDir uploads a directory and all its contents, returning a revision ID. +// This is a convenience method for validating the directory path and calling CreateRevisionFromPaths with a single directory path. +func (c *HTTPClient) CreateRevisionFromDir(ctx context.Context, dirPath string, opts UploadOptions) (UploadResult, error) { + info, err := os.Stat(dirPath) + if err != nil { + return UploadResult{}, uploadrevision.NewFileAccessError(dirPath, err) + } + + if !info.IsDir() { + return UploadResult{}, fmt.Errorf("the provided path is not a directory: %s", dirPath) + } + + return c.CreateRevisionFromPaths(ctx, []string{dirPath}, opts) +} + +// CreateRevisionFromFile uploads a single file, returning a revision ID. +// This is a convenience method for validating the file path and calling CreateRevisionFromPaths with a single file path. +func (c *HTTPClient) CreateRevisionFromFile(ctx context.Context, filePath string, opts UploadOptions) (UploadResult, error) { + info, err := os.Stat(filePath) + if err != nil { + return UploadResult{}, uploadrevision.NewFileAccessError(filePath, err) + } + + if !info.Mode().IsRegular() { + return UploadResult{}, fmt.Errorf("the provided path is not a regular file: %s", filePath) + } + + return c.CreateRevisionFromPaths(ctx, []string{filePath}, opts) +} diff --git a/pkg/apiclients/fileupload/client_integration_test.go b/pkg/apiclients/fileupload/client_integration_test.go new file mode 100644 index 000000000..73d74fa39 --- /dev/null +++ b/pkg/apiclients/fileupload/client_integration_test.go @@ -0,0 +1,85 @@ +//go:build integration + +package fileupload_test + +import ( + "path/filepath" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + "github.com/snyk/go-application-framework/pkg/apiclients/util" +) + +func TestUploadFileIntegration(t *testing.T) { + setup := util.NewIntegrationTestSetup(t) + fileUploadClient := newFileUploadClient(setup) + + files := []uploadrevision.LoadedFile{ + {Path: "src/main.go", Content: "package main"}, + } + + dir := util.CreateTmpFiles(t, files) + + res, err := fileUploadClient.CreateRevisionFromFile(t.Context(), filepath.Join(dir.Name(), files[0].Path), fileupload.UploadOptions{}) + if err != nil { + t.Errorf("failed to create fileupload revision: %s", err.Error()) + } + assert.NotEqual(t, uuid.Nil, res.RevisionID) +} + +func TestUploadDirectoryIntegration(t *testing.T) { + setup := util.NewIntegrationTestSetup(t) + fileUploadClient := newFileUploadClient(setup) + + files := []uploadrevision.LoadedFile{ + {Path: "src/main.go", Content: "package main"}, + {Path: "src/utils.go", Content: "package utils"}, + {Path: "README.md", Content: "# Project"}, + } + + dir := util.CreateTmpFiles(t, files) + + res, err := fileUploadClient.CreateRevisionFromDir(t.Context(), dir.Name(), fileupload.UploadOptions{}) + if err != nil { + t.Errorf("failed to create fileupload revision: %s", err.Error()) + } + assert.NotEqual(t, uuid.Nil, res.RevisionID) +} + +func TestUploadLargeFileIntegration(t *testing.T) { + setup := util.NewIntegrationTestSetup(t) + fileUploadClient := newFileUploadClient(setup) + + content := generateFileOfSizeMegabytes(t, 30) + files := []uploadrevision.LoadedFile{ + {Path: "src/main.go", Content: content}, + } + + dir := util.CreateTmpFiles(t, files) + + res, err := fileUploadClient.CreateRevisionFromFile(t.Context(), filepath.Join(dir.Name(), files[0].Path), fileupload.UploadOptions{}) + if err != nil { + t.Errorf("failed to create fileupload revision: %s", err.Error()) + } + assert.NotEqual(t, uuid.Nil, res.RevisionID) +} + +func newFileUploadClient(setup *util.IntegrationTestSetup) fileupload.Client { + return fileupload.NewClient( + setup.Client, + fileupload.Config{ + BaseURL: setup.Config.BaseURL, + OrgID: setup.Config.OrgID, + }, + ) +} + +func generateFileOfSizeMegabytes(t *testing.T, megabytes int) string { + t.Helper() + content := make([]byte, megabytes*1024*1024) + return string(content) +} diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go new file mode 100644 index 000000000..ebd94939a --- /dev/null +++ b/pkg/apiclients/fileupload/client_test.go @@ -0,0 +1,684 @@ +package fileupload_test + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/filters" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" +) + +// CreateTmpFiles is an utility function used to create temporary files in tests. +func createTmpFiles(t *testing.T, files []uploadrevision.LoadedFile) (dir *os.File) { + t.Helper() + + tempDir := t.TempDir() + dir, err := os.Open(tempDir) + if err != nil { + panic(err) + } + + for _, file := range files { + fullPath := filepath.Join(tempDir, file.Path) + + parentDir := filepath.Dir(fullPath) + if err := os.MkdirAll(parentDir, 0o755); err != nil { + panic(err) + } + + f, err := os.Create(fullPath) + if err != nil { + panic(err) + } + + if _, err := f.WriteString(file.Content); err != nil { + f.Close() + panic(err) + } + f.Close() + } + + t.Cleanup(func() { + if dir != nil { + dir.Close() + } + }) + + return dir +} + +func Test_CreateRevisionFromPaths(t *testing.T) { + llcfg := uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 10, + FileSizeLimit: 100, + TotalPayloadSizeLimit: 10_000, + FilePathLengthLimit: 20, + }, + } + + allowList := filters.AllowList{ + ConfigFiles: []string{"go.mod"}, + Extensions: []string{".txt", ".go", ".md"}, + } + + t.Run("mixed files and directories", func(t *testing.T) { + allFiles := []uploadrevision.LoadedFile{ + {Path: "src/main.go", Content: "package main"}, + {Path: "src/utils.go", Content: "package utils"}, + {Path: "config.yaml", Content: "version: 1"}, + {Path: "README.md", Content: "# Project"}, + } + + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) + + paths := []string{ + filepath.Join(dir.Name(), "src"), // Directory + filepath.Join(dir.Name(), "README.md"), // Individual file + } + + res, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + require.Len(t, uploadedFiles, 3) // 2 from src/ + 1 README.md + + uploadedPaths := make([]string, len(uploadedFiles)) + for i, f := range uploadedFiles { + uploadedPaths[i] = f.Path + } + assert.Contains(t, uploadedPaths, "main.go") + assert.Contains(t, uploadedPaths, "utils.go") + assert.Contains(t, uploadedPaths, "README.md") + }) + + t.Run("get filters error", func(t *testing.T) { + allFiles := []uploadrevision.LoadedFile{ + {Path: "src/main.go", Content: "package main"}, + {Path: "src/utils.go", Content: "package utils"}, + {Path: "config.yaml", Content: "version: 1"}, + {Path: "README.md", Content: "# Project"}, + } + + ctx, _, client, dir := setupTest(t, llcfg, allFiles, filters.AllowList{}, assert.AnError) + + paths := []string{ + filepath.Join(dir.Name(), "src"), // Directory + filepath.Join(dir.Name(), "README.md"), // Individual file + } + + _, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) + require.ErrorContains(t, err, "failed to load deeproxy filters") + }) + + t.Run("error handling with better context", func(t *testing.T) { + ctx, _, client, _ := setupTest(t, llcfg, []uploadrevision.LoadedFile{}, allowList, nil) + + paths := []string{ + "/nonexistent/file.go", + "/another/missing/path.txt", + } + + _, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) + require.Error(t, err) + var fileAccessErr *uploadrevision.FileAccessError + assert.ErrorAs(t, err, &fileAccessErr) + assert.Equal(t, "/nonexistent/file.go", fileAccessErr.FilePath) + assert.ErrorContains(t, fileAccessErr.Err, "no such file or directory") + }) +} + +func Test_CreateRevisionFromDir(t *testing.T) { + llcfg := uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 2, + FileSizeLimit: 100, + TotalPayloadSizeLimit: 10_000, + FilePathLengthLimit: 20, + }, + } + + allowList := filters.AllowList{ + ConfigFiles: []string{"go.mod"}, + Extensions: []string{".txt", ".go", ".md"}, + } + + t.Run("uploading a shallow directory", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "file1.txt", + Content: "content1", + }, + { + Path: "file2.txt", + Content: "content2", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory with nested files", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "src/main.go", + Content: "package main\n\nfunc main() {}", + }, + { + Path: "src/utils/helper.go", + Content: "package utils\n\nfunc Helper() {}", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory exceeding the file count limit for a single upload", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "file1.txt", + Content: "root level file", + }, + { + Path: "src/main.go", + Content: "package main\n\nfunc main() {}", + }, + { + Path: "src/utils/helper.go", + Content: "package utils\n\nfunc Helper() {}", + }, + { + Path: "docs/README.md", + Content: "# Project Documentation", + }, + { + Path: "src/go.mod", + Content: "foo bar", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory with file exceeding the file size limit", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "file2.txt", + Content: "foo", + }, + } + additionalFiles := []uploadrevision.LoadedFile{ + { + Path: "file1.txt", + Content: "foo bar", + }, + } + + allFiles := make([]uploadrevision.LoadedFile, 0, 2) + allFiles = append(allFiles, expectedFiles...) + allFiles = append(allFiles, additionalFiles...) + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 2, + FileSizeLimit: 6, + TotalPayloadSizeLimit: 100, + FilePathLengthLimit: 20, + }, + }, allFiles, allowList, nil) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + var fileSizeErr *uploadrevision.FileSizeLimitError + assert.Len(t, res.FilteredFiles, 1) + ff := res.FilteredFiles[0] + assert.Contains(t, ff.Path, "file1.txt") + assert.ErrorAs(t, ff.Reason, &fileSizeErr) + assert.Equal(t, "file1.txt", fileSizeErr.FilePath) + assert.Equal(t, int64(6), fileSizeErr.Limit) + assert.Equal(t, int64(7), fileSizeErr.FileSize) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory exceeding total payload size limit triggers batching", func(t *testing.T) { + // Create files that together exceed the payload size limit but not the count limit + // Each file is 30 bytes, limit is 70 bytes, so 3 files (90 bytes) should be split into 2 batches + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "file1.txt", + Content: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", // 30 bytes + }, + { + Path: "file2.txt", + Content: "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyy", // 30 bytes + }, + { + Path: "file3.txt", + Content: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", // 30 bytes + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 10, // High enough to not trigger count-based batching + FileSizeLimit: 50, // Each file is under this + TotalPayloadSizeLimit: 70, // 70 bytes - forces batching by size + FilePathLengthLimit: 20, + }, + }, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + // Success proves size-based batching works - without it, the low-level client + // would reject the 90-byte payload (limit: 70 bytes). + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading large individual files near payload limit", func(t *testing.T) { + // Tests edge case where individual files are large relative to the payload limit. + // File1: 150 bytes, File2: 80 bytes, File3: 60 bytes; Limit: 200 bytes + // Expected batches: [File1], [File2], [File3] - each file in its own batch + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "large1.txt", + Content: string(make([]byte, 150)), + }, + { + Path: "large2.txt", + Content: string(make([]byte, 80)), + }, + { + Path: "large3.txt", + Content: string(make([]byte, 60)), + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 10, + FileSizeLimit: 160, + TotalPayloadSizeLimit: 200, + FilePathLengthLimit: 20, + }, + }, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading files with variable sizes triggers optimal batching", func(t *testing.T) { + // Tests realistic scenario with mixed file sizes. + // Files: 10, 60, 5, 70, 45 bytes; Limit: 100 bytes + // Expected batching: [10+60+5=75], [70], [45] + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "tiny.txt", + Content: string(make([]byte, 10)), + }, + { + Path: "medium.txt", + Content: string(make([]byte, 60)), + }, + { + Path: "small.txt", + Content: string(make([]byte, 5)), + }, + { + Path: "large.txt", + Content: string(make([]byte, 70)), + }, + { + Path: "mid.txt", + Content: string(make([]byte, 45)), + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 10, + FileSizeLimit: 80, + TotalPayloadSizeLimit: 100, + FilePathLengthLimit: 20, + }, + }, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading directory where both size and count limits would be reached", func(t *testing.T) { + // Tests scenario where both limits are approached. + // 8 files of 30 bytes each = 240 bytes total + // FileCountLimit: 10, TotalPayloadSizeLimit: 200 bytes + // Should batch by size first: [file1-6=180], [file7-8=60] + expectedFiles := make([]uploadrevision.LoadedFile, 8) + for i := 0; i < 8; i++ { + expectedFiles[i] = uploadrevision.LoadedFile{ + Path: fmt.Sprintf("file%d.txt", i), + Content: string(make([]byte, 30)), + } + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 10, + FileSizeLimit: 50, + TotalPayloadSizeLimit: 200, + FilePathLengthLimit: 20, + }, + }, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory applies filtering", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "src/main.go", + Content: "package main\n\nfunc main() {}", + }, + { + Path: "src/utils/helper.go", + Content: "package utils\n\nfunc Helper() {}", + }, + { + Path: "src/go.mod", + Content: "foo bar", + }, + } + additionalFiles := []uploadrevision.LoadedFile{ + { + Path: "src/script.js", + Content: "console.log('hi')", + }, + { + Path: "src/package.json", + Content: "{}", + }, + } + //nolint:gocritic // Not an issue for tests. + allFiles := append(expectedFiles, additionalFiles...) + + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Len(t, res.FilteredFiles, 2) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory with filtering disabled", func(t *testing.T) { + allFiles := []uploadrevision.LoadedFile{ + { + Path: "src/main.go", + Content: "package main\n\nfunc main() {}", + }, + { + Path: "src/utils/helper.go", + Content: "package utils\n\nfunc Helper() {}", + }, + { + Path: "src/go.mod", + Content: "foo bar", + }, + { + Path: "src/script.js", + Content: "console.log('hi')", + }, + { + Path: "src/package.json", + Content: "{}", + }, + } + + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{SkipDeeproxyFiltering: true}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, allFiles, uploadedFiles) + }) +} + +func Test_CreateRevisionFromFile(t *testing.T) { + llcfg := uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 2, + FileSizeLimit: 100, + TotalPayloadSizeLimit: 10_000, + FilePathLengthLimit: 20, + }, + } + + allowList := filters.AllowList{ + ConfigFiles: []string{"go.mod"}, + Extensions: []string{".txt", ".go", ".md"}, + } + + t.Run("uploading a file", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "file1.txt", + Content: "content1", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a file exceeding the file size limit", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "file1.txt", + Content: "foo bar", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 1, + FileSizeLimit: 6, + TotalPayloadSizeLimit: 10_000, + FilePathLengthLimit: 20, + }, + }, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + require.NoError(t, err) + + var fileSizeErr *uploadrevision.FileSizeLimitError + assert.Len(t, res.FilteredFiles, 1) + ff := res.FilteredFiles[0] + assert.Contains(t, ff.Path, "file1.txt") + assert.ErrorAs(t, ff.Reason, &fileSizeErr) + assert.Equal(t, "file1.txt", fileSizeErr.FilePath) + assert.Equal(t, int64(6), fileSizeErr.Limit) + assert.Equal(t, int64(7), fileSizeErr.FileSize) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, nil, uploadedFiles) + }) + + t.Run("uploading a file exceeding the file path limit", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "file1.txt", + Content: "foo bar", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ + Limits: uploadrevision.Limits{ + FileCountLimit: 1, + FileSizeLimit: 10, + TotalPayloadSizeLimit: 10_000, + FilePathLengthLimit: 5, + }, + }, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + require.NoError(t, err) + + var filePathErr *uploadrevision.FilePathLengthLimitError + assert.Len(t, res.FilteredFiles, 1) + ff := res.FilteredFiles[0] + assert.Contains(t, ff.Path, "file1.txt") + assert.ErrorAs(t, ff.Reason, &filePathErr) + assert.Equal(t, "file1.txt", filePathErr.FilePath) + assert.Equal(t, 5, filePathErr.Limit) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, nil, uploadedFiles) + }) + + t.Run("uploading a file applies filtering", func(t *testing.T) { + allFiles := []uploadrevision.LoadedFile{ + { + Path: "script.js", + Content: "console.log('hi')", + }, + } + + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) + + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js"), fileupload.UploadOptions{}) + require.NoError(t, err) + + assert.Len(t, res.FilteredFiles, 1) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, nil, uploadedFiles) + }) + + t.Run("uploading a file with filtering disabled", func(t *testing.T) { + expectedFiles := []uploadrevision.LoadedFile{ + { + Path: "script.js", + Content: "console.log('hi')", + }, + } + + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) + + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js"), fileupload.UploadOptions{SkipDeeproxyFiltering: true}) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) +} + +func expectEqualFiles(t *testing.T, expectedFiles, uploadedFiles []uploadrevision.LoadedFile) { + t.Helper() + + require.Equal(t, len(expectedFiles), len(uploadedFiles)) + + slices.SortFunc(expectedFiles, func(fileA, fileB uploadrevision.LoadedFile) int { + return strings.Compare(fileA.Path, fileB.Path) + }) + + slices.SortFunc(uploadedFiles, func(fileA, fileB uploadrevision.LoadedFile) int { + return strings.Compare(fileA.Path, fileB.Path) + }) + + for i := range uploadedFiles { + assert.Equal(t, expectedFiles[i].Path, uploadedFiles[i].Path) + assert.Equal(t, expectedFiles[i].Content, uploadedFiles[i].Content) + } +} + +func setupTest( + t *testing.T, + llcfg uploadrevision.FakeClientConfig, + files []uploadrevision.LoadedFile, + allowList filters.AllowList, + filtersErr error, +) (context.Context, *uploadrevision.FakeSealableClient, *fileupload.HTTPClient, *os.File) { + t.Helper() + + ctx := context.Background() + orgID := uuid.New() + + fakeSealeableClient := uploadrevision.NewFakeSealableClient(llcfg) + fakeFiltersClient := filters.NewFakeClient(allowList, filtersErr) + client := fileupload.NewClient( + nil, + fileupload.Config{ + OrgID: orgID, + }, + fileupload.WithUploadRevisionSealableClient(fakeSealeableClient), + fileupload.WithFiltersClient(fakeFiltersClient), + ) + + dir := createTmpFiles(t, files) + + return ctx, fakeSealeableClient, client, dir +} diff --git a/pkg/apiclients/fileupload/errors.go b/pkg/apiclients/fileupload/errors.go new file mode 100644 index 000000000..df44bc518 --- /dev/null +++ b/pkg/apiclients/fileupload/errors.go @@ -0,0 +1,36 @@ +package fileupload + +import "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + +// Aliasing uploadRevisionSealableClient errors so that they're scoped to the fileupload package as well. + +// Sentinel errors for common conditions. +var ( + ErrNoFilesProvided = uploadrevision.ErrNoFilesProvided + ErrEmptyOrgID = uploadrevision.ErrEmptyOrgID + ErrEmptyRevisionID = uploadrevision.ErrEmptyRevisionID +) + +// FileSizeLimitError indicates a file exceeds the maximum allowed size. +type FileSizeLimitError = uploadrevision.FileSizeLimitError + +// FilePathLengthLimitError indicates a file's path exceeds the maximum allowed size. +type FilePathLengthLimitError = uploadrevision.FilePathLengthLimitError + +// FileCountLimitError indicates too many files were provided. +type FileCountLimitError = uploadrevision.FileCountLimitError + +// TotalPayloadSizeLimitError indicates the total size of all files exceeds the maximum allowed payload size. +type TotalPayloadSizeLimitError = uploadrevision.TotalPayloadSizeLimitError + +// FileAccessError indicates a file access permission issue. +type FileAccessError = uploadrevision.FileAccessError + +// SpecialFileError indicates a path points to a special file (device, pipe, socket, etc.) instead of a regular file. +type SpecialFileError = uploadrevision.SpecialFileError + +// HTTPError indicates an HTTP request/response error. +type HTTPError = uploadrevision.HTTPError + +// MultipartError indicates an issue with multipart request handling. +type MultipartError = uploadrevision.MultipartError diff --git a/pkg/apiclients/fileupload/fake_client.go b/pkg/apiclients/fileupload/fake_client.go new file mode 100644 index 000000000..7ed7ebb43 --- /dev/null +++ b/pkg/apiclients/fileupload/fake_client.go @@ -0,0 +1,99 @@ +package fileupload + +import ( + "context" + "fmt" + "os" + + "github.com/google/uuid" + + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" +) + +type FakeClient struct { + revisions map[RevisionID][]string + err error + uploadCount int // Tracks how many uploads have occurred + lastRevision RevisionID +} + +var _ Client = (*FakeClient)(nil) + +// NewFakeClient creates a new fake client. +func NewFakeClient() *FakeClient { + return &FakeClient{ + revisions: make(map[RevisionID][]string), + } +} + +// WithError configures the fake to return an error. +func (f *FakeClient) WithError(err error) *FakeClient { + f.err = err + return f +} + +func (f *FakeClient) CreateRevisionFromDir(ctx context.Context, dirPath string, opts UploadOptions) (UploadResult, error) { + if f.err != nil { + return UploadResult{}, f.err + } + + info, err := os.Stat(dirPath) + if err != nil { + return UploadResult{}, uploadrevision.NewFileAccessError(dirPath, err) + } + + if !info.IsDir() { + return UploadResult{}, fmt.Errorf("the provided path is not a directory: %s", dirPath) + } + + return f.CreateRevisionFromPaths(ctx, []string{dirPath}, opts) +} + +func (f *FakeClient) CreateRevisionFromFile(ctx context.Context, filePath string, opts UploadOptions) (UploadResult, error) { + if f.err != nil { + return UploadResult{}, f.err + } + + info, err := os.Stat(filePath) + if err != nil { + return UploadResult{}, uploadrevision.NewFileAccessError(filePath, err) + } + + if !info.Mode().IsRegular() { + return UploadResult{}, fmt.Errorf("the provided path is not a regular file: %s", filePath) + } + + return f.CreateRevisionFromPaths(ctx, []string{filePath}, opts) +} + +func (f *FakeClient) CreateRevisionFromPaths(ctx context.Context, paths []string, opts UploadOptions) (UploadResult, error) { + if f.err != nil { + return UploadResult{}, f.err + } + + revID := uuid.New() + f.revisions[revID] = append([]string(nil), paths...) + f.uploadCount++ + f.lastRevision = revID + + return UploadResult{RevisionID: revID, UploadedFilesCount: len(paths)}, nil +} + +func (f *FakeClient) GetRevisionPaths(revID RevisionID) []string { + return f.revisions[revID] +} + +// UploadOccurred returns true if at least one upload has been performed. +func (f *FakeClient) UploadOccurred() bool { + return f.uploadCount > 0 +} + +// GetUploadCount returns the number of uploads that have occurred. +func (f *FakeClient) GetUploadCount() int { + return f.uploadCount +} + +// GetLastRevisionID returns the ID of the most recent revision created. +func (f *FakeClient) GetLastRevisionID() RevisionID { + return f.lastRevision +} diff --git a/pkg/apiclients/fileupload/files/list_sources.go b/pkg/apiclients/fileupload/files/list_sources.go new file mode 100644 index 000000000..7f31d65a8 --- /dev/null +++ b/pkg/apiclients/fileupload/files/list_sources.go @@ -0,0 +1,21 @@ +package listsources + +import ( + "fmt" + + "github.com/rs/zerolog" + + "github.com/snyk/go-application-framework/pkg/utils" +) + +// ForPath returns a channel that notifies each file in the path that doesn't match the filter rules. +func ForPath(path string, logger *zerolog.Logger, maxThreads int) (<-chan string, error) { + filter := utils.NewFileFilter(path, logger, utils.WithThreadNumber(maxThreads)) + rules, err := filter.GetRules([]string{".gitignore", ".dcignore", ".snyk"}) + if err != nil { + return nil, fmt.Errorf("failed to get rules: %w", err) + } + + results := filter.GetFilteredFiles(filter.GetAllFiles(), rules) + return results, nil +} diff --git a/pkg/apiclients/fileupload/files/list_sources_test.go b/pkg/apiclients/fileupload/files/list_sources_test.go new file mode 100644 index 000000000..80e63867d --- /dev/null +++ b/pkg/apiclients/fileupload/files/list_sources_test.go @@ -0,0 +1,51 @@ +package listsources_test + +import ( + "fmt" + "io" + "path/filepath" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + listsources "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/files" +) + +func Test_ListsSources_Simplest(t *testing.T) { + sourcesDir := filepath.Join("testdata", "simplest") + + files, err := listSourcesForPath(sourcesDir) + require.NoError(t, err) + assert.Len(t, files, 2, "Expecting 2 files") + assert.Contains(t, files, filepath.Join(sourcesDir, "package.json")) + assert.Contains(t, files, filepath.Join(sourcesDir, "src", "index.js")) +} + +func Test_ListsSources_WithIgnores(t *testing.T) { + sourcesDir := filepath.Join("testdata", "with-ignores") + + files, err := listSourcesForPath(sourcesDir) + require.NoError(t, err) + + assert.Len(t, files, 3, "Expecting 3 files") + assert.Contains(t, files, filepath.Join(sourcesDir, ".gitignore")) + assert.Contains(t, files, filepath.Join(sourcesDir, "package.json")) + assert.Contains(t, files, filepath.Join(sourcesDir, "src", "with-ignores.js")) +} + +func listSourcesForPath(sourcesDir string) ([]string, error) { + mockLogger := zerolog.New(io.Discard) + filesCh, err := listsources.ForPath(sourcesDir, &mockLogger, 2) + if err != nil { + return nil, fmt.Errorf("failed to list sources: %w", err) + } + + files := []string{} + for file := range filesCh { + files = append(files, file) + } + + return files, nil +} diff --git a/pkg/apiclients/fileupload/files/testdata/simplest/package.json b/pkg/apiclients/fileupload/files/testdata/simplest/package.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/pkg/apiclients/fileupload/files/testdata/simplest/package.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/pkg/apiclients/fileupload/files/testdata/simplest/src/index.js b/pkg/apiclients/fileupload/files/testdata/simplest/src/index.js new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/apiclients/fileupload/files/testdata/with-ignores/.gitignore b/pkg/apiclients/fileupload/files/testdata/with-ignores/.gitignore new file mode 100644 index 000000000..344b2f249 --- /dev/null +++ b/pkg/apiclients/fileupload/files/testdata/with-ignores/.gitignore @@ -0,0 +1 @@ +**/gitignored.js diff --git a/pkg/apiclients/fileupload/files/testdata/with-ignores/package.json b/pkg/apiclients/fileupload/files/testdata/with-ignores/package.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/pkg/apiclients/fileupload/files/testdata/with-ignores/package.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/pkg/apiclients/fileupload/files/testdata/with-ignores/src/with-ignores.js b/pkg/apiclients/fileupload/files/testdata/with-ignores/src/with-ignores.js new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/apiclients/fileupload/filter.go b/pkg/apiclients/fileupload/filter.go new file mode 100644 index 000000000..3ad664d09 --- /dev/null +++ b/pkg/apiclients/fileupload/filter.go @@ -0,0 +1,17 @@ +package fileupload + +import "os" + +type fileToFilter struct { + Path string + Stat os.FileInfo +} + +// FilteredFile represents a file that was filtered. +// It includes the filtered file's path and the reason it was filtered. +type FilteredFile struct { + Path string + Reason error +} + +type filter func(fileToFilter) *FilteredFile diff --git a/pkg/apiclients/fileupload/filters/client.go b/pkg/apiclients/fileupload/filters/client.go new file mode 100644 index 000000000..78ed2faba --- /dev/null +++ b/pkg/apiclients/fileupload/filters/client.go @@ -0,0 +1,78 @@ +package filters + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/google/uuid" +) + +// AllowList represents the response structure from the deeproxy filters API. +type AllowList struct { + ConfigFiles []string `json:"configFiles"` + Extensions []string `json:"extensions"` +} + +// Client defines the interface for the filters client. +type Client interface { + GetFilters(ctx context.Context, orgID uuid.UUID) (AllowList, error) +} + +// DeeproxyClient is the deeproxy implementation of the Client interface. +type DeeproxyClient struct { + httpClient *http.Client + cfg Config +} + +// Config contains the configuration for the filters client. +type Config struct { + BaseURL string + IsFedRamp bool +} + +var _ Client = (*DeeproxyClient)(nil) + +// NewDeeproxyClient creates a new DeeproxyClient with the given configuration and options. +func NewDeeproxyClient(cfg Config, opts ...Opt) *DeeproxyClient { + c := &DeeproxyClient{ + cfg: cfg, + httpClient: http.DefaultClient, + } + + for _, opt := range opts { + opt(c) + } + + return c +} + +// GetFilters returns the deeproxy filters in the form of an AllowList. +func (c *DeeproxyClient) GetFilters(ctx context.Context, orgID uuid.UUID) (AllowList, error) { + var allowList AllowList + + url := getFilterURL(c.cfg.BaseURL, orgID, c.cfg.IsFedRamp) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) + if err != nil { + return allowList, fmt.Errorf("failed to create deeproxy filters request: %w", err) + } + + req.Header.Set("snyk-org-name", orgID.String()) + + resp, err := c.httpClient.Do(req) + if err != nil { + return allowList, fmt.Errorf("error making deeproxy filters request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return allowList, fmt.Errorf("unexpected response code: %s", resp.Status) + } + + if err := json.NewDecoder(resp.Body).Decode(&allowList); err != nil { + return allowList, fmt.Errorf("failed to decode deeproxy filters response: %w", err) + } + + return allowList, nil +} diff --git a/pkg/apiclients/fileupload/filters/client_test.go b/pkg/apiclients/fileupload/filters/client_test.go new file mode 100644 index 000000000..6278dafaf --- /dev/null +++ b/pkg/apiclients/fileupload/filters/client_test.go @@ -0,0 +1,77 @@ +package filters //nolint:testpackage // Testing private utility functions. + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClients(t *testing.T) { + tests := []struct { + getClient func(t *testing.T, orgID uuid.UUID, expectedAllow AllowList) (Client, func()) + clientName string + }{ + { + clientName: "deeproxyClient", + getClient: func(t *testing.T, orgID uuid.UUID, expectedAllow AllowList) (Client, func()) { + t.Helper() + + s := setupServer(t, orgID, expectedAllow) + cleanup := func() { + s.Close() + } + c := NewDeeproxyClient(Config{BaseURL: s.URL, IsFedRamp: true}, WithHTTPClient(s.Client())) + + return c, cleanup + }, + }, + { + clientName: "fakeClient", + getClient: func(t *testing.T, _ uuid.UUID, expectedAllow AllowList) (Client, func()) { + t.Helper() + + c := NewFakeClient(expectedAllow, nil) + + return c, func() {} + }, + }, + } + + for _, testData := range tests { + t.Run(testData.clientName+": GetFilters", func(t *testing.T) { + orgID := uuid.New() + expectedAllow := AllowList{ + ConfigFiles: []string{"package.json"}, + Extensions: []string{".ts", ".js"}, + } + client, cleanup := testData.getClient(t, orgID, expectedAllow) + defer cleanup() + + allow, err := client.GetFilters(t.Context(), orgID) + require.NoError(t, err) + + assert.Equal(t, expectedAllow, allow) + }) + } +} + +func setupServer(t *testing.T, orgID uuid.UUID, expectedAllow AllowList) *httptest.Server { + t.Helper() + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expectedURL := getFilterURL("", orgID, true) + assert.Equal(t, expectedURL, r.URL.Path) + assert.Equal(t, orgID.String(), r.Header.Get("snyk-org-name")) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(expectedAllow); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + return + } + })) + ts.Start() + return ts +} diff --git a/pkg/apiclients/fileupload/filters/fake_client.go b/pkg/apiclients/fileupload/filters/fake_client.go new file mode 100644 index 000000000..7245ff7e0 --- /dev/null +++ b/pkg/apiclients/fileupload/filters/fake_client.go @@ -0,0 +1,25 @@ +package filters + +import ( + "context" + + "github.com/google/uuid" +) + +type FakeClient struct { + getFilters func(ctx context.Context, orgID uuid.UUID) (AllowList, error) +} + +var _ Client = (*FakeClient)(nil) + +func NewFakeClient(allowList AllowList, err error) *FakeClient { + return &FakeClient{ + getFilters: func(ctx context.Context, orgID uuid.UUID) (AllowList, error) { + return allowList, err + }, + } +} + +func (f *FakeClient) GetFilters(ctx context.Context, orgID uuid.UUID) (AllowList, error) { + return f.getFilters(ctx, orgID) +} diff --git a/pkg/apiclients/fileupload/filters/opts.go b/pkg/apiclients/fileupload/filters/opts.go new file mode 100644 index 000000000..0ce885788 --- /dev/null +++ b/pkg/apiclients/fileupload/filters/opts.go @@ -0,0 +1,13 @@ +package filters + +import "net/http" + +// Opt is a function that configures an deeproxyClient instance. +type Opt func(*DeeproxyClient) + +// WithHTTPClient sets a custom HTTP client for the filters client. +func WithHTTPClient(httpClient *http.Client) Opt { + return func(c *DeeproxyClient) { + c.httpClient = httpClient + } +} diff --git a/pkg/apiclients/fileupload/filters/utils.go b/pkg/apiclients/fileupload/filters/utils.go new file mode 100644 index 000000000..457998f00 --- /dev/null +++ b/pkg/apiclients/fileupload/filters/utils.go @@ -0,0 +1,17 @@ +package filters + +import ( + "fmt" + "strings" + + "github.com/google/uuid" +) + +func getFilterURL(baseURL string, orgID uuid.UUID, isFedRamp bool) string { + if isFedRamp { + return fmt.Sprintf("%s/hidden/orgs/%s/code/filters", baseURL, orgID) + } + + deeproxyURL := strings.ReplaceAll(baseURL, "api", "deeproxy") + return fmt.Sprintf("%s/filters", deeproxyURL) +} diff --git a/pkg/apiclients/fileupload/filters/utils_test.go b/pkg/apiclients/fileupload/filters/utils_test.go new file mode 100644 index 000000000..62f68fef4 --- /dev/null +++ b/pkg/apiclients/fileupload/filters/utils_test.go @@ -0,0 +1,20 @@ +package filters //nolint:testpackage // Testing private utility functions. + +import ( + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" +) + +var orgID = uuid.MustParse("738ef92e-21cc-4a11-8c13-388d89272f4b") + +func Test_getBaseUrl_notFedramp(t *testing.T) { + actualURL := getFilterURL("https://api.snyk.io", orgID, false) + assert.Equal(t, "https://deeproxy.snyk.io/filters", actualURL) +} + +func Test_getBaseUrl_fedramp(t *testing.T) { + actualURL := getFilterURL("https://api.snyk.io", orgID, true) + assert.Equal(t, "https://api.snyk.io/hidden/orgs/738ef92e-21cc-4a11-8c13-388d89272f4b/code/filters", actualURL) +} diff --git a/pkg/apiclients/fileupload/opts.go b/pkg/apiclients/fileupload/opts.go new file mode 100644 index 000000000..c56f7e6af --- /dev/null +++ b/pkg/apiclients/fileupload/opts.go @@ -0,0 +1,32 @@ +package fileupload + +import ( + "github.com/rs/zerolog" + + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/filters" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" +) + +// Option allows customizing the Client during construction. +type Option func(*HTTPClient) + +// WithUploadRevisionSealableClient allows injecting a custom low-level client (primarily for testing). +func WithUploadRevisionSealableClient(client uploadrevision.SealableClient) Option { + return func(c *HTTPClient) { + c.uploadRevisionSealableClient = client + } +} + +// WithFiltersClient allows injecting a custom low-level client (primarily for testing). +func WithFiltersClient(client filters.Client) Option { + return func(c *HTTPClient) { + c.filtersClient = client + } +} + +// WithLogger allows injecting a custom logger instance. +func WithLogger(logger *zerolog.Logger) Option { + return func(h *HTTPClient) { + h.logger = logger + } +} diff --git a/pkg/apiclients/fileupload/types.go b/pkg/apiclients/fileupload/types.go new file mode 100644 index 000000000..bfde04d4a --- /dev/null +++ b/pkg/apiclients/fileupload/types.go @@ -0,0 +1,35 @@ +package fileupload + +import ( + "sync" + + "github.com/puzpuzpuz/xsync" + + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" +) + +// OrgID represents an organization identifier. +type OrgID = uploadrevision.OrgID + +// RevisionID represents a revision identifier. +type RevisionID = uploadrevision.RevisionID + +// Filters holds the filtering configuration for file uploads with thread-safe maps. +type Filters struct { + supportedExtensions *xsync.MapOf[string, bool] + supportedConfigFiles *xsync.MapOf[string, bool] + once sync.Once + initErr error +} + +// UploadOptions configures the behavior of file upload operations. +type UploadOptions struct { + SkipDeeproxyFiltering bool +} + +// UploadResult respresents the result of the upload. +type UploadResult struct { + RevisionID RevisionID // The ID of the revision which was created. + UploadedFilesCount int // The number of uploaded files. + FilteredFiles []FilteredFile // The list of files which were filtered. +} diff --git a/pkg/apiclients/fileupload/uploadrevision/client.go b/pkg/apiclients/fileupload/uploadrevision/client.go new file mode 100644 index 000000000..95d2dd2e0 --- /dev/null +++ b/pkg/apiclients/fileupload/uploadrevision/client.go @@ -0,0 +1,300 @@ +package uploadrevision + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + + "github.com/google/uuid" + "github.com/snyk/error-catalog-golang-public/snyk_errors" +) + +// SealableClient defines the interface for file upload API operations. +type SealableClient interface { + CreateRevision(ctx context.Context, orgID OrgID) (*ResponseBody, error) + UploadFiles(ctx context.Context, orgID OrgID, revisionID RevisionID, files []UploadFile) error + SealRevision(ctx context.Context, orgID OrgID, revisionID RevisionID) (*SealResponseBody, error) + + GetLimits() Limits +} + +// This will force go to complain if the type doesn't satisfy the interface. +var _ SealableClient = (*HTTPSealableClient)(nil) + +// Config contains the configuration for the file upload client. +type Config struct { + BaseURL string +} + +// HTTPSealableClient implements the SealableClient interface for file upload operations via HTTP API. +type HTTPSealableClient struct { + cfg Config + httpClient *http.Client +} + +// apiVersion specifies the API version to use for requests. +const apiVersion = "2024-10-15" + +const ( + fileSizeLimit = 50_000_000 // 50MB - maximum size per individual file + fileCountLimit = 300_000 // 300,000 - maximum number of files per request + totalPayloadSizeLimit = 200_000_000 // 200MB - maximum total uncompressed payload size per request + filePathLengthLimit = 256 // 256 - maximum length of file names +) + +// NewClient creates a new file upload client with the given configuration and options. +func NewClient(cfg Config, opts ...Opt) *HTTPSealableClient { + httpClient := &http.Client{ + Transport: http.DefaultTransport, + } + c := HTTPSealableClient{cfg, httpClient} + + for _, opt := range opts { + opt(&c) + } + + return &c +} + +// CreateRevision creates a new upload revision for the specified organization. +func (c *HTTPSealableClient) CreateRevision(ctx context.Context, orgID OrgID) (*ResponseBody, error) { + if orgID == uuid.Nil { + return nil, ErrEmptyOrgID + } + + body := RequestBody{ + Data: RequestData{ + Attributes: RequestAttributes{ + RevisionType: RevisionTypeSnapshot, + }, + Type: ResourceTypeUploadRevision, + }, + } + buff := bytes.NewBuffer(nil) + if err := json.NewEncoder(buff).Encode(body); err != nil { + return nil, fmt.Errorf("failed to encode request body: %w", err) + } + + url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions?version=%s", c.cfg.BaseURL, orgID, apiVersion) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, buff) + if err != nil { + return nil, fmt.Errorf("failed to create revision request: %w", err) + } + req.Header.Set(ContentType, "application/vnd.api+json") + + res, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("error making create revision request: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusCreated { + return nil, handleUnexpectedStatusCodes(res.Body, res.StatusCode, res.Status, "create upload revision") + } + + var respBody ResponseBody + if err := json.NewDecoder(res.Body).Decode(&respBody); err != nil { + return nil, fmt.Errorf("failed to decode upload revision response: %w", err) + } + + return &respBody, nil +} + +// UploadFiles uploads the provided files to the specified revision. It will not close the file descriptors. +func (c *HTTPSealableClient) UploadFiles(ctx context.Context, orgID OrgID, revisionID RevisionID, files []UploadFile) error { + if orgID == uuid.Nil { + return ErrEmptyOrgID + } + + if revisionID == uuid.Nil { + return ErrEmptyRevisionID + } + + if err := validateFiles(files); err != nil { + return err + } + + // Create pipe for multipart data + pipeReader, pipeWriter := io.Pipe() + defer pipeReader.Close() + + mpartWriter := multipart.NewWriter(pipeWriter) + + go streamFilesToPipe(pipeWriter, mpartWriter, files) + body := compressRequestBody(pipeReader) + + // Load body bytes into memmory so go can determine the Content-Length + // and not send the request chunked + bts, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("failed to create upload files request: %w", err) + } + + url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions/%s/files?version=%s", c.cfg.BaseURL, orgID, revisionID, apiVersion) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bts)) + if err != nil { + return fmt.Errorf("failed to create upload files request: %w", err) + } + req.Header.Set(ContentType, mpartWriter.FormDataContentType()) + req.Header.Set(ContentEncoding, "gzip") + + res, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("error making upload files request: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusNoContent { + return handleUnexpectedStatusCodes(res.Body, res.StatusCode, res.Status, "upload files") + } + + return nil +} + +// streamFilesToPipe writes files to the multipart form. +func streamFilesToPipe(pipeWriter *io.PipeWriter, mpartWriter *multipart.Writer, files []UploadFile) { + var streamError error + defer func() { + if closeErr := mpartWriter.Close(); closeErr != nil && streamError == nil { + streamError = closeErr + } + pipeWriter.CloseWithError(streamError) + }() + + for _, file := range files { + // Create form file part + part, err := mpartWriter.CreateFormFile(file.Path, file.Path) + if err != nil { + streamError = NewMultipartError(file.Path, err) + return + } + + if _, err := io.Copy(part, file.File); err != nil { + streamError = fmt.Errorf("failed to copy file content for %s: %w", file.Path, err) + return + } + } +} + +// validateFiles validates the files before upload. +func validateFiles(files []UploadFile) error { + if len(files) > fileCountLimit { + return NewFileCountLimitError(len(files), fileCountLimit) + } + + if len(files) == 0 { + return ErrNoFilesProvided + } + + var totalPayloadSize int64 + for _, file := range files { + if len(file.Path) > filePathLengthLimit { + return NewFilePathLengthLimitError(file.Path, len(file.Path), filePathLengthLimit) + } + + fileInfo, err := file.File.Stat() + if err != nil { + return NewFileAccessError(file.Path, err) + } + + if !fileInfo.Mode().IsRegular() { + return NewSpecialFileError(file.Path, fileInfo.Mode()) + } + + if fileInfo.Size() > fileSizeLimit { + return NewFileSizeLimitError(file.Path, fileInfo.Size(), fileSizeLimit) + } + + totalPayloadSize += fileInfo.Size() + } + + if totalPayloadSize > totalPayloadSizeLimit { + return NewTotalPayloadSizeLimitError(totalPayloadSize, totalPayloadSizeLimit) + } + + return nil +} + +// SealRevision seals the specified upload revision, marking it as complete. +func (c *HTTPSealableClient) SealRevision(ctx context.Context, orgID OrgID, revisionID RevisionID) (*SealResponseBody, error) { + if orgID == uuid.Nil { + return nil, ErrEmptyOrgID + } + + if revisionID == uuid.Nil { + return nil, ErrEmptyRevisionID + } + + body := SealRequestBody{ + Data: SealRequestData{ + ID: revisionID, + Attributes: SealRequestAttributes{ + Sealed: true, + }, + Type: ResourceTypeUploadRevision, + }, + } + buff := bytes.NewBuffer(nil) + if err := json.NewEncoder(buff).Encode(body); err != nil { + return nil, fmt.Errorf("failed to encode request body: %w", err) + } + + url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions/%s?version=%s", c.cfg.BaseURL, orgID, revisionID, apiVersion) + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, url, buff) + if err != nil { + return nil, fmt.Errorf("failed to create seal request: %w", err) + } + req.Header.Set(ContentType, "application/vnd.api+json") + + res, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("error making seal revision request: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, handleUnexpectedStatusCodes(res.Body, res.StatusCode, res.Status, "seal upload revision") + } + + var respBody SealResponseBody + if err := json.NewDecoder(res.Body).Decode(&respBody); err != nil { + return nil, fmt.Errorf("failed to decode upload revision response: %w", err) + } + + return &respBody, nil +} + +func handleUnexpectedStatusCodes(body io.ReadCloser, statusCode int, status, operation string) error { + bts, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + if len(bts) > 0 { + snykErrorList, parseErr := snyk_errors.FromJSONAPIErrorBytes(bts) + if parseErr == nil && len(snykErrorList) > 0 && snykErrorList[0].Title != "" { + errsToJoin := []error{} + for i := range snykErrorList { + errsToJoin = append(errsToJoin, snykErrorList[i]) + } + return fmt.Errorf("api error during %s: %w", operation, errors.Join(errsToJoin...)) + } + } + + return NewHTTPError(statusCode, status, operation, bts) +} + +// GetLimits returns the upload Limits defined in the low level client. +func (c *HTTPSealableClient) GetLimits() Limits { + return Limits{ + FileCountLimit: fileCountLimit, + FileSizeLimit: fileSizeLimit, + TotalPayloadSizeLimit: totalPayloadSizeLimit, + FilePathLengthLimit: filePathLengthLimit, + } +} diff --git a/pkg/apiclients/fileupload/uploadrevision/client_test.go b/pkg/apiclients/fileupload/uploadrevision/client_test.go new file mode 100644 index 000000000..e54c1017a --- /dev/null +++ b/pkg/apiclients/fileupload/uploadrevision/client_test.go @@ -0,0 +1,565 @@ +package uploadrevision_test + +import ( + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "io/fs" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path" + "strings" + "testing" + "testing/fstest" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" +) + +var ( + orgID = uuid.MustParse("9102b78b-c28d-4392-a39f-08dd26fd9622") + revID = uuid.MustParse("ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f") +) + +func TestClient_CreateRevision(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + resp, err := c.CreateRevision(context.Background(), orgID) + + require.NoError(t, err) + expectedID := uuid.MustParse("a7d975fb-2076-49b7-bc1f-31c395c3ce93") + assert.Equal(t, expectedID, resp.Data.ID) +} + +func TestClient_CreateRevision_EmptyOrgID(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + resp, err := c.CreateRevision(context.Background(), uuid.Nil) + + assert.Error(t, err) + assert.Nil(t, resp) + assert.ErrorIs(t, err, uploadrevision.ErrEmptyOrgID) +} + +func TestClient_CreateRevision_ServerError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + c := uploadrevision.NewClient(uploadrevision.Config{ + BaseURL: srv.URL, + }) + + resp, err := c.CreateRevision(context.Background(), orgID) + + assert.Nil(t, resp) + var httpErr *uploadrevision.HTTPError + assert.ErrorAs(t, err, &httpErr) + assert.Equal(t, http.StatusInternalServerError, httpErr.StatusCode) + assert.Equal(t, "create upload revision", httpErr.Operation) +} + +func TestClient_UploadFiles(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + mockFS := fstest.MapFS{ + "foo/bar": {Data: []byte("asdf")}, + } + fd, err := mockFS.Open("foo/bar") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision.UploadFile{ + {Path: "foo/bar", File: fd}, + }) + + require.NoError(t, err) +} + +func TestClient_UploadFiles_MultipleFiles(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + mockFS := fstest.MapFS{ + "file1.txt": {Data: []byte("content1")}, + "file2.json": {Data: []byte("content2")}, + } + + file1, err := mockFS.Open("file1.txt") + require.NoError(t, err) + file2, err := mockFS.Open("file2.json") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision.UploadFile{ + {Path: "file1.txt", File: file1}, + {Path: "file2.json", File: file2}, + }) + + require.NoError(t, err) +} + +func TestClient_UploadFiles_EmptyOrgID(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + mockFS := fstest.MapFS{ + "test.txt": {Data: []byte("content")}, + } + file, err := mockFS.Open("test.txt") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + uuid.Nil, // empty orgID + revID, + []uploadrevision.UploadFile{ + {Path: "test.txt", File: file}, + }) + + assert.Error(t, err) + assert.ErrorIs(t, err, uploadrevision.ErrEmptyOrgID) +} + +func TestClient_UploadFiles_EmptyRevisionID(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + mockFS := fstest.MapFS{ + "test.txt": {Data: []byte("content")}, + } + file, err := mockFS.Open("test.txt") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + orgID, + uuid.Nil, // empty revisionID + []uploadrevision.UploadFile{ + {Path: "test.txt", File: file}, + }) + + assert.Error(t, err) + assert.ErrorIs(t, err, uploadrevision.ErrEmptyRevisionID) +} + +func TestClient_UploadFiles_FileSizeLimit(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + largeContent := make([]byte, c.GetLimits().FileSizeLimit+1) + mockFS := fstest.MapFS{ + "large_file.txt": {Data: largeContent}, + } + + file, err := mockFS.Open("large_file.txt") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision.UploadFile{ + {Path: "large_file.txt", File: file}, + }) + + assert.Error(t, err) + var fileSizeErr *uploadrevision.FileSizeLimitError + assert.ErrorAs(t, err, &fileSizeErr) + assert.Equal(t, "large_file.txt", fileSizeErr.FilePath) + assert.Equal(t, c.GetLimits().FileSizeLimit+1, fileSizeErr.FileSize) + assert.Equal(t, c.GetLimits().FileSizeLimit, fileSizeErr.Limit) +} + +func TestClient_UploadFiles_FileCountLimit(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + files := make([]uploadrevision.UploadFile, c.GetLimits().FileCountLimit+1) + mockFS := fstest.MapFS{} + + for i := range c.GetLimits().FileCountLimit + 1 { + filename := fmt.Sprintf("file%d.txt", i) + mockFS[filename] = &fstest.MapFile{Data: []byte("content")} + + file, err := mockFS.Open(filename) + require.NoError(t, err) + + files[i] = uploadrevision.UploadFile{ + Path: filename, + File: file, + } + } + + err := c.UploadFiles(context.Background(), orgID, revID, files) + + assert.Error(t, err) + var fileCountErr *uploadrevision.FileCountLimitError + assert.ErrorAs(t, err, &fileCountErr) + assert.Equal(t, c.GetLimits().FileCountLimit+1, fileCountErr.Count) + assert.Equal(t, c.GetLimits().FileCountLimit, fileCountErr.Limit) +} + +func TestClient_UploadFiles_FilePathLengthLimit(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + // Create a file path that exceeds the limit + longFilePath := strings.Repeat("a", c.GetLimits().FilePathLengthLimit+1) + + mockFS := fstest.MapFS{ + "short_file.txt": {Data: []byte("content")}, + } + + file, err := mockFS.Open("short_file.txt") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision.UploadFile{ + {Path: longFilePath, File: file}, + }) + + assert.Error(t, err) + var filePathLengthErr *uploadrevision.FilePathLengthLimitError + assert.ErrorAs(t, err, &filePathLengthErr) + assert.Equal(t, longFilePath, filePathLengthErr.FilePath) + assert.Equal(t, c.GetLimits().FilePathLengthLimit+1, filePathLengthErr.Length) + assert.Equal(t, c.GetLimits().FilePathLengthLimit, filePathLengthErr.Limit) +} + +func TestClient_UploadFiles_FilePathLengthExactlyAtLimit(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + // Create a file name that is exactly at the limit + filePathAtLimit := strings.Repeat("a", c.GetLimits().FilePathLengthLimit) + + mockFS := fstest.MapFS{ + "short_file.txt": {Data: []byte("content")}, + } + + file, err := mockFS.Open("short_file.txt") + require.NoError(t, err) + + // This should not error since the file path is exactly at the limit + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision.UploadFile{ + {Path: filePathAtLimit, File: file}, + }) + + assert.NoError(t, err) +} + +func TestClient_UploadFiles_TotalPayloadSizeLimit(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + // Create multiple files that individually are under the size limit, + // but together exceed the total payload size limit + mockFS := fstest.MapFS{} + files := []uploadrevision.UploadFile{} + + // Use files that are 30MB each (under the 50MB individual limit) + // 8 files = 240MB > 200MB total limit + fileSize := int64(30_000_000) + numFiles := 8 + + for i := range numFiles { + filename := fmt.Sprintf("file%d.txt", i) + mockFS[filename] = &fstest.MapFile{Data: make([]byte, fileSize)} + + file, err := mockFS.Open(filename) + require.NoError(t, err) + + files = append(files, uploadrevision.UploadFile{ + Path: filename, + File: file, + }) + } + + err := c.UploadFiles(context.Background(), orgID, revID, files) + + assert.Error(t, err) + var totalSizeErr *uploadrevision.TotalPayloadSizeLimitError + assert.ErrorAs(t, err, &totalSizeErr) + assert.Equal(t, fileSize*int64(numFiles), totalSizeErr.TotalSize) + assert.Equal(t, c.GetLimits().TotalPayloadSizeLimit, totalSizeErr.Limit) +} + +func TestClient_UploadFiles_TotalPayloadSizeExactlyAtLimit(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + // Test boundary: exactly 200MB (should succeed) + mockFS := fstest.MapFS{} + files := []uploadrevision.UploadFile{} + + // Create files that sum exactly to 200MB + // 4 files of 50MB each = 200MB exactly + fileSize := int64(50_000_000) + numFiles := 4 + + for i := 0; i < numFiles; i++ { + filename := fmt.Sprintf("file%d.txt", i) + mockFS[filename] = &fstest.MapFile{Data: make([]byte, fileSize)} + + file, err := mockFS.Open(filename) + require.NoError(t, err) + + files = append(files, uploadrevision.UploadFile{ + Path: filename, + File: file, + }) + } + + err := c.UploadFiles(context.Background(), orgID, revID, files) + + // Should succeed - exactly at limit is allowed + assert.NoError(t, err) +} + +func TestClient_UploadFiles_IndividualFileSizeExactlyAtLimit(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + // Test boundary: individual file exactly 50MB (should succeed) + mockFS := fstest.MapFS{ + "exact_limit.bin": {Data: make([]byte, c.GetLimits().FileSizeLimit)}, + } + + file, err := mockFS.Open("exact_limit.bin") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), orgID, revID, []uploadrevision.UploadFile{ + {Path: "exact_limit.bin", File: file}, + }) + + // Should succeed - exactly at limit is allowed + assert.NoError(t, err) +} + +func TestClient_UploadFiles_SpecialFileError(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + tests := []struct { + name string + setupFS func() (fstest.MapFS, string) + setupRealFile func() string + }{ + { + name: "directory file", + setupFS: func() (fstest.MapFS, string) { + return fstest.MapFS{ + "test-directory": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, "test-directory" + }, + }, + { + name: "device file", + setupRealFile: func() string { + return "/dev/null" + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var file fs.File + var filePath string + var err error + + if tt.setupFS != nil { + mockFS, path := tt.setupFS() + filePath = path + file, err = mockFS.Open(path) + require.NoError(t, err) + } else if tt.setupRealFile != nil { + filePath = tt.setupRealFile() + + realFile, openErr := os.Open(filePath) + require.NoError(t, openErr) + defer realFile.Close() + file = realFile + } + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision.UploadFile{ + {Path: filePath, File: file}, + }) + + assert.Error(t, err) + + var sfe *uploadrevision.SpecialFileError + assert.ErrorAs(t, err, &sfe) + assert.Equal(t, filePath, sfe.FilePath) + }) + } +} + +func TestClient_UploadFiles_Symlink(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + tmpDir := t.TempDir() + tmpFile := path.Join(tmpDir, "temp-regular-file") + + err := os.WriteFile(tmpFile, []byte("foo bar"), 0o600) + require.NoError(t, err) + + tmpSlnPth := path.Join(tmpDir, "temp-symlink") + err = os.Symlink(tmpFile, tmpSlnPth) + require.NoError(t, err) + + tmpSln, err := os.Open(tmpSlnPth) + require.NoError(t, err) + defer tmpSln.Close() + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision.UploadFile{ + {Path: tmpSlnPth, File: tmpSln}, + }) + + assert.NoError(t, err) +} + +func TestClient_UploadFiles_EmptyFileList(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + err := c.UploadFiles(context.Background(), orgID, revID, []uploadrevision.UploadFile{}) + + assert.Error(t, err) + assert.ErrorIs(t, err, uploadrevision.ErrNoFilesProvided) +} + +func TestClient_SealRevision(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + resp, err := c.SealRevision(context.Background(), orgID, revID) + + require.NoError(t, err) + assert.Equal(t, revID, resp.Data.ID) + assert.True(t, resp.Data.Attributes.Sealed) +} + +func TestClient_SealRevision_EmptyOrgID(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + resp, err := c.SealRevision(context.Background(), + uuid.Nil, // empty orgID + revID, + ) + + assert.Error(t, err) + assert.ErrorIs(t, err, uploadrevision.ErrEmptyOrgID) + assert.Nil(t, resp) +} + +func TestClient_SealRevision_EmptyRevisionID(t *testing.T) { + c := uploadrevision.NewClient(uploadrevision.Config{}) + + resp, err := c.SealRevision(context.Background(), + orgID, + uuid.Nil, // empty revisionID + ) + + assert.Error(t, err) + assert.ErrorIs(t, err, uploadrevision.ErrEmptyRevisionID) + assert.Nil(t, resp) +} + +func setupTestServer(t *testing.T) (*httptest.Server, *uploadrevision.HTTPSealableClient) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "2024-10-15", r.URL.Query().Get("version")) + + switch { + // Create revision + case r.Method == http.MethodPost && + r.URL.Path == "/hidden/orgs/9102b78b-c28d-4392-a39f-08dd26fd9622/upload_revisions": + + assert.Equal(t, "application/vnd.api+json", r.Header.Get("Content-Type")) + + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{ + "data": { + "attributes": { + "revision_type": "snapshot", + "sealed": false + }, + "id": "a7d975fb-2076-49b7-bc1f-31c395c3ce93", + "type": "upload_revision" + } + }`)) + + // Upload files + case r.Method == http.MethodPost && + r.URL.Path == "/hidden/orgs/9102b78b-c28d-4392-a39f-08dd26fd9622/upload_revisions/ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f/files": + + assert.Equal(t, "gzip", r.Header.Get("Content-Encoding")) + assert.Contains(t, r.Header.Get("Content-Type"), "multipart/form-data") + + contentType := r.Header.Get("Content-Type") + _, params, err := mime.ParseMediaType(contentType) + require.NoError(t, err) + boundary := params["boundary"] + require.NotEmpty(t, boundary, "multipart boundary should be present") + + gzipReader, err := gzip.NewReader(r.Body) + require.NoError(t, err) + reader := multipart.NewReader(gzipReader, boundary) + + for { + _, err := reader.NextPart() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + } + + w.WriteHeader(http.StatusNoContent) + + // Seal revision + case r.Method == http.MethodPatch && + r.URL.Path == "/hidden/orgs/9102b78b-c28d-4392-a39f-08dd26fd9622/upload_revisions/ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f": + + assert.Equal(t, "application/vnd.api+json", r.Header.Get("Content-Type")) + + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{ + "data": { + "attributes": { + "revision_type": "snapshot", + "sealed": true + }, + "id": "ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f", + "type": "upload_revision" + } + }`)) + + default: + t.Errorf("Unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + + client := uploadrevision.NewClient(uploadrevision.Config{ + BaseURL: srv.URL, + }) + + return srv, client +} diff --git a/pkg/apiclients/fileupload/uploadrevision/compression.go b/pkg/apiclients/fileupload/uploadrevision/compression.go new file mode 100644 index 000000000..04899883c --- /dev/null +++ b/pkg/apiclients/fileupload/uploadrevision/compression.go @@ -0,0 +1,67 @@ +package uploadrevision + +import ( + "compress/gzip" + "io" + "net/http" +) + +// CompressionRoundTripper is an http.RoundTripper that automatically compresses +// request bodies using gzip compression. It wraps another RoundTripper and adds +// Content-Encoding: gzip header while removing Content-Length to allow proper +// compression handling. +type CompressionRoundTripper struct { + defaultRoundTripper http.RoundTripper +} + +// NewCompressionRoundTripper creates a new CompressionRoundTripper that wraps +// the provided RoundTripper. If drt is nil, http.DefaultTransport is used. +// All HTTP requests with a body will be automatically compressed using gzip. +func NewCompressionRoundTripper(drt http.RoundTripper) *CompressionRoundTripper { + rt := drt + if rt == nil { + rt = http.DefaultTransport + } + return &CompressionRoundTripper{rt} +} + +// compressRequestBody wraps the given reader with gzip compression. +func compressRequestBody(body io.Reader) io.ReadCloser { + pipeReader, pipeWriter := io.Pipe() + + go func() { + var err error + gzWriter := gzip.NewWriter(pipeWriter) + + _, err = io.Copy(gzWriter, body) + + if closeErr := gzWriter.Close(); closeErr != nil && err == nil { + err = closeErr + } + pipeWriter.CloseWithError(err) + }() + + return pipeReader +} + +// RoundTrip implements the http.RoundTripper interface. It compresses the request +// body using gzip if a body is present, sets the Content-Encoding header to "gzip", +// and removes the Content-Length header to allow Go's HTTP client to calculate +// the correct length after compression. Requests without a body are passed through +// unchanged to the wrapped RoundTripper. +func (crt *CompressionRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + if r.Body == nil || r.Body == http.NoBody { + //nolint:wrapcheck // No need to wrap the error here. + return crt.defaultRoundTripper.RoundTrip(r) + } + + compressedBody := compressRequestBody(r.Body) + + r.Body = compressedBody + r.Header.Set(ContentEncoding, "gzip") + r.Header.Del(ContentLength) + r.ContentLength = -1 // Let Go calculate the length + + //nolint:wrapcheck // No need to wrap the error here. + return crt.defaultRoundTripper.RoundTrip(r) +} diff --git a/pkg/apiclients/fileupload/uploadrevision/compression_test.go b/pkg/apiclients/fileupload/uploadrevision/compression_test.go new file mode 100644 index 000000000..84caed70e --- /dev/null +++ b/pkg/apiclients/fileupload/uploadrevision/compression_test.go @@ -0,0 +1,120 @@ +package uploadrevision_test + +import ( + "compress/gzip" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" +) + +func TestCompressionRoundTripper_RoundTrip(t *testing.T) { + t.Run("request without body", func(t *testing.T) { + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify no compression headers are set + assert.Empty(t, r.Header.Get("Content-Encoding")) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + crt := uploadrevision.NewCompressionRoundTripper(http.DefaultTransport) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, http.NoBody) + require.NoError(t, err) + + resp, err := crt.RoundTrip(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + resp.Body.Close() + }) + + t.Run("request with body gets compressed", func(t *testing.T) { + ctx := context.Background() + originalBody := "Hello, World! This is some test data that should be compressed." + var receivedBody []byte + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "gzip", r.Header.Get("Content-Encoding")) + assert.Empty(t, r.Header.Get("Content-Length")) + assert.Equal(t, int64(-1), r.ContentLength) + + gzipReader, err := gzip.NewReader(r.Body) + require.NoError(t, err) + defer gzipReader.Close() + + receivedBody, err = io.ReadAll(gzipReader) + require.NoError(t, err) + + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + crt := uploadrevision.NewCompressionRoundTripper(http.DefaultTransport) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, server.URL, strings.NewReader(originalBody)) + require.NoError(t, err) + + resp, err := crt.RoundTrip(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + resp.Body.Close() + + assert.Equal(t, originalBody, string(receivedBody)) + }) + + t.Run("preserves existing headers except Content-Length", func(t *testing.T) { + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "application/json", r.Header.Get("Content-Type")) + assert.Equal(t, "Bearer token123", r.Header.Get("Authorization")) + assert.Equal(t, "gzip", r.Header.Get("Content-Encoding")) + assert.Empty(t, r.Header.Get("Content-Length")) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + crt := uploadrevision.NewCompressionRoundTripper(http.DefaultTransport) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, server.URL, strings.NewReader(`{"key":"value"}`)) + require.NoError(t, err) + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer token123") + req.Header.Set("Content-Length", "15") + + resp, err := crt.RoundTrip(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + resp.Body.Close() + }) + + t.Run("wraps underlying transport errors", func(t *testing.T) { + ctx := context.Background() + failingTransport := &failingRoundTripper{err: assert.AnError} + crt := uploadrevision.NewCompressionRoundTripper(failingTransport) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://example.com", strings.NewReader("test")) + require.NoError(t, err) + + _, err = crt.RoundTrip(req) + assert.Error(t, err) + assert.ErrorIs(t, err, assert.AnError) + }) +} + +// failingRoundTripper is a test helper that always returns an error. +type failingRoundTripper struct { + err error +} + +func (f *failingRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, f.err +} diff --git a/pkg/apiclients/fileupload/uploadrevision/errors.go b/pkg/apiclients/fileupload/uploadrevision/errors.go new file mode 100644 index 000000000..7c96d44f8 --- /dev/null +++ b/pkg/apiclients/fileupload/uploadrevision/errors.go @@ -0,0 +1,174 @@ +package uploadrevision + +import ( + "errors" + "fmt" + "os" +) + +// Sentinel errors for common conditions. +var ( + ErrNoFilesProvided = errors.New("no files provided for upload") + ErrEmptyOrgID = errors.New("organization ID cannot be empty") + ErrEmptyRevisionID = errors.New("revision ID cannot be empty") +) + +// FileSizeLimitError indicates a file exceeds the maximum allowed size. +type FileSizeLimitError struct { + FilePath string + FileSize int64 + Limit int64 +} + +func (e *FileSizeLimitError) Error() string { + return fmt.Sprintf("file %s size %d exceeds limit of %d bytes", e.FilePath, e.FileSize, e.Limit) +} + +// FileCountLimitError indicates too many files were provided. +type FileCountLimitError struct { + Count int + Limit int +} + +func (e *FileCountLimitError) Error() string { + return fmt.Sprintf("too many files: %d exceeds limit of %d", e.Count, e.Limit) +} + +// TotalPayloadSizeLimitError indicates the total size of all files exceeds the maximum allowed payload size. +type TotalPayloadSizeLimitError struct { + TotalSize int64 + Limit int64 +} + +func (e *TotalPayloadSizeLimitError) Error() string { + return fmt.Sprintf("total payload size %d bytes exceeds limit of %d bytes", e.TotalSize, e.Limit) +} + +// FilePathLengthLimitError indicates a file path exceeds the maximum allowed length. +type FilePathLengthLimitError struct { + FilePath string + Length int + Limit int +} + +func (e *FilePathLengthLimitError) Error() string { + return fmt.Sprintf("file name %s length %d exceeds limit of %d characters", e.FilePath, e.Length, e.Limit) +} + +// FileAccessError indicates a file cannot be accessed or read. +type FileAccessError struct { + FilePath string + Err error +} + +func (e *FileAccessError) Error() string { + return fmt.Sprintf("file %s cannot be accessed: %v", e.FilePath, e.Err) +} + +func (e *FileAccessError) Unwrap() error { + return e.Err +} + +// HTTPError represents an HTTP error response. +type HTTPError struct { + StatusCode int + Status string + Operation string + Body []byte +} + +func (e *HTTPError) Error() string { + return fmt.Sprintf("unsuccessful request to %s: %s", e.Operation, e.Status) +} + +// MultipartError indicates an error creating multipart form data. +type MultipartError struct { + FilePath string + Err error +} + +func (e *MultipartError) Error() string { + return fmt.Sprintf("failed to create multipart form for %s: %v", e.FilePath, e.Err) +} + +func (e *MultipartError) Unwrap() error { + return e.Err +} + +// SpecialFileError indicates a path points to a special file (device, pipe, socket, etc.) instead of a regular file. +type SpecialFileError struct { + FilePath string + Mode os.FileMode +} + +func (e *SpecialFileError) Error() string { + return fmt.Sprintf("path %s is not a regular file (mode: %s)", e.FilePath, e.Mode) +} + +// NewFileSizeLimitError creates a new FileSizeLimitError with the given parameters. +func NewFileSizeLimitError(filePath string, fileSize, limit int64) *FileSizeLimitError { + return &FileSizeLimitError{ + FilePath: filePath, + FileSize: fileSize, + Limit: limit, + } +} + +// NewFileCountLimitError creates a new FileCountLimitError with the given parameters. +func NewFileCountLimitError(count, limit int) *FileCountLimitError { + return &FileCountLimitError{ + Count: count, + Limit: limit, + } +} + +// NewTotalPayloadSizeLimitError creates a new TotalPayloadSizeLimitError with the given parameters. +func NewTotalPayloadSizeLimitError(totalSize, limit int64) *TotalPayloadSizeLimitError { + return &TotalPayloadSizeLimitError{ + TotalSize: totalSize, + Limit: limit, + } +} + +// NewFileAccessError creates a new FileAccessError with the given parameters. +func NewFileAccessError(filePath string, err error) *FileAccessError { + return &FileAccessError{ + FilePath: filePath, + Err: err, + } +} + +// NewHTTPError creates a new HTTPError with the given parameters. +func NewHTTPError(statusCode int, status, operation string, body []byte) *HTTPError { + return &HTTPError{ + StatusCode: statusCode, + Status: status, + Operation: operation, + Body: body, + } +} + +// NewMultipartError creates a new MultipartError with the given parameters. +func NewMultipartError(filePath string, err error) *MultipartError { + return &MultipartError{ + FilePath: filePath, + Err: err, + } +} + +// NewSpecialFileError creates a new SpecialFileError with the given path and mode. +func NewSpecialFileError(path string, mode os.FileMode) *SpecialFileError { + return &SpecialFileError{ + FilePath: path, + Mode: mode, + } +} + +// NewFilePathLengthLimitError creates a new FilePathLengthLimitError with the given parameters. +func NewFilePathLengthLimitError(filePath string, length, limit int) *FilePathLengthLimitError { + return &FilePathLengthLimitError{ + FilePath: filePath, + Length: length, + Limit: limit, + } +} diff --git a/pkg/apiclients/fileupload/uploadrevision/fake_client.go b/pkg/apiclients/fileupload/uploadrevision/fake_client.go new file mode 100644 index 000000000..0a1ed46d7 --- /dev/null +++ b/pkg/apiclients/fileupload/uploadrevision/fake_client.go @@ -0,0 +1,146 @@ +package uploadrevision + +import ( + "context" + "fmt" + "io" + + "github.com/google/uuid" +) + +type LoadedFile struct { + Path string + Content string +} + +// revisionState holds the in-memory state for a single revision. +type revisionState struct { + orgID OrgID + sealed bool + files []LoadedFile +} + +// FakeSealableClient is a mock implementation of the SealableClient for testing. +// It tracks revisions in memory and enforces the revision lifecycle (create -> upload -> seal). +type FakeSealableClient struct { + cfg FakeClientConfig + revisions map[RevisionID]*revisionState +} + +type FakeClientConfig struct { + Limits +} + +var _ SealableClient = (*FakeSealableClient)(nil) + +// NewFakeSealableClient creates a new instance of the fake client. +func NewFakeSealableClient(cfg FakeClientConfig) *FakeSealableClient { + return &FakeSealableClient{ + cfg: cfg, + revisions: make(map[RevisionID]*revisionState), + } +} + +func (f *FakeSealableClient) CreateRevision(_ context.Context, orgID OrgID) (*ResponseBody, error) { + newRevisionID := uuid.New() + f.revisions[newRevisionID] = &revisionState{ + orgID: orgID, + sealed: false, + } + + return &ResponseBody{ + Data: ResponseData{ + ID: newRevisionID, + }, + }, nil +} + +func (f *FakeSealableClient) UploadFiles(_ context.Context, orgID OrgID, revisionID RevisionID, files []UploadFile) error { + rev, ok := f.revisions[revisionID] + if !ok { + return fmt.Errorf("revision %s not found", revisionID) + } + + if rev.orgID != orgID { + return fmt.Errorf("orgID mismatch for revision %s", revisionID) + } + + if rev.sealed { + return fmt.Errorf("revision %s is sealed and cannot be modified", revisionID) + } + + if len(files) > f.cfg.FileCountLimit { + return NewFileCountLimitError(len(files), f.cfg.FileCountLimit) + } + + if len(files) == 0 { + return ErrNoFilesProvided + } + + var totalPayloadSize int64 + for _, file := range files { + fileInfo, err := file.File.Stat() + if err != nil { + return NewFileAccessError(file.Path, err) + } + + if !fileInfo.Mode().IsRegular() { + return NewSpecialFileError(file.Path, fileInfo.Mode()) + } + + if fileInfo.Size() > f.cfg.FileSizeLimit { + return NewFileSizeLimitError(file.Path, fileInfo.Size(), f.cfg.FileSizeLimit) + } + + totalPayloadSize += fileInfo.Size() + } + + if totalPayloadSize > f.cfg.TotalPayloadSizeLimit { + return NewTotalPayloadSizeLimitError(totalPayloadSize, f.cfg.TotalPayloadSizeLimit) + } + + for _, file := range files { + bts, err := io.ReadAll(file.File) + if err != nil { + return err + } + rev.files = append(rev.files, LoadedFile{ + Path: file.Path, + Content: string(bts), + }) + } + return nil +} + +func (f *FakeSealableClient) SealRevision(_ context.Context, orgID OrgID, revisionID RevisionID) (*SealResponseBody, error) { + rev, ok := f.revisions[revisionID] + if !ok { + return nil, fmt.Errorf("revision %s not found", revisionID) + } + + if rev.orgID != orgID { + return nil, fmt.Errorf("orgID mismatch for revision %s", revisionID) + } + + rev.sealed = true + return &SealResponseBody{}, nil +} + +// GetSealedRevisionFiles is a test helper to retrieve files for a sealed revision. +// It is not part of the SealableClient interface. +func (f *FakeSealableClient) GetSealedRevisionFiles(revisionID RevisionID) ([]LoadedFile, error) { + rev, ok := f.revisions[revisionID] + if !ok { + return nil, fmt.Errorf("revision %s not found", revisionID) + } + + if !rev.sealed { + return nil, fmt.Errorf("revision %s is not sealed", revisionID) + } + + return rev.files, nil +} + +func (f *FakeSealableClient) GetLimits() Limits { + return f.cfg.Limits +} diff --git a/pkg/apiclients/fileupload/uploadrevision/opts.go b/pkg/apiclients/fileupload/uploadrevision/opts.go new file mode 100644 index 000000000..9b1cfbbf0 --- /dev/null +++ b/pkg/apiclients/fileupload/uploadrevision/opts.go @@ -0,0 +1,13 @@ +package uploadrevision + +import "net/http" + +// Opt is a function that configures an HTTPSealableClient instance. +type Opt func(*HTTPSealableClient) + +// WithHTTPClient sets a custom HTTP client for the file upload client. +func WithHTTPClient(httpClient *http.Client) Opt { + return func(c *HTTPSealableClient) { + c.httpClient = httpClient + } +} diff --git a/pkg/apiclients/fileupload/uploadrevision/opts_test.go b/pkg/apiclients/fileupload/uploadrevision/opts_test.go new file mode 100644 index 000000000..e69b3dc9c --- /dev/null +++ b/pkg/apiclients/fileupload/uploadrevision/opts_test.go @@ -0,0 +1,46 @@ +package uploadrevision_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" +) + +type CustomRoundTripper struct{} + +func (crt *CustomRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + r.Header.Set("foo", "bar") + return http.DefaultTransport.RoundTrip(r) +} + +func Test_WithHTTPClient(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fooValue := r.Header.Get("foo") + assert.Equal(t, "bar", fooValue) + + resp, err := json.Marshal(uploadrevision.ResponseBody{}) + require.NoError(t, err) + + w.WriteHeader(http.StatusCreated) + w.Write(resp) + })) + defer srv.Close() + customClient := srv.Client() + customClient.Transport = &CustomRoundTripper{} + + llc := uploadrevision.NewClient(uploadrevision.Config{ + BaseURL: srv.URL, + }, uploadrevision.WithHTTPClient(customClient)) + + _, err := llc.CreateRevision(context.Background(), uuid.New()) + + require.NoError(t, err) +} diff --git a/pkg/apiclients/fileupload/uploadrevision/types.go b/pkg/apiclients/fileupload/uploadrevision/types.go new file mode 100644 index 000000000..9d4563e14 --- /dev/null +++ b/pkg/apiclients/fileupload/uploadrevision/types.go @@ -0,0 +1,138 @@ +package uploadrevision + +import ( + "io/fs" + + "github.com/google/uuid" +) + +// OrgID represents an organization identifier. +type OrgID = uuid.UUID + +// RevisionID represents a revision identifier. +type RevisionID = uuid.UUID + +// RevisionType represents the type of revision being created. +type RevisionType string + +const ( + // RevisionTypeSnapshot represents a snapshot revision type. + RevisionTypeSnapshot RevisionType = "snapshot" +) + +// ResourceType represents the type of resource in API requests. +type ResourceType string + +const ( + // ResourceTypeUploadRevision represents an upload revision resource type. + ResourceTypeUploadRevision ResourceType = "upload_revision" +) + +// RequestAttributes contains the attributes for creating an upload revision. +type RequestAttributes struct { + RevisionType RevisionType `json:"revision_type"` //nolint:tagliatelle // API expects snake_case +} + +// RequestData contains the data payload for creating an upload revision. +type RequestData struct { + Attributes RequestAttributes `json:"attributes"` + Type ResourceType `json:"type"` +} + +// RequestBody contains the complete request body for creating an upload revision. +type RequestBody struct { + Data RequestData `json:"data"` +} + +// ResponseAttributes contains the attributes returned when creating an upload revision. +type ResponseAttributes struct { + RevisionType RevisionType `json:"revision_type"` //nolint:tagliatelle // API expects snake_case + Sealed bool `json:"sealed"` +} + +// ResponseData contains the data returned when creating an upload revision. +type ResponseData struct { + ID RevisionID `json:"id"` + Type ResourceType `json:"type"` + Attributes ResponseAttributes `json:"attributes"` +} + +// ResponseBody contains the complete response body when creating an upload revision. +type ResponseBody struct { + Data ResponseData `json:"data"` +} + +// SealRequestAttributes contains the attributes for sealing an upload revision. +type SealRequestAttributes struct { + Sealed bool `json:"sealed"` +} + +// SealRequestData contains the data payload for sealing an upload revision. +type SealRequestData struct { + ID RevisionID `json:"id"` + Type ResourceType `json:"type"` + Attributes SealRequestAttributes `json:"attributes"` +} + +// SealRequestBody contains the complete request body for sealing an upload revision. +type SealRequestBody struct { + Data SealRequestData `json:"data"` +} + +// SealResponseAttributes contains the attributes returned when sealing an upload revision. +type SealResponseAttributes struct { + RevisionType RevisionType `json:"revision_type"` //nolint:tagliatelle // API expects snake_case + Sealed bool `json:"sealed"` +} + +// SealResponseData contains the data returned when sealing an upload revision. +type SealResponseData struct { + ID RevisionID `json:"id"` + Type ResourceType `json:"type"` + Attributes SealResponseAttributes `json:"attributes"` +} + +// SealResponseBody contains the complete response body when sealing an upload revision. +type SealResponseBody struct { + Data SealResponseData `json:"data"` +} + +// ResponseError represents an error in an API response. +type ResponseError struct { + ID string `json:"id"` + Title string `json:"title"` + Status string `json:"status"` + Detail string `json:"detail"` +} + +// ErrorResponseBody contains the complete error response body. +type ErrorResponseBody struct { + Errors []ResponseError `json:"errors"` +} + +// UploadFile represents a file to be uploaded, containing both the path and file handle. +type UploadFile struct { + Path string // The path of the uploaded file, relative to the root directory. + File fs.File +} + +const ( + // ContentType is the HTTP header name for content type. + ContentType = "Content-Type" + // ContentEncoding is the HTTP header name for content encoding. + ContentEncoding = "Content-Encoding" + // ContentLength is the HTTP header name for content length. + ContentLength = "Content-Length" +) + +// Limits contains the limits enforced by the low level client. +type Limits struct { + // FileCountLimit specifies the maximum number of files allowed in a single upload. + FileCountLimit int + // FileSizeLimit specifies the maximum allowed file size in bytes. + FileSizeLimit int64 + // TotalPayloadSizeLimit specifies the maximum total uncompressed payload size in bytes. + TotalPayloadSizeLimit int64 + // FilePathLengthLimit specifies the maximum allowed file name length in characters. + FilePathLengthLimit int +} From a94a8dd22d79681c1fdf46aaf9f65a6d6c89b1cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:08:33 +0100 Subject: [PATCH 02/15] chore: remove helper that adds additional dependency --- pkg/apiclients/fileupload/client.go | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/pkg/apiclients/fileupload/client.go b/pkg/apiclients/fileupload/client.go index be978239c..cab16d22c 100644 --- a/pkg/apiclients/fileupload/client.go +++ b/pkg/apiclients/fileupload/client.go @@ -12,13 +12,11 @@ import ( "github.com/google/uuid" "github.com/puzpuzpuz/xsync" "github.com/rs/zerolog" - "github.com/snyk/go-application-framework/pkg/configuration" - "github.com/snyk/go-application-framework/pkg/utils" - "github.com/snyk/go-application-framework/pkg/workflow" listsources "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/files" "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/filters" "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + "github.com/snyk/go-application-framework/pkg/utils" ) // Config contains configuration for the file upload client. @@ -80,22 +78,6 @@ func NewClient(httpClient *http.Client, cfg Config, opts ...Option) *HTTPClient return client } -// NewClientFromInvocationContext creates a new file upload client from a workflow.InvocationContext. -// This is a convenience function that extracts the necessary configuration and HTTP client -// from the invocation context. -func NewClientFromInvocationContext(ictx workflow.InvocationContext, orgID uuid.UUID) Client { - cfg := ictx.GetConfiguration() - return NewClient( - ictx.GetNetworkAccess().GetHttpClient(), - Config{ - BaseURL: cfg.GetString(configuration.API_URL), - OrgID: orgID, - IsFedRamp: cfg.GetBool(configuration.IS_FEDRAMP), - }, - WithLogger(ictx.GetEnhancedLogger()), - ) -} - func (c *HTTPClient) loadFilters(ctx context.Context) error { c.filters.once.Do(func() { filtersResp, err := c.filtersClient.GetFilters(ctx, c.cfg.OrgID) From 30538d8ad6be249b308b12beee291b805238a716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 14 Nov 2025 19:57:54 +0100 Subject: [PATCH 03/15] chore: fix windows tests --- pkg/apiclients/fileupload/client_test.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index ebd94939a..0646d7138 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "path" "path/filepath" "slices" "strings" @@ -523,7 +522,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { } ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, filepath.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -548,7 +547,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { }, }, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, filepath.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) require.NoError(t, err) var fileSizeErr *uploadrevision.FileSizeLimitError @@ -581,7 +580,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { }, }, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, filepath.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) require.NoError(t, err) var filePathErr *uploadrevision.FilePathLengthLimitError @@ -607,7 +606,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, filepath.Join(dir.Name(), "script.js"), fileupload.UploadOptions{}) require.NoError(t, err) assert.Len(t, res.FilteredFiles, 1) @@ -626,7 +625,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js"), fileupload.UploadOptions{SkipDeeproxyFiltering: true}) + res, err := client.CreateRevisionFromFile(ctx, filepath.Join(dir.Name(), "script.js"), fileupload.UploadOptions{SkipDeeproxyFiltering: true}) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) From 3c4b88fe4800b01b65037e823a1f00e9dfc312a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 14 Nov 2025 20:05:05 +0100 Subject: [PATCH 04/15] chore: fix windows tests --- pkg/apiclients/fileupload/client_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index 0646d7138..edf23f2e1 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -126,16 +126,18 @@ func Test_CreateRevisionFromPaths(t *testing.T) { t.Run("error handling with better context", func(t *testing.T) { ctx, _, client, _ := setupTest(t, llcfg, []uploadrevision.LoadedFile{}, allowList, nil) + nonexistentPath := filepath.Join(string(filepath.Separator), "nonexistent", "file.go") + anotherMissingPath := filepath.Join(string(filepath.Separator), "another", "missing", "path.txt") paths := []string{ - "/nonexistent/file.go", - "/another/missing/path.txt", + nonexistentPath, + anotherMissingPath, } _, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) require.Error(t, err) var fileAccessErr *uploadrevision.FileAccessError assert.ErrorAs(t, err, &fileAccessErr) - assert.Equal(t, "/nonexistent/file.go", fileAccessErr.FilePath) + assert.Equal(t, nonexistentPath, fileAccessErr.FilePath) assert.ErrorContains(t, fileAccessErr.Err, "no such file or directory") }) } From 8e4554c8df4608b929c637061caed3519c7f9c1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 14 Nov 2025 21:05:46 +0100 Subject: [PATCH 05/15] Revert "chore: fix windows tests" This reverts commit 3c4b88fe4800b01b65037e823a1f00e9dfc312a1. --- pkg/apiclients/fileupload/client_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index edf23f2e1..0646d7138 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -126,18 +126,16 @@ func Test_CreateRevisionFromPaths(t *testing.T) { t.Run("error handling with better context", func(t *testing.T) { ctx, _, client, _ := setupTest(t, llcfg, []uploadrevision.LoadedFile{}, allowList, nil) - nonexistentPath := filepath.Join(string(filepath.Separator), "nonexistent", "file.go") - anotherMissingPath := filepath.Join(string(filepath.Separator), "another", "missing", "path.txt") paths := []string{ - nonexistentPath, - anotherMissingPath, + "/nonexistent/file.go", + "/another/missing/path.txt", } _, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) require.Error(t, err) var fileAccessErr *uploadrevision.FileAccessError assert.ErrorAs(t, err, &fileAccessErr) - assert.Equal(t, nonexistentPath, fileAccessErr.FilePath) + assert.Equal(t, "/nonexistent/file.go", fileAccessErr.FilePath) assert.ErrorContains(t, fileAccessErr.Err, "no such file or directory") }) } From fad8eaa5c697f614897918629058e427e8ded066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 14 Nov 2025 21:05:51 +0100 Subject: [PATCH 06/15] Revert "chore: fix windows tests" This reverts commit 30538d8ad6be249b308b12beee291b805238a716. --- pkg/apiclients/fileupload/client_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index 0646d7138..ebd94939a 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path" "path/filepath" "slices" "strings" @@ -522,7 +523,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { } ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, filepath.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -547,7 +548,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { }, }, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, filepath.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) require.NoError(t, err) var fileSizeErr *uploadrevision.FileSizeLimitError @@ -580,7 +581,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { }, }, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, filepath.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) require.NoError(t, err) var filePathErr *uploadrevision.FilePathLengthLimitError @@ -606,7 +607,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, filepath.Join(dir.Name(), "script.js"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js"), fileupload.UploadOptions{}) require.NoError(t, err) assert.Len(t, res.FilteredFiles, 1) @@ -625,7 +626,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, filepath.Join(dir.Name(), "script.js"), fileupload.UploadOptions{SkipDeeproxyFiltering: true}) + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js"), fileupload.UploadOptions{SkipDeeproxyFiltering: true}) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) From de6d3f7efbfbed80a6f5554f544c3515cc5a30bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 14 Nov 2025 21:21:19 +0100 Subject: [PATCH 07/15] fix: test on windows --- pkg/apiclients/fileupload/client_test.go | 56 ++++++++++++++---------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index ebd94939a..4a7ac18bb 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -19,6 +19,16 @@ import ( "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" ) +var mainpath = filepath.Join("src", "main.go") +var utilspath = filepath.Join("src", "utils.go") +var helperpath = filepath.Join("src", "utils", "helper.go") +var docpath = filepath.Join("docs", "README.md") +var gomodpath = filepath.Join("src", "go.mod") +var scriptpath = filepath.Join("src", "script.js") +var packagelockpath = filepath.Join("src", "package.json") +var nonexistpath = filepath.Join("nonexistent", "file.go") +var missingpath = filepath.Join("another", "missing", "path.txt") + // CreateTmpFiles is an utility function used to create temporary files in tests. func createTmpFiles(t *testing.T, files []uploadrevision.LoadedFile) (dir *os.File) { t.Helper() @@ -75,8 +85,8 @@ func Test_CreateRevisionFromPaths(t *testing.T) { t.Run("mixed files and directories", func(t *testing.T) { allFiles := []uploadrevision.LoadedFile{ - {Path: "src/main.go", Content: "package main"}, - {Path: "src/utils.go", Content: "package utils"}, + {Path: mainpath, Content: "package main"}, + {Path: utilspath, Content: "package utils"}, {Path: "config.yaml", Content: "version: 1"}, {Path: "README.md", Content: "# Project"}, } @@ -107,8 +117,8 @@ func Test_CreateRevisionFromPaths(t *testing.T) { t.Run("get filters error", func(t *testing.T) { allFiles := []uploadrevision.LoadedFile{ - {Path: "src/main.go", Content: "package main"}, - {Path: "src/utils.go", Content: "package utils"}, + {Path: mainpath, Content: "package main"}, + {Path: utilspath, Content: "package utils"}, {Path: "config.yaml", Content: "version: 1"}, {Path: "README.md", Content: "# Project"}, } @@ -128,15 +138,15 @@ func Test_CreateRevisionFromPaths(t *testing.T) { ctx, _, client, _ := setupTest(t, llcfg, []uploadrevision.LoadedFile{}, allowList, nil) paths := []string{ - "/nonexistent/file.go", - "/another/missing/path.txt", + nonexistpath, + missingpath, } _, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) require.Error(t, err) var fileAccessErr *uploadrevision.FileAccessError assert.ErrorAs(t, err, &fileAccessErr) - assert.Equal(t, "/nonexistent/file.go", fileAccessErr.FilePath) + assert.Equal(t, nonexistpath, fileAccessErr.FilePath) assert.ErrorContains(t, fileAccessErr.Err, "no such file or directory") }) } @@ -181,11 +191,11 @@ func Test_CreateRevisionFromDir(t *testing.T) { t.Run("uploading a directory with nested files", func(t *testing.T) { expectedFiles := []uploadrevision.LoadedFile{ { - Path: "src/main.go", + Path: filepath.Join("src", "main.go"), Content: "package main\n\nfunc main() {}", }, { - Path: "src/utils/helper.go", + Path: filepath.Join("src", "utils", "helper.go"), Content: "package utils\n\nfunc Helper() {}", }, } @@ -207,19 +217,19 @@ func Test_CreateRevisionFromDir(t *testing.T) { Content: "root level file", }, { - Path: "src/main.go", + Path: mainpath, Content: "package main\n\nfunc main() {}", }, { - Path: "src/utils/helper.go", + Path: helperpath, Content: "package utils\n\nfunc Helper() {}", }, { - Path: "docs/README.md", + Path: docpath, Content: "# Project Documentation", }, { - Path: "src/go.mod", + Path: gomodpath, Content: "foo bar", }, } @@ -427,25 +437,25 @@ func Test_CreateRevisionFromDir(t *testing.T) { t.Run("uploading a directory applies filtering", func(t *testing.T) { expectedFiles := []uploadrevision.LoadedFile{ { - Path: "src/main.go", + Path: mainpath, Content: "package main\n\nfunc main() {}", }, { - Path: "src/utils/helper.go", + Path: helperpath, Content: "package utils\n\nfunc Helper() {}", }, { - Path: "src/go.mod", + Path: gomodpath, Content: "foo bar", }, } additionalFiles := []uploadrevision.LoadedFile{ { - Path: "src/script.js", + Path: scriptpath, Content: "console.log('hi')", }, { - Path: "src/package.json", + Path: packagelockpath, Content: "{}", }, } @@ -466,23 +476,23 @@ func Test_CreateRevisionFromDir(t *testing.T) { t.Run("uploading a directory with filtering disabled", func(t *testing.T) { allFiles := []uploadrevision.LoadedFile{ { - Path: "src/main.go", + Path: mainpath, Content: "package main\n\nfunc main() {}", }, { - Path: "src/utils/helper.go", + Path: helperpath, Content: "package utils\n\nfunc Helper() {}", }, { - Path: "src/go.mod", + Path: gomodpath, Content: "foo bar", }, { - Path: "src/script.js", + Path: scriptpath, Content: "console.log('hi')", }, { - Path: "src/package.json", + Path: packagelockpath, Content: "{}", }, } From 3f9432caae67865b88cf26fdc622b57ec56b9677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 14 Nov 2025 21:45:05 +0100 Subject: [PATCH 08/15] fix: test on windows --- pkg/apiclients/fileupload/client_test.go | 1 - .../fileupload/uploadrevision/client_test.go | 13 +++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index 4a7ac18bb..f264ca5cd 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -147,7 +147,6 @@ func Test_CreateRevisionFromPaths(t *testing.T) { var fileAccessErr *uploadrevision.FileAccessError assert.ErrorAs(t, err, &fileAccessErr) assert.Equal(t, nonexistpath, fileAccessErr.FilePath) - assert.ErrorContains(t, fileAccessErr.Err, "no such file or directory") }) } diff --git a/pkg/apiclients/fileupload/uploadrevision/client_test.go b/pkg/apiclients/fileupload/uploadrevision/client_test.go index e54c1017a..5e9ba2e37 100644 --- a/pkg/apiclients/fileupload/uploadrevision/client_test.go +++ b/pkg/apiclients/fileupload/uploadrevision/client_test.go @@ -13,6 +13,7 @@ import ( "net/http/httptest" "os" "path" + "runtime" "strings" "testing" "testing/fstest" @@ -364,12 +365,20 @@ func TestClient_UploadFiles_SpecialFileError(t *testing.T) { }, "test-directory" }, }, - { + } + + // on non windows os test this case + if runtime.GOOS != "windows" { + tests = append(tests, struct { + name string + setupFS func() (fstest.MapFS, string) + setupRealFile func() string + }{ name: "device file", setupRealFile: func() string { return "/dev/null" }, - }, + }) } for _, tt := range tests { From cdf286183ef77c155c8ed0a3176c8c194b1b292d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Tue, 18 Nov 2025 17:53:18 +0100 Subject: [PATCH 09/15] chore: remove filter from public interface and default disable --- pkg/apiclients/fileupload/client.go | 33 +++--- .../fileupload/client_integration_test.go | 9 +- pkg/apiclients/fileupload/client_test.go | 107 +++--------------- pkg/apiclients/fileupload/fake_client.go | 10 +- pkg/apiclients/fileupload/types.go | 2 +- 5 files changed, 44 insertions(+), 117 deletions(-) diff --git a/pkg/apiclients/fileupload/client.go b/pkg/apiclients/fileupload/client.go index cab16d22c..bddba6219 100644 --- a/pkg/apiclients/fileupload/client.go +++ b/pkg/apiclients/fileupload/client.go @@ -21,9 +21,8 @@ import ( // Config contains configuration for the file upload client. type Config struct { - BaseURL string - OrgID OrgID - IsFedRamp bool + BaseURL string + OrgID OrgID } // HTTPClient provides high-level file upload functionality. @@ -37,9 +36,9 @@ type HTTPClient struct { // Client defines the interface for the high level file upload client. type Client interface { - CreateRevisionFromPaths(ctx context.Context, paths []string, opts UploadOptions) (UploadResult, error) - CreateRevisionFromDir(ctx context.Context, dirPath string, opts UploadOptions) (UploadResult, error) - CreateRevisionFromFile(ctx context.Context, filePath string, opts UploadOptions) (UploadResult, error) + CreateRevisionFromPaths(ctx context.Context, paths []string) (UploadResult, error) + CreateRevisionFromDir(ctx context.Context, dirPath string) (UploadResult, error) + CreateRevisionFromFile(ctx context.Context, filePath string) (UploadResult, error) } var _ Client = (*HTTPClient)(nil) @@ -71,7 +70,7 @@ func NewClient(httpClient *http.Client, cfg Config, opts ...Option) *HTTPClient if client.filtersClient == nil { client.filtersClient = filters.NewDeeproxyClient(filters.Config{ BaseURL: cfg.BaseURL, - IsFedRamp: cfg.IsFedRamp, + IsFedRamp: false, //cfg.IsFedRamp, }, filters.WithHTTPClient(httpClient)) } @@ -152,7 +151,7 @@ func (c *HTTPClient) addPathsToRevision( revisionID RevisionID, rootPath string, pathsChan <-chan string, - opts UploadOptions, + opts uploadOptions, ) (UploadResult, error) { res := UploadResult{ RevisionID: revisionID, @@ -224,7 +223,7 @@ func (c *HTTPClient) createRevision(ctx context.Context) (RevisionID, error) { } // addFileToRevision adds a single file to an existing revision. -func (c *HTTPClient) addFileToRevision(ctx context.Context, revisionID RevisionID, filePath string, opts UploadOptions) (UploadResult, error) { +func (c *HTTPClient) addFileToRevision(ctx context.Context, revisionID RevisionID, filePath string, opts uploadOptions) (UploadResult, error) { writableChan := make(chan string, 1) writableChan <- filePath close(writableChan) @@ -233,7 +232,7 @@ func (c *HTTPClient) addFileToRevision(ctx context.Context, revisionID RevisionI } // addDirToRevision adds a directory and all its contents to an existing revision. -func (c *HTTPClient) addDirToRevision(ctx context.Context, revisionID RevisionID, dirPath string, opts UploadOptions) (UploadResult, error) { +func (c *HTTPClient) addDirToRevision(ctx context.Context, revisionID RevisionID, dirPath string, opts uploadOptions) (UploadResult, error) { sources, err := listsources.ForPath(dirPath, c.logger, runtime.NumCPU()) if err != nil { return UploadResult{}, fmt.Errorf("failed to list files in directory %s: %w", dirPath, err) @@ -253,7 +252,11 @@ func (c *HTTPClient) sealRevision(ctx context.Context, revisionID RevisionID) er // CreateRevisionFromPaths uploads multiple paths (files or directories), returning a revision ID. // This is a convenience method that creates, uploads, and seals a revision. -func (c *HTTPClient) CreateRevisionFromPaths(ctx context.Context, paths []string, opts UploadOptions) (UploadResult, error) { +func (c *HTTPClient) CreateRevisionFromPaths(ctx context.Context, paths []string) (UploadResult, error) { + opts := uploadOptions{ + SkipDeeproxyFiltering: true, + } + res := UploadResult{ FilteredFiles: make([]FilteredFile, 0), } @@ -300,7 +303,7 @@ func (c *HTTPClient) CreateRevisionFromPaths(ctx context.Context, paths []string // CreateRevisionFromDir uploads a directory and all its contents, returning a revision ID. // This is a convenience method for validating the directory path and calling CreateRevisionFromPaths with a single directory path. -func (c *HTTPClient) CreateRevisionFromDir(ctx context.Context, dirPath string, opts UploadOptions) (UploadResult, error) { +func (c *HTTPClient) CreateRevisionFromDir(ctx context.Context, dirPath string) (UploadResult, error) { info, err := os.Stat(dirPath) if err != nil { return UploadResult{}, uploadrevision.NewFileAccessError(dirPath, err) @@ -310,12 +313,12 @@ func (c *HTTPClient) CreateRevisionFromDir(ctx context.Context, dirPath string, return UploadResult{}, fmt.Errorf("the provided path is not a directory: %s", dirPath) } - return c.CreateRevisionFromPaths(ctx, []string{dirPath}, opts) + return c.CreateRevisionFromPaths(ctx, []string{dirPath}) } // CreateRevisionFromFile uploads a single file, returning a revision ID. // This is a convenience method for validating the file path and calling CreateRevisionFromPaths with a single file path. -func (c *HTTPClient) CreateRevisionFromFile(ctx context.Context, filePath string, opts UploadOptions) (UploadResult, error) { +func (c *HTTPClient) CreateRevisionFromFile(ctx context.Context, filePath string) (UploadResult, error) { info, err := os.Stat(filePath) if err != nil { return UploadResult{}, uploadrevision.NewFileAccessError(filePath, err) @@ -325,5 +328,5 @@ func (c *HTTPClient) CreateRevisionFromFile(ctx context.Context, filePath string return UploadResult{}, fmt.Errorf("the provided path is not a regular file: %s", filePath) } - return c.CreateRevisionFromPaths(ctx, []string{filePath}, opts) + return c.CreateRevisionFromPaths(ctx, []string{filePath}) } diff --git a/pkg/apiclients/fileupload/client_integration_test.go b/pkg/apiclients/fileupload/client_integration_test.go index 73d74fa39..4fba3f99e 100644 --- a/pkg/apiclients/fileupload/client_integration_test.go +++ b/pkg/apiclients/fileupload/client_integration_test.go @@ -9,9 +9,10 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" + "github.com/snyk/go-application-framework/pkg/apiclients/util" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload" "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" - "github.com/snyk/go-application-framework/pkg/apiclients/util" ) func TestUploadFileIntegration(t *testing.T) { @@ -24,7 +25,7 @@ func TestUploadFileIntegration(t *testing.T) { dir := util.CreateTmpFiles(t, files) - res, err := fileUploadClient.CreateRevisionFromFile(t.Context(), filepath.Join(dir.Name(), files[0].Path), fileupload.UploadOptions{}) + res, err := fileUploadClient.CreateRevisionFromFile(t.Context(), filepath.Join(dir.Name(), files[0].Path)) if err != nil { t.Errorf("failed to create fileupload revision: %s", err.Error()) } @@ -43,7 +44,7 @@ func TestUploadDirectoryIntegration(t *testing.T) { dir := util.CreateTmpFiles(t, files) - res, err := fileUploadClient.CreateRevisionFromDir(t.Context(), dir.Name(), fileupload.UploadOptions{}) + res, err := fileUploadClient.CreateRevisionFromDir(t.Context(), dir.Name()) if err != nil { t.Errorf("failed to create fileupload revision: %s", err.Error()) } @@ -61,7 +62,7 @@ func TestUploadLargeFileIntegration(t *testing.T) { dir := util.CreateTmpFiles(t, files) - res, err := fileUploadClient.CreateRevisionFromFile(t.Context(), filepath.Join(dir.Name(), files[0].Path), fileupload.UploadOptions{}) + res, err := fileUploadClient.CreateRevisionFromFile(t.Context(), filepath.Join(dir.Name(), files[0].Path)) if err != nil { t.Errorf("failed to create fileupload revision: %s", err.Error()) } diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index f264ca5cd..bb8a696e2 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -98,7 +98,7 @@ func Test_CreateRevisionFromPaths(t *testing.T) { filepath.Join(dir.Name(), "README.md"), // Individual file } - res, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromPaths(ctx, paths) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -115,25 +115,6 @@ func Test_CreateRevisionFromPaths(t *testing.T) { assert.Contains(t, uploadedPaths, "README.md") }) - t.Run("get filters error", func(t *testing.T) { - allFiles := []uploadrevision.LoadedFile{ - {Path: mainpath, Content: "package main"}, - {Path: utilspath, Content: "package utils"}, - {Path: "config.yaml", Content: "version: 1"}, - {Path: "README.md", Content: "# Project"}, - } - - ctx, _, client, dir := setupTest(t, llcfg, allFiles, filters.AllowList{}, assert.AnError) - - paths := []string{ - filepath.Join(dir.Name(), "src"), // Directory - filepath.Join(dir.Name(), "README.md"), // Individual file - } - - _, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) - require.ErrorContains(t, err, "failed to load deeproxy filters") - }) - t.Run("error handling with better context", func(t *testing.T) { ctx, _, client, _ := setupTest(t, llcfg, []uploadrevision.LoadedFile{}, allowList, nil) @@ -142,7 +123,7 @@ func Test_CreateRevisionFromPaths(t *testing.T) { missingpath, } - _, err := client.CreateRevisionFromPaths(ctx, paths, fileupload.UploadOptions{}) + _, err := client.CreateRevisionFromPaths(ctx, paths) require.Error(t, err) var fileAccessErr *uploadrevision.FileAccessError assert.ErrorAs(t, err, &fileAccessErr) @@ -178,7 +159,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { } ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -200,7 +181,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { } ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -234,7 +215,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { } ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -269,7 +250,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { }, }, allFiles, allowList, nil) - res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) var fileSizeErr *uploadrevision.FileSizeLimitError @@ -312,7 +293,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { }, }, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -350,7 +331,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { }, }, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -394,7 +375,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { }, }, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -424,7 +405,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { }, }, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -433,45 +414,6 @@ func Test_CreateRevisionFromDir(t *testing.T) { expectEqualFiles(t, expectedFiles, uploadedFiles) }) - t.Run("uploading a directory applies filtering", func(t *testing.T) { - expectedFiles := []uploadrevision.LoadedFile{ - { - Path: mainpath, - Content: "package main\n\nfunc main() {}", - }, - { - Path: helperpath, - Content: "package utils\n\nfunc Helper() {}", - }, - { - Path: gomodpath, - Content: "foo bar", - }, - } - additionalFiles := []uploadrevision.LoadedFile{ - { - Path: scriptpath, - Content: "console.log('hi')", - }, - { - Path: packagelockpath, - Content: "{}", - }, - } - //nolint:gocritic // Not an issue for tests. - allFiles := append(expectedFiles, additionalFiles...) - - ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) - - res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{}) - require.NoError(t, err) - - assert.Len(t, res.FilteredFiles, 2) - uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) - require.NoError(t, err) - expectEqualFiles(t, expectedFiles, uploadedFiles) - }) - t.Run("uploading a directory with filtering disabled", func(t *testing.T) { allFiles := []uploadrevision.LoadedFile{ { @@ -498,7 +440,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) - res, err := client.CreateRevisionFromDir(ctx, dir.Name(), fileupload.UploadOptions{SkipDeeproxyFiltering: true}) + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -532,7 +474,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { } ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) @@ -557,7 +499,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { }, }, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) require.NoError(t, err) var fileSizeErr *uploadrevision.FileSizeLimitError @@ -590,7 +532,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { }, }, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt"), fileupload.UploadOptions{}) + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) require.NoError(t, err) var filePathErr *uploadrevision.FilePathLengthLimitError @@ -606,25 +548,6 @@ func Test_CreateRevisionFromFile(t *testing.T) { expectEqualFiles(t, nil, uploadedFiles) }) - t.Run("uploading a file applies filtering", func(t *testing.T) { - allFiles := []uploadrevision.LoadedFile{ - { - Path: "script.js", - Content: "console.log('hi')", - }, - } - - ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) - - res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js"), fileupload.UploadOptions{}) - require.NoError(t, err) - - assert.Len(t, res.FilteredFiles, 1) - uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) - require.NoError(t, err) - expectEqualFiles(t, nil, uploadedFiles) - }) - t.Run("uploading a file with filtering disabled", func(t *testing.T) { expectedFiles := []uploadrevision.LoadedFile{ { @@ -635,7 +558,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) - res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js"), fileupload.UploadOptions{SkipDeeproxyFiltering: true}) + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js")) require.NoError(t, err) assert.Empty(t, res.FilteredFiles) diff --git a/pkg/apiclients/fileupload/fake_client.go b/pkg/apiclients/fileupload/fake_client.go index 7ed7ebb43..a41e04bab 100644 --- a/pkg/apiclients/fileupload/fake_client.go +++ b/pkg/apiclients/fileupload/fake_client.go @@ -32,7 +32,7 @@ func (f *FakeClient) WithError(err error) *FakeClient { return f } -func (f *FakeClient) CreateRevisionFromDir(ctx context.Context, dirPath string, opts UploadOptions) (UploadResult, error) { +func (f *FakeClient) CreateRevisionFromDir(ctx context.Context, dirPath string) (UploadResult, error) { if f.err != nil { return UploadResult{}, f.err } @@ -46,10 +46,10 @@ func (f *FakeClient) CreateRevisionFromDir(ctx context.Context, dirPath string, return UploadResult{}, fmt.Errorf("the provided path is not a directory: %s", dirPath) } - return f.CreateRevisionFromPaths(ctx, []string{dirPath}, opts) + return f.CreateRevisionFromPaths(ctx, []string{dirPath}) } -func (f *FakeClient) CreateRevisionFromFile(ctx context.Context, filePath string, opts UploadOptions) (UploadResult, error) { +func (f *FakeClient) CreateRevisionFromFile(ctx context.Context, filePath string) (UploadResult, error) { if f.err != nil { return UploadResult{}, f.err } @@ -63,10 +63,10 @@ func (f *FakeClient) CreateRevisionFromFile(ctx context.Context, filePath string return UploadResult{}, fmt.Errorf("the provided path is not a regular file: %s", filePath) } - return f.CreateRevisionFromPaths(ctx, []string{filePath}, opts) + return f.CreateRevisionFromPaths(ctx, []string{filePath}) } -func (f *FakeClient) CreateRevisionFromPaths(ctx context.Context, paths []string, opts UploadOptions) (UploadResult, error) { +func (f *FakeClient) CreateRevisionFromPaths(ctx context.Context, paths []string) (UploadResult, error) { if f.err != nil { return UploadResult{}, f.err } diff --git a/pkg/apiclients/fileupload/types.go b/pkg/apiclients/fileupload/types.go index bfde04d4a..1344675ff 100644 --- a/pkg/apiclients/fileupload/types.go +++ b/pkg/apiclients/fileupload/types.go @@ -23,7 +23,7 @@ type Filters struct { } // UploadOptions configures the behavior of file upload operations. -type UploadOptions struct { +type uploadOptions struct { SkipDeeproxyFiltering bool } From 9069c2f9211de554512dab0e535753e8bee7ad58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Tue, 18 Nov 2025 18:17:25 +0100 Subject: [PATCH 10/15] fix: linter and remove unused test --- .../fileupload/client_integration_test.go | 86 ------------------- pkg/apiclients/fileupload/client_test.go | 33 ++++--- .../fileupload/filters/client_test.go | 2 +- .../fileupload/filters/fake_client.go | 4 +- .../fileupload/uploadrevision/client_test.go | 6 +- .../fileupload/uploadrevision/opts_test.go | 3 +- 6 files changed, 25 insertions(+), 109 deletions(-) delete mode 100644 pkg/apiclients/fileupload/client_integration_test.go diff --git a/pkg/apiclients/fileupload/client_integration_test.go b/pkg/apiclients/fileupload/client_integration_test.go deleted file mode 100644 index 4fba3f99e..000000000 --- a/pkg/apiclients/fileupload/client_integration_test.go +++ /dev/null @@ -1,86 +0,0 @@ -//go:build integration - -package fileupload_test - -import ( - "path/filepath" - "testing" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - - "github.com/snyk/go-application-framework/pkg/apiclients/util" - - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" -) - -func TestUploadFileIntegration(t *testing.T) { - setup := util.NewIntegrationTestSetup(t) - fileUploadClient := newFileUploadClient(setup) - - files := []uploadrevision.LoadedFile{ - {Path: "src/main.go", Content: "package main"}, - } - - dir := util.CreateTmpFiles(t, files) - - res, err := fileUploadClient.CreateRevisionFromFile(t.Context(), filepath.Join(dir.Name(), files[0].Path)) - if err != nil { - t.Errorf("failed to create fileupload revision: %s", err.Error()) - } - assert.NotEqual(t, uuid.Nil, res.RevisionID) -} - -func TestUploadDirectoryIntegration(t *testing.T) { - setup := util.NewIntegrationTestSetup(t) - fileUploadClient := newFileUploadClient(setup) - - files := []uploadrevision.LoadedFile{ - {Path: "src/main.go", Content: "package main"}, - {Path: "src/utils.go", Content: "package utils"}, - {Path: "README.md", Content: "# Project"}, - } - - dir := util.CreateTmpFiles(t, files) - - res, err := fileUploadClient.CreateRevisionFromDir(t.Context(), dir.Name()) - if err != nil { - t.Errorf("failed to create fileupload revision: %s", err.Error()) - } - assert.NotEqual(t, uuid.Nil, res.RevisionID) -} - -func TestUploadLargeFileIntegration(t *testing.T) { - setup := util.NewIntegrationTestSetup(t) - fileUploadClient := newFileUploadClient(setup) - - content := generateFileOfSizeMegabytes(t, 30) - files := []uploadrevision.LoadedFile{ - {Path: "src/main.go", Content: content}, - } - - dir := util.CreateTmpFiles(t, files) - - res, err := fileUploadClient.CreateRevisionFromFile(t.Context(), filepath.Join(dir.Name(), files[0].Path)) - if err != nil { - t.Errorf("failed to create fileupload revision: %s", err.Error()) - } - assert.NotEqual(t, uuid.Nil, res.RevisionID) -} - -func newFileUploadClient(setup *util.IntegrationTestSetup) fileupload.Client { - return fileupload.NewClient( - setup.Client, - fileupload.Config{ - BaseURL: setup.Config.BaseURL, - OrgID: setup.Config.OrgID, - }, - ) -} - -func generateFileOfSizeMegabytes(t *testing.T, megabytes int) string { - t.Helper() - content := make([]byte, megabytes*1024*1024) - return string(content) -} diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index bb8a696e2..575fd0d5b 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -91,7 +91,7 @@ func Test_CreateRevisionFromPaths(t *testing.T) { {Path: "README.md", Content: "# Project"}, } - ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList) paths := []string{ filepath.Join(dir.Name(), "src"), // Directory @@ -116,7 +116,7 @@ func Test_CreateRevisionFromPaths(t *testing.T) { }) t.Run("error handling with better context", func(t *testing.T) { - ctx, _, client, _ := setupTest(t, llcfg, []uploadrevision.LoadedFile{}, allowList, nil) + ctx, _, client, _ := setupTest(t, llcfg, []uploadrevision.LoadedFile{}, allowList) paths := []string{ nonexistpath, @@ -157,7 +157,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { Content: "content2", }, } - ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList) res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) @@ -179,7 +179,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { Content: "package utils\n\nfunc Helper() {}", }, } - ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList) res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) @@ -213,7 +213,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { Content: "foo bar", }, } - ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList) res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) @@ -248,7 +248,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { TotalPayloadSizeLimit: 100, FilePathLengthLimit: 20, }, - }, allFiles, allowList, nil) + }, allFiles, allowList) res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) @@ -291,7 +291,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { TotalPayloadSizeLimit: 70, // 70 bytes - forces batching by size FilePathLengthLimit: 20, }, - }, expectedFiles, allowList, nil) + }, expectedFiles, allowList) res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) @@ -329,7 +329,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { TotalPayloadSizeLimit: 200, FilePathLengthLimit: 20, }, - }, expectedFiles, allowList, nil) + }, expectedFiles, allowList) res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) @@ -373,7 +373,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { TotalPayloadSizeLimit: 100, FilePathLengthLimit: 20, }, - }, expectedFiles, allowList, nil) + }, expectedFiles, allowList) res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) @@ -403,7 +403,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { TotalPayloadSizeLimit: 200, FilePathLengthLimit: 20, }, - }, expectedFiles, allowList, nil) + }, expectedFiles, allowList) res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) @@ -438,7 +438,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { }, } - ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList, nil) + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList) res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) @@ -472,7 +472,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { Content: "content1", }, } - ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList) res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) require.NoError(t, err) @@ -497,7 +497,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { TotalPayloadSizeLimit: 10_000, FilePathLengthLimit: 20, }, - }, expectedFiles, allowList, nil) + }, expectedFiles, allowList) res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) require.NoError(t, err) @@ -530,7 +530,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { TotalPayloadSizeLimit: 10_000, FilePathLengthLimit: 5, }, - }, expectedFiles, allowList, nil) + }, expectedFiles, allowList) res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) require.NoError(t, err) @@ -556,7 +556,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { }, } - ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList, nil) + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList) res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js")) require.NoError(t, err) @@ -592,7 +592,6 @@ func setupTest( llcfg uploadrevision.FakeClientConfig, files []uploadrevision.LoadedFile, allowList filters.AllowList, - filtersErr error, ) (context.Context, *uploadrevision.FakeSealableClient, *fileupload.HTTPClient, *os.File) { t.Helper() @@ -600,7 +599,7 @@ func setupTest( orgID := uuid.New() fakeSealeableClient := uploadrevision.NewFakeSealableClient(llcfg) - fakeFiltersClient := filters.NewFakeClient(allowList, filtersErr) + fakeFiltersClient := filters.NewFakeClient(allowList) client := fileupload.NewClient( nil, fileupload.Config{ diff --git a/pkg/apiclients/fileupload/filters/client_test.go b/pkg/apiclients/fileupload/filters/client_test.go index 6278dafaf..2a30d8cde 100644 --- a/pkg/apiclients/fileupload/filters/client_test.go +++ b/pkg/apiclients/fileupload/filters/client_test.go @@ -35,7 +35,7 @@ func TestClients(t *testing.T) { getClient: func(t *testing.T, _ uuid.UUID, expectedAllow AllowList) (Client, func()) { t.Helper() - c := NewFakeClient(expectedAllow, nil) + c := NewFakeClient(expectedAllow) return c, func() {} }, diff --git a/pkg/apiclients/fileupload/filters/fake_client.go b/pkg/apiclients/fileupload/filters/fake_client.go index 7245ff7e0..4d2d26d71 100644 --- a/pkg/apiclients/fileupload/filters/fake_client.go +++ b/pkg/apiclients/fileupload/filters/fake_client.go @@ -12,10 +12,10 @@ type FakeClient struct { var _ Client = (*FakeClient)(nil) -func NewFakeClient(allowList AllowList, err error) *FakeClient { +func NewFakeClient(allowList AllowList) *FakeClient { return &FakeClient{ getFilters: func(ctx context.Context, orgID uuid.UUID) (AllowList, error) { - return allowList, err + return allowList, nil }, } } diff --git a/pkg/apiclients/fileupload/uploadrevision/client_test.go b/pkg/apiclients/fileupload/uploadrevision/client_test.go index 5e9ba2e37..64f4dbbcc 100644 --- a/pkg/apiclients/fileupload/uploadrevision/client_test.go +++ b/pkg/apiclients/fileupload/uploadrevision/client_test.go @@ -504,7 +504,7 @@ func setupTestServer(t *testing.T) (*httptest.Server, *uploadrevision.HTTPSealab assert.Equal(t, "application/vnd.api+json", r.Header.Get("Content-Type")) w.WriteHeader(http.StatusCreated) - w.Write([]byte(`{ + _, err := w.Write([]byte(`{ "data": { "attributes": { "revision_type": "snapshot", @@ -514,6 +514,7 @@ func setupTestServer(t *testing.T) (*httptest.Server, *uploadrevision.HTTPSealab "type": "upload_revision" } }`)) + assert.NoError(t, err) // Upload files case r.Method == http.MethodPost && @@ -549,7 +550,7 @@ func setupTestServer(t *testing.T) (*httptest.Server, *uploadrevision.HTTPSealab assert.Equal(t, "application/vnd.api+json", r.Header.Get("Content-Type")) w.WriteHeader(http.StatusOK) - w.Write([]byte(`{ + _, err := w.Write([]byte(`{ "data": { "attributes": { "revision_type": "snapshot", @@ -559,6 +560,7 @@ func setupTestServer(t *testing.T) (*httptest.Server, *uploadrevision.HTTPSealab "type": "upload_revision" } }`)) + assert.NoError(t, err) default: t.Errorf("Unexpected request: %s %s", r.Method, r.URL.Path) diff --git a/pkg/apiclients/fileupload/uploadrevision/opts_test.go b/pkg/apiclients/fileupload/uploadrevision/opts_test.go index e69b3dc9c..a35617c73 100644 --- a/pkg/apiclients/fileupload/uploadrevision/opts_test.go +++ b/pkg/apiclients/fileupload/uploadrevision/opts_test.go @@ -30,7 +30,8 @@ func Test_WithHTTPClient(t *testing.T) { require.NoError(t, err) w.WriteHeader(http.StatusCreated) - w.Write(resp) + _, err = w.Write(resp) + assert.NoError(t, err) })) defer srv.Close() customClient := srv.Client() From 19b18c40199e1411e0e968ad688af12a80329f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Tue, 18 Nov 2025 18:48:45 +0100 Subject: [PATCH 11/15] chore: reduce public interface --- pkg/apiclients/fileupload/client.go | 4 ++-- pkg/apiclients/fileupload/{files => }/list_sources.go | 6 +++--- pkg/apiclients/fileupload/{files => }/list_sources_test.go | 6 ++---- .../fileupload/{files => }/testdata/simplest/package.json | 0 .../fileupload/{files => }/testdata/simplest/src/index.js | 0 .../fileupload/{files => }/testdata/with-ignores/.gitignore | 0 .../{files => }/testdata/with-ignores/package.json | 0 .../{files => }/testdata/with-ignores/src/with-ignores.js | 0 8 files changed, 7 insertions(+), 9 deletions(-) rename pkg/apiclients/fileupload/{files => }/list_sources.go (76%) rename pkg/apiclients/fileupload/{files => }/list_sources_test.go (87%) rename pkg/apiclients/fileupload/{files => }/testdata/simplest/package.json (100%) rename pkg/apiclients/fileupload/{files => }/testdata/simplest/src/index.js (100%) rename pkg/apiclients/fileupload/{files => }/testdata/with-ignores/.gitignore (100%) rename pkg/apiclients/fileupload/{files => }/testdata/with-ignores/package.json (100%) rename pkg/apiclients/fileupload/{files => }/testdata/with-ignores/src/with-ignores.js (100%) diff --git a/pkg/apiclients/fileupload/client.go b/pkg/apiclients/fileupload/client.go index bddba6219..14f380531 100644 --- a/pkg/apiclients/fileupload/client.go +++ b/pkg/apiclients/fileupload/client.go @@ -13,7 +13,6 @@ import ( "github.com/puzpuzpuz/xsync" "github.com/rs/zerolog" - listsources "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/files" "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/filters" "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" "github.com/snyk/go-application-framework/pkg/utils" @@ -233,7 +232,8 @@ func (c *HTTPClient) addFileToRevision(ctx context.Context, revisionID RevisionI // addDirToRevision adds a directory and all its contents to an existing revision. func (c *HTTPClient) addDirToRevision(ctx context.Context, revisionID RevisionID, dirPath string, opts uploadOptions) (UploadResult, error) { - sources, err := listsources.ForPath(dirPath, c.logger, runtime.NumCPU()) + //nolint:contextcheck // will be considered later + sources, err := forPath(dirPath, c.logger, runtime.NumCPU()) if err != nil { return UploadResult{}, fmt.Errorf("failed to list files in directory %s: %w", dirPath, err) } diff --git a/pkg/apiclients/fileupload/files/list_sources.go b/pkg/apiclients/fileupload/list_sources.go similarity index 76% rename from pkg/apiclients/fileupload/files/list_sources.go rename to pkg/apiclients/fileupload/list_sources.go index 7f31d65a8..9bb0689c4 100644 --- a/pkg/apiclients/fileupload/files/list_sources.go +++ b/pkg/apiclients/fileupload/list_sources.go @@ -1,4 +1,4 @@ -package listsources +package fileupload import ( "fmt" @@ -8,8 +8,8 @@ import ( "github.com/snyk/go-application-framework/pkg/utils" ) -// ForPath returns a channel that notifies each file in the path that doesn't match the filter rules. -func ForPath(path string, logger *zerolog.Logger, maxThreads int) (<-chan string, error) { +// forPath returns a channel that notifies each file in the path that doesn't match the filter rules. +func forPath(path string, logger *zerolog.Logger, maxThreads int) (<-chan string, error) { filter := utils.NewFileFilter(path, logger, utils.WithThreadNumber(maxThreads)) rules, err := filter.GetRules([]string{".gitignore", ".dcignore", ".snyk"}) if err != nil { diff --git a/pkg/apiclients/fileupload/files/list_sources_test.go b/pkg/apiclients/fileupload/list_sources_test.go similarity index 87% rename from pkg/apiclients/fileupload/files/list_sources_test.go rename to pkg/apiclients/fileupload/list_sources_test.go index 80e63867d..a473a2090 100644 --- a/pkg/apiclients/fileupload/files/list_sources_test.go +++ b/pkg/apiclients/fileupload/list_sources_test.go @@ -1,4 +1,4 @@ -package listsources_test +package fileupload import ( "fmt" @@ -9,8 +9,6 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - listsources "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/files" ) func Test_ListsSources_Simplest(t *testing.T) { @@ -37,7 +35,7 @@ func Test_ListsSources_WithIgnores(t *testing.T) { func listSourcesForPath(sourcesDir string) ([]string, error) { mockLogger := zerolog.New(io.Discard) - filesCh, err := listsources.ForPath(sourcesDir, &mockLogger, 2) + filesCh, err := forPath(sourcesDir, &mockLogger, 2) if err != nil { return nil, fmt.Errorf("failed to list sources: %w", err) } diff --git a/pkg/apiclients/fileupload/files/testdata/simplest/package.json b/pkg/apiclients/fileupload/testdata/simplest/package.json similarity index 100% rename from pkg/apiclients/fileupload/files/testdata/simplest/package.json rename to pkg/apiclients/fileupload/testdata/simplest/package.json diff --git a/pkg/apiclients/fileupload/files/testdata/simplest/src/index.js b/pkg/apiclients/fileupload/testdata/simplest/src/index.js similarity index 100% rename from pkg/apiclients/fileupload/files/testdata/simplest/src/index.js rename to pkg/apiclients/fileupload/testdata/simplest/src/index.js diff --git a/pkg/apiclients/fileupload/files/testdata/with-ignores/.gitignore b/pkg/apiclients/fileupload/testdata/with-ignores/.gitignore similarity index 100% rename from pkg/apiclients/fileupload/files/testdata/with-ignores/.gitignore rename to pkg/apiclients/fileupload/testdata/with-ignores/.gitignore diff --git a/pkg/apiclients/fileupload/files/testdata/with-ignores/package.json b/pkg/apiclients/fileupload/testdata/with-ignores/package.json similarity index 100% rename from pkg/apiclients/fileupload/files/testdata/with-ignores/package.json rename to pkg/apiclients/fileupload/testdata/with-ignores/package.json diff --git a/pkg/apiclients/fileupload/files/testdata/with-ignores/src/with-ignores.js b/pkg/apiclients/fileupload/testdata/with-ignores/src/with-ignores.js similarity index 100% rename from pkg/apiclients/fileupload/files/testdata/with-ignores/src/with-ignores.js rename to pkg/apiclients/fileupload/testdata/with-ignores/src/with-ignores.js From c8eb46609d68da2171c9dbc7e471f97bbd70e1ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Tue, 18 Nov 2025 19:06:55 +0100 Subject: [PATCH 12/15] chore: reduce public api by moving filters to internal --- .../api}/fileupload/filters/client.go | 0 .../api}/fileupload/filters/client_test.go | 0 .../api}/fileupload/filters/fake_client.go | 0 .../api}/fileupload/filters/opts.go | 0 .../api}/fileupload/filters/utils.go | 0 .../api}/fileupload/filters/utils_test.go | 0 pkg/apiclients/fileupload/client.go | 2 +- pkg/apiclients/fileupload/client_test.go | 6 ++---- pkg/apiclients/fileupload/opts.go | 8 -------- 9 files changed, 3 insertions(+), 13 deletions(-) rename {pkg/apiclients => internal/api}/fileupload/filters/client.go (100%) rename {pkg/apiclients => internal/api}/fileupload/filters/client_test.go (100%) rename {pkg/apiclients => internal/api}/fileupload/filters/fake_client.go (100%) rename {pkg/apiclients => internal/api}/fileupload/filters/opts.go (100%) rename {pkg/apiclients => internal/api}/fileupload/filters/utils.go (100%) rename {pkg/apiclients => internal/api}/fileupload/filters/utils_test.go (100%) diff --git a/pkg/apiclients/fileupload/filters/client.go b/internal/api/fileupload/filters/client.go similarity index 100% rename from pkg/apiclients/fileupload/filters/client.go rename to internal/api/fileupload/filters/client.go diff --git a/pkg/apiclients/fileupload/filters/client_test.go b/internal/api/fileupload/filters/client_test.go similarity index 100% rename from pkg/apiclients/fileupload/filters/client_test.go rename to internal/api/fileupload/filters/client_test.go diff --git a/pkg/apiclients/fileupload/filters/fake_client.go b/internal/api/fileupload/filters/fake_client.go similarity index 100% rename from pkg/apiclients/fileupload/filters/fake_client.go rename to internal/api/fileupload/filters/fake_client.go diff --git a/pkg/apiclients/fileupload/filters/opts.go b/internal/api/fileupload/filters/opts.go similarity index 100% rename from pkg/apiclients/fileupload/filters/opts.go rename to internal/api/fileupload/filters/opts.go diff --git a/pkg/apiclients/fileupload/filters/utils.go b/internal/api/fileupload/filters/utils.go similarity index 100% rename from pkg/apiclients/fileupload/filters/utils.go rename to internal/api/fileupload/filters/utils.go diff --git a/pkg/apiclients/fileupload/filters/utils_test.go b/internal/api/fileupload/filters/utils_test.go similarity index 100% rename from pkg/apiclients/fileupload/filters/utils_test.go rename to internal/api/fileupload/filters/utils_test.go diff --git a/pkg/apiclients/fileupload/client.go b/pkg/apiclients/fileupload/client.go index 14f380531..7fe4fd384 100644 --- a/pkg/apiclients/fileupload/client.go +++ b/pkg/apiclients/fileupload/client.go @@ -13,7 +13,7 @@ import ( "github.com/puzpuzpuz/xsync" "github.com/rs/zerolog" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/filters" + "github.com/snyk/go-application-framework/internal/api/fileupload/filters" "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" "github.com/snyk/go-application-framework/pkg/utils" ) diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index 575fd0d5b..c5fdfd082 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/snyk/go-application-framework/internal/api/fileupload/filters" "github.com/snyk/go-application-framework/pkg/apiclients/fileupload" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/filters" "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" ) @@ -591,7 +591,7 @@ func setupTest( t *testing.T, llcfg uploadrevision.FakeClientConfig, files []uploadrevision.LoadedFile, - allowList filters.AllowList, + _ filters.AllowList, ) (context.Context, *uploadrevision.FakeSealableClient, *fileupload.HTTPClient, *os.File) { t.Helper() @@ -599,14 +599,12 @@ func setupTest( orgID := uuid.New() fakeSealeableClient := uploadrevision.NewFakeSealableClient(llcfg) - fakeFiltersClient := filters.NewFakeClient(allowList) client := fileupload.NewClient( nil, fileupload.Config{ OrgID: orgID, }, fileupload.WithUploadRevisionSealableClient(fakeSealeableClient), - fileupload.WithFiltersClient(fakeFiltersClient), ) dir := createTmpFiles(t, files) diff --git a/pkg/apiclients/fileupload/opts.go b/pkg/apiclients/fileupload/opts.go index c56f7e6af..f341ab7ef 100644 --- a/pkg/apiclients/fileupload/opts.go +++ b/pkg/apiclients/fileupload/opts.go @@ -3,7 +3,6 @@ package fileupload import ( "github.com/rs/zerolog" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/filters" "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" ) @@ -17,13 +16,6 @@ func WithUploadRevisionSealableClient(client uploadrevision.SealableClient) Opti } } -// WithFiltersClient allows injecting a custom low-level client (primarily for testing). -func WithFiltersClient(client filters.Client) Option { - return func(c *HTTPClient) { - c.filtersClient = client - } -} - // WithLogger allows injecting a custom logger instance. func WithLogger(logger *zerolog.Logger) Option { return func(h *HTTPClient) { From 1550c07f4197168f67ce43c1ad0cae9d6d9530ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Tue, 18 Nov 2025 19:17:38 +0100 Subject: [PATCH 13/15] chore: reduce public api surface by moving upload revision --- .../api}/fileupload/uploadrevision/client.go | 0 .../fileupload/uploadrevision/client_test.go | 88 +++++++-------- .../fileupload/uploadrevision/compression.go | 0 .../uploadrevision/compression_test.go | 2 +- .../api}/fileupload/uploadrevision/errors.go | 0 .../fileupload/uploadrevision/fake_client.go | 0 .../api}/fileupload/uploadrevision/opts.go | 0 .../fileupload/uploadrevision/opts_test.go | 8 +- .../api}/fileupload/uploadrevision/types.go | 0 pkg/apiclients/fileupload/batch.go | 2 +- pkg/apiclients/fileupload/client.go | 18 ++-- pkg/apiclients/fileupload/client_test.go | 102 +++++++++--------- pkg/apiclients/fileupload/errors.go | 4 +- pkg/apiclients/fileupload/fake_client.go | 2 +- pkg/apiclients/fileupload/opts.go | 2 +- pkg/apiclients/fileupload/types.go | 2 +- 16 files changed, 116 insertions(+), 114 deletions(-) rename {pkg/apiclients => internal/api}/fileupload/uploadrevision/client.go (100%) rename {pkg/apiclients => internal/api}/fileupload/uploadrevision/client_test.go (85%) rename {pkg/apiclients => internal/api}/fileupload/uploadrevision/compression.go (100%) rename {pkg/apiclients => internal/api}/fileupload/uploadrevision/compression_test.go (97%) rename {pkg/apiclients => internal/api}/fileupload/uploadrevision/errors.go (100%) rename {pkg/apiclients => internal/api}/fileupload/uploadrevision/fake_client.go (100%) rename {pkg/apiclients => internal/api}/fileupload/uploadrevision/opts.go (100%) rename {pkg/apiclients => internal/api}/fileupload/uploadrevision/opts_test.go (77%) rename {pkg/apiclients => internal/api}/fileupload/uploadrevision/types.go (100%) diff --git a/pkg/apiclients/fileupload/uploadrevision/client.go b/internal/api/fileupload/uploadrevision/client.go similarity index 100% rename from pkg/apiclients/fileupload/uploadrevision/client.go rename to internal/api/fileupload/uploadrevision/client.go diff --git a/pkg/apiclients/fileupload/uploadrevision/client_test.go b/internal/api/fileupload/uploadrevision/client_test.go similarity index 85% rename from pkg/apiclients/fileupload/uploadrevision/client_test.go rename to internal/api/fileupload/uploadrevision/client_test.go index 64f4dbbcc..70d7efb8c 100644 --- a/pkg/apiclients/fileupload/uploadrevision/client_test.go +++ b/internal/api/fileupload/uploadrevision/client_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + uploadrevision2 "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" ) var ( @@ -42,27 +42,27 @@ func TestClient_CreateRevision(t *testing.T) { } func TestClient_CreateRevision_EmptyOrgID(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) resp, err := c.CreateRevision(context.Background(), uuid.Nil) assert.Error(t, err) assert.Nil(t, resp) - assert.ErrorIs(t, err, uploadrevision.ErrEmptyOrgID) + assert.ErrorIs(t, err, uploadrevision2.ErrEmptyOrgID) } func TestClient_CreateRevision_ServerError(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) })) - c := uploadrevision.NewClient(uploadrevision.Config{ + c := uploadrevision2.NewClient(uploadrevision2.Config{ BaseURL: srv.URL, }) resp, err := c.CreateRevision(context.Background(), orgID) assert.Nil(t, resp) - var httpErr *uploadrevision.HTTPError + var httpErr *uploadrevision2.HTTPError assert.ErrorAs(t, err, &httpErr) assert.Equal(t, http.StatusInternalServerError, httpErr.StatusCode) assert.Equal(t, "create upload revision", httpErr.Operation) @@ -81,7 +81,7 @@ func TestClient_UploadFiles(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []uploadrevision.UploadFile{ + []uploadrevision2.UploadFile{ {Path: "foo/bar", File: fd}, }) @@ -105,7 +105,7 @@ func TestClient_UploadFiles_MultipleFiles(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []uploadrevision.UploadFile{ + []uploadrevision2.UploadFile{ {Path: "file1.txt", File: file1}, {Path: "file2.json", File: file2}, }) @@ -114,7 +114,7 @@ func TestClient_UploadFiles_MultipleFiles(t *testing.T) { } func TestClient_UploadFiles_EmptyOrgID(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) mockFS := fstest.MapFS{ "test.txt": {Data: []byte("content")}, @@ -125,16 +125,16 @@ func TestClient_UploadFiles_EmptyOrgID(t *testing.T) { err = c.UploadFiles(context.Background(), uuid.Nil, // empty orgID revID, - []uploadrevision.UploadFile{ + []uploadrevision2.UploadFile{ {Path: "test.txt", File: file}, }) assert.Error(t, err) - assert.ErrorIs(t, err, uploadrevision.ErrEmptyOrgID) + assert.ErrorIs(t, err, uploadrevision2.ErrEmptyOrgID) } func TestClient_UploadFiles_EmptyRevisionID(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) mockFS := fstest.MapFS{ "test.txt": {Data: []byte("content")}, @@ -145,16 +145,16 @@ func TestClient_UploadFiles_EmptyRevisionID(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, uuid.Nil, // empty revisionID - []uploadrevision.UploadFile{ + []uploadrevision2.UploadFile{ {Path: "test.txt", File: file}, }) assert.Error(t, err) - assert.ErrorIs(t, err, uploadrevision.ErrEmptyRevisionID) + assert.ErrorIs(t, err, uploadrevision2.ErrEmptyRevisionID) } func TestClient_UploadFiles_FileSizeLimit(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) largeContent := make([]byte, c.GetLimits().FileSizeLimit+1) mockFS := fstest.MapFS{ @@ -167,12 +167,12 @@ func TestClient_UploadFiles_FileSizeLimit(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []uploadrevision.UploadFile{ + []uploadrevision2.UploadFile{ {Path: "large_file.txt", File: file}, }) assert.Error(t, err) - var fileSizeErr *uploadrevision.FileSizeLimitError + var fileSizeErr *uploadrevision2.FileSizeLimitError assert.ErrorAs(t, err, &fileSizeErr) assert.Equal(t, "large_file.txt", fileSizeErr.FilePath) assert.Equal(t, c.GetLimits().FileSizeLimit+1, fileSizeErr.FileSize) @@ -180,9 +180,9 @@ func TestClient_UploadFiles_FileSizeLimit(t *testing.T) { } func TestClient_UploadFiles_FileCountLimit(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) - files := make([]uploadrevision.UploadFile, c.GetLimits().FileCountLimit+1) + files := make([]uploadrevision2.UploadFile, c.GetLimits().FileCountLimit+1) mockFS := fstest.MapFS{} for i := range c.GetLimits().FileCountLimit + 1 { @@ -192,7 +192,7 @@ func TestClient_UploadFiles_FileCountLimit(t *testing.T) { file, err := mockFS.Open(filename) require.NoError(t, err) - files[i] = uploadrevision.UploadFile{ + files[i] = uploadrevision2.UploadFile{ Path: filename, File: file, } @@ -201,14 +201,14 @@ func TestClient_UploadFiles_FileCountLimit(t *testing.T) { err := c.UploadFiles(context.Background(), orgID, revID, files) assert.Error(t, err) - var fileCountErr *uploadrevision.FileCountLimitError + var fileCountErr *uploadrevision2.FileCountLimitError assert.ErrorAs(t, err, &fileCountErr) assert.Equal(t, c.GetLimits().FileCountLimit+1, fileCountErr.Count) assert.Equal(t, c.GetLimits().FileCountLimit, fileCountErr.Limit) } func TestClient_UploadFiles_FilePathLengthLimit(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) // Create a file path that exceeds the limit longFilePath := strings.Repeat("a", c.GetLimits().FilePathLengthLimit+1) @@ -223,12 +223,12 @@ func TestClient_UploadFiles_FilePathLengthLimit(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []uploadrevision.UploadFile{ + []uploadrevision2.UploadFile{ {Path: longFilePath, File: file}, }) assert.Error(t, err) - var filePathLengthErr *uploadrevision.FilePathLengthLimitError + var filePathLengthErr *uploadrevision2.FilePathLengthLimitError assert.ErrorAs(t, err, &filePathLengthErr) assert.Equal(t, longFilePath, filePathLengthErr.FilePath) assert.Equal(t, c.GetLimits().FilePathLengthLimit+1, filePathLengthErr.Length) @@ -253,7 +253,7 @@ func TestClient_UploadFiles_FilePathLengthExactlyAtLimit(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []uploadrevision.UploadFile{ + []uploadrevision2.UploadFile{ {Path: filePathAtLimit, File: file}, }) @@ -261,12 +261,12 @@ func TestClient_UploadFiles_FilePathLengthExactlyAtLimit(t *testing.T) { } func TestClient_UploadFiles_TotalPayloadSizeLimit(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) // Create multiple files that individually are under the size limit, // but together exceed the total payload size limit mockFS := fstest.MapFS{} - files := []uploadrevision.UploadFile{} + files := []uploadrevision2.UploadFile{} // Use files that are 30MB each (under the 50MB individual limit) // 8 files = 240MB > 200MB total limit @@ -280,7 +280,7 @@ func TestClient_UploadFiles_TotalPayloadSizeLimit(t *testing.T) { file, err := mockFS.Open(filename) require.NoError(t, err) - files = append(files, uploadrevision.UploadFile{ + files = append(files, uploadrevision2.UploadFile{ Path: filename, File: file, }) @@ -289,7 +289,7 @@ func TestClient_UploadFiles_TotalPayloadSizeLimit(t *testing.T) { err := c.UploadFiles(context.Background(), orgID, revID, files) assert.Error(t, err) - var totalSizeErr *uploadrevision.TotalPayloadSizeLimitError + var totalSizeErr *uploadrevision2.TotalPayloadSizeLimitError assert.ErrorAs(t, err, &totalSizeErr) assert.Equal(t, fileSize*int64(numFiles), totalSizeErr.TotalSize) assert.Equal(t, c.GetLimits().TotalPayloadSizeLimit, totalSizeErr.Limit) @@ -301,7 +301,7 @@ func TestClient_UploadFiles_TotalPayloadSizeExactlyAtLimit(t *testing.T) { // Test boundary: exactly 200MB (should succeed) mockFS := fstest.MapFS{} - files := []uploadrevision.UploadFile{} + files := []uploadrevision2.UploadFile{} // Create files that sum exactly to 200MB // 4 files of 50MB each = 200MB exactly @@ -315,7 +315,7 @@ func TestClient_UploadFiles_TotalPayloadSizeExactlyAtLimit(t *testing.T) { file, err := mockFS.Open(filename) require.NoError(t, err) - files = append(files, uploadrevision.UploadFile{ + files = append(files, uploadrevision2.UploadFile{ Path: filename, File: file, }) @@ -339,7 +339,7 @@ func TestClient_UploadFiles_IndividualFileSizeExactlyAtLimit(t *testing.T) { file, err := mockFS.Open("exact_limit.bin") require.NoError(t, err) - err = c.UploadFiles(context.Background(), orgID, revID, []uploadrevision.UploadFile{ + err = c.UploadFiles(context.Background(), orgID, revID, []uploadrevision2.UploadFile{ {Path: "exact_limit.bin", File: file}, }) @@ -348,7 +348,7 @@ func TestClient_UploadFiles_IndividualFileSizeExactlyAtLimit(t *testing.T) { } func TestClient_UploadFiles_SpecialFileError(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) tests := []struct { name string @@ -404,13 +404,13 @@ func TestClient_UploadFiles_SpecialFileError(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []uploadrevision.UploadFile{ + []uploadrevision2.UploadFile{ {Path: filePath, File: file}, }) assert.Error(t, err) - var sfe *uploadrevision.SpecialFileError + var sfe *uploadrevision2.SpecialFileError assert.ErrorAs(t, err, &sfe) assert.Equal(t, filePath, sfe.FilePath) }) @@ -438,7 +438,7 @@ func TestClient_UploadFiles_Symlink(t *testing.T) { err = c.UploadFiles(context.Background(), orgID, revID, - []uploadrevision.UploadFile{ + []uploadrevision2.UploadFile{ {Path: tmpSlnPth, File: tmpSln}, }) @@ -446,12 +446,12 @@ func TestClient_UploadFiles_Symlink(t *testing.T) { } func TestClient_UploadFiles_EmptyFileList(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) - err := c.UploadFiles(context.Background(), orgID, revID, []uploadrevision.UploadFile{}) + err := c.UploadFiles(context.Background(), orgID, revID, []uploadrevision2.UploadFile{}) assert.Error(t, err) - assert.ErrorIs(t, err, uploadrevision.ErrNoFilesProvided) + assert.ErrorIs(t, err, uploadrevision2.ErrNoFilesProvided) } func TestClient_SealRevision(t *testing.T) { @@ -466,7 +466,7 @@ func TestClient_SealRevision(t *testing.T) { } func TestClient_SealRevision_EmptyOrgID(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) resp, err := c.SealRevision(context.Background(), uuid.Nil, // empty orgID @@ -474,12 +474,12 @@ func TestClient_SealRevision_EmptyOrgID(t *testing.T) { ) assert.Error(t, err) - assert.ErrorIs(t, err, uploadrevision.ErrEmptyOrgID) + assert.ErrorIs(t, err, uploadrevision2.ErrEmptyOrgID) assert.Nil(t, resp) } func TestClient_SealRevision_EmptyRevisionID(t *testing.T) { - c := uploadrevision.NewClient(uploadrevision.Config{}) + c := uploadrevision2.NewClient(uploadrevision2.Config{}) resp, err := c.SealRevision(context.Background(), orgID, @@ -487,11 +487,11 @@ func TestClient_SealRevision_EmptyRevisionID(t *testing.T) { ) assert.Error(t, err) - assert.ErrorIs(t, err, uploadrevision.ErrEmptyRevisionID) + assert.ErrorIs(t, err, uploadrevision2.ErrEmptyRevisionID) assert.Nil(t, resp) } -func setupTestServer(t *testing.T) (*httptest.Server, *uploadrevision.HTTPSealableClient) { +func setupTestServer(t *testing.T) (*httptest.Server, *uploadrevision2.HTTPSealableClient) { t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "2024-10-15", r.URL.Query().Get("version")) @@ -568,7 +568,7 @@ func setupTestServer(t *testing.T) (*httptest.Server, *uploadrevision.HTTPSealab } })) - client := uploadrevision.NewClient(uploadrevision.Config{ + client := uploadrevision2.NewClient(uploadrevision2.Config{ BaseURL: srv.URL, }) diff --git a/pkg/apiclients/fileupload/uploadrevision/compression.go b/internal/api/fileupload/uploadrevision/compression.go similarity index 100% rename from pkg/apiclients/fileupload/uploadrevision/compression.go rename to internal/api/fileupload/uploadrevision/compression.go diff --git a/pkg/apiclients/fileupload/uploadrevision/compression_test.go b/internal/api/fileupload/uploadrevision/compression_test.go similarity index 97% rename from pkg/apiclients/fileupload/uploadrevision/compression_test.go rename to internal/api/fileupload/uploadrevision/compression_test.go index 84caed70e..892c4fba0 100644 --- a/pkg/apiclients/fileupload/uploadrevision/compression_test.go +++ b/internal/api/fileupload/uploadrevision/compression_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" ) func TestCompressionRoundTripper_RoundTrip(t *testing.T) { diff --git a/pkg/apiclients/fileupload/uploadrevision/errors.go b/internal/api/fileupload/uploadrevision/errors.go similarity index 100% rename from pkg/apiclients/fileupload/uploadrevision/errors.go rename to internal/api/fileupload/uploadrevision/errors.go diff --git a/pkg/apiclients/fileupload/uploadrevision/fake_client.go b/internal/api/fileupload/uploadrevision/fake_client.go similarity index 100% rename from pkg/apiclients/fileupload/uploadrevision/fake_client.go rename to internal/api/fileupload/uploadrevision/fake_client.go diff --git a/pkg/apiclients/fileupload/uploadrevision/opts.go b/internal/api/fileupload/uploadrevision/opts.go similarity index 100% rename from pkg/apiclients/fileupload/uploadrevision/opts.go rename to internal/api/fileupload/uploadrevision/opts.go diff --git a/pkg/apiclients/fileupload/uploadrevision/opts_test.go b/internal/api/fileupload/uploadrevision/opts_test.go similarity index 77% rename from pkg/apiclients/fileupload/uploadrevision/opts_test.go rename to internal/api/fileupload/uploadrevision/opts_test.go index a35617c73..99d201996 100644 --- a/pkg/apiclients/fileupload/uploadrevision/opts_test.go +++ b/internal/api/fileupload/uploadrevision/opts_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + uploadrevision2 "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" ) type CustomRoundTripper struct{} @@ -26,7 +26,7 @@ func Test_WithHTTPClient(t *testing.T) { fooValue := r.Header.Get("foo") assert.Equal(t, "bar", fooValue) - resp, err := json.Marshal(uploadrevision.ResponseBody{}) + resp, err := json.Marshal(uploadrevision2.ResponseBody{}) require.NoError(t, err) w.WriteHeader(http.StatusCreated) @@ -37,9 +37,9 @@ func Test_WithHTTPClient(t *testing.T) { customClient := srv.Client() customClient.Transport = &CustomRoundTripper{} - llc := uploadrevision.NewClient(uploadrevision.Config{ + llc := uploadrevision2.NewClient(uploadrevision2.Config{ BaseURL: srv.URL, - }, uploadrevision.WithHTTPClient(customClient)) + }, uploadrevision2.WithHTTPClient(customClient)) _, err := llc.CreateRevision(context.Background(), uuid.New()) diff --git a/pkg/apiclients/fileupload/uploadrevision/types.go b/internal/api/fileupload/uploadrevision/types.go similarity index 100% rename from pkg/apiclients/fileupload/uploadrevision/types.go rename to internal/api/fileupload/uploadrevision/types.go diff --git a/pkg/apiclients/fileupload/batch.go b/pkg/apiclients/fileupload/batch.go index 7f0f55fac..0153a6b84 100644 --- a/pkg/apiclients/fileupload/batch.go +++ b/pkg/apiclients/fileupload/batch.go @@ -6,7 +6,7 @@ import ( "os" "path/filepath" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" ) // uploadBatch manages a batch of files for upload. diff --git a/pkg/apiclients/fileupload/client.go b/pkg/apiclients/fileupload/client.go index 7fe4fd384..054286e84 100644 --- a/pkg/apiclients/fileupload/client.go +++ b/pkg/apiclients/fileupload/client.go @@ -14,7 +14,7 @@ import ( "github.com/rs/zerolog" "github.com/snyk/go-application-framework/internal/api/fileupload/filters" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + uploadrevision2 "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" "github.com/snyk/go-application-framework/pkg/utils" ) @@ -26,7 +26,7 @@ type Config struct { // HTTPClient provides high-level file upload functionality. type HTTPClient struct { - uploadRevisionSealableClient uploadrevision.SealableClient + uploadRevisionSealableClient uploadrevision2.SealableClient filtersClient filters.Client cfg Config filters Filters @@ -61,9 +61,9 @@ func NewClient(httpClient *http.Client, cfg Config, opts ...Option) *HTTPClient } if client.uploadRevisionSealableClient == nil { - client.uploadRevisionSealableClient = uploadrevision.NewClient(uploadrevision.Config{ + client.uploadRevisionSealableClient = uploadrevision2.NewClient(uploadrevision2.Config{ BaseURL: cfg.BaseURL, - }, uploadrevision.WithHTTPClient(httpClient)) + }, uploadrevision2.WithHTTPClient(httpClient)) } if client.filtersClient == nil { @@ -162,7 +162,7 @@ func (c *HTTPClient) addPathsToRevision( if ff.Stat.Size() > fileSizeLimit { return &FilteredFile{ Path: ff.Path, - Reason: uploadrevision.NewFileSizeLimitError(ff.Stat.Name(), ff.Stat.Size(), fileSizeLimit), + Reason: uploadrevision2.NewFileSizeLimitError(ff.Stat.Name(), ff.Stat.Size(), fileSizeLimit), } } @@ -174,7 +174,7 @@ func (c *HTTPClient) addPathsToRevision( if len(ff.Path) > filePathLengthLimit { return &FilteredFile{ Path: ff.Path, - Reason: uploadrevision.NewFilePathLengthLimitError(ff.Path, len(ff.Path), filePathLengthLimit), + Reason: uploadrevision2.NewFilePathLengthLimitError(ff.Path, len(ff.Path), filePathLengthLimit), } } @@ -270,7 +270,7 @@ func (c *HTTPClient) CreateRevisionFromPaths(ctx context.Context, paths []string for _, pth := range paths { info, err := os.Stat(pth) if err != nil { - return UploadResult{}, uploadrevision.NewFileAccessError(pth, err) + return UploadResult{}, uploadrevision2.NewFileAccessError(pth, err) } if info.IsDir() { @@ -306,7 +306,7 @@ func (c *HTTPClient) CreateRevisionFromPaths(ctx context.Context, paths []string func (c *HTTPClient) CreateRevisionFromDir(ctx context.Context, dirPath string) (UploadResult, error) { info, err := os.Stat(dirPath) if err != nil { - return UploadResult{}, uploadrevision.NewFileAccessError(dirPath, err) + return UploadResult{}, uploadrevision2.NewFileAccessError(dirPath, err) } if !info.IsDir() { @@ -321,7 +321,7 @@ func (c *HTTPClient) CreateRevisionFromDir(ctx context.Context, dirPath string) func (c *HTTPClient) CreateRevisionFromFile(ctx context.Context, filePath string) (UploadResult, error) { info, err := os.Stat(filePath) if err != nil { - return UploadResult{}, uploadrevision.NewFileAccessError(filePath, err) + return UploadResult{}, uploadrevision2.NewFileAccessError(filePath, err) } if !info.Mode().IsRegular() { diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index c5fdfd082..2f78669b4 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -15,8 +15,8 @@ import ( "github.com/stretchr/testify/require" "github.com/snyk/go-application-framework/internal/api/fileupload/filters" + uploadrevision2 "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" "github.com/snyk/go-application-framework/pkg/apiclients/fileupload" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" ) var mainpath = filepath.Join("src", "main.go") @@ -30,7 +30,7 @@ var nonexistpath = filepath.Join("nonexistent", "file.go") var missingpath = filepath.Join("another", "missing", "path.txt") // CreateTmpFiles is an utility function used to create temporary files in tests. -func createTmpFiles(t *testing.T, files []uploadrevision.LoadedFile) (dir *os.File) { +func createTmpFiles(t *testing.T, files []uploadrevision2.LoadedFile) (dir *os.File) { t.Helper() tempDir := t.TempDir() @@ -69,8 +69,8 @@ func createTmpFiles(t *testing.T, files []uploadrevision.LoadedFile) (dir *os.Fi } func Test_CreateRevisionFromPaths(t *testing.T) { - llcfg := uploadrevision.FakeClientConfig{ - Limits: uploadrevision.Limits{ + llcfg := uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ FileCountLimit: 10, FileSizeLimit: 100, TotalPayloadSizeLimit: 10_000, @@ -84,7 +84,7 @@ func Test_CreateRevisionFromPaths(t *testing.T) { } t.Run("mixed files and directories", func(t *testing.T) { - allFiles := []uploadrevision.LoadedFile{ + allFiles := []uploadrevision2.LoadedFile{ {Path: mainpath, Content: "package main"}, {Path: utilspath, Content: "package utils"}, {Path: "config.yaml", Content: "version: 1"}, @@ -116,7 +116,7 @@ func Test_CreateRevisionFromPaths(t *testing.T) { }) t.Run("error handling with better context", func(t *testing.T) { - ctx, _, client, _ := setupTest(t, llcfg, []uploadrevision.LoadedFile{}, allowList) + ctx, _, client, _ := setupTest(t, llcfg, []uploadrevision2.LoadedFile{}, allowList) paths := []string{ nonexistpath, @@ -125,15 +125,15 @@ func Test_CreateRevisionFromPaths(t *testing.T) { _, err := client.CreateRevisionFromPaths(ctx, paths) require.Error(t, err) - var fileAccessErr *uploadrevision.FileAccessError + var fileAccessErr *uploadrevision2.FileAccessError assert.ErrorAs(t, err, &fileAccessErr) assert.Equal(t, nonexistpath, fileAccessErr.FilePath) }) } func Test_CreateRevisionFromDir(t *testing.T) { - llcfg := uploadrevision.FakeClientConfig{ - Limits: uploadrevision.Limits{ + llcfg := uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ FileCountLimit: 2, FileSizeLimit: 100, TotalPayloadSizeLimit: 10_000, @@ -147,7 +147,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { } t.Run("uploading a shallow directory", func(t *testing.T) { - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: "file1.txt", Content: "content1", @@ -169,7 +169,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { }) t.Run("uploading a directory with nested files", func(t *testing.T) { - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: filepath.Join("src", "main.go"), Content: "package main\n\nfunc main() {}", @@ -191,7 +191,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { }) t.Run("uploading a directory exceeding the file count limit for a single upload", func(t *testing.T) { - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: "file1.txt", Content: "root level file", @@ -225,24 +225,24 @@ func Test_CreateRevisionFromDir(t *testing.T) { }) t.Run("uploading a directory with file exceeding the file size limit", func(t *testing.T) { - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: "file2.txt", Content: "foo", }, } - additionalFiles := []uploadrevision.LoadedFile{ + additionalFiles := []uploadrevision2.LoadedFile{ { Path: "file1.txt", Content: "foo bar", }, } - allFiles := make([]uploadrevision.LoadedFile, 0, 2) + allFiles := make([]uploadrevision2.LoadedFile, 0, 2) allFiles = append(allFiles, expectedFiles...) allFiles = append(allFiles, additionalFiles...) - ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ - Limits: uploadrevision.Limits{ + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ FileCountLimit: 2, FileSizeLimit: 6, TotalPayloadSizeLimit: 100, @@ -253,7 +253,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { res, err := client.CreateRevisionFromDir(ctx, dir.Name()) require.NoError(t, err) - var fileSizeErr *uploadrevision.FileSizeLimitError + var fileSizeErr *uploadrevision2.FileSizeLimitError assert.Len(t, res.FilteredFiles, 1) ff := res.FilteredFiles[0] assert.Contains(t, ff.Path, "file1.txt") @@ -270,7 +270,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { t.Run("uploading a directory exceeding total payload size limit triggers batching", func(t *testing.T) { // Create files that together exceed the payload size limit but not the count limit // Each file is 30 bytes, limit is 70 bytes, so 3 files (90 bytes) should be split into 2 batches - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: "file1.txt", Content: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", // 30 bytes @@ -284,8 +284,8 @@ func Test_CreateRevisionFromDir(t *testing.T) { Content: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", // 30 bytes }, } - ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ - Limits: uploadrevision.Limits{ + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ FileCountLimit: 10, // High enough to not trigger count-based batching FileSizeLimit: 50, // Each file is under this TotalPayloadSizeLimit: 70, // 70 bytes - forces batching by size @@ -308,7 +308,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { // Tests edge case where individual files are large relative to the payload limit. // File1: 150 bytes, File2: 80 bytes, File3: 60 bytes; Limit: 200 bytes // Expected batches: [File1], [File2], [File3] - each file in its own batch - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: "large1.txt", Content: string(make([]byte, 150)), @@ -322,8 +322,8 @@ func Test_CreateRevisionFromDir(t *testing.T) { Content: string(make([]byte, 60)), }, } - ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ - Limits: uploadrevision.Limits{ + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ FileCountLimit: 10, FileSizeLimit: 160, TotalPayloadSizeLimit: 200, @@ -344,7 +344,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { // Tests realistic scenario with mixed file sizes. // Files: 10, 60, 5, 70, 45 bytes; Limit: 100 bytes // Expected batching: [10+60+5=75], [70], [45] - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: "tiny.txt", Content: string(make([]byte, 10)), @@ -366,8 +366,8 @@ func Test_CreateRevisionFromDir(t *testing.T) { Content: string(make([]byte, 45)), }, } - ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ - Limits: uploadrevision.Limits{ + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ FileCountLimit: 10, FileSizeLimit: 80, TotalPayloadSizeLimit: 100, @@ -389,15 +389,15 @@ func Test_CreateRevisionFromDir(t *testing.T) { // 8 files of 30 bytes each = 240 bytes total // FileCountLimit: 10, TotalPayloadSizeLimit: 200 bytes // Should batch by size first: [file1-6=180], [file7-8=60] - expectedFiles := make([]uploadrevision.LoadedFile, 8) + expectedFiles := make([]uploadrevision2.LoadedFile, 8) for i := 0; i < 8; i++ { - expectedFiles[i] = uploadrevision.LoadedFile{ + expectedFiles[i] = uploadrevision2.LoadedFile{ Path: fmt.Sprintf("file%d.txt", i), Content: string(make([]byte, 30)), } } - ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ - Limits: uploadrevision.Limits{ + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ FileCountLimit: 10, FileSizeLimit: 50, TotalPayloadSizeLimit: 200, @@ -415,7 +415,7 @@ func Test_CreateRevisionFromDir(t *testing.T) { }) t.Run("uploading a directory with filtering disabled", func(t *testing.T) { - allFiles := []uploadrevision.LoadedFile{ + allFiles := []uploadrevision2.LoadedFile{ { Path: mainpath, Content: "package main\n\nfunc main() {}", @@ -451,8 +451,8 @@ func Test_CreateRevisionFromDir(t *testing.T) { } func Test_CreateRevisionFromFile(t *testing.T) { - llcfg := uploadrevision.FakeClientConfig{ - Limits: uploadrevision.Limits{ + llcfg := uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ FileCountLimit: 2, FileSizeLimit: 100, TotalPayloadSizeLimit: 10_000, @@ -466,7 +466,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { } t.Run("uploading a file", func(t *testing.T) { - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: "file1.txt", Content: "content1", @@ -484,14 +484,14 @@ func Test_CreateRevisionFromFile(t *testing.T) { }) t.Run("uploading a file exceeding the file size limit", func(t *testing.T) { - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: "file1.txt", Content: "foo bar", }, } - ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ - Limits: uploadrevision.Limits{ + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ FileCountLimit: 1, FileSizeLimit: 6, TotalPayloadSizeLimit: 10_000, @@ -502,7 +502,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) require.NoError(t, err) - var fileSizeErr *uploadrevision.FileSizeLimitError + var fileSizeErr *uploadrevision2.FileSizeLimitError assert.Len(t, res.FilteredFiles, 1) ff := res.FilteredFiles[0] assert.Contains(t, ff.Path, "file1.txt") @@ -517,14 +517,14 @@ func Test_CreateRevisionFromFile(t *testing.T) { }) t.Run("uploading a file exceeding the file path limit", func(t *testing.T) { - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: "file1.txt", Content: "foo bar", }, } - ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision.FakeClientConfig{ - Limits: uploadrevision.Limits{ + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ FileCountLimit: 1, FileSizeLimit: 10, TotalPayloadSizeLimit: 10_000, @@ -535,7 +535,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) require.NoError(t, err) - var filePathErr *uploadrevision.FilePathLengthLimitError + var filePathErr *uploadrevision2.FilePathLengthLimitError assert.Len(t, res.FilteredFiles, 1) ff := res.FilteredFiles[0] assert.Contains(t, ff.Path, "file1.txt") @@ -549,7 +549,7 @@ func Test_CreateRevisionFromFile(t *testing.T) { }) t.Run("uploading a file with filtering disabled", func(t *testing.T) { - expectedFiles := []uploadrevision.LoadedFile{ + expectedFiles := []uploadrevision2.LoadedFile{ { Path: "script.js", Content: "console.log('hi')", @@ -568,16 +568,16 @@ func Test_CreateRevisionFromFile(t *testing.T) { }) } -func expectEqualFiles(t *testing.T, expectedFiles, uploadedFiles []uploadrevision.LoadedFile) { +func expectEqualFiles(t *testing.T, expectedFiles, uploadedFiles []uploadrevision2.LoadedFile) { t.Helper() require.Equal(t, len(expectedFiles), len(uploadedFiles)) - slices.SortFunc(expectedFiles, func(fileA, fileB uploadrevision.LoadedFile) int { + slices.SortFunc(expectedFiles, func(fileA, fileB uploadrevision2.LoadedFile) int { return strings.Compare(fileA.Path, fileB.Path) }) - slices.SortFunc(uploadedFiles, func(fileA, fileB uploadrevision.LoadedFile) int { + slices.SortFunc(uploadedFiles, func(fileA, fileB uploadrevision2.LoadedFile) int { return strings.Compare(fileA.Path, fileB.Path) }) @@ -589,16 +589,16 @@ func expectEqualFiles(t *testing.T, expectedFiles, uploadedFiles []uploadrevisio func setupTest( t *testing.T, - llcfg uploadrevision.FakeClientConfig, - files []uploadrevision.LoadedFile, + llcfg uploadrevision2.FakeClientConfig, + files []uploadrevision2.LoadedFile, _ filters.AllowList, -) (context.Context, *uploadrevision.FakeSealableClient, *fileupload.HTTPClient, *os.File) { +) (context.Context, *uploadrevision2.FakeSealableClient, *fileupload.HTTPClient, *os.File) { t.Helper() ctx := context.Background() orgID := uuid.New() - fakeSealeableClient := uploadrevision.NewFakeSealableClient(llcfg) + fakeSealeableClient := uploadrevision2.NewFakeSealableClient(llcfg) client := fileupload.NewClient( nil, fileupload.Config{ diff --git a/pkg/apiclients/fileupload/errors.go b/pkg/apiclients/fileupload/errors.go index df44bc518..0c35bc39a 100644 --- a/pkg/apiclients/fileupload/errors.go +++ b/pkg/apiclients/fileupload/errors.go @@ -1,6 +1,8 @@ package fileupload -import "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" +import ( + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" +) // Aliasing uploadRevisionSealableClient errors so that they're scoped to the fileupload package as well. diff --git a/pkg/apiclients/fileupload/fake_client.go b/pkg/apiclients/fileupload/fake_client.go index a41e04bab..4cd59a087 100644 --- a/pkg/apiclients/fileupload/fake_client.go +++ b/pkg/apiclients/fileupload/fake_client.go @@ -7,7 +7,7 @@ import ( "github.com/google/uuid" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" ) type FakeClient struct { diff --git a/pkg/apiclients/fileupload/opts.go b/pkg/apiclients/fileupload/opts.go index f341ab7ef..e9403d1a4 100644 --- a/pkg/apiclients/fileupload/opts.go +++ b/pkg/apiclients/fileupload/opts.go @@ -3,7 +3,7 @@ package fileupload import ( "github.com/rs/zerolog" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" ) // Option allows customizing the Client during construction. diff --git a/pkg/apiclients/fileupload/types.go b/pkg/apiclients/fileupload/types.go index 1344675ff..2df32e6a0 100644 --- a/pkg/apiclients/fileupload/types.go +++ b/pkg/apiclients/fileupload/types.go @@ -5,7 +5,7 @@ import ( "github.com/puzpuzpuz/xsync" - "github.com/snyk/go-application-framework/pkg/apiclients/fileupload/uploadrevision" + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" ) // OrgID represents an organization identifier. From c847c0df52552a6e3febd1508f94bb5ac454c763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Wed, 19 Nov 2025 17:17:59 +0100 Subject: [PATCH 14/15] chore: reduce public interface by moving Filter type --- internal/api/fileupload/types.go | 15 +++++++++++++++ pkg/apiclients/fileupload/client.go | 23 ++++++++++++----------- pkg/apiclients/fileupload/types.go | 12 ------------ 3 files changed, 27 insertions(+), 23 deletions(-) create mode 100644 internal/api/fileupload/types.go diff --git a/internal/api/fileupload/types.go b/internal/api/fileupload/types.go new file mode 100644 index 000000000..de71702cc --- /dev/null +++ b/internal/api/fileupload/types.go @@ -0,0 +1,15 @@ +package fileupload + +import ( + "sync" + + "github.com/puzpuzpuz/xsync" +) + +// Filters holds the filtering configuration for file uploads with thread-safe maps. +type Filters struct { + SupportedExtensions *xsync.MapOf[string, bool] + SupportedConfigFiles *xsync.MapOf[string, bool] + Once sync.Once + InitErr error +} diff --git a/pkg/apiclients/fileupload/client.go b/pkg/apiclients/fileupload/client.go index 054286e84..fd4b6769d 100644 --- a/pkg/apiclients/fileupload/client.go +++ b/pkg/apiclients/fileupload/client.go @@ -13,6 +13,7 @@ import ( "github.com/puzpuzpuz/xsync" "github.com/rs/zerolog" + fileuploadinternal "github.com/snyk/go-application-framework/internal/api/fileupload" "github.com/snyk/go-application-framework/internal/api/fileupload/filters" uploadrevision2 "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" "github.com/snyk/go-application-framework/pkg/utils" @@ -29,7 +30,7 @@ type HTTPClient struct { uploadRevisionSealableClient uploadrevision2.SealableClient filtersClient filters.Client cfg Config - filters Filters + filters fileuploadinternal.Filters logger *zerolog.Logger } @@ -46,9 +47,9 @@ var _ Client = (*HTTPClient)(nil) func NewClient(httpClient *http.Client, cfg Config, opts ...Option) *HTTPClient { client := &HTTPClient{ cfg: cfg, - filters: Filters{ - supportedExtensions: xsync.NewMapOf[bool](), - supportedConfigFiles: xsync.NewMapOf[bool](), + filters: fileuploadinternal.Filters{ + SupportedExtensions: xsync.NewMapOf[bool](), + SupportedConfigFiles: xsync.NewMapOf[bool](), }, } @@ -77,15 +78,15 @@ func NewClient(httpClient *http.Client, cfg Config, opts ...Option) *HTTPClient } func (c *HTTPClient) loadFilters(ctx context.Context) error { - c.filters.once.Do(func() { + c.filters.Once.Do(func() { filtersResp, err := c.filtersClient.GetFilters(ctx, c.cfg.OrgID) if err != nil { - c.filters.initErr = err + c.filters.InitErr = err return } for _, ext := range filtersResp.Extensions { - c.filters.supportedExtensions.Store(ext, true) + c.filters.SupportedExtensions.Store(ext, true) } for _, configFile := range filtersResp.ConfigFiles { // .gitignore and .dcignore should not be uploaded @@ -93,10 +94,10 @@ func (c *HTTPClient) loadFilters(ctx context.Context) error { if configFile == ".gitignore" || configFile == ".dcignore" { continue } - c.filters.supportedConfigFiles.Store(configFile, true) + c.filters.SupportedConfigFiles.Store(configFile, true) } }) - return c.filters.initErr + return c.filters.InitErr } // createDeeproxyFilter creates a filter function based on the current deeproxy filtering configuration. @@ -108,8 +109,8 @@ func (c *HTTPClient) createDeeproxyFilter(ctx context.Context) (filter, error) { return func(ff fileToFilter) *FilteredFile { fileExt := filepath.Ext(ff.Stat.Name()) fileName := filepath.Base(ff.Stat.Name()) - _, isSupportedExtension := c.filters.supportedExtensions.Load(fileExt) - _, isSupportedConfigFile := c.filters.supportedConfigFiles.Load(fileName) + _, isSupportedExtension := c.filters.SupportedExtensions.Load(fileExt) + _, isSupportedConfigFile := c.filters.SupportedConfigFiles.Load(fileName) if !isSupportedExtension && !isSupportedConfigFile { var reason error diff --git a/pkg/apiclients/fileupload/types.go b/pkg/apiclients/fileupload/types.go index 2df32e6a0..2618f4b48 100644 --- a/pkg/apiclients/fileupload/types.go +++ b/pkg/apiclients/fileupload/types.go @@ -1,10 +1,6 @@ package fileupload import ( - "sync" - - "github.com/puzpuzpuz/xsync" - "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" ) @@ -14,14 +10,6 @@ type OrgID = uploadrevision.OrgID // RevisionID represents a revision identifier. type RevisionID = uploadrevision.RevisionID -// Filters holds the filtering configuration for file uploads with thread-safe maps. -type Filters struct { - supportedExtensions *xsync.MapOf[string, bool] - supportedConfigFiles *xsync.MapOf[string, bool] - once sync.Once - initErr error -} - // UploadOptions configures the behavior of file upload operations. type uploadOptions struct { SkipDeeproxyFiltering bool From eb1e36566e9ca7d3b44433816cb8fc1de7ccf14d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Wed, 19 Nov 2025 17:22:03 +0100 Subject: [PATCH 15/15] chore: return interface instead of struct --- pkg/apiclients/fileupload/client.go | 2 +- pkg/apiclients/fileupload/client_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/apiclients/fileupload/client.go b/pkg/apiclients/fileupload/client.go index fd4b6769d..a8b6876e1 100644 --- a/pkg/apiclients/fileupload/client.go +++ b/pkg/apiclients/fileupload/client.go @@ -44,7 +44,7 @@ type Client interface { var _ Client = (*HTTPClient)(nil) // NewClient creates a new high-level file upload client. -func NewClient(httpClient *http.Client, cfg Config, opts ...Option) *HTTPClient { +func NewClient(httpClient *http.Client, cfg Config, opts ...Option) Client { client := &HTTPClient{ cfg: cfg, filters: fileuploadinternal.Filters{ diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go index 2f78669b4..726137909 100644 --- a/pkg/apiclients/fileupload/client_test.go +++ b/pkg/apiclients/fileupload/client_test.go @@ -592,7 +592,7 @@ func setupTest( llcfg uploadrevision2.FakeClientConfig, files []uploadrevision2.LoadedFile, _ filters.AllowList, -) (context.Context, *uploadrevision2.FakeSealableClient, *fileupload.HTTPClient, *os.File) { +) (context.Context, *uploadrevision2.FakeSealableClient, fileupload.Client, *os.File) { t.Helper() ctx := context.Background()