diff --git a/README.md b/README.md index 22a796a..2f4783c 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Currently works with npm, PyPI, pub.dev, Composer, and Cargo, which all include | NuGet | .NET | Yes | ✓ | | Composer | PHP | Yes | ✓ | | Conan | C/C++ | | ✓ | -| Conda | Python/R | | ✓ | +| Conda | Python/R | Yes | ✓ | | CRAN | R | | ✓ | | Container | Docker/OCI | | ✓ | | Debian | Debian/Ubuntu | | ✓ | diff --git a/docs/configuration.md b/docs/configuration.md index 6004b5f..577b98c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -209,7 +209,7 @@ Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages. -Currently supported for npm, PyPI, pub.dev, Composer, Cargo, and NuGet. These ecosystems include publish timestamps in their metadata. +Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, and Conda. These ecosystems include publish timestamps in their metadata. ## Docker diff --git a/internal/handler/conda.go b/internal/handler/conda.go index f929194..46d8704 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -1,8 +1,13 @@ package handler import ( + "encoding/json" + "io" "net/http" "strings" + "time" + + "github.com/git-pkgs/purl" ) const ( @@ -31,9 +36,9 @@ func (h *CondaHandler) Routes() http.Handler { mux := http.NewServeMux() // Channel index (repodata) - mux.HandleFunc("GET /{channel}/{arch}/repodata.json", h.proxyUpstream) + mux.HandleFunc("GET /{channel}/{arch}/repodata.json", h.handleRepodata) mux.HandleFunc("GET /{channel}/{arch}/repodata.json.bz2", h.proxyUpstream) - mux.HandleFunc("GET /{channel}/{arch}/current_repodata.json", h.proxyUpstream) + mux.HandleFunc("GET /{channel}/{arch}/current_repodata.json", h.handleRepodata) // Package downloads (cache these) mux.HandleFunc("GET /{channel}/{arch}/{filename}", h.handleDownload) @@ -119,6 +124,114 @@ func (h *CondaHandler) parseFilename(filename string) (name, version string) { return name, version } +// handleRepodata proxies repodata.json, applying cooldown filtering when enabled. +func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + h.proxyUpstream(w, r) + return + } + + upstreamURL := h.upstreamURL + r.URL.Path + + h.proxy.Logger.Debug("fetching repodata for cooldown filtering", "url", upstreamURL) + + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { + http.Error(w, "failed to create request", http.StatusInternalServerError) + return + } + req.Header.Set("Accept-Encoding", "gzip") + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + h.proxy.Logger.Error("upstream request failed", "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + return + } + + body, err := ReadMetadata(resp.Body) + if err != nil { + http.Error(w, "failed to read response", http.StatusInternalServerError) + return + } + + filtered, err := h.applyCooldownFiltering(body) + if err != nil { + h.proxy.Logger.Warn("failed to filter repodata, proxying original", "error", err) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + return + } + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(filtered) +} + +// condaTimestampDivisor converts Conda's millisecond timestamps to seconds. +const condaTimestampDivisor = 1000 + +// applyCooldownFiltering removes entries from repodata.json that were +// published too recently based on their timestamp field. +func (h *CondaHandler) applyCooldownFiltering(body []byte) ([]byte, error) { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return body, nil + } + + var repodata map[string]any + if err := json.Unmarshal(body, &repodata); err != nil { + return nil, err + } + + for _, key := range []string{"packages", "packages.conda"} { + packages, ok := repodata[key].(map[string]any) + if !ok { + continue + } + + for filename, entry := range packages { + entryMap, ok := entry.(map[string]any) + if !ok { + continue + } + + ts, ok := entryMap["timestamp"].(float64) + if !ok || ts == 0 { + continue + } + + publishedAt := time.Unix(int64(ts)/condaTimestampDivisor, 0) + + name, _ := entryMap["name"].(string) + if name == "" { + continue + } + + packagePURL := purl.MakePURLString("conda", name, "") + + if !h.proxy.Cooldown.IsAllowed("conda", packagePURL, publishedAt) { + version, _ := entryMap["version"].(string) + h.proxy.Logger.Info("cooldown: filtering conda package", + "name", name, "version", version, "filename", filename) + delete(packages, filename) + } + } + } + + return json.Marshal(repodata) +} + // proxyUpstream forwards a request to Anaconda without caching. func (h *CondaHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{"Accept-Encoding"}) diff --git a/internal/handler/conda_test.go b/internal/handler/conda_test.go index 457d201..5443161 100644 --- a/internal/handler/conda_test.go +++ b/internal/handler/conda_test.go @@ -1,8 +1,14 @@ package handler import ( + "encoding/json" "log/slog" + "net/http" + "net/http/httptest" "testing" + "time" + + "github.com/git-pkgs/proxy/internal/cooldown" ) func TestCondaParseFilename(t *testing.T) { @@ -49,3 +55,251 @@ func TestCondaIsPackageFile(t *testing.T) { } } } + +func TestCondaCooldownFiltering(t *testing.T) { + now := time.Now() + oldTimestamp := float64(now.Add(-7*24*time.Hour).UnixMilli()) + recentTimestamp := float64(now.Add(-1*time.Hour).UnixMilli()) + + repodata := map[string]any{ + "info": map[string]any{}, + "packages": map[string]any{ + "numpy-1.24.0-old.tar.bz2": map[string]any{ + "name": "numpy", + "version": "1.24.0", + "timestamp": oldTimestamp, + }, + "numpy-1.25.0-new.tar.bz2": map[string]any{ + "name": "numpy", + "version": "1.25.0", + "timestamp": recentTimestamp, + }, + }, + "packages.conda": map[string]any{ + "scipy-1.11.0-old.conda": map[string]any{ + "name": "scipy", + "version": "1.11.0", + "timestamp": oldTimestamp, + }, + "scipy-1.12.0-new.conda": map[string]any{ + "name": "scipy", + "version": "1.12.0", + "timestamp": recentTimestamp, + }, + }, + } + + body, err := json.Marshal(repodata) + if err != nil { + t.Fatal(err) + } + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{ + Default: "3d", + } + + h := &CondaHandler{ + proxy: proxy, + proxyURL: "http://localhost:8080", + } + + filtered, err := h.applyCooldownFiltering(body) + if err != nil { + t.Fatal(err) + } + + var result map[string]any + if err := json.Unmarshal(filtered, &result); err != nil { + t.Fatal(err) + } + + packages := result["packages"].(map[string]any) + if len(packages) != 1 { + t.Fatalf("expected 1 package in packages, got %d", len(packages)) + } + if _, ok := packages["numpy-1.24.0-old.tar.bz2"]; !ok { + t.Error("expected old numpy to survive filtering") + } + + condaPkgs := result["packages.conda"].(map[string]any) + if len(condaPkgs) != 1 { + t.Fatalf("expected 1 package in packages.conda, got %d", len(condaPkgs)) + } + if _, ok := condaPkgs["scipy-1.11.0-old.conda"]; !ok { + t.Error("expected old scipy to survive filtering") + } +} + +func TestCondaCooldownFilteringWithPackageOverride(t *testing.T) { + now := time.Now() + recentTimestamp := float64(now.Add(-2 * time.Hour).UnixMilli()) + + repodata := map[string]any{ + "info": map[string]any{}, + "packages": map[string]any{ + "special-1.0.0-build.tar.bz2": map[string]any{ + "name": "special", + "version": "1.0.0", + "timestamp": recentTimestamp, + }, + }, + "packages.conda": map[string]any{}, + } + + body, err := json.Marshal(repodata) + if err != nil { + t.Fatal(err) + } + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{ + Default: "3d", + Packages: map[string]string{"pkg:conda/special": "1h"}, + } + + h := &CondaHandler{ + proxy: proxy, + proxyURL: "http://localhost:8080", + } + + filtered, err := h.applyCooldownFiltering(body) + if err != nil { + t.Fatal(err) + } + + var result map[string]any + if err := json.Unmarshal(filtered, &result); err != nil { + t.Fatal(err) + } + + packages := result["packages"].(map[string]any) + if len(packages) != 1 { + t.Fatalf("expected 1 package (override allows it), got %d", len(packages)) + } +} + +func TestCondaCooldownFilteringNoTimestamp(t *testing.T) { + repodata := map[string]any{ + "info": map[string]any{}, + "packages": map[string]any{ + "old-pkg-1.0.0-build.tar.bz2": map[string]any{ + "name": "old-pkg", + "version": "1.0.0", + // no timestamp field + }, + }, + "packages.conda": map[string]any{}, + } + + body, err := json.Marshal(repodata) + if err != nil { + t.Fatal(err) + } + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{ + Default: "3d", + } + + h := &CondaHandler{ + proxy: proxy, + proxyURL: "http://localhost:8080", + } + + filtered, err := h.applyCooldownFiltering(body) + if err != nil { + t.Fatal(err) + } + + var result map[string]any + if err := json.Unmarshal(filtered, &result); err != nil { + t.Fatal(err) + } + + packages := result["packages"].(map[string]any) + if len(packages) != 1 { + t.Fatalf("entries without timestamp should pass through, got %d", len(packages)) + } +} + +func TestCondaHandleRepodataWithCooldown(t *testing.T) { + now := time.Now() + oldTimestamp := float64(now.Add(-7 * 24 * time.Hour).UnixMilli()) + recentTimestamp := float64(now.Add(-1 * time.Hour).UnixMilli()) + + repodataJSON, _ := json.Marshal(map[string]any{ + "info": map[string]any{}, + "packages": map[string]any{ + "old-1.0.0-build.tar.bz2": map[string]any{ + "name": "testpkg", "version": "1.0.0", "timestamp": oldTimestamp, + }, + "new-2.0.0-build.tar.bz2": map[string]any{ + "name": "testpkg", "version": "2.0.0", "timestamp": recentTimestamp, + }, + }, + "packages.conda": map[string]any{}, + }) + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(repodataJSON) + })) + defer upstream.Close() + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{ + Default: "3d", + } + + h := &CondaHandler{ + proxy: proxy, + upstreamURL: upstream.URL, + proxyURL: "http://proxy.local", + } + + req := httptest.NewRequest(http.MethodGet, "/conda-forge/noarch/repodata.json", nil) + req.SetPathValue("channel", "conda-forge") + req.SetPathValue("arch", "noarch") + w := httptest.NewRecorder() + h.handleRepodata(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + + var result map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + + packages := result["packages"].(map[string]any) + if len(packages) != 1 { + t.Fatalf("expected 1 package after filtering, got %d", len(packages)) + } +} + +func TestCondaHandleRepodataWithoutCooldown(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"info":{},"packages":{},"packages.conda":{}}`)) + })) + defer upstream.Close() + + h := &CondaHandler{ + proxy: &Proxy{Logger: slog.Default(), HTTPClient: http.DefaultClient}, + upstreamURL: upstream.URL, + proxyURL: "http://proxy.local", + } + + req := httptest.NewRequest(http.MethodGet, "/conda-forge/noarch/repodata.json", nil) + req.SetPathValue("channel", "conda-forge") + req.SetPathValue("arch", "noarch") + w := httptest.NewRecorder() + h.handleRepodata(w, req) + + // Without cooldown, should proxy directly (response comes from upstream) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } +}