diff --git a/README.md b/README.md index dff6964e..7ba3cb21 100644 --- a/README.md +++ b/README.md @@ -1592,6 +1592,8 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | `proxy_metadata_cache_served_bytes_total` | counter | `ecosystem` | Metadata bytes served from the metadata cache. | | `proxy_metadata_cache_size_bytes` | gauge | `ecosystem` | Size of cached metadata documents per ecosystem. | | `proxy_metadata_cache_entries` | gauge | `ecosystem` | Number of cached metadata documents per ecosystem. | +| `proxy_git_cache_size_bytes` | gauge | | Size of the cached Git repositories. Git LFS objects are artifacts and count under `proxy_cache_size_bytes`. | +| `proxy_git_cached_repositories` | gauge | | Number of cached Git repositories. | | `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) | | `proxy_metadata_evicted_total` | counter | `ecosystem` | Cached metadata documents evicted by the size limit | | `proxy_metadata_expired_by_change_feed_total` | counter | `ecosystem` | Cached metadata documents set to revalidate because an upstream [change feed](docs/configuration.md#change-feeds) listed them | diff --git a/docs/configuration.md b/docs/configuration.md index d887509a..44fe821c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -960,6 +960,28 @@ paths that differ only in case (`acme/widgets` and `acme/WIDGETS`) share one directory and are treated as one repository. Use a case-sensitive file system for an upstream whose repositories differ only in case. +The web UI lists the cached repositories at `/ui/git`, linked as "Git" in the +navigation: each one's upstream, clone URL, size, the newest commit of the +branch HEAD names (ID, subject and age), how many branches and tags it has, +when the upstream was last checked and a client last read it, how many +times clients read its refs (each clone, fetch and `git ls-remote` does), +and, while the last refresh failed, the kind of error (such as +`fetch_failed` or `too_large`). A repository the upstream refused shows no +commit, and one the `repos` allowlist no longer allows isn't listed, as `/git` +serves neither. Branches and commits are read with git, one repository at a +time and for at most two seconds per view, and kept until a fetch changes +the repository; the page says how many it hasn't read yet, and each reload +reads more. The request counts and errors are kept in memory and start over +when the proxy restarts. Like the rest of the UI the page has no +authentication, so it shows everyone who can reach it the names, default +branches and newest commit subjects of repositories from upstreams with +credentials. `/git` lets the same people clone them, but where `/ui` and +`/git` sit behind different access rules, the page shows more than `/git` +does. The `proxy_git_cache_size_bytes` and +`proxy_git_cached_repositories` gauges report the git cache's size and +repository count, apart from the artifact gauges, and clones and fetches +count as cache hits or misses under `ecosystem="git"`. + Git LFS objects are cached in the artifact cache, so they use the configured storage backend and retention rather than `cache_dir`. git-lfs finds the LFS API under the rewritten remote, at `/git/{name}/{repo}.git/info/lfs`, with no diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index 54c97d00..2bef68e7 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -98,6 +98,8 @@ type Cache struct { // foldCase is set where the cache directory ignores case, so keys that // differ only in case, which name one directory, are one repository. foldCase bool + // stateReads lets one RepositoryState read run git at a time. + stateReads chan struct{} ctx context.Context cancel context.CancelFunc @@ -142,6 +144,17 @@ type repo struct { lastUsed atomic.Int64 lastChecked atomic.Int64 usedWritten atomic.Int64 + // requests counts the times clients read the refs, and lastError is the + // last failed refresh, nil once one succeeds. Neither is kept across + // restarts. + requests atomic.Int64 + lastError atomic.Pointer[refreshError] + // state is the last RepositoryState read, and refsWritten counts the + // times the repository's refs or HEAD were written: by a clone, a fetch, + // a fetch of wanted commits and a change of the default branch. A state + // is current while the count is the one it was read at. + state atomic.Pointer[cachedState] + refsWritten atomic.Int64 } var errLocked = errors.New("locked by another process") @@ -213,6 +226,7 @@ func New(opts Options) (*Cache, error) { fetches: make(chan struct{}, maxConcurrentFetches), uploadPacks: make(chan struct{}, uploadPacks), refListings: make(chan struct{}, refListings), + stateReads: make(chan struct{}, 1), refCheckTimeout: defaultRefCheckTimeout, lowSpeedTime: defaultLowSpeedTime, maxRequestBody: defaultMaxRequestBody, diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index f467a3dd..683be053 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -151,9 +151,13 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { cancel() metrics.RecordUpstreamFetch(ecosystem, time.Since(start)) // A refresh cut short by Close is not an upstream failure. - if call.err != nil && c.ctx.Err() == nil { + switch { + case call.err == nil: + rp.lastError.Store(nil) + case c.ctx.Err() == nil: metrics.RecordUpstreamError(ecosystem, errorType(call.err), metrics.FailureDetail{URL: repoURL, Err: call.err}) c.logger.Warn("git cache: refresh failed", "repository", rp.key, "error", call.err) + rp.lastError.Store(&refreshError{kind: errorType(call.err), at: time.Now()}) } c.mu.Lock() rp.refreshing = nil @@ -223,7 +227,7 @@ func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remo if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, res, refreshFlags, branchesAndTags...); err != nil { return err } - if err := c.setHead(ctx, tmp, remote, ""); err != nil { + if _, err := c.setHead(ctx, tmp, remote, ""); err != nil { return err } size := dirSize(tmp) @@ -236,6 +240,7 @@ func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remo if err := os.Rename(tmp, rp.dir); err != nil { return fmt.Errorf("moving clone into place: %w", err) } + rp.refsWritten.Add(1) res.settle(rp, size) c.logger.Info("git cache: cloned repository", "repository", rp.key, "size", size) return nil @@ -262,6 +267,7 @@ func (c *Cache) update(ctx context.Context, rp *repo, repoURL string, remote rem } res := c.newReservation() err = c.fetch(ctx, rp.dir, repoURL, limit, res, refreshFlags, branchesAndTags...) + rp.refsWritten.Add(1) res.settle(rp, dirSize(rp.dir)) if err != nil { return err @@ -271,7 +277,11 @@ func (c *Cache) update(ctx context.Context, rp *repo, repoURL string, remote rem c.logger.Warn("git cache: pruning wanted refs failed", "repository", rp.key, "error", err) } head, _ := c.run(ctx, rp.dir, "symbolic-ref", "--quiet", "HEAD") - return c.setHead(ctx, rp.dir, remote, strings.TrimSpace(head)) + moved, err := c.setHead(ctx, rp.dir, remote, strings.TrimSpace(head)) + if moved { + rp.refsWritten.Add(1) + } + return err } // branchesAndTags are the refspecs of a refresh. A bare clone has none of its @@ -336,16 +346,17 @@ func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, res } // setHead points HEAD at the upstream's default branch, when the upstream -// names one and it is among the cached branches. -func (c *Cache) setHead(ctx context.Context, dir string, remote remoteRefs, current string) error { +// names one and it is among the cached branches. It reports whether it +// wrote HEAD. +func (c *Cache) setHead(ctx context.Context, dir string, remote remoteRefs, current string) (bool, error) { if remote.head == "" || remote.head == current { - return nil + return false, nil } if _, ok := remote.refs[remote.head]; !ok { - return nil + return false, nil } _, err := c.run(ctx, dir, "symbolic-ref", "HEAD", remote.head) - return err + return err == nil, err } // localRefs lists the cached branches and tags, and the names of the wanted diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index 5d809389..11d45777 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -60,6 +60,9 @@ func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) } rp := c.acquire(key) hit := rp.exists() + if opts.Refresh { + rp.requests.Add(1) + } var err error switch { case opts.Refresh || !hit: diff --git a/internal/gitcache/stats.go b/internal/gitcache/stats.go new file mode 100644 index 00000000..8e3b9ab7 --- /dev/null +++ b/internal/gitcache/stats.go @@ -0,0 +1,229 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +// RepositoryStats describes a cached repository. +type RepositoryStats struct { + // Key is the repository's path in the cache, such as + // github-1a2b3c4d/acme/widgets.git. It is spelled as the directory is, + // also where the cache folds keys, so it keeps the case of the clone + // that created it. + Key string + Size int64 + // LastChecked is when the upstream's refs were last compared, and + // LastUsed when a client last read the repository. + LastChecked, LastUsed time.Time + // Requests counts the times clients read the repository's refs, which + // each clone, fetch and ls-remote does, since the cache was opened. + Requests int64 + // LastError names the kind of error the last refresh failed with, such + // as fetch_failed or too_large, and LastErrorAt when. Both are empty + // once a refresh succeeds. + LastError string + LastErrorAt time.Time + // Refused is set while the upstream's last answer refused the + // repository or reported it missing, so it isn't served. + Refused bool +} + +// refreshError is the last failed refresh of a repository. +type refreshError struct { + kind string + at time.Time +} + +// Repositories describes the repositories in the cache, sorted by key. +func (c *Cache) Repositories() []RepositoryStats { + c.mu.Lock() + repos := make([]*repo, 0, len(c.repos)) + refused := make(map[*repo]bool) + for _, rp := range c.repos { + repos = append(repos, rp) + refused[rp] = rp.refused != nil + } + c.mu.Unlock() + + var out []RepositoryStats + for _, rp := range repos { + if !rp.exists() { + continue + } + s := RepositoryStats{ + Key: c.dirKey(rp), + Size: rp.size.Load(), + LastChecked: unixNanoTime(rp.lastChecked.Load()), + LastUsed: unixNanoTime(rp.lastUsed.Load()), + Requests: rp.requests.Load(), + Refused: refused[rp], + } + if e := rp.lastError.Load(); e != nil { + s.LastError, s.LastErrorAt = e.kind, e.at + } + out = append(out, s) + } + slices.SortFunc(out, func(a, b RepositoryStats) int { return strings.Compare(a.Key, b.Key) }) + return out +} + +// dirKey is the key of rp spelled as its directory is, which rp.key is not +// where keys are folded. +func (c *Cache) dirKey(rp *repo) string { + rel, err := filepath.Rel(c.dir, rp.dir) + if err != nil { + return rp.key + } + return filepath.ToSlash(rel) +} + +func unixNanoTime(n int64) time.Time { + if n == 0 { + return time.Time{} + } + return time.Unix(0, n) +} + +// RepositoryState is what a cached repository holds: the branch HEAD names +// with its newest commit, and how many branches and tags there are. +type RepositoryState struct { + Branch string + Commit string + CommitTime time.Time + // Subject is the first line of the commit message, at most + // maxSubjectLength bytes. + Subject string + Branches, Tags int +} + +// maxSubjectLength bounds RepositoryState.Subject. +const maxSubjectLength = 200 + +// headFields are the commit ID, time and subject readState asks git log for. +const headFields = 3 + +// stateRetryDelay is how long a failed state read is kept. +const stateRetryDelay = time.Minute + +// cachedState is a repository's state, or the error reading it failed +// with, as read when the repository's refs had been written written times. +// A failed read is kept until retryAt. +type cachedState struct { + written int64 + state RepositoryState + err error + retryAt time.Time +} + +// RepositoryState reads the state of the repository cached under key. It is +// read again only once the repository's refs or HEAD were written, not after +// a check of the upstream that found nothing new, so a page listing many +// repositories runs git for those that changed. A read that failed is kept +// for stateRetryDelay. +// +// One read runs at a time, however many callers ask, so listing a large +// cache doesn't start git processes without bound. A caller whose ctx ends +// before its read is done gets ctx's error; a state already read is +// returned without waiting. +func (c *Cache) RepositoryState(ctx context.Context, key string) (RepositoryState, error) { + c.mu.Lock() + rp := c.repos[c.repoKey(key)] + c.mu.Unlock() + if rp == nil || !rp.exists() { + return RepositoryState{}, ErrNotFound + } + if cached := rp.knownState(); cached != nil { + return cached.state, cached.err + } + if err := ctx.Err(); err != nil { + return RepositoryState{}, err + } + select { + case c.stateReads <- struct{}{}: + defer func() { <-c.stateReads }() + case <-ctx.Done(): + return RepositoryState{}, ctx.Err() + } + // The read this one waited for may have been of the same repository. + if cached := rp.knownState(); cached != nil { + return cached.state, cached.err + } + // Counted before the read: a write during it leaves this state stale, + // and it is then read again. + written := rp.refsWritten.Load() + state, err := c.readState(ctx, rp.dir) + if err != nil && ctx.Err() != nil { + // A read the caller gave up on says nothing about the repository. + return RepositoryState{}, ctx.Err() + } + rp.state.Store(&cachedState{written: written, state: state, err: err, retryAt: time.Now().Add(stateRetryDelay)}) + return state, err +} + +// knownState is the state read since rp's refs were last written, or nil if +// it has to be read. +func (rp *repo) knownState() *cachedState { + cached := rp.state.Load() + if cached == nil || cached.written != rp.refsWritten.Load() || + (cached.err != nil && !time.Now().Before(cached.retryAt)) { + return nil + } + return cached +} + +func (c *Cache) readState(ctx context.Context, dir string) (RepositoryState, error) { + var state RepositoryState + refs, err := c.run(ctx, dir, "for-each-ref", "--format=%(refname)", "refs/heads", "refs/tags") + if err != nil { + return state, err + } + for ref := range strings.Lines(refs) { + if strings.HasPrefix(ref, "refs/heads/") { + state.Branches++ + } else { + state.Tags++ + } + } + // Both exit 1, and print nothing, for a detached HEAD and for one with + // no commit yet; any other failure is an error. + branch, err := c.run(ctx, dir, "symbolic-ref", "--quiet", "--short", "HEAD") + if err != nil && !exitedWith(err, 1) { + return state, err + } + state.Branch = strings.TrimSpace(branch) + if _, err := c.run(ctx, dir, "rev-parse", "--verify", "--quiet", "HEAD"); exitedWith(err, 1) { + return state, nil + } else if err != nil { + return state, err + } + head, err := c.run(ctx, dir, "log", "-1", "--format=%H%x00%ct%x00%s", "HEAD", "--") + if err != nil { + return state, err + } + fields := strings.SplitN(strings.TrimSpace(head), "\x00", headFields) + if len(fields) != headFields { + return state, fmt.Errorf("git log: unexpected output %q", head) + } + state.Commit, state.Subject = fields[0], fields[2] + if unix, err := strconv.ParseInt(fields[1], 10, 64); err == nil { + state.CommitTime = time.Unix(unix, 0) + } + if len(state.Subject) > maxSubjectLength { + state.Subject = strings.ToValidUTF8(state.Subject[:maxSubjectLength], "") + "…" + } + return state, nil +} + +// exitedWith reports whether err is from a git process that exited with code. +func exitedWith(err error, code int) bool { + var exit *exec.ExitError + return errors.As(err, &exit) && exit.ExitCode() == code +} diff --git a/internal/gitcache/stats_test.go b/internal/gitcache/stats_test.go new file mode 100644 index 00000000..e159ee33 --- /dev/null +++ b/internal/gitcache/stats_test.go @@ -0,0 +1,302 @@ +package gitcache + +import ( + "context" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestRepositoriesDescribesCachedRepositories(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + if got := tc.Repositories(); len(got) != 0 { + t.Fatalf("empty cache lists %+v", got) + } + + start := time.Now().Add(-time.Second) + tc.clone(t, "repo") + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + got := tc.Repositories() + if len(got) != 1 { + t.Fatalf("repositories %+v, want one", got) + } + r := got[0] + if r.Key != "up/repo.git" || r.Size <= 0 || r.Requests != 2 || r.LastError != "" { + t.Errorf("repository %+v, want up/repo.git with a size, 2 requests and no error", r) + } + if r.LastChecked.Before(start) || r.LastUsed.Before(start) { + t.Errorf("checked %v, used %v, want both after %v", r.LastChecked, r.LastUsed, start) + } + + // A refresh that fails is reported until one succeeds. + upstream.server.Close() + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + r = tc.Repositories()[0] + if r.LastError != "fetch_failed" || r.LastErrorAt.Before(start) || r.Requests != 3 { + t.Errorf("after a failed refresh %+v, want fetch_failed and 3 requests", r) + } +} + +// TestRepositoriesReportsRefusal checks a repository the upstream refused is +// reported as refused until a check succeeds, unlike a failed fetch. +func TestRepositoriesReportsRefusal(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + if r := tc.Repositories()[0]; r.Refused { + t.Fatalf("fresh copy reported refused: %+v", r) + } + + upstream.status.Store(http.StatusForbidden) + _, _ = tryGit(t, "", "ls-remote", tc.url("repo")) + waitIdle(t, tc.Cache) + if r := tc.Repositories()[0]; !r.Refused { + t.Errorf("refused copy not reported refused: %+v", r) + } + + upstream.status.Store(0) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + if r := tc.Repositories()[0]; r.Refused { + t.Errorf("still reported refused after a check succeeded: %+v", r) + } +} + +// TestRepositoriesListsRepositoriesFromDisk checks a cache opened over a +// cache_dir lists what is already there. +func TestRepositoriesListsRepositoriesFromDisk(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + dir := t.TempDir() + first := newTestCache(t, upstream.server.URL, func(o *Options) { o.CacheDir = dir }) + first.clone(t, "repo") + waitIdle(t, first.Cache) + if err := first.Close(); err != nil { + t.Fatal(err) + } + + second := newTestCache(t, upstream.server.URL, func(o *Options) { o.CacheDir = dir }) + got := second.Repositories() + if len(got) != 1 || got[0].Key != "up/repo.git" || got[0].Size <= 0 || got[0].LastChecked.IsZero() || got[0].Requests != 0 { + t.Errorf("repositories %+v, want up/repo.git with its size and last check", got) + } +} + +func TestRepositoryState(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + head := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + got, err := tc.RepositoryState(context.Background(), "up/repo.git") + if err != nil { + t.Fatalf("RepositoryState: %v", err) + } + if got.Branch != "main" || got.Commit != head || got.Subject != "first" || got.Branches != 1 || got.Tags != 1 || + time.Since(got.CommitTime) > time.Hour { + t.Errorf("state %+v, want main at %s, first, 1 branch and 1 tag", got, head) + } + + // The state follows a refresh that fetched. + runGit(t, work, "checkout", "--quiet", "-b", "feature") + runGit(t, work, "push", "--quiet", "origin", "feature") + runGit(t, work, "checkout", "--quiet", "main") + head = upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + if got, err = tc.RepositoryState(context.Background(), "up/repo.git"); err != nil || got.Commit != head || got.Subject != "second" || got.Branches != 2 { + t.Errorf("after a push: %+v, %v; want main at %s, second, 2 branches", got, err, head) + } + + if _, err := tc.RepositoryState(context.Background(), "up/missing.git"); !errors.Is(err, ErrNotFound) { + t.Errorf("state of an uncached repository: %v, want ErrNotFound", err) + } +} + +// TestRepositoryStateIsKeptWhileRefsStay checks a check of the upstream that +// finds nothing new leaves the state as read, so a repository clients keep +// cloning isn't read again on every view, and that a new default branch, +// which moves HEAD but no ref, has it read again. +func TestRepositoryStateIsKeptWhileRefsStay(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, work, "push", "--quiet", "origin", "main:trunk") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + rp := tc.repo("up/repo.git") + if st, err := tc.RepositoryState(context.Background(), "up/repo.git"); err != nil || st.Branch != "main" { + t.Fatalf("state %+v, %v; want main", st, err) + } + read, checked := rp.state.Load(), rp.lastChecked.Load() + + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + if rp.lastChecked.Load() == checked { + t.Fatal("the second clone did not check the upstream") + } + if rp.knownState() != read { + t.Error("a check that found no change dropped the state") + } + + runGit(t, upstream.bare(), "symbolic-ref", "HEAD", "refs/heads/trunk") + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + if st, err := tc.RepositoryState(context.Background(), "up/repo.git"); err != nil || st.Branch != "trunk" { + t.Errorf("state after the default branch changed = %+v, %v; want trunk", st, err) + } +} + +// TestRepositoryStateIsReadAgainAfterAWantedFetch checks a commit fetched by +// ID, which writes a ref too, has the state read again. +func TestRepositoryStateIsReadAgainAfterAWantedFetch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + if _, err := tc.RepositoryState(context.Background(), "up/repo.git"); err != nil { + t.Fatal(err) + } + + if err := fetchByID(t, tc.url("repo"), sha); err != nil { + t.Fatalf("fetch by ID: %v", err) + } + waitIdle(t, tc.Cache) + if tc.repo("up/repo.git").knownState() != nil { + t.Error("state kept over a fetch of a wanted commit") + } +} + +// TestRepositoryStateFoldsCase checks that, where keys are folded, a +// repository's state is found under any spelling of its key. +func TestRepositoryStateFoldsCase(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.mu.Lock() + tc.foldCase = true + tc.mu.Unlock() + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + if _, err := tc.RepositoryState(context.Background(), "up/REPO.git"); err != nil { + t.Errorf("RepositoryState(up/REPO.git): %v", err) + } +} + +// TestRepositoriesKeepCaseWhereKeysAreFolded checks that, where keys are +// folded, a repository is listed under its directory's spelling, not the +// folded key, so its upstream and path match the configuration and the +// upstream's names. +func TestRepositoriesKeepCaseWhereKeysAreFolded(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("Repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.mu.Lock() + tc.foldCase = true + tc.mu.Unlock() + tc.clone(t, "Repo") + waitIdle(t, tc.Cache) + + got := tc.Repositories() + if len(got) != 1 || got[0].Key != "up/Repo.git" { + t.Fatalf("repositories %+v, want up/Repo.git", got) + } + if _, err := tc.RepositoryState(context.Background(), got[0].Key); err != nil { + t.Errorf("RepositoryState(%s): %v", got[0].Key, err) + } +} + +// TestRepositoryStateRetriesAFailedRead checks a read that fails, here on +// a branch whose commit is missing, is reported as an error and read again +// a while later even if the upstream isn't checked in between. +func TestRepositoryStateRetriesAFailedRead(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + rp := tc.repo("up/repo.git") + ref := filepath.Join(rp.dir, "refs", "heads", "main") + head := runGit(t, rp.dir, "rev-parse", "main") + if err := os.WriteFile(ref, []byte(strings.Repeat("1", len(head))+"\n"), 0o600); err != nil { + t.Fatal(err) + } + + if st, err := tc.RepositoryState(context.Background(), "up/repo.git"); err == nil { + t.Fatalf("state %+v of a branch with no commit, want an error", st) + } + if err := os.WriteFile(ref, []byte(head+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := tc.RepositoryState(context.Background(), "up/repo.git"); err == nil { + t.Error("failed read not kept for a while") + } + rp.state.Load().retryAt = time.Now().Add(-time.Second) + if st, err := tc.RepositoryState(context.Background(), "up/repo.git"); err != nil || st.Commit != head { + t.Errorf("state after the retry delay = %+v, %v; want %s", st, err, head) + } +} + +// TestRepositoryStateReadsOneAtATime checks a state that isn't cached waits +// for the read running before it, gives up with its context, and that a +// cached state needs neither. +func TestRepositoryStateReadsOneAtATime(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + tc.stateReads <- struct{}{} + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if st, err := tc.RepositoryState(ctx, "up/repo.git"); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("state read while another runs = %+v, %v; want the context's error", st, err) + } + if tc.repo("up/repo.git").state.Load() != nil { + t.Error("a read that gave up was kept") + } + <-tc.stateReads + want, err := tc.RepositoryState(context.Background(), "up/repo.git") + if err != nil || want.Commit == "" { + t.Fatalf("state once the other read is done = %+v, %v", want, err) + } + + tc.stateReads <- struct{}{} + defer func() { <-tc.stateReads }() + done, stop := context.WithCancel(context.Background()) + stop() + if got, err := tc.RepositoryState(done, "up/repo.git"); err != nil || got != want { + t.Errorf("cached state with a read running and the context done = %+v, %v; want %+v", got, err, want) + } +} + +// TestReadStateOfAnEmptyRepository checks a repository with no commits has +// a state, not an error. +func TestReadStateOfAnEmptyRepository(t *testing.T) { + upstream := newFakeUpstream(t) + tc := newTestCache(t, upstream.server.URL, nil) + dir := filepath.Join(t.TempDir(), "empty.git") + runGit(t, "", "init", "--quiet", "--bare", "--initial-branch=main", dir) + st, err := tc.readState(context.Background(), dir) + if err != nil || st.Branch != "main" || st.Commit != "" || st.Branches != 0 { + t.Errorf("state %+v, %v; want main with no commit", st, err) + } +} diff --git a/internal/gitcache/wants.go b/internal/gitcache/wants.go index 8755ba1b..0c42aa57 100644 --- a/internal/gitcache/wants.go +++ b/internal/gitcache/wants.go @@ -132,6 +132,7 @@ func (c *Cache) fetchWants(ctx context.Context, rp *repo, repoURL string, oids [ } res := c.newReservation() err = c.fetch(ctx, rp.dir, repoURL, limit, res, []string{"--atomic"}, refspecs...) + rp.refsWritten.Add(1) res.settle(rp, dirSize(rp.dir)) return err } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 85eeef46..af73eea1 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -264,6 +264,22 @@ var ( []string{"ecosystem"}, ) + // Git cache gauges. Repositories are kept apart from the artifact + // figures, which hold Git LFS objects under ecosystem git. + GitCacheSize = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "proxy_git_cache_size_bytes", + Help: "Size of the cached Git repositories in bytes", + }, + ) + + GitCachedRepositories = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "proxy_git_cached_repositories", + Help: "Number of cached Git repositories", + }, + ) + ResponseBytes = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "proxy_response_bytes_total", @@ -348,6 +364,8 @@ func init() { EcosystemVersions, MetadataCacheSize, MetadataCacheEntries, + GitCacheSize, + GitCachedRepositories, ResponseBytes, ClientRequests, ClientResponseBytes, @@ -523,6 +541,12 @@ func UpdateCacheStats(sizeBytes, artifactCount int64) { CachedArtifacts.Set(float64(artifactCount)) } +// UpdateGitCacheStats sets the git cache gauges. +func UpdateGitCacheStats(repositories int, sizeBytes int64) { + GitCachedRepositories.Set(float64(repositories)) + GitCacheSize.Set(float64(sizeBytes)) +} + // EcosystemStats is one ecosystem's row of the snapshot published as gauges. type EcosystemStats struct { Ecosystem string diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 25ae1cd4..8193462b 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -57,6 +57,23 @@ func TestUpdateCacheStats(t *testing.T) { // No panics = success } +func TestUpdateGitCacheStats(t *testing.T) { + UpdateGitCacheStats(3, 42) + if got := testutil.ToFloat64(GitCachedRepositories); got != 3 { + t.Errorf("proxy_git_cached_repositories = %v, want 3", got) + } + if got := testutil.ToFloat64(GitCacheSize); got != 42 { + t.Errorf("proxy_git_cache_size_bytes = %v, want 42", got) + } + snap, err := Gather() + if err != nil { + t.Fatal(err) + } + if len(snap.Samples("proxy_git_cached_repositories")) != 1 || len(snap.Samples("proxy_git_cache_size_bytes")) != 1 { + t.Error("git cache gauges are not registered") + } +} + func TestCircuitBreakerMetrics(t *testing.T) { UpdateCircuitBreakerState("npmjs.org", 0) // closed UpdateCircuitBreakerState("npmjs.org", 2) // open diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index 02e67e2c..cd15d7c1 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -27,6 +27,10 @@ var metricSurface = map[string]string{ "proxy_cached_artifacts_total": "Cached artifacts tile", "proxy_metadata_cache_size_bytes": "Metadata cache card", "proxy_metadata_cache_entries": "Metadata cache card", + // Not on /ui/analytics: the git page's summary shows these figures for + // the repositories it lists. + "proxy_git_cache_size_bytes": "/ui/git summary", + "proxy_git_cached_repositories": "/ui/git summary", // Request-path counters. "proxy_response_bytes_total": "Runtime: Served", diff --git a/internal/server/git_ui.go b/internal/server/git_ui.go new file mode 100644 index 00000000..661d48c1 --- /dev/null +++ b/internal/server/git_ui.go @@ -0,0 +1,154 @@ +package server + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/gitcache" + "github.com/git-pkgs/proxy/internal/metrics" +) + +// GitRepositoriesData is the git page: the repositories in the git cache. +type GitRepositoriesData struct { + Layout + Rows []GitRepositoryRow + // Summary counts the rows and their size. + Summary string + // Pending counts the rows whose branches and commits this view had no + // time left to read, and is empty when it read them all. + Pending string +} + +// GitRepositoryRow is one cached repository on the git page. +type GitRepositoryRow struct { + Upstream, Repo, CloneURL string + Size string + Requests string + LastChecked, LastUsed string + // LastError names the kind of error the last refresh failed with. + LastError, LastErrorAt string + // Branch, Commit, Subject and CommitAge describe the newest commit of + // the branch HEAD names, and Refs counts branches and tags. + Branch, Commit, Subject, CommitAge string + Refs string +} + +// gitStateBudget is how long one view of the git page spends reading what +// the repositories hold. The cache keeps what a view read, so the next one +// goes on where this one stopped. +const gitStateBudget = 2 * time.Second + +func (s *Server) handleGitRepositories(w http.ResponseWriter, r *http.Request) { + repos := s.git.Repositories() + ctx, cancel := context.WithTimeout(r.Context(), gitStateBudget) + defer cancel() + pending := 0 + state := func(key string) (gitcache.RepositoryState, bool) { + st, err := s.git.RepositoryState(ctx, key) + switch { + case err == nil: + case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): + pending++ + default: + s.logger.Warn("failed to read git repository state", "repository", key, "error", err) + } + return st, err == nil + } + rows, size := gitRepositoryRows(s.cfg.BaseURL, s.cfg.Upstream.Git, repos, state) + data := GitRepositoriesData{ + Layout: s.layoutFor(r), + Rows: rows, + Summary: plural(len(rows), "repository", "repositories") + " in the git cache, " + formatSize(size), + } + if pending > 0 { + data.Pending = plural(pending, "repository", "repositories") + } + if err := s.templates.Render(w, "git_repositories", data); err != nil { + s.logger.Error("failed to render git repositories", "error", err) + } +} + +// gitRepositoryRows names each cached repository by its upstream and path, +// and gives the URL it is cloned from through the proxy. Repositories of an +// upstream no longer configured, or that its repos allowlist no longer +// allows, are left out, and a repository the upstream refused shows no +// cached state, as /git serves none of them. state reads what a repository +// holds, reporting false when it can't. It also returns the rows' total size. +func gitRepositoryRows(baseURL string, upstreams map[string]config.GitUpstream, repos []gitcache.RepositoryStats, + state func(key string) (gitcache.RepositoryState, bool)) ([]GitRepositoryRow, int64) { + names := make(map[string]string, len(upstreams)) + for name, u := range upstreams { + names[gitcache.UpstreamDir(name, u.URL)] = name + } + rows := make([]GitRepositoryRow, 0, len(repos)) + var size int64 + for _, repo := range repos { + dir, path, _ := strings.Cut(repo.Key, "/") + name, ok := names[dir] + if !ok || !upstreams[name].AllowsRepo(strings.TrimSuffix(path, ".git")) { + continue + } + row := GitRepositoryRow{ + Upstream: name, + Repo: strings.TrimSuffix(path, ".git"), + CloneURL: strings.TrimSuffix(baseURL, "/") + "/git/" + escapePath(name+"/"+path), + Size: formatSize(repo.Size), + Requests: plural(int(repo.Requests), "request", "requests"), + LastChecked: formatTimeAgo(repo.LastChecked), + LastUsed: formatTimeAgo(repo.LastUsed), + LastError: repo.LastError, + LastErrorAt: formatTimeAgo(repo.LastErrorAt), + } + if st, ok := rowState(repo, state); ok { + row.Branch, row.Subject, row.CommitAge = st.Branch, st.Subject, formatTimeAgo(st.CommitTime) + row.Commit = st.Commit[:min(len(st.Commit), shortCommitLength)] + row.Refs = plural(st.Branches, "branch", "branches") + ", " + plural(st.Tags, "tag", "tags") + } + rows = append(rows, row) + size += repo.Size + } + return rows, size +} + +// rowState is what repo holds, unless the upstream refused it. +func rowState(repo gitcache.RepositoryStats, state func(key string) (gitcache.RepositoryState, bool)) (gitcache.RepositoryState, bool) { + if repo.Refused { + return gitcache.RepositoryState{}, false + } + return state(repo.Key) +} + +// escapePath escapes each segment of a slash-separated path. +func escapePath(path string) string { + segments := strings.Split(path, "/") + for i, s := range segments { + segments[i] = url.PathEscape(s) + } + return strings.Join(segments, "/") +} + +// shortCommitLength is how much of a commit ID the git page shows. +const shortCommitLength = 7 + +func plural(n int, one, many string) string { + if n == 1 { + return "1 " + one + } + return fmt.Sprintf("%d %s", n, many) +} + +// updateGitCacheStats publishes the git cache gauges. +func (s *Server) updateGitCacheStats() { + repos := s.git.Repositories() + var size int64 + for _, r := range repos { + size += r.Size + } + metrics.UpdateGitCacheStats(len(repos), size) +} diff --git a/internal/server/git_ui_test.go b/internal/server/git_ui_test.go new file mode 100644 index 00000000..d5d2c558 --- /dev/null +++ b/internal/server/git_ui_test.go @@ -0,0 +1,263 @@ +package server + +import ( + "bytes" + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/gitcache" + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestGitRepositoriesPage(t *testing.T) { + root := t.TempDir() + newGitTestRepo(t, root, "team/repo") + upstream := httptest.NewServer(gitBackend(t, root)) + defer upstream.Close() + cfg, _ := startGitTestServer(t, func(cfg *config.Config) { + cfg.Upstream.Git = map[string]config.GitUpstream{"up": {URL: upstream.URL}} + }) + + if status, body := request(t, http.MethodGet, cfg.BaseURL+"/ui/git", "", nil); status != http.StatusOK || !strings.Contains(body, "No repository is cached yet") { + t.Errorf("empty page: status %d\n%s", status, body) + } + gitCommand(t, "", "clone", "--quiet", cfg.BaseURL+"/git/up/team/repo.git", filepath.Join(t.TempDir(), "clone")) + + if _, body := request(t, http.MethodGet, cfg.BaseURL+"/ui/", "", nil); !strings.Contains(body, `href="/ui/git"`) { + t.Error("navigation doesn't link the git page") + } + status, body := request(t, http.MethodGet, cfg.BaseURL+"/ui/git", "", nil) + if status != http.StatusOK { + t.Fatalf("status %d", status) + } + for _, want := range []string{"1 repository in the git cache", ">up<", "team/repo", cfg.BaseURL + "/git/up/team/repo.git", "1 request", "main", "1 branch, 0 tags"} { + if !strings.Contains(body, want) { + t.Errorf("page lacks %q:\n%s", want, body) + } + } +} + +// TestGitRepositoriesPageSaysWhatIsNotReadYet checks a view that is out of +// time for reading commits lists the repositories without them and says so. +func TestGitRepositoriesPageSaysWhatIsNotReadYet(t *testing.T) { + cacheDir := t.TempDir() + upstreams := map[string]config.GitUpstream{"up": {URL: "https://example.com"}} + upstreamDir := gitcache.UpstreamDir("up", "https://example.com") + for _, name := range []string{"one", "two"} { + gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", filepath.Join(cacheDir, upstreamDir, "team", name+".git")) + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: cacheDir, RefreshWait: time.Second, StaleMaxAge: time.Hour, + Client: &http.Client{}, Logger: logger, Upstreams: map[string]string{"up": upstreamDir}, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cache.Close() }) + s := &Server{cfg: &config.Config{BaseURL: "http://proxy.example"}, git: cache, logger: logger, templates: &Templates{}} + s.cfg.Upstream.Git = upstreams + + view := func(ctx context.Context) string { + w := httptest.NewRecorder() + s.handleGitRepositories(w, httptest.NewRequest(http.MethodGet, "/ui/git", nil).WithContext(ctx)) + return w.Body.String() + } + done, stop := context.WithCancel(context.Background()) + stop() + body := view(done) + for _, want := range []string{"2 repositories in the git cache", "team/one", "team/two", "not read yet for 2 repositories"} { + if !strings.Contains(body, want) { + t.Errorf("page out of time lacks %q:\n%s", want, body) + } + } + if strings.Contains(body, "0 branches") { + t.Errorf("page out of time shows a state:\n%s", body) + } + body = view(context.Background()) + if !strings.Contains(body, "0 branches, 0 tags") || strings.Contains(body, "not read yet") { + t.Errorf("page with time shows no state, or says one is missing:\n%s", body) + } +} + +// TestGitRepositoriesPageLogsAStateItCannotRead checks a repository git +// fails to read is still listed, and the log says why it shows no commit. +func TestGitRepositoriesPageLogsAStateItCannotRead(t *testing.T) { + cacheDir := t.TempDir() + upstreamDir := gitcache.UpstreamDir("up", "https://example.com") + repoDir := filepath.Join(cacheDir, upstreamDir, "team", "broken.git") + gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", repoDir) + // A branch naming a commit the repository doesn't have. + if err := os.WriteFile(filepath.Join(repoDir, "refs", "heads", "main"), []byte(strings.Repeat("1", 40)+"\n"), 0o600); err != nil { + t.Fatal(err) + } + var logs bytes.Buffer + logger := slog.New(slog.NewTextHandler(&logs, nil)) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: cacheDir, RefreshWait: time.Second, StaleMaxAge: time.Hour, + Client: &http.Client{}, Logger: logger, Upstreams: map[string]string{"up": upstreamDir}, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cache.Close() }) + s := &Server{cfg: &config.Config{BaseURL: "http://proxy.example"}, git: cache, logger: logger, templates: &Templates{}} + s.cfg.Upstream.Git = map[string]config.GitUpstream{"up": {URL: "https://example.com"}} + + w := httptest.NewRecorder() + s.handleGitRepositories(w, httptest.NewRequest(http.MethodGet, "/ui/git", nil)) + if body := w.Body.String(); !strings.Contains(body, "team/broken") || strings.Contains(body, "not read yet") { + t.Errorf("page doesn't list the repository, or says it has yet to read it:\n%s", body) + } + for _, want := range []string{"level=WARN", "repository=" + upstreamDir + "/team/broken.git", "error="} { + if !strings.Contains(logs.String(), want) { + t.Errorf("log lacks %q:\n%s", want, logs.String()) + } + } +} + +// TestGitRepositoryRowsKeepCase checks an upstream, a repos pattern and a +// repository with uppercase letters are listed as they are spelled. Where +// cache_dir ignores case, as on macOS and Windows, the cache tracks them +// under a lowercased key. +func TestGitRepositoryRowsKeepCase(t *testing.T) { + cacheDir := t.TempDir() + upstreams := map[string]config.GitUpstream{"Up": {URL: "https://example.com", Repos: []string{"Team/*"}}} + upstreamDir := gitcache.UpstreamDir("Up", "https://example.com") + gitCommand(t, "", "init", "--quiet", "--bare", filepath.Join(cacheDir, upstreamDir, "Team", "Repo.git")) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: cacheDir, RefreshWait: time.Second, StaleMaxAge: time.Hour, + Client: &http.Client{}, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), Upstreams: map[string]string{"Up": upstreamDir}, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cache.Close() }) + + rows, _ := gitRepositoryRows("http://proxy.example", upstreams, cache.Repositories(), func(key string) (gitcache.RepositoryState, bool) { + st, err := cache.RepositoryState(context.Background(), key) + return st, err == nil + }) + if len(rows) != 1 || rows[0].Upstream != "Up" || rows[0].Repo != "Team/Repo" || + rows[0].CloneURL != "http://proxy.example/git/Up/Team/Repo.git" || rows[0].Refs == "" { + t.Errorf("rows %+v, want Up / Team/Repo with its clone URL and state", rows) + } +} + +func TestGitRepositoriesPageNeedsUpstreamGit(t *testing.T) { + cfg, _ := startGitTestServer(t, func(*config.Config) {}) + if status, _ := request(t, http.MethodGet, cfg.BaseURL+"/ui/git", "", nil); status != http.StatusNotFound { + t.Errorf("status %d, want 404 without upstream.git", status) + } + if _, body := request(t, http.MethodGet, cfg.BaseURL+"/ui/", "", nil); strings.Contains(body, `href="/ui/git"`) { + t.Error("navigation links the git page without upstream.git") + } +} + +func TestGitRepositoryRows(t *testing.T) { + upstreams := map[string]config.GitUpstream{"github": {URL: "https://github.com"}} + dir := gitcache.UpstreamDir("github", "https://github.com") + failed := time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC) + state := func(key string) (gitcache.RepositoryState, bool) { + return gitcache.RepositoryState{ + Branch: "main", Commit: "0123456789abcdef0123456789abcdef01234567", Subject: "Fix the build", + CommitTime: time.Now().Add(-3 * time.Hour), Branches: 2, Tags: 1, + }, key == dir+"/acme/widgets.git" + } + rows, size := gitRepositoryRows("http://proxy.example", upstreams, []gitcache.RepositoryStats{ + {Key: dir + "/acme/widgets.git", Size: 3 << 20, Requests: 1, LastError: "fetch_failed", LastErrorAt: failed}, + {Key: "gone-12345678/x.git"}, + }, state) + if len(rows) != 1 || size != 3<<20 { + t.Fatalf("rows %+v, size %d; want the configured upstream's one and its size", rows, size) + } + r := rows[0] + if r.Upstream != "github" || r.Repo != "acme/widgets" || r.CloneURL != "http://proxy.example/git/github/acme/widgets.git" || + r.Size != "3.0 MB" || r.Requests != "1 request" || r.LastError != "fetch_failed" || + r.Branch != "main" || r.Commit != "0123456" || r.Subject != "Fix the build" || r.CommitAge != "3 hours ago" || r.Refs != "2 branches, 1 tag" { + t.Errorf("row %+v", r) + } +} + +// TestGitRepositoryRowsHideWhatGitRefuses checks the page leaves out +// repositories the repos allowlist no longer allows, and shows no cached +// commit of one the upstream refused, as /git serves neither. +func TestGitRepositoryRowsHideWhatGitRefuses(t *testing.T) { + upstreams := map[string]config.GitUpstream{"github": {URL: "https://github.com", Repos: []string{"acme/*"}}} + dir := gitcache.UpstreamDir("github", "https://github.com") + state := func(string) (gitcache.RepositoryState, bool) { + return gitcache.RepositoryState{Branch: "main", Commit: "0123456789", Subject: "Secret plans", Branches: 1}, true + } + rows, size := gitRepositoryRows("http://proxy.example", upstreams, []gitcache.RepositoryStats{ + {Key: dir + "/acme/widgets.git", Size: 1, Refused: true, LastError: "refused"}, + {Key: dir + "/acme/gadgets.git", Size: 2, LastError: "fetch_failed"}, + {Key: dir + "/other/secret.git", Size: 4}, + }, state) + if len(rows) != 2 || size != 3 { + t.Fatalf("rows %+v, size %d; want the two allowed repositories", rows, size) + } + if r := rows[0]; r.Repo != "acme/widgets" || r.Commit != "" || r.Subject != "" || r.Branch != "" || r.Refs != "" || r.LastError != "refused" { + t.Errorf("refused row %+v, want no cached state but the error", r) + } + if r := rows[1]; r.Repo != "acme/gadgets" || r.Subject != "Secret plans" { + t.Errorf("row after a failed fetch %+v, want its cached state", r) + } +} + +// TestGitRepositoryRowsEscapeCloneURL checks path segments such as Azure +// DevOps project names, which may hold spaces, '#' or '%', are escaped. +func TestGitRepositoryRowsEscapeCloneURL(t *testing.T) { + upstreams := map[string]config.GitUpstream{"azure": {URL: "https://dev.azure.com"}} + dir := gitcache.UpstreamDir("azure", "https://dev.azure.com") + none := func(string) (gitcache.RepositoryState, bool) { return gitcache.RepositoryState{}, false } + rows, _ := gitRepositoryRows("http://proxy.example/", upstreams, []gitcache.RepositoryStats{ + {Key: dir + "/org/My Project/_git/My Repo#1?x%.git"}, + }, none) + want := "http://proxy.example/git/azure/org/My%20Project/_git/My%20Repo%231%3Fx%25.git" + if len(rows) != 1 || rows[0].CloneURL != want { + t.Errorf("rows %+v, want clone URL %s", rows, want) + } +} + +// TestGitCacheGauges checks the git cache has gauges of its own, published +// even when the database can't be read. +func TestGitCacheGauges(t *testing.T) { + cacheDir := t.TempDir() + upstreamDir := gitcache.UpstreamDir("up", "https://example.com") + gitCommand(t, "", "init", "--quiet", "--bare", filepath.Join(cacheDir, upstreamDir, "team", "repo.git")) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: cacheDir, RefreshWait: time.Second, StaleMaxAge: time.Hour, + Client: &http.Client{}, Logger: logger, Upstreams: map[string]string{"up": upstreamDir}, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cache.Close() }) + db, err := database.Create(filepath.Join(t.TempDir(), "proxy.db")) + if err != nil { + t.Fatal(err) + } + _ = db.Close() + + metrics.UpdateGitCacheStats(0, 0) + s := &Server{db: db, git: cache, logger: logger} + s.updateGauges() + if got := testutil.ToFloat64(metrics.GitCachedRepositories); got != 1 { + t.Errorf("proxy_git_cached_repositories = %v, want 1", got) + } + if got := testutil.ToFloat64(metrics.GitCacheSize); got <= 0 { + t.Errorf("proxy_git_cache_size_bytes = %v, want the repository's size", got) + } +} diff --git a/internal/server/layout.go b/internal/server/layout.go index 1a63d2ff..798fcd9d 100644 --- a/internal/server/layout.go +++ b/internal/server/layout.go @@ -19,6 +19,8 @@ type Layout struct { CacheDelete bool // Projects shows the project inventory pages. Projects bool + // Git shows the page listing the git cache's repositories. + Git bool } func (s *Server) layoutFor(r *http.Request) Layout { @@ -28,5 +30,6 @@ func (s *Server) layoutFor(r *http.Request) Layout { CanonicalPath: r.URL.Path, CacheDelete: s.cfg.CacheDelete, Projects: s.cfg.Projects.Enabled, + Git: s.git != nil, } } diff --git a/internal/server/server.go b/internal/server/server.go index df0dd25e..b219ed2f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -383,6 +383,9 @@ func (s *Server) serve(listener net.Listener) error { ui.Delete("/api/cache/{ecosystem}/*", s.handleCacheDelete) } s.mountProjectUI(ui) + if s.git != nil { + ui.Get("/git", s.handleGitRepositories) + } }) r.Get("/", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/ui/", http.StatusFound) @@ -710,8 +713,7 @@ func (s *Server) updateCacheStatsMetrics(ctx context.Context) { defer ticker.Stop() for { - s.updateCacheStats() - s.updateMetadataCacheStats() + s.updateGauges() select { case <-ctx.Done(): return @@ -720,6 +722,16 @@ func (s *Server) updateCacheStatsMetrics(ctx context.Context) { } } +// updateGauges republishes the cache gauges. Each group is updated on its +// own, so one failing leaves the others current. +func (s *Server) updateGauges() { + s.updateCacheStats() + s.updateMetadataCacheStats() + if s.git != nil { + s.updateGitCacheStats() + } +} + // updateMetadataCacheStats republishes the metadata cache gauges from a fresh // read of the snapshot the analytics page also serves. It runs separately // from updateCacheStats so a failure in one leaves the other's gauges current. diff --git a/internal/server/templates/layout/header.html b/internal/server/templates/layout/header.html index 8b4d94f0..a5740103 100644 --- a/internal/server/templates/layout/header.html +++ b/internal/server/templates/layout/header.html @@ -54,6 +54,7 @@ {{end}} {{define "nav_links"}} +{{if .Git}}Git{{end}} {{if .Projects}}Projects{{end}} Analytics Install diff --git a/internal/server/templates/pages/git_repositories.html b/internal/server/templates/pages/git_repositories.html new file mode 100644 index 00000000..fb9bee2c --- /dev/null +++ b/internal/server/templates/pages/git_repositories.html @@ -0,0 +1,41 @@ +{{define "title"}}Git repositories - git-pkgs proxy{{end}} + +{{define "content"}} +
+

Git repositories

+

{{.Summary}}

+ {{if .Pending}}

Branches and commits not read yet for {{.Pending}}. Reload to see more.

{{end}} +
+ +{{if .Rows}} +
+
+ {{range .Rows}} +
+
+
{{.Upstream}} / {{.Repo}}
+
{{.CloneURL}}
+ {{if .Commit}} +
{{.Branch}}{{if .Branch}} @ {{end}}{{.Commit}} {{.Subject}}{{if .CommitAge}} · {{.CommitAge}}{{end}}
+ {{end}} + {{if .LastError}} +
Last refresh failed: {{.LastError}}{{if .LastErrorAt}}, {{.LastErrorAt}}{{end}}
+ {{end}} +
+
+ {{if .Refs}}{{.Refs}}{{end}} + {{.Size}} + {{.Requests}} + {{if .LastChecked}}checked {{.LastChecked}}{{end}} + {{if .LastUsed}}used {{.LastUsed}}{{end}} +
+
+ {{end}} +
+
+{{else}} +
+

No repository is cached yet

+
+{{end}} +{{end}}