From f8d528f1192e96094188c9308e03504ecfb3d979 Mon Sep 17 00:00:00 2001 From: tsan88 Date: Fri, 4 Sep 2026 15:42:19 +0700 Subject: [PATCH 1/2] feat(api): bound event list responses by size and time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /api/events and /api/events/preview returned every event stored for the requested type/project — FindOptions already had Limit and Offset, but the handlers never filled them in. On a busy instance one preview request weighed 25.8 MB, and the frontend then filtered that in the browser. Both endpoints now accept limit, offset/page, from, to and window (15m, 24h, "7d"; "all" opts out of the configured default), and report what was applied in the response meta so a client can tell a truncated response from an exhausted one. Defaults are deliberately conservative and non-breaking: 1000 events per response, no time window. An operator can set ui.default_window (or UI_DEFAULT_WINDOW) to also narrow lists by time. --- buggregator.yaml.example | 6 ++ internal/app/app.go | 29 ++++++- internal/app/config.go | 38 +++++++++ internal/event/store.go | 5 ++ internal/server/http/api.go | 142 ++++++++++++++++++++++++++++--- internal/server/http/api_test.go | 113 +++++++++++++++++++++++- internal/storage/sqlite.go | 11 +++ 7 files changed, 331 insertions(+), 13 deletions(-) diff --git a/buggregator.yaml.example b/buggregator.yaml.example index a2bbbc4..fe449f7 100644 --- a/buggregator.yaml.example +++ b/buggregator.yaml.example @@ -21,6 +21,12 @@ storage: mode: ${STORAGE_MODE:memory} # "memory" (default, lost on restart) or "filesystem" path: ${STORAGE_PATH:./storage} # Directory for filesystem mode +# UI event lists: how much /api/events and /api/events/preview return +ui: + default_limit: ${UI_DEFAULT_LIMIT:1000} # Events returned when the request has no limit param + max_limit: ${UI_MAX_LIMIT:5000} # Ceiling for an explicit ?limit= + default_window: ${UI_DEFAULT_WINDOW:} # Default time window, e.g. "1h", "24h", "7d". Empty = no window + # Prometheus metrics metrics: enabled: ${METRICS_ENABLED:false} # Set to true to expose Prometheus metrics diff --git a/internal/app/app.go b/internal/app/app.go index bb62682..13860dd 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "os/signal" + "strings" "syscall" "github.com/buggregator/go-buggregator/internal/auth" @@ -105,7 +106,7 @@ func (a *App) Run() { } // Register core API routes (settings endpoint is public, others go through auth middleware). - httpserver.RegisterAPI(mux, store, a.registry.Previews(), eventService, a.cfg.Version, a.db, a.cfg.Modules.EnabledTypes(), authSettings, authMiddleware) + httpserver.RegisterAPI(mux, store, a.registry.Previews(), eventService, a.cfg.Version, a.db, a.cfg.Modules.EnabledTypes(), authSettings, authMiddleware, a.listLimits()) // Register attachment API endpoints. httpserver.RegisterAttachmentAPI(mux, a.db, a.attachments) @@ -232,3 +233,29 @@ func (a *App) Run() { _ = srv.Shutdown(context.Background()) tcpManager.Wait() } + +// listLimits turns the ui config section into list limits. An unreadable +// default_window must not take the service down: the window is dropped and the +// reason is logged. +func (a *App) listLimits() httpserver.ListLimits { + lim := httpserver.ListLimits{ + DefaultLimit: a.cfg.UI.DefaultLimit, + MaxLimit: a.cfg.UI.MaxLimit, + } + + switch w := strings.TrimSpace(a.cfg.UI.DefaultWindow); w { + case "", "all", "0": + lim.DefaultWindow = 0 + default: + d, err := httpserver.ParseWindow(w) + if err != nil || d <= 0 { + slog.Warn("ui.default_window is not a duration, no time window applied", "value", w, "err", err) + d = 0 + } + lim.DefaultWindow = d + } + + slog.Info("event list limits", "default_limit", lim.DefaultLimit, + "max_limit", lim.MaxLimit, "default_window", lim.DefaultWindow) + return lim +} diff --git a/internal/app/config.go b/internal/app/config.go index b482656..e4b50b7 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -6,6 +6,7 @@ import ( "log/slog" "os" "regexp" + "strconv" "strings" "gopkg.in/yaml.v3" @@ -23,6 +24,17 @@ type AuthConfig struct { JWTSecret string `yaml:"jwt_secret"` // Secret for signing internal JWT tokens. Required when auth is enabled. } +// UIConfig bounds the event lists served to the UI. +// +// /api/events and /api/events/preview used to return everything stored for a +// project. The defaults below (1000 events, no time window) only cap the +// response size; set default_window to also narrow it by time, e.g. "24h". +type UIConfig struct { + DefaultLimit int `yaml:"default_limit"` // events returned without a limit param (default 1000) + MaxLimit int `yaml:"max_limit"` // ceiling for an explicit limit (default 5000) + DefaultWindow string `yaml:"default_window"` // time window without from/to: "24h", "7d"; empty or "all" = no window +} + // Config holds application configuration. type Config struct { Server ServerConfig `yaml:"server"` @@ -32,6 +44,7 @@ type Config struct { Metrics MetricsConfig `yaml:"metrics"` MCP MCPConfig `yaml:"mcp"` Auth AuthConfig `yaml:"auth"` + UI UIConfig `yaml:"ui"` Modules ModulesConfig `yaml:"modules"` Webhooks []WebhookDef `yaml:"webhooks"` Projects []ProjectDef `yaml:"projects"` @@ -207,6 +220,11 @@ func LoadConfig() Config { cfg.Auth.Scopes = coalesce(os.Getenv("AUTH_SCOPES"), fileCfg.Auth.Scopes, "openid,email,profile") cfg.Auth.JWTSecret = coalesce(os.Getenv("AUTH_JWT_SECRET"), fileCfg.Auth.JWTSecret) + // UI list limits. + cfg.UI.DefaultLimit = coalesceInt(atoiOrZero(os.Getenv("UI_DEFAULT_LIMIT")), fileCfg.UI.DefaultLimit, 1000) + cfg.UI.MaxLimit = coalesceInt(atoiOrZero(os.Getenv("UI_MAX_LIMIT")), fileCfg.UI.MaxLimit, 5000) + cfg.UI.DefaultWindow = coalesce(os.Getenv("UI_DEFAULT_WINDOW"), fileCfg.UI.DefaultWindow) + // CORS origins. cfg.Server.CORSOrigins = fileCfg.Server.CORSOrigins if env := os.Getenv("CORS_ORIGINS"); env != "" { @@ -318,6 +336,26 @@ func expandEnvVars(input string) string { }) } +// coalesceInt is coalesce for numbers: zero counts as "not set". +func coalesceInt(values ...int) int { + for _, v := range values { + if v > 0 { + return v + } + } + return 0 +} + +// atoiOrZero lets an env variable be passed to coalesceInt in one expression: a +// non-empty but non-numeric value counts as "not set". +func atoiOrZero(s string) int { + n, err := strconv.Atoi(strings.TrimSpace(s)) + if err != nil || n < 0 { + return 0 + } + return n +} + func coalesce(values ...string) string { for _, v := range values { if v != "" { diff --git a/internal/event/store.go b/internal/event/store.go index 11d68cd..6573b97 100644 --- a/internal/event/store.go +++ b/internal/event/store.go @@ -8,6 +8,11 @@ type FindOptions struct { Project string Limit int Offset int + + // From and To bound the selection by event time (unix seconds with a + // fraction, same as Event.Timestamp). Zero means "no bound". + From float64 + To float64 } // DeleteOptions configures batch deletion. diff --git a/internal/server/http/api.go b/internal/server/http/api.go index 6f76954..163d2a4 100644 --- a/internal/server/http/api.go +++ b/internal/server/http/api.go @@ -4,6 +4,9 @@ import ( "database/sql" "encoding/json" "net/http" + "strconv" + "strings" + "time" "github.com/buggregator/go-buggregator/internal/event" ) @@ -14,9 +17,132 @@ type AuthSettings struct { LoginURL string } +// ListLimits bounds what /api/events and /api/events/preview return. +// +// Both endpoints used to return every event matching type/project: Limit and +// Offset existed in FindOptions but were never filled in. On a busy project +// that is a very large response — measured on a production instance, a single +// project preview weighed 25.8 MB — which the frontend then filters in the +// browser. +type ListLimits struct { + DefaultLimit int // how many events to return when limit is absent; 0 = unlimited + MaxLimit int // ceiling for an explicit limit; 0 = no ceiling + DefaultWindow time.Duration // time window applied when neither from/to nor window is given; 0 = no window +} + +// parseListOptions builds FindOptions from query parameters. +// +// type, project — unchanged; +// limit — how many events to return (capped by MaxLimit); +// offset / page — offset (page is counted from limit, 1-based); +// from, to — window bounds: unix seconds or RFC3339 ("2026-09-01T10:00:00Z"); +// window — window relative to now: "24h", "15m", "7d"; +// "all" or "0" opt out of DefaultWindow. +func parseListOptions(r *http.Request, lim ListLimits) event.FindOptions { + q := r.URL.Query() + opts := event.FindOptions{ + Type: q.Get("type"), + Project: q.Get("project"), + Limit: lim.DefaultLimit, + } + + if v := q.Get("limit"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + opts.Limit = n + } + } + if lim.MaxLimit > 0 && opts.Limit > lim.MaxLimit { + opts.Limit = lim.MaxLimit + } + + if v := q.Get("offset"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + opts.Offset = n + } + } else if v := q.Get("page"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 1 { + opts.Offset = (n - 1) * opts.Limit + } + } + + opts.From = parseTimeParam(q.Get("from")) + opts.To = parseTimeParam(q.Get("to")) + + // The default window only applies when the caller set no bounds itself. + if opts.From == 0 && opts.To == 0 { + switch w := strings.TrimSpace(q.Get("window")); w { + case "": + if lim.DefaultWindow > 0 { + opts.From = epochSeconds(time.Now().Add(-lim.DefaultWindow)) + } + case "all", "0": + // explicit opt-out + default: + if d, err := ParseWindow(w); err == nil && d > 0 { + opts.From = epochSeconds(time.Now().Add(-d)) + } else if lim.DefaultWindow > 0 { + opts.From = epochSeconds(time.Now().Add(-lim.DefaultWindow)) + } + } + } + + return opts +} + +// parseTimeParam reads a window bound: unix seconds (fraction allowed) or +// RFC3339. Returns 0 for an empty or unreadable value, which leaves the bound +// unset instead of failing the request. +func parseTimeParam(v string) float64 { + v = strings.TrimSpace(v) + if v == "" { + return 0 + } + if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 { + return f + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, v); err == nil { + return epochSeconds(t) + } + } + return 0 +} + +// ParseWindow is time.ParseDuration plus the "d" suffix for days, which Go does +// not support but is the most common unit in a UI ("7d"). +func ParseWindow(v string) (time.Duration, error) { + if strings.HasSuffix(v, "d") { + if n, err := strconv.ParseFloat(strings.TrimSuffix(v, "d"), 64); err == nil { + return time.Duration(n * float64(24*time.Hour)), nil + } + } + return time.ParseDuration(v) +} + +func epochSeconds(t time.Time) float64 { + return float64(t.UnixMicro()) / 1e6 +} + +// listMeta reports the applied limits, so a client can tell a truncated +// response from an exhausted one. +func listMeta(opts event.FindOptions, returned int) map[string]any { + meta := map[string]any{ + "limit": opts.Limit, + "offset": opts.Offset, + "returned": returned, + } + if opts.From > 0 { + meta["from"] = opts.From + } + if opts.To > 0 { + meta["to"] = opts.To + } + return meta +} + // RegisterAPI registers core API routes on the given mux. // authMiddleware wraps protected routes; pass a no-op when auth is disabled. -func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewRegistry, es *EventService, version string, db *sql.DB, enabledEvents []string, authSettings AuthSettings, authMiddleware func(http.Handler) http.Handler) { +func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewRegistry, es *EventService, version string, db *sql.DB, enabledEvents []string, authSettings AuthSettings, authMiddleware func(http.Handler) http.Handler, listLimits ListLimits) { // Public routes (no auth required). mux.HandleFunc("GET /api/version", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, map[string]string{"version": version}) @@ -40,10 +166,7 @@ func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewR // List events. protect("GET /api/events", func(w http.ResponseWriter, r *http.Request) { - opts := event.FindOptions{ - Type: r.URL.Query().Get("type"), - Project: r.URL.Query().Get("project"), - } + opts := parseListOptions(r, listLimits) events, err := store.FindAll(r.Context(), opts) if err != nil { writeError(w, err.Error(), http.StatusInternalServerError) @@ -52,15 +175,12 @@ func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewR if events == nil { events = []event.Event{} } - writeJSON(w, map[string]any{"data": events, "meta": map[string]any{}}) + writeJSON(w, map[string]any{"data": events, "meta": listMeta(opts, len(events))}) }) // List event previews. protect("GET /api/events/preview", func(w http.ResponseWriter, r *http.Request) { - opts := event.FindOptions{ - Type: r.URL.Query().Get("type"), - Project: r.URL.Query().Get("project"), - } + opts := parseListOptions(r, listLimits) events, err := store.FindAll(r.Context(), opts) if err != nil { writeError(w, err.Error(), http.StatusInternalServerError) @@ -70,7 +190,7 @@ func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewR for _, ev := range events { result = append(result, previews.BuildPreview(ev)) } - writeJSON(w, map[string]any{"data": result, "meta": map[string]any{}}) + writeJSON(w, map[string]any{"data": result, "meta": listMeta(opts, len(result))}) }) // Get single event. diff --git a/internal/server/http/api_test.go b/internal/server/http/api_test.go index 4310625..7a4f58a 100644 --- a/internal/server/http/api_test.go +++ b/internal/server/http/api_test.go @@ -5,8 +5,10 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strconv" "strings" "testing" + "time" "github.com/buggregator/go-buggregator/internal/event" "github.com/buggregator/go-buggregator/internal/module" @@ -16,6 +18,12 @@ import ( ) func setupAPI(t *testing.T) (*http.ServeMux, *storage.SQLiteStore) { + t.Helper() + // Empty limits keep the historical behaviour: return everything. + return setupAPILimits(t, serverhttp.ListLimits{}) +} + +func setupAPILimits(t *testing.T, limits serverhttp.ListLimits) (*http.ServeMux, *storage.SQLiteStore) { t.Helper() db, err := storage.Open(":memory:") if err != nil { @@ -43,7 +51,7 @@ func setupAPI(t *testing.T) (*http.ServeMux, *storage.SQLiteStore) { mux := http.NewServeMux() noopMiddleware := func(next http.Handler) http.Handler { return next } - serverhttp.RegisterAPI(mux, store, event.NewPreviewRegistry(), es, "test-version", db, []string{"sentry", "ray"}, serverhttp.AuthSettings{}, noopMiddleware) + serverhttp.RegisterAPI(mux, store, event.NewPreviewRegistry(), es, "test-version", db, []string{"sentry", "ray"}, serverhttp.AuthSettings{}, noopMiddleware, limits) return mux, store } @@ -259,3 +267,106 @@ func TestAPI_Projects(t *testing.T) { t.Errorf("key = %v", proj["key"]) } } + +func TestAPI_Events_Limits(t *testing.T) { + store := func(t *testing.T, mux *http.ServeMux, s *storage.SQLiteStore, n int, ts float64) { + t.Helper() + ctx := context.Background() + for i := 0; i < n; i++ { + if err := s.Store(ctx, event.Event{ + UUID: "uuid-" + strconv.Itoa(i) + "-" + strconv.FormatFloat(ts, 'f', 0, 64), + Type: "sentry", + Payload: json.RawMessage(`{"message":"error"}`), + Timestamp: ts + float64(i), + Project: "default", + }); err != nil { + t.Fatal(err) + } + } + } + list := func(t *testing.T, mux *http.ServeMux, query string) (int, map[string]any) { + t.Helper() + r := httptest.NewRequest("GET", "/api/events"+query, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, r) + + var resp struct { + Data []event.Event `json:"data"` + Meta map[string]any `json:"meta"` + } + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode: %v", err) + } + return len(resp.Data), resp.Meta + } + + now := float64(time.Now().Unix()) + + t.Run("default limit caps the response", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 5, MaxLimit: 10}) + store(t, mux, s, 12, now) + + if n, meta := list(t, mux, ""); n != 5 || meta["limit"] != float64(5) { + t.Fatalf("got %d events, meta %v; want 5", n, meta) + } + }) + + t.Run("explicit limit is capped by max_limit", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 5, MaxLimit: 10}) + store(t, mux, s, 12, now) + + if n, _ := list(t, mux, "?limit=1000"); n != 10 { + t.Fatalf("got %d events, want 10 (max_limit)", n) + } + }) + + t.Run("default window hides old events", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 100, DefaultWindow: time.Hour}) + store(t, mux, s, 3, now-24*3600) // yesterday + store(t, mux, s, 2, now-60) // a minute ago + + if n, _ := list(t, mux, ""); n != 2 { + t.Fatalf("got %d events in the last hour, want 2", n) + } + // window=all opts out of the default window. + if n, _ := list(t, mux, "?window=all"); n != 5 { + t.Fatalf("got %d events for window=all, want 5", n) + } + // An explicit window overrides the default one. + if n, _ := list(t, mux, "?window=7d"); n != 5 { + t.Fatalf("got %d events for window=7d, want 5", n) + } + }) + + t.Run("explicit from/to bounds", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 100, DefaultWindow: time.Hour}) + store(t, mux, s, 4, 1700000000) // long before the default window + + from := strconv.FormatFloat(1700000000, 'f', 0, 64) + to := strconv.FormatFloat(1700000002, 'f', 0, 64) + if n, _ := list(t, mux, "?from="+from+"&to="+to); n != 3 { + t.Fatalf("got %d events for [%s..%s], want 3", n, from, to) + } + if n, _ := list(t, mux, "?from=2023-01-01"); n != 4 { + t.Fatalf("got %d events for an ISO from, want 4", n) + } + }) +} + +func TestParseWindow(t *testing.T) { + cases := map[string]time.Duration{ + "15m": 15 * time.Minute, + "24h": 24 * time.Hour, + "7d": 7 * 24 * time.Hour, + "1.5d": 36 * time.Hour, + } + for in, want := range cases { + got, err := serverhttp.ParseWindow(in) + if err != nil || got != want { + t.Errorf("ParseWindow(%q) = %v, %v; want %v", in, got, err, want) + } + } + if _, err := serverhttp.ParseWindow("yesterday"); err == nil { + t.Error("ParseWindow(\"yesterday\") should fail") + } +} diff --git a/internal/storage/sqlite.go b/internal/storage/sqlite.go index 6c13dda..4a214a5 100644 --- a/internal/storage/sqlite.go +++ b/internal/storage/sqlite.go @@ -62,6 +62,17 @@ func (s *SQLiteStore) FindAll(ctx context.Context, opts event.FindOptions) ([]ev conditions = append(conditions, "project = ?") args = append(args, opts.Project) } + // Window bounds are compared numerically: timestamp is stored as text + // (fmt.Sprintf("%.6f")), and a string comparison would break as soon as the + // number of digits in unix seconds changes. + if opts.From > 0 { + conditions = append(conditions, "CAST(timestamp AS REAL) >= ?") + args = append(args, opts.From) + } + if opts.To > 0 { + conditions = append(conditions, "CAST(timestamp AS REAL) <= ?") + args = append(args, opts.To) + } if len(conditions) > 0 { query += " WHERE " + strings.Join(conditions, " AND ") } From 95a9ec7c65d22605b7892d2ee33a9e3ae338f5cf Mon Sep 17 00:00:00 2001 From: tsan88 Date: Fri, 4 Sep 2026 15:45:01 +0700 Subject: [PATCH 2/2] feat(sentry): filter exceptions, traces and logs by project, environment and period MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sentry endpoints could only be narrowed by level, handled and trace_id. project_id and environment are written for every error event but had no reader, so picking a project in the UI still listed every project's errors — including in the grouped view, which is what makes it useful on a shared instance. There was no way to narrow anything by time at all, so reaching a particular hour meant paging through the entire list. Adds: - project and environment filters on /api/sentry/exceptions (both the chronological and the grouped view); - environment and release filters on /api/sentry/traces (columns that were also written but never read); - from / to / window ("15m", "24h", "7d") on exceptions, traces, logs and counts. /api/sentry/counts now respects the same project and period as the lists it labels, and counts transactions rather than traces — that is what the traces list shows and the table that carries a timestamp to filter on. Filters only apply when the parameters are present, so existing clients see no change. --- modules/sentry/api.go | 77 ++++++++++++++++++++++++++ modules/sentry/api_exceptions.go | 22 ++++++++ modules/sentry/api_logs.go | 2 + modules/sentry/api_service_map.go | 21 ++++++- modules/sentry/api_test.go | 92 +++++++++++++++++++++++++++++++ modules/sentry/api_traces.go | 28 ++++++++-- 6 files changed, 235 insertions(+), 7 deletions(-) diff --git a/modules/sentry/api.go b/modules/sentry/api.go index 296e005..cf05669 100644 --- a/modules/sentry/api.go +++ b/modules/sentry/api.go @@ -5,6 +5,10 @@ import ( "encoding/json" "net/http" "strconv" + "strings" + "time" + + httpserver "github.com/buggregator/go-buggregator/internal/server/http" ) func registerAPI(mux *http.ServeMux, db *sql.DB) { @@ -41,6 +45,79 @@ func handleClearAll(db *sql.DB) http.HandlerFunc { } } +// timeWindow reads the period bounds from the query parameters. +// +// from, to — unix seconds or RFC3339 ("2026-09-01T10:00:00Z", "2026-09-01"); +// window — a window relative to now: "24h", "15m", "7d". +// +// A zero result means "unbounded". None of the sentry endpoints could be +// narrowed by time before, so on a stream of tens of thousands of events a day +// the only way to reach a particular hour was to page through the whole list. +func timeWindow(r *http.Request) (from, to time.Time) { + q := r.URL.Query() + from = parseWhen(q.Get("from")) + to = parseWhen(q.Get("to")) + + if from.IsZero() && to.IsZero() { + if w := strings.TrimSpace(q.Get("window")); w != "" && w != "all" && w != "0" { + if d, err := httpserver.ParseWindow(w); err == nil && d > 0 { + from = time.Now().UTC().Add(-d) + } + } + } + return from, to +} + +// parseWhen reads a single bound: unix seconds or one of the ISO forms. +func parseWhen(v string) time.Time { + v = strings.TrimSpace(v) + if v == "" { + return time.Time{} + } + if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 { + return time.Unix(int64(f), 0).UTC() + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, v); err == nil { + return t.UTC() + } + } + return time.Time{} +} + +// appendTimeConditions adds the period bounds to a WHERE clause. +// +// The stored format differs per column, hence the iso flag: +// - received_at, first_seen, last_seen — 'YYYY-MM-DD HH:MM:SS' (datetime('now')); +// - start_ts, end_ts of transactions and spans — ISO with T and Z. +// +// The comparison is done on strings: for both formats lexicographic order +// matches chronological order, and the indexes on these columns are textual. +func appendTimeConditions(conditions []string, args []any, r *http.Request, column string, iso bool) ([]string, []any) { + from, to := timeWindow(r) + layout := "2006-01-02 15:04:05" + if iso { + layout = "2006-01-02T15:04:05" + } + if !from.IsZero() { + conditions = append(conditions, column+" >= ?") + args = append(args, from.UTC().Format(layout)) + } + if !to.IsZero() { + conditions = append(conditions, column+" <= ?") + args = append(args, to.UTC().Format(layout)) + } + return conditions, args +} + +// whereOf builds a WHERE clause from conditions (empty string when there are none). +func whereOf(conditions []string) string { + if len(conditions) == 0 { + return "" + } + return " WHERE " + strings.Join(conditions, " AND ") +} + // pagination extracts page/limit from query params with defaults. func pagination(r *http.Request, defaultLimit int) (limit, offset int) { limit = defaultLimit diff --git a/modules/sentry/api_exceptions.go b/modules/sentry/api_exceptions.go index ff1d915..61f28d2 100644 --- a/modules/sentry/api_exceptions.go +++ b/modules/sentry/api_exceptions.go @@ -55,6 +55,14 @@ func handleExceptionsGrouped(db *sql.DB, w http.ResponseWriter, r *http.Request) conditions = append(conditions, "e.level = ?") args = append(args, v) } + if v := q.Get("environment"); v != "" { + conditions = append(conditions, "e.environment = ?") + args = append(args, v) + } + if v := q.Get("project"); v != "" { + conditions = append(conditions, "e.project_id = ?") + args = append(args, v) + } if v := q.Get("handled"); v != "" { if v == "true" { conditions = append(conditions, "e.handled = 1") @@ -63,6 +71,9 @@ func handleExceptionsGrouped(db *sql.DB, w http.ResponseWriter, r *http.Request) } } + // Period: from/to/window. received_at is covered by idx_sentry_errors_received_at. + conditions, args = appendTimeConditions(conditions, args, r, "e.received_at", false) + where := "" if len(conditions) > 0 { where = " WHERE " + strings.Join(conditions, " AND ") @@ -142,6 +153,14 @@ func handleExceptionsChronological(db *sql.DB, w http.ResponseWriter, r *http.Re conditions = append(conditions, "e.level = ?") args = append(args, v) } + if v := q.Get("environment"); v != "" { + conditions = append(conditions, "e.environment = ?") + args = append(args, v) + } + if v := q.Get("project"); v != "" { + conditions = append(conditions, "e.project_id = ?") + args = append(args, v) + } if v := q.Get("handled"); v != "" { if v == "true" { conditions = append(conditions, "e.handled = 1") @@ -150,6 +169,9 @@ func handleExceptionsChronological(db *sql.DB, w http.ResponseWriter, r *http.Re } } + // Period: from/to/window. received_at is covered by idx_sentry_errors_received_at. + conditions, args = appendTimeConditions(conditions, args, r, "e.received_at", false) + where := "" if len(conditions) > 0 { where = " WHERE " + strings.Join(conditions, " AND ") diff --git a/modules/sentry/api_logs.go b/modules/sentry/api_logs.go index a24fb65..f0704c3 100644 --- a/modules/sentry/api_logs.go +++ b/modules/sentry/api_logs.go @@ -30,6 +30,8 @@ func handleLogsList(db *sql.DB) http.HandlerFunc { conditions = append(conditions, "l.trace_id = ?") args = append(args, v) } + // Period: log_ts is stored in ISO form and covered by idx_sentry_logs_ts. + conditions, args = appendTimeConditions(conditions, args, r, "l.log_ts", true) where := "" if len(conditions) > 0 { diff --git a/modules/sentry/api_service_map.go b/modules/sentry/api_service_map.go index e750287..cf52338 100644 --- a/modules/sentry/api_service_map.go +++ b/modules/sentry/api_service_map.go @@ -133,9 +133,24 @@ func labelForNode(address, opType string) string { func handleCounts(db *sql.DB) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var exceptions, traces, logs int - db.QueryRow(`SELECT COUNT(*) FROM sentry_error_events`).Scan(&exceptions) - db.QueryRow(`SELECT COUNT(*) FROM sentry_traces`).Scan(&traces) - db.QueryRow(`SELECT COUNT(*) FROM sentry_logs`).Scan(&logs) + // The counters follow the same project and period as the lists they label. + // They used to count the whole database, so the project selected in the UI + // had no effect on the numbers next to the tabs. + var errConds, txnConds, logConds []string + var errArgs, txnArgs, logArgs []any + errConds, errArgs = appendTimeConditions(errConds, errArgs, r, "received_at", false) + txnConds, txnArgs = appendTimeConditions(txnConds, txnArgs, r, "start_ts", true) + logConds, logArgs = appendTimeConditions(logConds, logArgs, r, "log_ts", true) + if v := r.URL.Query().Get("project"); v != "" { + errConds = append(errConds, "project_id = ?") + errArgs = append(errArgs, v) + } + + db.QueryRow(`SELECT COUNT(*) FROM sentry_error_events`+whereOf(errConds), errArgs...).Scan(&exceptions) + // Counted over sentry_transactions rather than sentry_traces: that is what + // the traces list shows, and it is the table carrying a timestamp to filter on. + db.QueryRow(`SELECT COUNT(*) FROM sentry_transactions`+whereOf(txnConds), txnArgs...).Scan(&traces) + db.QueryRow(`SELECT COUNT(*) FROM sentry_logs`+whereOf(logConds), logArgs...).Scan(&logs) apiJSON(w, map[string]any{ "exceptions": exceptions, diff --git a/modules/sentry/api_test.go b/modules/sentry/api_test.go index bec6cd7..45c08d1 100644 --- a/modules/sentry/api_test.go +++ b/modules/sentry/api_test.go @@ -463,3 +463,95 @@ func TestAPIEmptyDatabase(t *testing.T) { }) } } + +// The exceptions list could not be narrowed to a project: the column is filled +// in on write, but the handler had no reader for it, so selecting a project in +// the UI showed every project's errors. Same for environment, and for the +// period there was no filter at all. +func TestAPIExceptionsFilters(t *testing.T) { + db := setupTestDB(t) + mux := http.NewServeMux() + registerAPI(mux, db) + + seed := func(eventID, env, project string) { + t.Helper() + payload, _ := json.Marshal(map[string]any{ + "event_id": eventID, + "level": "error", + "environment": env, + "exception": map[string]any{"values": []map[string]any{{"type": "RuntimeException", "value": eventID}}}, + }) + var ev ErrorEvent + if err := json.Unmarshal(payload, &ev); err != nil { + t.Fatal(err) + } + if _, err := storeErrorEvent(db, &ev, payload, project); err != nil { + t.Fatalf("seed %s: %v", eventID, err) + } + } + seed("evt-prod-a", "production", "shop") + seed("evt-prod-b", "production", "shop") + seed("evt-stage", "staging", "shop") + seed("evt-other", "production", "billing") + + count := func(query string) int { + t.Helper() + req := httptest.NewRequest("GET", "/api/sentry/exceptions"+query, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("status = %d for %q", w.Code, query) + } + var resp map[string]any + json.Unmarshal(w.Body.Bytes(), &resp) + return len(resp["data"].([]any)) + } + + cases := map[string]struct { + query string + want int + }{ + "no filters": {"", 4}, + "by project": {"?project=shop", 3}, + "by environment": {"?environment=staging", 1}, + "project and env": {"?project=shop&environment=production", 2}, + "grouped by project": {"?grouped=true&project=billing", 1}, + "within window": {"?window=24h", 4}, + "outside window": {"?from=2030-01-01", 0}, + "grouped in window": {"?grouped=true&from=2030-01-01", 0}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + if got := count(tc.query); got != tc.want { + t.Errorf("%q returned %d events, want %d", tc.query, got, tc.want) + } + }) + } +} + +// The tab counters counted the whole database, so they disagreed with the lists +// they label as soon as a project or a period was selected. +func TestAPICountsFollowFilters(t *testing.T) { + mux := http.NewServeMux() + seedTestData(t, mux) + + counts := func(query string) map[string]any { + t.Helper() + req := httptest.NewRequest("GET", "/api/sentry/counts"+query, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + var resp map[string]any + json.Unmarshal(w.Body.Bytes(), &resp) + return resp + } + + if got := counts("?project=default")["exceptions"]; int(got.(float64)) != 3 { + t.Errorf("exceptions for the seeded project = %v, want 3", got) + } + if got := counts("?project=nonexistent")["exceptions"]; int(got.(float64)) != 0 { + t.Errorf("exceptions for an unknown project = %v, want 0", got) + } + if got := counts("?from=2030-01-01")["exceptions"]; int(got.(float64)) != 0 { + t.Errorf("exceptions in a future window = %v, want 0", got) + } +} diff --git a/modules/sentry/api_traces.go b/modules/sentry/api_traces.go index a0a4d0a..a4e2a35 100644 --- a/modules/sentry/api_traces.go +++ b/modules/sentry/api_traces.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "net/http" + "strings" "time" ) @@ -11,17 +12,36 @@ func handleTracesList(db *sql.DB) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { limit, offset := pagination(r, 50) - countQuery := `SELECT COUNT(*) FROM sentry_transactions` + // Period: start_ts of a transaction is in ISO form, index idx_sentry_txn_start. + // Plus environment and release filters: both columns are populated but had + // no reader. + var conditions []string + var args []any + conditions, args = appendTimeConditions(conditions, args, r, "t.start_ts", true) + if v := r.URL.Query().Get("environment"); v != "" { + conditions = append(conditions, "t.environment = ?") + args = append(args, v) + } + if v := r.URL.Query().Get("release"); v != "" { + conditions = append(conditions, "t.release = ?") + args = append(args, v) + } + where := "" + if len(conditions) > 0 { + where = " WHERE " + strings.Join(conditions, " AND ") + } + var total int - db.QueryRow(countQuery).Scan(&total) + db.QueryRow(`SELECT COUNT(*) FROM sentry_transactions t`+where, args...).Scan(&total) + queryArgs := append(append([]any{}, args...), limit, offset) rows, err := db.Query( `SELECT t.trace_id, t.id, t.transaction_name, t.op, t.status, t.duration_ms, tr.span_count, tr.error_count, t.start_ts FROM sentry_transactions t - JOIN sentry_traces tr ON tr.trace_id = t.trace_id + JOIN sentry_traces tr ON tr.trace_id = t.trace_id`+where+` ORDER BY t.start_ts DESC - LIMIT ? OFFSET ?`, limit, offset, + LIMIT ? OFFSET ?`, queryArgs..., ) if err != nil { apiError(w, err.Error(), http.StatusInternalServerError)