From aecbf521e7424913636d6920bb79d577d4927b44 Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Fri, 28 Nov 2025 15:58:04 +0200 Subject: [PATCH 1/9] Add `stat_statements_jit` metric. It queries `pg_stat_statements` to identify queries with high percent of execution time spent in JIT. --- internal/metrics/metrics.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/metrics/metrics.yaml b/internal/metrics/metrics.yaml index e42bd9ba0..676d7e9be 100644 --- a/internal/metrics/metrics.yaml +++ b/internal/metrics/metrics.yaml @@ -2760,6 +2760,22 @@ metrics: pg_stat_statements where dbid = (select oid from pg_database where datname = current_database()) + stat_statements_jit: + description: > + This metric collects statistics from the `pg_stat_statements` extension, focusing on the total JIT generation time. + It provides insights into query performance, including total JIT generation time and total execution times. + This metric is useful for monitoring query performance and identifying queries with high percent of execution time spent in JIT. + init_sql: CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + sqls: + 13: | + select /*pgwatch_generated*/ + (extract(epoch from now()) * 1e9)::int8 as epoch_ns, + coalesce(sum(jit_generation_time)::numeric, 0)::double precision as total_jit_time, + coalesce(round(sum(total_exec_time)::numeric, 3), 0)::double precision as total_time + from + pg_stat_statements + where + dbid = (select oid from pg_database where datname = current_database()) stat_statements_no_query_text: description: > This metric collects statistics from the `pg_stat_statements` extension without including the query text. From a554becdd9b7f4b2359f491f818bf45da8efe36e Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Fri, 28 Nov 2025 15:59:22 +0200 Subject: [PATCH 2/9] Add `JIT %` panel to `Health Check` dashboard. It queries the `stat_statements_jit` metric table to identify the percentage of jit in the current total query execution time within the specified grafana time window. --- grafana/postgres/v12/0-health-check.json | 150 ++++++++++++++++++++++- 1 file changed, 148 insertions(+), 2 deletions(-) diff --git a/grafana/postgres/v12/0-health-check.json b/grafana/postgres/v12/0-health-check.json index 80db1bbff..ec0336e30 100644 --- a/grafana/postgres/v12/0-health-check.json +++ b/grafana/postgres/v12/0-health-check.json @@ -4979,6 +4979,152 @@ "title": "Max. XMIN horizon age", "type": "stat" }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "description": "Percentage of total query execution time spent in JIT. (requires stat_statements_jit metric)", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": 0 + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 50, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^pct$/", + "limit": 1, + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "editorMode": "code", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "365d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "stat_statements_jit", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select \n $__timeGroup(time, $__interval),\n (sum(total_jit - total_jit_lag) / sum(total_time - total_time_lag)) * 100 as pct\nfrom\n(\n select\n time,\n (data->>'total_jit_time')::float8 as total_jit, lag((data->>'total_jit_time')::float8) over w as total_jit_lag,\n (data->>'total_time')::float8 as total_time, lag((data->>'total_time')::float8) over w as total_time_lag\n from\n stat_statements_jit\n where \n dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n)\nwhere total_time > total_time_lag\ngroup by 1\norder by 1 desc\nlimit 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [], + "type": "last" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "JIT %", + "type": "stat" + }, { "fieldConfig": { "defaults": {}, @@ -5011,8 +5157,8 @@ "overrides": [] }, "gridPos": { - "h": 5, - "w": 13, + "h": 7, + "w": 12, "x": 0, "y": 20 }, From 917446876a2f209a9978c691fbf7f6c061608e45 Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Fri, 28 Nov 2025 18:06:51 +0200 Subject: [PATCH 3/9] Add `Top n by JIT %` panel to `Global Health` dashboard. It queries the `stat_statements_jit` metric table to identify the percentage of jit in the current total query execution time for each dbname within the specified grafana time window. --- grafana/postgres/v12/global-health.json | 199 +++++++++++++++++++++++- 1 file changed, 197 insertions(+), 2 deletions(-) diff --git a/grafana/postgres/v12/global-health.json b/grafana/postgres/v12/global-health.json index c250d3f9e..6782cc458 100644 --- a/grafana/postgres/v12/global-health.json +++ b/grafana/postgres/v12/global-health.json @@ -1966,15 +1966,210 @@ ], "type": "table" }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "description": "Top $top_limit sources by the percentage of total query execution time spent in JIT. (requires stat_statements_jit metric).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "pct" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 5 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 2 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 5 + } + ] + } + }, + { + "id": "displayName", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dbname" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Go to 'System Stats' dash", + "url": "/d/system-stats?var-dbname=${__value.text}" + } + ] + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 22 + }, + "id": 17, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select \n dbname, pct\nfrom (\n select \n dbname,\n (sum(total_jit - total_jit_lag) / sum(total_time - total_time_lag)) * 100 as pct\n from\n (\n select\n dbname,\n (data->>'total_jit_time')::float8 as total_jit, lag((data->>'total_jit_time')::float8) over w as total_jit_lag,\n (data->>'total_time')::float8 as total_time, lag((data->>'total_time')::float8) over w as total_time_lag\n from stat_statements_jit\n where $__timeFilter(time)\n window w as (order by time)\n )\n where total_time > total_time_lag\n group by 1\n)\nwhere pct > 0\norder by 2 desc\nlimit $top_limit", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top $top_limit by JIT %", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { - "h": 4, + "h": 5, "w": 8, - "x": 0, + "x": 8, "y": 22 }, "id": 13, From d5892eb57221f400551935f9cbd7ceeb39c45283 Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Fri, 28 Nov 2025 20:32:56 +0200 Subject: [PATCH 4/9] Add `Auto` option to `$online_interval` variable. The `Auto` option will contain a value identical to the current time range, so we can partially avoid the imprecision of `$__timeGroup()` --- grafana/postgres/v12/0-health-check.json | 57 +++++++++++++++++------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/grafana/postgres/v12/0-health-check.json b/grafana/postgres/v12/0-health-check.json index ec0336e30..e6a20f656 100644 --- a/grafana/postgres/v12/0-health-check.json +++ b/grafana/postgres/v12/0-health-check.json @@ -4987,7 +4987,7 @@ "description": "Percentage of total query execution time spent in JIT. (requires stat_statements_jit metric)", "fieldConfig": { "defaults": { - "decimals": 1, + "decimals": 5, "mappings": [ { "options": { @@ -5077,7 +5077,7 @@ "orderByTime": "ASC", "policy": "default", "rawQuery": true, - "rawSql": "select \n $__timeGroup(time, $__interval),\n (sum(total_jit - total_jit_lag) / sum(total_time - total_time_lag)) * 100 as pct\nfrom\n(\n select\n time,\n (data->>'total_jit_time')::float8 as total_jit, lag((data->>'total_jit_time')::float8) over w as total_jit_lag,\n (data->>'total_time')::float8 as total_time, lag((data->>'total_time')::float8) over w as total_time_lag\n from\n stat_statements_jit\n where \n dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n)\nwhere total_time > total_time_lag\ngroup by 1\norder by 1 desc\nlimit 1", + "rawSql": "select \n $__timeGroup(time, $online_interval),\n (sum(total_jit - total_jit_lag) / sum(total_time - total_time_lag)) * 100 as pct\nfrom\n(\n select\n time,\n (data->>'total_jit_time')::float8 as total_jit, lag((data->>'total_jit_time')::float8) over w as total_jit_lag,\n (data->>'total_time')::float8 as total_time, lag((data->>'total_time')::float8) over w as total_time_lag\n from\n stat_statements_jit\n where \n dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n)\nwhere total_time > total_time_lag\ngroup by 1\norder by 1 desc\nlimit 1", "refId": "A", "resultFormat": "time_series", "select": [ @@ -5205,14 +5205,14 @@ "type": "query" }, { - "auto": false, - "auto_count": 30, + "auto": true, + "auto_count": 1, "auto_min": "10s", "current": { - "text": "10m", - "value": "10m" + "text": "$__auto", + "value": "$__auto" }, - "label": "Max. age for 'online' metrics", + "hide": 3, "name": "online_interval", "options": [ { @@ -5222,23 +5222,48 @@ }, { "selected": false, - "text": "3m", - "value": "3m" + "text": "10m", + "value": "10m" }, { "selected": false, - "text": "5m", - "value": "5m" + "text": "30m", + "value": "30m" }, { - "selected": true, - "text": "10m", - "value": "10m" + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" }, { "selected": false, - "text": "15m", - "value": "15m" + "text": "30d", + "value": "30d" } ], "query": "1m,3m,5m,10m,15m", From 4638a28bb558b54a5ec4596600d43493db72456e Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Fri, 28 Nov 2025 20:52:08 +0200 Subject: [PATCH 5/9] Revert removing the label of `$online_interval` --- grafana/postgres/v12/0-health-check.json | 1 + 1 file changed, 1 insertion(+) diff --git a/grafana/postgres/v12/0-health-check.json b/grafana/postgres/v12/0-health-check.json index e6a20f656..62da1c8cf 100644 --- a/grafana/postgres/v12/0-health-check.json +++ b/grafana/postgres/v12/0-health-check.json @@ -5213,6 +5213,7 @@ "value": "$__auto" }, "hide": 3, + "label": "Max. age for 'online' metrics", "name": "online_interval", "options": [ { From 5e66cf00ff268801b463fd04356d4b592bdadbcb Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Mon, 1 Dec 2025 10:56:45 +0200 Subject: [PATCH 6/9] Update JIT Panel to order by biggest value. --- grafana/postgres/v12/0-health-check.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana/postgres/v12/0-health-check.json b/grafana/postgres/v12/0-health-check.json index 62da1c8cf..20b616783 100644 --- a/grafana/postgres/v12/0-health-check.json +++ b/grafana/postgres/v12/0-health-check.json @@ -5077,7 +5077,7 @@ "orderByTime": "ASC", "policy": "default", "rawQuery": true, - "rawSql": "select \n $__timeGroup(time, $online_interval),\n (sum(total_jit - total_jit_lag) / sum(total_time - total_time_lag)) * 100 as pct\nfrom\n(\n select\n time,\n (data->>'total_jit_time')::float8 as total_jit, lag((data->>'total_jit_time')::float8) over w as total_jit_lag,\n (data->>'total_time')::float8 as total_time, lag((data->>'total_time')::float8) over w as total_time_lag\n from\n stat_statements_jit\n where \n dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n)\nwhere total_time > total_time_lag\ngroup by 1\norder by 1 desc\nlimit 1", + "rawSql": "select \n $__timeGroup(time, $online_interval),\n (sum(total_jit - total_jit_lag) / sum(total_time - total_time_lag)) * 100 as pct\nfrom\n(\n select\n time,\n (data->>'total_jit_time')::float8 as total_jit, lag((data->>'total_jit_time')::float8) over w as total_jit_lag,\n (data->>'total_time')::float8 as total_time, lag((data->>'total_time')::float8) over w as total_time_lag\n from\n stat_statements_jit\n where \n dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n)\nwhere total_time > total_time_lag\ngroup by 1\norder by 2 desc\nlimit 1", "refId": "A", "resultFormat": "time_series", "select": [ From 7488cd8f403a3c2ea855e268272403a8bf255b5e Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Mon, 1 Dec 2025 11:33:45 +0200 Subject: [PATCH 7/9] require pg 15 at least for `stat_statements_jit` metric. the minimum version where `jit_generation_time` field is available in `pg_stat_statements`. --- internal/metrics/metrics.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/metrics/metrics.yaml b/internal/metrics/metrics.yaml index 676d7e9be..e41f651aa 100644 --- a/internal/metrics/metrics.yaml +++ b/internal/metrics/metrics.yaml @@ -2767,7 +2767,7 @@ metrics: This metric is useful for monitoring query performance and identifying queries with high percent of execution time spent in JIT. init_sql: CREATE EXTENSION IF NOT EXISTS pg_stat_statements; sqls: - 13: | + 15: | select /*pgwatch_generated*/ (extract(epoch from now()) * 1e9)::int8 as epoch_ns, coalesce(sum(jit_generation_time)::numeric, 0)::double precision as total_jit_time, From 041510b64e3147e8eb70c8d8fc55a9360fd0ebf1 Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Mon, 1 Dec 2025 11:47:27 +0200 Subject: [PATCH 8/9] Adjust num of decimals and threshold in JIT panel. --- grafana/postgres/v12/0-health-check.json | 6 +++--- grafana/postgres/v12/global-health.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/grafana/postgres/v12/0-health-check.json b/grafana/postgres/v12/0-health-check.json index 20b616783..b11bc7111 100644 --- a/grafana/postgres/v12/0-health-check.json +++ b/grafana/postgres/v12/0-health-check.json @@ -4987,7 +4987,7 @@ "description": "Percentage of total query execution time spent in JIT. (requires stat_statements_jit metric)", "fieldConfig": { "defaults": { - "decimals": 5, + "decimals": 2, "mappings": [ { "options": { @@ -5010,11 +5010,11 @@ }, { "color": "orange", - "value": 2 + "value": 1 }, { "color": "red", - "value": 5 + "value": 25 } ] }, diff --git a/grafana/postgres/v12/global-health.json b/grafana/postgres/v12/global-health.json index 6782cc458..943555431 100644 --- a/grafana/postgres/v12/global-health.json +++ b/grafana/postgres/v12/global-health.json @@ -2012,7 +2012,7 @@ }, { "id": "decimals", - "value": 5 + "value": 2 }, { "id": "custom.cellOptions", From 3800c5e114da74776eba75b12271122fd4681442 Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Mon, 1 Dec 2025 18:15:46 +0200 Subject: [PATCH 9/9] Use table view instead of time series for JIT panel in `0-health-check`. --- grafana/postgres/v12/0-health-check.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/grafana/postgres/v12/0-health-check.json b/grafana/postgres/v12/0-health-check.json index b11bc7111..66e3d666b 100644 --- a/grafana/postgres/v12/0-health-check.json +++ b/grafana/postgres/v12/0-health-check.json @@ -19,7 +19,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1, + "id": 21, "links": [], "panels": [ { @@ -5056,7 +5056,7 @@ "uid": "pgwatch-metrics" }, "editorMode": "code", - "format": "time_series", + "format": "table", "group": [], "groupBy": [ { @@ -5077,7 +5077,7 @@ "orderByTime": "ASC", "policy": "default", "rawQuery": true, - "rawSql": "select \n $__timeGroup(time, $online_interval),\n (sum(total_jit - total_jit_lag) / sum(total_time - total_time_lag)) * 100 as pct\nfrom\n(\n select\n time,\n (data->>'total_jit_time')::float8 as total_jit, lag((data->>'total_jit_time')::float8) over w as total_jit_lag,\n (data->>'total_time')::float8 as total_time, lag((data->>'total_time')::float8) over w as total_time_lag\n from\n stat_statements_jit\n where \n dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n)\nwhere total_time > total_time_lag\ngroup by 1\norder by 2 desc\nlimit 1", + "rawSql": "select\n (sum(total_jit - total_jit_lag) / sum(total_time - total_time_lag)) * 100 as pct\nfrom\n(\n select\n time,\n (data->>'total_jit_time')::float8 as total_jit, lag((data->>'total_jit_time')::float8) over w as total_jit_lag,\n (data->>'total_time')::float8 as total_time, lag((data->>'total_time')::float8) over w as total_time_lag\n from\n stat_statements_jit\n where \n dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n)\nwhere total_time > total_time_lag", "refId": "A", "resultFormat": "time_series", "select": [ @@ -5205,12 +5205,12 @@ "type": "query" }, { - "auto": true, - "auto_count": 1, + "auto": false, + "auto_count": 30, "auto_min": "10s", "current": { - "text": "$__auto", - "value": "$__auto" + "text": "30m", + "value": "30m" }, "hide": 3, "label": "Max. age for 'online' metrics",