diff --git a/Dockerfile b/Dockerfile index 6d0c98f3d..ea68ff411 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,6 +64,7 @@ ENV PYTHONUNBUFFERED=1 ENV VIRTUAL_ENV=/app/venv ENV PATH="/venv/bin:$PATH" ENV PYTHONPATH=$PYTHONPATH:.:/app/src +ENV MPLBACKEND=Agg WORKDIR /app diff --git a/playbooks/pyproject.toml b/playbooks/pyproject.toml index 7aa615778..25bc1590c 100644 --- a/playbooks/pyproject.toml +++ b/playbooks/pyproject.toml @@ -17,7 +17,7 @@ include_trailing_comma = true python = "^3.7.1" Flask = "^2.0.2" prometheus-api-client = "^0.5.4" -pygal = "^3.0.0" +matplotlib = "^3.10.8" tinycss = "^0.4" cssselect = "^1.1.0" rsa = "^4.8" diff --git a/playbooks/robusta_playbooks/node_cpu_analysis.py b/playbooks/robusta_playbooks/node_cpu_analysis.py index a3497a507..087d05f42 100644 --- a/playbooks/robusta_playbooks/node_cpu_analysis.py +++ b/playbooks/robusta_playbooks/node_cpu_analysis.py @@ -1,10 +1,9 @@ import logging import textwrap -import pygal - from robusta.api import ( FLOAT_PRECISION_LIMIT, + BarChart, DividerBlock, FileBlock, HeaderBlock, @@ -13,6 +12,7 @@ NodeCpuAnalyzer, NodeEvent, PrometheusParams, + TreemapChart, action, charts_style, ) @@ -41,7 +41,7 @@ def node_cpu_enricher(event: NodeEvent, params: PrometheusParams): per_pod_request = analyzer.get_per_pod_cpu_request() all_pod_names = list(set(per_pod_usage_unbounded.keys()).union(per_pod_request.keys())) - treemap = pygal.Treemap(style=charts_style()) + treemap = TreemapChart(style=charts_style()) treemap.title = f"CPU Usage on Node {node.metadata.name}" treemap.value_formatter = lambda x: f"{int(x * 100)}%" treemap.add("Non-container usage", [non_container_cpu_usage]) @@ -50,7 +50,7 @@ def node_cpu_enricher(event: NodeEvent, params: PrometheusParams): treemap.add(pod_name, [cpu_usage]) MISSING_VALUE = -0.001 - bar_chart = pygal.Bar(x_label_rotation=-40, style=charts_style()) + bar_chart = BarChart(x_label_rotation=-40, style=charts_style()) bar_chart.title = f"Actual Vs Requested vCPUs on Node {node.metadata.name}" bar_chart.x_labels = all_pod_names bar_chart.value_formatter = lambda x: f"{x:.2f} vCPU" if x != MISSING_VALUE else "no data" diff --git a/poetry.lock b/poetry.lock index e2e04cee7..2b3409873 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1188,29 +1188,6 @@ files = [ {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, ] -[[package]] -name = "importlib-metadata" -version = "8.7.1" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.9" -files = [ - {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, - {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, -] - -[package.dependencies] -zipp = ">=3.20" - -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=3.4)"] -perf = ["ipython"] -test = ["flufl.flake8", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] -type = ["mypy (<1.19)", "pytest-mypy (>=1.0.1)"] - [[package]] name = "iniconfig" version = "2.3.0" @@ -2304,27 +2281,6 @@ typing-extensions = ">3.10,<4.6.0 || >4.6.0" [package.extras] dev = ["build", "coverage", "furo", "invoke", "mypy", "pytest", "pytest-cov", "pytest-mypy-testing", "ruff", "sphinx", "sphinx-autodoc-typehints", "tox", "twine", "wheel"] -[[package]] -name = "pygal" -version = "3.1.0" -description = "A Python svg graph plotting library" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pygal-3.1.0-py3-none-any.whl", hash = "sha256:4e923490f3490c90c481f4535fa3adcda20ff374257ab9d8ae897f91b632c0bb"}, - {file = "pygal-3.1.0.tar.gz", hash = "sha256:fbdee7351a7423e7907fb8a9c3b77305f6b5678cb2e6fd0db36a8825e42955ec"}, -] - -[package.dependencies] -importlib-metadata = "*" - -[package.extras] -docs = ["pygal-sphinx-directives", "sphinx", "sphinx-rtd-theme"] -lxml = ["lxml"] -moulinrouge = ["flask", "pygal-maps-ch", "pygal-maps-fr", "pygal-maps-world"] -png = ["cairosvg"] -test = ["cairosvg", "coveralls", "lxml", "pyquery", "pytest", "pytest-cov", "ruff (>=0.5.6)"] - [[package]] name = "pygments" version = "2.19.2" @@ -4026,4 +3982,4 @@ all = ["Flask", "better-exceptions", "datadog-api-client", "grafana-api", "kafka [metadata] lock-version = "2.0" python-versions = ">=3.10, <3.12" -content-hash = "c5d7cdbbba8b3be724a8ffe61ea0fc095df31655b409f77693144aeb79bb9842" +content-hash = "2c929f840307368d61129ff121ad4414cbfc5f2e6312421bd147419e4aed2bb1" diff --git a/pyproject.toml b/pyproject.toml index ac38ca29a..0be4fad5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,10 @@ datadog-api-client = { version = "^1.2.0", optional = true } dpath = "^2.0.5" websocket-client = "1.3.3" prometheus-client = "^0.12.0" -pygal = "^3.0.0" +# Chart renderer for Prometheus/resource graphs (PSF-based permissive license; +# replaced LGPL-licensed pygal). Already present transitively via +# prometheus-api-client, so this adds no new packages to the image. +matplotlib = "^3.10.8" pyyaml = "^6.0" pytz = "^2021.3" poetry-core = "1.1.0a7" diff --git a/src/robusta/api/__init__.py b/src/robusta/api/__init__.py index 99bfe2b75..d3228378d 100644 --- a/src/robusta/api/__init__.py +++ b/src/robusta/api/__init__.py @@ -191,6 +191,7 @@ ScanType, SlackAnnotations, ) +from robusta.core.reporting.charts import BarChart, ChartStyle, TreemapChart, XYChart from robusta.core.reporting.custom_rendering import RendererType, charts_style, render_value from robusta.core.reporting.finding_subjects import KubeObjFindingSubject, PodFindingSubject from robusta.core.reporting.holmes import HolmesRequest, HolmesResult, HolmesResultsBlock diff --git a/src/robusta/core/playbooks/prometheus_enrichment_utils.py b/src/robusta/core/playbooks/prometheus_enrichment_utils.py index f619983e5..e7b1f3f51 100644 --- a/src/robusta/core/playbooks/prometheus_enrichment_utils.py +++ b/src/robusta/core/playbooks/prometheus_enrichment_utils.py @@ -6,7 +6,6 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union import humanize -import pygal from hikaru.model.rel_1_26 import Node from prometrix import PrometheusQueryResult from pydantic import BaseModel @@ -19,7 +18,8 @@ ) from robusta.core.model.env_vars import FLOAT_PRECISION_LIMIT, PROMETHEUS_REQUEST_TIMEOUT_SECONDS from robusta.core.reporting.blocks import GraphBlock, PrometheusBlock, PrometheusBlockLineData -from robusta.core.reporting.custom_rendering import PlotCustomCSS, charts_style +from robusta.core.reporting.charts import XYChart +from robusta.core.reporting.custom_rendering import charts_style from robusta.integrations.prometheus.utils import get_prometheus_connect ResourceKey = Tuple[ResourceChartResourceType, ResourceChartItemType] @@ -162,7 +162,7 @@ def create_chart_from_prometheus_query( filter_prom_jobs: bool = False, hide_legends: Optional[bool] = False, metrics_legends_labels: Optional[List[str]] = None, -) -> Tuple[pygal.Graph, PrometheusBlock]: +) -> Tuple[XYChart, PrometheusBlock]: starts_at: datetime ends_at: datetime if not alert_starts_at: @@ -197,7 +197,8 @@ def create_chart_from_prometheus_query( f"Unsupported query result for robusta chart, Type received: {prometheus_query_result.result_type}, type supported 'matrix'" ) - # fix a pygal bug which causes infinite loops due to rounding errors with floating points + # sentinels for the min/max scan below; HIGHEST_END is a timestamp far past + # any real sample, so the first data point always replaces it # TODO: change min_time time before Jan 19 3001 HIGHEST_END = 32536799999 LOWEST_START = 0 @@ -298,15 +299,9 @@ def create_chart_from_prometheus_query( graph_plot_color_list = [plot_data.color for plot_data in plot_data_list] graph_plot_color_list.extend(["#1e0047", "#2a0065"]) - config = pygal.Config() - custom_css = PlotCustomCSS().get_css_file_path() - config.css.append(f"file://{custom_css}") - chart = pygal.XY( - config, - show_dots=True, + chart = XYChart( style=charts_style(graph_colors=tuple(graph_plot_color_list)), truncate_legend=15, - include_x_axis=include_x_axis, width=1280, height=500, show_legend=hide_legends is not True, @@ -328,7 +323,7 @@ def create_chart_from_prometheus_query( # Fix for the case when the request and limit has the same value. # 6 pixels where chosen as minimum distance based on current width and height of the slack graph - delta = (chart.range[1] - chart.range[0]) * 6 / chart.config.height + delta = (chart.range[1] - chart.range[0]) * 6 / chart.height # Limit delta to a maximum of 2% of the Y-axis range (to prefent significant deviation) delta = min(delta, (chart.range[1] - chart.range[0]) * 0.02) @@ -405,7 +400,7 @@ def build_chart_from_prometheus_result( prometheus_query_result: PrometheusQueryResult, chart_title: Optional[str] = "Prometheus Chart", values_format: Optional[ChartValuesFormat] = None, -) -> pygal.Graph: +) -> XYChart: if prometheus_query_result.result_type != "matrix": raise ValueError(f"Expected 'matrix' result_type, got '{prometheus_query_result.result_type}'") @@ -475,19 +470,12 @@ def build_chart_from_prometheus_result( if min_time == HIGHEST_END: raise ValueError("No valid data points found in time series.") - config = pygal.Config() - custom_css = PlotCustomCSS().get_css_file_path() - config.css.append(f"file://{custom_css}") - graph_colors = [plot_data.color for plot_data in plot_data_list] graph_colors.extend(["#1e0047", "#2a0065"]) - chart = pygal.XY( - config, - show_dots=True, + chart = XYChart( style=charts_style(graph_colors=tuple(graph_colors)), truncate_legend=15, - include_x_axis=True, width=1280, height=500, show_legend=True, diff --git a/src/robusta/core/reporting/charts.py b/src/robusta/core/reporting/charts.py new file mode 100644 index 000000000..44e6401c5 --- /dev/null +++ b/src/robusta/core/reporting/charts.py @@ -0,0 +1,545 @@ +"""SVG chart rendering for Robusta enrichments. + +Charts are produced as SVG bytes and handed to sinks as FileBlock/GraphBlock +contents; chat sinks that cannot display SVG rasterize them via +``robusta.core.reporting.utils.convert_svg_to_png``. + +The classes here expose a small pygal-shaped surface (``add()`` + ``render()`` +plus a handful of assignable attributes) because that is the shape the chart +builders and playbooks were written against. + +Implementation notes: + +- matplotlib is driven through the object-oriented ``Figure``/``FigureCanvasSVG`` + API rather than ``pyplot``. There is no global figure registry to leak and no + backend to select, which matters in a long-lived server process. +- Text is rendered as paths (matplotlib's default ``svg.fonttype``), so the + rasterizer does not need any font installed in the container to produce + correct output. +- The SVG's ``width``/``height`` are rewritten to exact pixel values, because + matplotlib emits points and the sinks rely on a known pixel size. +""" +import io +import re +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional, Sequence, Tuple + +import matplotlib +from matplotlib.backends.backend_svg import FigureCanvasSVG +from matplotlib.figure import Figure +from matplotlib.patches import Rectangle + +DEFAULT_GRAPH_COLORS = ("#9747FF", "#FF5959", "#0DC291", "#2a0065", "#1e0047") + +# matplotlib works in inches; every size in this module is expressed in pixels +# and converted with this constant so the call sites can keep using pixels. +_DPI = 100 + +# matplotlib salts generated SVG element ids with a fresh uuid4 per render unless +# told otherwise, which would make two renders of the same chart differ. A fixed +# salt keeps output byte-identical for identical input. +_SVG_HASH_SALT = "robusta" + + +@dataclass +class ChartStyle: + """Visual configuration for a chart. + + Field names mirror the pygal ``Style`` this replaced, so playbooks that + build a style and pass it through keep working unchanged. + """ + + background: str = "#FFFFFF" + plot_background: str = "#FFFFFF" + foreground: str = "#607D8B" + foreground_strong: str = "#607D8B" + foreground_subtle: str = "#607D8B" + guide_stroke_color: str = "#E7EBEB" + major_guide_stroke_color: str = "#E7EBEB" + title_color: str = "#11383A" + label_color: str = "#3f3f3f" + opacity: float = 0.9 + colors: Tuple[str, ...] = DEFAULT_GRAPH_COLORS + + def color_at(self, index: int) -> str: + if not self.colors: + return DEFAULT_GRAPH_COLORS[index % len(DEFAULT_GRAPH_COLORS)] + return self.colors[index % len(self.colors)] + + +@dataclass +class _Series: + label: str + values: Sequence + stroke_style: Optional[Dict] = None + show_dots: bool = True + dots_size: Optional[float] = None + stroke: bool = True + color: Optional[str] = None + extra: Dict = field(default_factory=dict) + + +def _truncate(label: str, limit: Optional[int]) -> str: + """Match pygal's legend truncation: cut to ``limit`` chars including the ellipsis.""" + label = (label or "").replace("\n", " ").strip() + if not limit or limit <= 0 or len(label) <= limit: + return label + return label[: max(limit - 1, 0)] + "…" + + +def _dashes_from_stroke_style(stroke_style: Optional[Dict]) -> Optional[Tuple[float, ...]]: + """Translate an SVG ``stroke-dasharray`` into a matplotlib dash tuple.""" + if not stroke_style: + return None + dasharray = stroke_style.get("dasharray") + if not dasharray: + return None + parts = [p.strip() for p in str(dasharray).replace(",", " ").split() if p.strip()] + try: + nums = [float(p) for p in parts] + except ValueError: + return None + if not nums: + return None + # a single value means "on and off for the same length" + return tuple(nums) if len(nums) > 1 else (nums[0], nums[0]) + + +def _linewidth_from_stroke_style(stroke_style: Optional[Dict], default: float = 1.6) -> float: + """pygal stroke widths are on a much coarser scale than matplotlib points.""" + if not stroke_style: + return default + width = stroke_style.get("width") + if width is None: + return default + try: + return max(float(width) / 5.0, 0.8) + except (TypeError, ValueError): + return default + + +def _force_svg_pixel_size(svg: bytes, width: int, height: int) -> bytes: + """Pin the root ```` to an exact pixel size. + + matplotlib emits ``width="921.6pt"``, which rasterizers scale by their own + pt->px ratio. Sinks (and their tests) expect an exact pixel size, so the + attributes are rewritten while the viewBox is left alone. + """ + head = svg[:1024] + head = re.sub(rb'\bwidth="[^"]*"', b'width="%d"' % width, head, count=1) + head = re.sub(rb'\bheight="[^"]*"', b'height="%d"' % height, head, count=1) + return head + svg[1024:] + + +class _BaseChart: + """Common figure setup, title handling and SVG serialization.""" + + def __init__(self, style: Optional[ChartStyle] = None, width: int = 800, height: int = 600): + self.style = style or ChartStyle() + self.width = width + self.height = height + self.title: Optional[str] = None + self._series: List[_Series] = [] + + def add(self, label: str, values, **kwargs) -> None: + self._series.append( + _Series( + label=label, + values=values, + stroke_style=kwargs.pop("stroke_style", None), + show_dots=kwargs.pop("show_dots", True), + dots_size=kwargs.pop("dots_size", None), + stroke=kwargs.pop("stroke", True), + color=kwargs.pop("color", None), + extra=kwargs, + ) + ) + + def _new_figure(self) -> Tuple[Figure, "object"]: + fig = Figure(figsize=(self.width / _DPI, self.height / _DPI), dpi=_DPI) + fig.patch.set_facecolor(self.style.background) + ax = fig.add_subplot(111) + ax.set_facecolor(self.style.plot_background) + return fig, ax + + def _apply_title(self, ax) -> None: + if self.title: + ax.set_title(self.title, color=self.style.title_color, fontsize=11, pad=12) + + def _draw(self, fig, ax) -> None: # pragma: no cover - overridden + raise NotImplementedError + + def render(self) -> bytes: + """Render the chart and return SVG bytes.""" + fig, ax = self._new_figure() + try: + self._draw(fig, ax) + buf = io.BytesIO() + FigureCanvasSVG(fig) # attaching the canvas is what makes savefig emit SVG + with matplotlib.rc_context({"svg.hashsalt": _SVG_HASH_SALT}): + fig.savefig( + buf, + format="svg", + facecolor=fig.get_facecolor(), + # drop the embedded creation date so identical input renders + # to identical bytes + metadata={"Date": None}, + ) + return _force_svg_pixel_size(buf.getvalue(), self.width, self.height) + finally: + fig.clear() + + +class XYChart(_BaseChart): + """Line chart over ``(x, y)`` pairs - the Prometheus time-series graph.""" + + def __init__( + self, + style: Optional[ChartStyle] = None, + width: int = 1280, + height: int = 500, + show_legend: bool = True, + truncate_legend: Optional[int] = 15, + legend_at_bottom_columns: int = 5, + x_label_rotation: int = 35, + ): + super().__init__(style=style, width=width, height=height) + self.show_legend = show_legend + self.truncate_legend = truncate_legend + self.legend_at_bottom_columns = legend_at_bottom_columns + self.x_label_rotation = x_label_rotation + self.range: Optional[Tuple[float, float]] = None + self.y_labels: Optional[List[float]] = None + self.x_labels: Optional[List[float]] = None + self.value_formatter: Callable = str + self.x_value_formatter: Callable = str + self.show_minor_y_labels: bool = True + + def _x_tick_positions(self, x_min: float, x_max: float) -> List[float]: + if self.x_labels: + return list(self.x_labels) + # pygal spaces roughly a label per ~140px of plot width + count = max(int(self.width / 140), 2) + if x_max <= x_min: + return [x_min] + step = (x_max - x_min) / (count - 1) + return [x_min + i * step for i in range(count)] + + def _draw(self, fig, ax) -> None: + style = self.style + + x_values = [x for s in self._series for (x, _) in s.values] + x_min, x_max = (min(x_values), max(x_values)) if x_values else (0, 1) + + for index, series in enumerate(self._series): + color = series.color or style.color_at(index) + xs = [point[0] for point in series.values] + ys = [point[1] for point in series.values] + line_kwargs = { + "color": color, + "alpha": style.opacity, + "linewidth": _linewidth_from_stroke_style(series.stroke_style), + "linestyle": "-" if series.stroke else "none", + "marker": "o" if series.show_dots else "None", + "markersize": (series.dots_size or 2.5) if series.show_dots else 0, + "markerfacecolor": color, + "markeredgecolor": color, + "label": _truncate(series.label, self.truncate_legend), + "solid_capstyle": "round", + } + # matplotlib rejects an explicit dashes=None, so only set it when dashed + # a lone sample has no segment to draw, so it would be invisible with + # markers off - the alert pipeline disables them for every series + if len(series.values) == 1 and not series.show_dots: + line_kwargs["marker"] = "o" + line_kwargs["markersize"] = series.dots_size or 3.5 + + dashes = _dashes_from_stroke_style(series.stroke_style) + if dashes: + line_kwargs["dashes"] = dashes + ax.plot(xs, ys, **line_kwargs) + + if self.range: + y_min, y_max = self.range[0], self.range[1] + if y_max <= y_min: + # every sample shared one value - a metric flat at zero is the + # common case - and the callers derive the range from the maximum, + # so it collapses. Give the axis height rather than let matplotlib + # warn about a singular transform and pick its own bounds. + y_max = y_min + 1.0 + ax.set_ylim(y_min, y_max) + if x_max > x_min: + ax.set_xlim(x_min, x_max) + else: + # a query that returned a single sample gives a zero-width range; + # widen it so matplotlib does not warn about a singular transform + ax.set_xlim(x_min - 1, x_max + 1) + + if self.y_labels is not None: + # the same collapse makes every computed tick identical; one is enough + y_labels = list(self.y_labels) + if len(y_labels) > 1 and len(set(y_labels)) == 1: + y_labels = y_labels[:1] + ax.set_yticks(y_labels) + ax.set_yticklabels([self.value_formatter(v) for v in y_labels]) + if not self.show_minor_y_labels: + ax.set_yticks(list(self.y_labels or [])) + + if x_values: + ticks = self._x_tick_positions(x_min, x_max) + ax.set_xticks(ticks) + ax.set_xticklabels( + [self.x_value_formatter(t) for t in ticks], + rotation=self.x_label_rotation, + ha="right", + rotation_mode="anchor", + ) + else: + # no series at all: the placeholder x-range would otherwise label + # every tick with the epoch + ax.set_xticks([]) + + ax.grid(True, which="major", color=style.guide_stroke_color, linewidth=1, zorder=0) + ax.set_axisbelow(True) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + for side in ("left", "bottom"): + ax.spines[side].set_color(style.foreground) + ax.spines[side].set_linewidth(0.8) + ax.tick_params(colors=style.label_color, labelsize=8, length=0) + + self._apply_title(ax) + fig.subplots_adjust(left=0.075, right=0.985, top=0.9, bottom=0.26) + + if self.show_legend and self._series: + # anchored to the figure, not the axes, so the legend sits flush + # against the bottom edge instead of leaving a dead band below it + legend = fig.legend( + loc="lower center", + bbox_to_anchor=(0.5, 0.005), + ncol=max(self.legend_at_bottom_columns, 1), + frameon=False, + fontsize=8, + handlelength=1.4, + handleheight=0.9, + handletextpad=0.5, + columnspacing=1.6, + borderpad=0, + ) + for text in legend.get_texts(): + text.set_color(style.label_color) + + +class BarChart(_BaseChart): + """Grouped vertical bars over categorical x labels.""" + + def __init__( + self, + style: Optional[ChartStyle] = None, + width: int = 800, + height: int = 600, + x_label_rotation: int = -40, + truncate_legend: Optional[int] = None, + ): + super().__init__(style=style, width=width, height=height) + self.x_label_rotation = x_label_rotation + self.truncate_legend = truncate_legend + self.x_labels: List[str] = [] + self.value_formatter: Callable = str + + def _draw(self, fig, ax) -> None: + style = self.style + categories = list(self.x_labels or []) + group_count = max(len(self._series), 1) + bar_width = 0.8 / group_count + positions = range(len(categories)) + + for index, series in enumerate(self._series): + color = series.color or style.color_at(index) + offset = (index - (group_count - 1) / 2) * bar_width + # negative sentinels mean "no data" - draw nothing for those slots + heights = [v if v is not None and v >= 0 else 0 for v in series.values] + ax.bar( + [p + offset for p in positions], + heights, + width=bar_width, + color=color, + alpha=style.opacity, + label=_truncate(series.label, self.truncate_legend), + zorder=3, + ) + + ax.set_xticks(list(positions)) + ax.set_xticklabels( + [_truncate(c, 22) for c in categories], + rotation=self.x_label_rotation, + ha="left" if self.x_label_rotation < 0 else "right", + rotation_mode="anchor", + ) + ax.yaxis.set_major_formatter(lambda value, _pos: self.value_formatter(value)) + ax.grid(True, axis="y", color=style.guide_stroke_color, linewidth=1, zorder=0) + ax.set_axisbelow(True) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + for side in ("left", "bottom"): + ax.spines[side].set_color(style.foreground) + ax.spines[side].set_linewidth(0.8) + ax.tick_params(colors=style.label_color, labelsize=8, length=0) + + self._apply_title(ax) + fig.subplots_adjust(left=0.11, right=0.97, top=0.9, bottom=0.24) + + if self._series: + legend = fig.legend( + loc="lower center", + bbox_to_anchor=(0.5, 0.005), + ncol=min(len(self._series), 4), + frameon=False, + fontsize=8, + handlelength=0.9, + handleheight=0.9, + handletextpad=0.5, + borderpad=0, + ) + for text in legend.get_texts(): + text.set_color(style.label_color) + + +def _worst_ratio(row: List[float], length: float) -> float: + """Worst aspect ratio among the rectangles in ``row`` laid out along ``length``. + + This is the squarify heuristic: keep adding rectangles to a row while this + value improves, and break to a new row once it gets worse. Lower is squarer. + """ + total = sum(row) + if total <= 0 or length <= 0: + return float("inf") + largest, smallest = max(row), min(row) + if smallest <= 0: + return float("inf") + return max((length**2) * largest / (total**2), (total**2) / ((length**2) * smallest)) + + +def _layout_row(row: List[float], x: float, y: float, dx: float, dy: float): + """Place one row of areas along the shorter side; return rects + leftover space.""" + covered = sum(row) + rects = [] + if dx >= dy: + width = covered / dy if dy else 0 + offset = y + for area in row: + height = area / width if width else 0 + rects.append((x, offset, width, height)) + offset += height + return rects, (x + width, y, dx - width, dy) + height = covered / dx if dx else 0 + offset = x + for area in row: + width = area / height if height else 0 + rects.append((offset, y, width, height)) + offset += width + return rects, (x, y + height, dx, dy - height) + + +def squarify(areas: List[float], x: float, y: float, dx: float, dy: float): + """Squarified treemap layout. ``areas`` must be sorted descending and already + normalized so that their sum equals ``dx * dy``. Rects come back in input order.""" + remaining = list(areas) + rects: List[Tuple[float, float, float, float]] = [] + row: List[float] = [] + while remaining: + length = min(dx, dy) + if not row: + row.append(remaining.pop(0)) + continue + if _worst_ratio(row, length) >= _worst_ratio(row + [remaining[0]], length): + row.append(remaining.pop(0)) + else: + placed, (x, y, dx, dy) = _layout_row(row, x, y, dx, dy) + rects.extend(placed) + row = [] + if row: + placed, _ = _layout_row(row, x, y, dx, dy) + rects.extend(placed) + return rects + + +class TreemapChart(_BaseChart): + """Treemap using the squarified layout, one rectangle per added series.""" + + def __init__(self, style: Optional[ChartStyle] = None, width: int = 800, height: int = 600): + super().__init__(style=style, width=width, height=height) + self.value_formatter: Callable = str + + def _draw(self, fig, ax) -> None: + style = self.style + entries = [] + for index, series in enumerate(self._series): + values = series.values if isinstance(series.values, (list, tuple)) else [series.values] + total = sum(v for v in values if isinstance(v, (int, float)) and v > 0) + if total > 0: + entries.append((total, series.label, series.color or style.color_at(index))) + + ax.set_axis_off() + self._apply_title(ax) + if not entries: + return + + # squarify needs descending areas; keep the original colour/label pairing + entries.sort(key=lambda e: e[0], reverse=True) + grand_total = sum(e[0] for e in entries) + canvas_w, canvas_h = 100.0, 100.0 + normalized = [e[0] * canvas_w * canvas_h / grand_total for e in entries] + rects = squarify(normalized, 0.0, 0.0, canvas_w, canvas_h) + + for (rx, ry, rw, rh), (value, label, color) in zip(rects, entries): + ax.add_patch( + Rectangle( + (rx, ry), + rw, + rh, + facecolor=color, + alpha=style.opacity, + edgecolor=style.background, + linewidth=1.5, + ) + ) + # label only where the text actually fits; the canvas is 100 units + # wide, and at 7pt roughly 0.78 units are needed per character + if rw > 10 and rh > 6: + max_chars = int(rw / 0.78) + ax.text( + rx + rw / 2, + ry + rh / 2, + f"{_truncate(label, max_chars)}\n{self.value_formatter(value)}", + ha="center", + va="center", + fontsize=7, + color="#FFFFFF", + linespacing=1.4, + ) + + ax.set_xlim(0, canvas_w) + ax.set_ylim(0, canvas_h) + ax.invert_yaxis() # first (largest) entry lands top-left, as pygal did + + legend_handles = [ + Rectangle((0, 0), 1, 1, facecolor=color, alpha=style.opacity) for _, _, color in entries + ] + legend = ax.legend( + legend_handles, + [_truncate(label, 24) for _, label, _ in entries], + loc="upper center", + bbox_to_anchor=(0.5, -0.02), + ncol=min(len(entries), 4), + frameon=False, + fontsize=8, + handlelength=0.9, + handleheight=0.9, + handletextpad=0.5, + borderpad=0, + ) + for text in legend.get_texts(): + text.set_color(style.label_color) + + fig.subplots_adjust(left=0.03, right=0.97, top=0.91, bottom=0.14) diff --git a/src/robusta/core/reporting/custom_rendering.py b/src/robusta/core/reporting/custom_rendering.py index 3dfddf43c..331b999bf 100644 --- a/src/robusta/core/reporting/custom_rendering.py +++ b/src/robusta/core/reporting/custom_rendering.py @@ -1,9 +1,8 @@ -import logging from datetime import datetime -from typing import Tuple, Optional -import tempfile +from typing import Tuple from robusta.core.model.env_vars import DEFAULT_TIMEZONE +from robusta.core.reporting.charts import DEFAULT_GRAPH_COLORS, ChartStyle class RendererType: @@ -18,59 +17,8 @@ def render_value(renderer: RendererType, value): def charts_style( - graph_colors: Tuple = ("#9747FF", "#FF5959", "#0DC291", "#2a0065", "#1e0047"), -): - from pygal.style import Style - - return Style( - background="#FFFFFF", - plot_background="#FFFFFF", - value_background="rgba(229, 229, 229, 1)", - foreground="#607D8B", - foreground_strong="#607D8B", - foreground_subtle="#607D8B", - guide_stroke_dasharray="0,0", - major_guide_stroke_dasharray="0,0", - guide_stroke_color="#E7EBEB", - major_guide_stroke_color="#E7EBEB", - opacity=".6", - opacity_hover=".9", - transition="400ms ease-in", - colors=graph_colors, - ) - - -class PlotCustomCSS: - _css_file_path = None - - def __init__(self): - if PlotCustomCSS._css_file_path is None: - try: - custom_css = ''' - {{ id }}.title { - fill: #11383A; - } - - {{ id }}.legends .legend text { - fill: #3f3f3f; - } - - {{ id }}.axis.y text { - fill: #3f3f3f; - } - - {{ id }}.axis.x text { - fill: #3f3f3f; - } - ''' - - with tempfile.NamedTemporaryFile(delete=False, suffix='.css') as f: - f.write(custom_css.encode('utf-8')) - f.flush() - PlotCustomCSS._css_file_path = f.name - except Exception as e: - logging.error(f"Error during initializing PlotCustomCSS: {e}", exc_info=True) - PlotCustomCSS._css_file_path = None - - def get_css_file_path(self) -> Optional[str]: - return self._css_file_path + graph_colors: Tuple = DEFAULT_GRAPH_COLORS, +) -> ChartStyle: + """Robusta's chart palette. Kept as a function (and exported through + ``robusta.api``) so playbooks can build a styled chart in one call.""" + return ChartStyle(colors=tuple(graph_colors)) diff --git a/tests/test_ai_integration.py b/tests/test_ai_integration.py index 86b056b4c..23b8bb623 100644 --- a/tests/test_ai_integration.py +++ b/tests/test_ai_integration.py @@ -17,7 +17,7 @@ def assert_valid_png(png_bytes: bytes): image = Image.open(BytesIO(png_bytes)) image.load() # force full decoding, not just the header assert image.format == "PNG" - # graph-tool charts are rendered at the fixed pygal size used by the pipeline + # graph-tool charts are rendered at the fixed chart size used by the pipeline assert image.size == (1280, 500) diff --git a/tests/test_charts.py b/tests/test_charts.py new file mode 100644 index 000000000..af157502d --- /dev/null +++ b/tests/test_charts.py @@ -0,0 +1,328 @@ +""" +Tests for the matplotlib-backed chart renderers that produce Robusta's +Prometheus/resource graphs. + +These cover the contract the rest of the pipeline depends on: SVG bytes out, at +an exact pixel size, rasterizable by resvg, with Robusta's styling applied. +""" +import warnings +from io import BytesIO + +import pytest +from PIL import Image + +from robusta.core.reporting.charts import BarChart, ChartStyle, TreemapChart, XYChart, _truncate, squarify +from robusta.core.reporting.custom_rendering import charts_style +from robusta.core.reporting.utils import convert_svg_to_png + +SVG_ROOT = b" XYChart: + chart = XYChart(**kwargs) + chart.title = "test chart" + chart.add("series-a", [(0, 1.0), (1, 2.0), (2, 3.0)]) + return chart + + +# --- XY chart ------------------------------------------------------------------- + + +def test_xy_chart_renders_svg(): + svg = line_chart().render() + + assert SVG_ROOT in svg[:512] + assert svg.rstrip().endswith(b"") + + +@pytest.mark.parametrize("width,height", [(1280, 500), (400, 300), (800, 600)]) +def test_xy_chart_rasterizes_at_exact_requested_size(width, height): + """matplotlib emits points, not pixels; the renderer pins the root to an + exact pixel size so sinks get a predictable raster.""" + svg = line_chart(width=width, height=height).render() + + assert png_size(svg) == (width, height) + + +def test_render_is_deterministic(): + """Identical input must produce identical bytes - no embedded timestamp.""" + assert line_chart().render() == line_chart().render() + + +def test_value_formatter_applied_to_y_axis(): + """Y-axis ticks are labelled through value_formatter (Bytes, CPUUsage, ...). + + Glyphs are emitted as paths, so the formatter itself is spied on rather than + the rendered text. + """ + seen = [] + chart = line_chart() + chart.range = (0, 100) + chart.y_labels = [0, 50, 100] + chart.value_formatter = lambda v: seen.append(v) or f"{v} pct" + + chart.render() + + assert seen == [0, 50, 100] + + +def test_hidden_legend_is_not_rendered(): + with_legend = line_chart(show_legend=True).render() + without_legend = line_chart(show_legend=False).render() + + assert len(without_legend) < len(with_legend) + + +def test_dashed_series_renders(): + """stroke_style dasharray must survive translation into matplotlib dashes.""" + chart = XYChart() + chart.add( + "dashed", + [(0, 1.0), (1, 2.0)], + stroke_style={"width": 8, "dasharray": "8", "linecap": "round"}, + show_dots=False, + ) + + assert png_size(chart.render()) == (1280, 500) + + +def test_empty_chart_still_renders(): + """A query returning no series must not blow up the enrichment.""" + chart = XYChart() + chart.title = "no data" + chart.y_labels = [] + chart.show_minor_y_labels = False + + assert png_size(chart.render()) == (1280, 500) + + +def test_empty_chart_draws_no_x_ticks(): + """With no series there is no real x-range, so the placeholder range must not + get labelled - it would render every tick as the epoch.""" + formatted = [] + chart = XYChart() + chart.x_value_formatter = lambda t: formatted.append(t) or "tick" + + chart.render() + + assert formatted == [] + + +def test_single_point_is_visible_even_with_dots_disabled(): + """The alert pipeline builds every series with show_dots=False. One sample has + no segment to draw, so without a marker the chart would come out blank.""" + chart = XYChart() + chart.add("solo", [(1755300000, 42.0)], show_dots=False) + + png = convert_svg_to_png(chart.render()) + image = Image.open(BytesIO(png)).convert("RGB") + colors = {c for _, c in (image.getcolors(maxcolors=1_000_000) or [])} + # the default first palette colour is #9747FF - some trace of it must survive + assert any(r > 100 and b > 180 and g < 120 for r, g, b in colors), "the lone sample was not drawn" + + +def test_all_zero_series_does_not_warn_about_singular_axis(): + """A metric flat at zero collapses the callers' derived range to (0, 0), which + is still truthy - the axis must be given height rather than handed to + matplotlib as a singular transform.""" + chart = XYChart() + chart.add("quiet", [(0, 0.0), (1, 0.0), (2, 0.0)]) + chart.range = (0, 0.0) + chart.y_labels = [0, 0, 0, 0, 0] + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + svg = chart.render() + + assert png_size(svg) == (1280, 500) + + +def test_collapsed_range_collapses_duplicate_y_labels(): + """The same collapse makes every derived tick identical; five stacked zeros + would be drawn on top of each other.""" + seen = [] + chart = XYChart() + chart.add("quiet", [(0, 0.0), (1, 0.0)]) + chart.range = (0, 0.0) + chart.y_labels = [0, 0, 0, 0, 0] + chart.value_formatter = lambda v: seen.append(v) or str(v) + + chart.render() + + assert seen == [0], "duplicate ticks should collapse to a single label" + + +def test_distinct_y_labels_are_left_alone(): + seen = [] + chart = line_chart() + chart.range = (0, 100) + chart.y_labels = [0, 25, 50, 75, 100] + chart.value_formatter = lambda v: seen.append(v) or str(v) + + chart.render() + + assert seen == [0, 25, 50, 75, 100] + + +def test_single_point_does_not_warn_about_singular_axis(): + """One sample gives a zero-width x-range; the limits must be widened rather + than left for matplotlib to complain about.""" + chart = XYChart() + chart.add("solo", [(1755300000, 42.0)]) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + svg = chart.render() + + assert png_size(svg) == (1280, 500) + + +# --- label truncation ----------------------------------------------------------- + + +def test_truncate_matches_pygal_semantics(): + # limit is inclusive of the ellipsis, as pygal's truncate_legend was + assert _truncate("checkout-api-7d9f8b6c4-hk2xl", 15) == "checkout-api-7…" + assert len(_truncate("checkout-api-7d9f8b6c4-hk2xl", 15)) == 15 + + +def test_truncate_leaves_short_labels_alone(): + assert _truncate("short", 15) == "short" + + +def test_truncate_flattens_multiline_labels(): + """Series labels are built by joining metric values with newlines.""" + assert _truncate("pod-a\nprod", None) == "pod-a prod" + + +# --- bar chart ------------------------------------------------------------------ + + +def test_bar_chart_renders_with_missing_values(): + """node_cpu_analysis passes a negative sentinel for pods with no request.""" + chart = BarChart(style=charts_style()) + chart.title = "actual vs requested" + chart.x_labels = ["pod-a", "pod-b", "pod-c"] + chart.value_formatter = lambda v: f"{v:.2f} vCPU" + chart.add("Actual CPU Usage", [1.4, 0.2, 0.1]) + chart.add("CPU Request", [1.0, 0.1, -0.001]) + + assert png_size(chart.render()) == (800, 600) + + +# --- treemap -------------------------------------------------------------------- + + +def test_treemap_renders(): + chart = TreemapChart(style=charts_style()) + chart.title = "cpu by pod" + chart.value_formatter = lambda x: f"{int(x * 100)}%" + for index, value in enumerate([0.4, 0.25, 0.15, 0.1, 0.1]): + chart.add(f"pod-{index}", [value]) + + assert png_size(chart.render()) == (800, 600) + + +def test_treemap_ignores_non_positive_values(): + """Free-CPU style entries can legitimately compute to zero or below.""" + chart = TreemapChart() + chart.add("real", [0.5]) + chart.add("zero", [0]) + chart.add("negative", [-0.2]) + + assert png_size(chart.render()) == (800, 600) + + +def test_treemap_with_no_positive_values_renders_empty(): + chart = TreemapChart() + chart.title = "nothing to show" + chart.add("zero", [0]) + + assert png_size(chart.render()) == (800, 600) + + +# --- squarify layout ------------------------------------------------------------ + +def squarified(values, width=100.0, height=100.0): + ordered = sorted(values, reverse=True) + total = sum(ordered) + normalized = [v * width * height / total for v in ordered] + return squarify(normalized, 0.0, 0.0, width, height), ordered + + +def test_squarify_produces_one_rect_per_value(): + rects, ordered = squarified([5, 3, 2, 1, 1]) + + assert len(rects) == len(ordered) + + +def test_squarify_areas_are_proportional_to_values(): + values = [5, 3, 2, 1, 1] + rects, ordered = squarified(values) + + total = sum(ordered) + for (_, _, w, h), value in zip(rects, ordered): + expected_share = value / total + assert w * h / (100.0 * 100.0) == pytest.approx(expected_share, rel=1e-6) + + +def test_squarify_rects_stay_inside_the_canvas(): + rects, _ = squarified([5, 3, 2, 1, 1]) + + for x, y, w, h in rects: + assert x >= -1e-9 and y >= -1e-9 + assert x + w <= 100.0 + 1e-6 + assert y + h <= 100.0 + 1e-6 + + +def test_squarify_rects_do_not_overlap(): + rects, _ = squarified([5, 3, 2, 1, 1, 4, 6]) + + for i, a in enumerate(rects): + for b in rects[i + 1:]: + separated = ( + a[0] + a[2] <= b[0] + 1e-6 + or b[0] + b[2] <= a[0] + 1e-6 + or a[1] + a[3] <= b[1] + 1e-6 + or b[1] + b[3] <= a[1] + 1e-6 + ) + assert separated, f"{a} overlaps {b}" + + +def test_squarify_tiles_the_whole_canvas(): + rects, _ = squarified([5, 3, 2, 1, 1]) + + assert sum(w * h for _, _, w, h in rects) == pytest.approx(100.0 * 100.0, rel=1e-6) + + +def test_squarify_handles_a_single_value(): + rects, _ = squarified([1]) + + assert rects == [(0.0, 0.0, 100.0, 100.0)] + + +# --- style ---------------------------------------------------------------------- + + +def test_charts_style_uses_supplied_colors(): + style = charts_style(graph_colors=("#111111", "#222222")) + + assert style.colors == ("#111111", "#222222") + assert style.color_at(0) == "#111111" + assert style.color_at(2) == "#111111", "palette must cycle" + + +def test_charts_style_has_a_default_palette(): + assert charts_style().colors[0] == "#9747FF" + + +def test_chart_style_falls_back_when_palette_is_empty(): + assert ChartStyle(colors=()).color_at(0) == "#9747FF" diff --git a/tests/test_svg_conversion.py b/tests/test_svg_conversion.py index b060c1aad..d51289f5a 100644 --- a/tests/test_svg_conversion.py +++ b/tests/test_svg_conversion.py @@ -1,8 +1,8 @@ """ Tests for the SVG -> raster-image conversion used by chat sinks. -Robusta renders Prometheus/resource graphs as pygal SVG charts and rasterizes them -before sending to chat platforms that cannot display SVG: +Robusta renders Prometheus/resource graphs as matplotlib SVG charts and rasterizes +them before sending to chat platforms that cannot display SVG: - convert_svg_to_png / add_pngs_for_all_svgs (Slack, Discord, Mattermost, RocketChat, Jira, Webex, Telegram, Pushover, Zulip, Yandex) - MsTeamsAdaptiveCardFilesImage (MS Teams needs base64 JPEG data-URLs) @@ -11,7 +11,6 @@ """ from io import BytesIO -import pygal import pytest from PIL import Image from prometrix import PrometheusQueryResult @@ -19,6 +18,7 @@ from robusta.core.model.base_params import ChartValuesFormat from robusta.core.playbooks.prometheus_enrichment_utils import build_chart_from_prometheus_result from robusta.core.reporting.blocks import FileBlock, MarkdownBlock +from robusta.core.reporting.charts import XYChart from robusta.core.reporting.utils import add_pngs_for_all_svgs, convert_svg_to_png from robusta.integrations.msteams.msteams_adaptive_card_files_image import MsTeamsAdaptiveCardFilesImage @@ -49,17 +49,17 @@ def make_prometheus_matrix(series_count: int = 2, points: int = 10) -> dict: @pytest.fixture -def simple_pygal_svg() -> bytes: - chart = pygal.Line(width=400, height=300) +def simple_chart_svg() -> bytes: + chart = XYChart(width=400, height=300) chart.title = "simple chart" - chart.add("series-a", [1, 2, 3, 2, 5]) - chart.add("series-b", [5, 4, 3, 4, 1]) + chart.add("series-a", [(0, 1), (1, 2), (2, 3), (3, 2), (4, 5)]) + chart.add("series-b", [(0, 5), (1, 4), (2, 3), (3, 4), (4, 1)]) return chart.render() @pytest.fixture def robusta_styled_chart_svg() -> bytes: - """An SVG built through the real chart pipeline, custom CSS injection included.""" + """An SVG built through the real chart pipeline, Robusta styling included.""" query_result = PrometheusQueryResult(data=make_prometheus_matrix()) chart = build_chart_from_prometheus_result( query_result, @@ -69,8 +69,8 @@ def robusta_styled_chart_svg() -> bytes: return chart.render() -def test_convert_simple_pygal_svg_to_png(simple_pygal_svg): - png = convert_svg_to_png(simple_pygal_svg) +def test_convert_simple_chart_svg_to_png(simple_chart_svg): + png = convert_svg_to_png(simple_chart_svg) assert png is not None assert png[:8] == PNG_MAGIC @@ -80,7 +80,7 @@ def test_convert_simple_pygal_svg_to_png(simple_pygal_svg): def test_convert_robusta_styled_chart_to_png(robusta_styled_chart_svg): - """The chart pipeline injects custom CSS into the pygal SVG; the rasterizer must + """The chart pipeline emits SVG with text as paths; the rasterizer must handle it. This is the closest unit-level reproduction of what every graph-bearing alert notification goes through on its way to a chat sink.""" png = convert_svg_to_png(robusta_styled_chart_svg) @@ -89,7 +89,7 @@ def test_convert_robusta_styled_chart_to_png(robusta_styled_chart_svg): assert png[:8] == PNG_MAGIC image = decode_image(png) assert image.format == "PNG" - # pygal charts in the alert pipeline are rendered at a fixed 1280x500 + # charts in the alert pipeline are rendered at a fixed 1280x500 assert image.size == (1280, 500) @@ -97,8 +97,8 @@ def test_invalid_svg_returns_none(): assert convert_svg_to_png(b"this is not svg at all") is None -def test_add_pngs_for_all_svgs_appends_png_twin(simple_pygal_svg): - svg_block = FileBlock("chart.svg", simple_pygal_svg) +def test_add_pngs_for_all_svgs_appends_png_twin(simple_chart_svg): + svg_block = FileBlock("chart.svg", simple_chart_svg) text_block = FileBlock("log.txt", b"some log") markdown_block = MarkdownBlock("not a file") original_blocks = [svg_block, text_block, markdown_block] @@ -144,8 +144,8 @@ def test_msteams_svg_becomes_jpeg_data_url(robusta_styled_chart_svg): assert image.size == (1280, 500) -def test_msteams_png_becomes_jpeg_data_url(simple_pygal_svg): - png_bytes = convert_svg_to_png(simple_pygal_svg) +def test_msteams_png_becomes_jpeg_data_url(simple_chart_svg): + png_bytes = convert_svg_to_png(simple_chart_svg) png_block = FileBlock("chart.png", png_bytes) image_set = MsTeamsAdaptiveCardFilesImage.create_files_for_presentation([png_block])