diff --git a/tools/analysis/ping-patterns/README.md b/tools/analysis/ping-patterns/README.md deleted file mode 100644 index ef21983059..0000000000 --- a/tools/analysis/ping-patterns/README.md +++ /dev/null @@ -1,71 +0,0 @@ -This directory contains a tool to analyze the patterns of baseline and metrics pings received on a per-client basis in Fenix. These pattern analyses are then summarized in a single plot showing various patterns and issues found in the data. - -## Requirements - -A recent version of [`matplotlib`](https://matplotlib.org). - -## Collecting input data - -The input data is created using the following Redash query: - -```sql -SELECT - client_info.client_id, - DATE(submission_timestamp) AS date, - 'metrics' as ping_type, - client_info.app_display_version AS app_version, - client_info.telemetry_sdk_build AS telemetry_sdk_build, - client_info.android_sdk_version AS sdk, - ping_info.start_time AS start_time, - ping_info.end_time AS end_time, - 0 AS duration, - ping_info.seq AS seq -FROM - org_mozilla_fenix_nightly.metrics -WHERE - DATE(submission_timestamp) > "2019-11-01" -UNION ALL -SELECT - client_info.client_id, - DATE(submission_timestamp) AS date, - 'baseline' as ping_type, - client_info.app_display_version AS app_version, - client_info.telemetry_sdk_build AS telemetry_sdk_build, - client_info.android_sdk_version AS sdk, - ping_info.start_time AS start_time, - ping_info.end_time AS end_time, - metrics.timespan.glean_baseline_duration.value AS duration, - ping_info.seq AS seq -FROM - org_mozilla_fenix_nightly.baseline -WHERE - DATE(submission_timestamp) > "2019-11-01" -``` - -This query is also available [here](https://sql.telemetry.mozilla.org/queries/66682). - -Save the result of the query as a `.csv` file to use as input to this script. - -## Configuring the script - -Configuration is performed by editing the `config.py` script. There are comments as to what the fields do there. - -## Running the script - -To run the script, pass the input dataset and the output directory on the commandline. - -```bash -$ ./ping-patterns.py dataset.csv plots -``` - -## Output - -The output directory will contain one `.svg` file per client with a timeline of the baseline and metrics pings received from that client. - -Baseline pings are in blue. The thick part of the line represents the active session as returned in the `baseline.duration` metric. - -Metrics pings are in red. Issues found with metrics pings are notated with a number to the right. Hover over the number to display a tooltip with further information about the issue. - -Gray vertical lines are midnight local time. Dashed gray vertical lines are 04:00 local time. - -Green vertical lines indicate the first ping coming from an interesting revision that contains a related fix. diff --git a/tools/analysis/ping-patterns/config.py b/tools/analysis/ping-patterns/config.py deleted file mode 100644 index e537ecbbcd..0000000000 --- a/tools/analysis/ping-patterns/config.py +++ /dev/null @@ -1,30 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -""" -Configuration constants for the analysis script. -""" - - -import datetime - - -FIRST_DATE = datetime.datetime.fromisoformat("2019-11-01") -""" -The earliest date to include in the analysis. This helps to remove clients with -wildly incorrect clocks. -""" - -FIXES = [ - ("Double scheduling of metrics ping", 191115), # Double-schedule of metrics ping - ("Proguard rule to retain lifetime API", 191116), # Proguard rule - ("Avoid reflection in the lifetime API", 191120), # Non-reflection-based API -] -""" -A list of Fenix revisions to highlight in the output. Each entry is a tuple -`(description, version)` where `version` is in Fenix nightly version format: -YYMMDD. -""" - -FIXES = sorted(FIXES, key=lambda x: x[1]) diff --git a/tools/analysis/ping-patterns/ping-patterns.py b/tools/analysis/ping-patterns/ping-patterns.py deleted file mode 100755 index 959e4a114b..0000000000 --- a/tools/analysis/ping-patterns/ping-patterns.py +++ /dev/null @@ -1,505 +0,0 @@ -#!/usr/bin/env python3 - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - - -import argparse -import csv -import datetime -import os -import sys -import xml.etree.ElementTree as ET - - -from matplotlib import pyplot as plt - - -from config import * - - -""" -Get the data from this query: -https://sql.telemetry.mozilla.org/queries/66682 -""" - -# TODO: This script is likely to be obsoleted by bug 1602824 - - -ROW_HEIGHT = 10 - - -ZERO_LENGTH = 1 -DUPLICATE_TIME = 2 -NO_BASELINE = 3 -MISSING_SEQ = 4 -DUPLICATE_SEQ = 5 -TOO_LATE = 6 -FAR_FROM_4AM = 7 -MAX_NOTES = 8 - -NOTE_SUMMARIES = { - ZERO_LENGTH: "metrics ping had start/end time of < 1 minute", - DUPLICATE_TIME: "2 or more metrics pings were collected within the same minute", - NO_BASELINE: "a metrics ping was collected with no baseline ping since the last metrics ping", - MISSING_SEQ: "the seq number is not contiguous with the previous ping", - DUPLICATE_SEQ: "the same seq number was used more than once", - TOO_LATE: "the metrics ping was collected more than 24 hours after the last baseline ping", - FAR_FROM_4AM: "the metrics ping was sent more than an hour from 4am local time", -} - - -def load_data(filename): - """ - Load the csv file and convert it to a list of dicts. - """ - print("Loading CSV") - data = [] - with open(filename) as fd: - reader = csv.reader(fd) - column_names = next(reader) - for row in reader: - data.append(dict((name, value) for (name, value) in zip(column_names, row))) - return data - - -def parse_version(build_id): - """ - Parse the "date-like" string out of the Fenix Nightly version convention. - - Returns `None` if no version found. - """ - if build_id.startswith('"'): - build_id = build_id[1:-1] - if build_id.startswith("Nightly"): - parts = build_id.split() - date = int(parts[1]) - return date - return None - - -def filter_data(data): - """ - Remove pings that are too old. - """ - now = datetime.datetime.now() + datetime.timedelta(days=1) - result = [ - x - for x in data - if ( - get_local_time(x["start_time"]) >= FIRST_DATE - and get_local_time(x["end_time"]) >= FIRST_DATE - and get_local_time(x["start_time"]) < now - and get_local_time(x["end_time"]) < now - ) - ] - - print(f"Removed {len(data)-len(result)}/{len(data)} pings with out-of-range dates") - return result - - -def annotate_data(data): - """ - Add some derived values to the data set. - """ - print("Annotating CSV") - for ping in data: - ping["start_time_tz"] = get_timezone(ping["start_time"]) - ping["end_time_tz"] = get_timezone(ping["end_time"]) - ping["start_time_local"] = get_local_time(ping["start_time"]) - ping["end_time_local"] = get_local_time(ping["end_time"]) - ping["start_time_hour"] = get_fractional_hour(ping["start_time_local"]) - ping["end_time_hour"] = get_fractional_hour(ping["end_time_local"]) - ping["version_date"] = parse_version(ping["app_version"]) - ping["notes"] = set() - - -def sort_data_by_client_id(data): - """ - Reorganize the data so it is grouped by client id. - """ - data_by_client_id = {} - for row in data: - client_id = row.get("client_id") - data_by_client_id.setdefault(client_id, []) - data_by_client_id[client_id].append(row) - return data_by_client_id - - -def get_timezone(date_string): - """ - Get the timezone offset from a Glean timestamp. - """ - return date_string[-6:] - - -def get_local_time(date_string): - """ - Get just the local time from the Glean timestamp. - """ - return datetime.datetime.fromisoformat(date_string[:-6]) - - -def get_fractional_hour(dt): - """ - Convert the timestamp to a "fractional hour" (hours since the UNIX epoch) - which is useful for plotting. - """ - return dt.timestamp() / 360.0 - - -def has_timezone_change(client_data): - """ - Determine if the client had a timezone change in their history. These are - excluded from the analysis for now because it's a complicated corner case. - """ - timezones = set() - for entry in client_data: - timezones.add(entry["start_time_tz"]) - timezones.add(entry["end_time_tz"]) - return len(timezones) > 1 - - -def organize_plot(data): - """ - Organize the data into rows so no two timespans overlap. - """ - rows = [] - - for entry in data: - # Find the first row will the entry will fit, otherwise, create a new - # row - for row in rows: - if entry["start_time_local"] > row[-1]["end_time_local"]: - row.append(entry) - break - else: - rows.append([entry]) - - return rows - - -def draw_line(parent, x1, x2, y1, y2, **kwargs): - """ - Draw an SVG line. It is adjusted so it's length is at least 0.5 pixels, - otherwise it will disappear during rendering. - """ - diff = abs(x2 - x1) - 0.5 - if diff < 0: - x1 -= diff / 2.0 - x2 += diff / 2.0 - attrs = {"x1": str(x1), "x2": str(x2), "y1": str(y1), "y2": str(y2)} - kwargs = dict((k.replace("_", "-"), v) for (k, v) in kwargs.items()) - attrs.update(kwargs) - return ET.SubElement(parent, "line", attrs) - - -def draw_text(parent, x, y, text, **kwargs): - """ - Draw SVG text. - """ - title = kwargs.pop("title", None) - - attrs = {"x": str(x), "y": str(y), "font-family": "sans-serif", "font-size": "10px"} - kwargs = dict((k.replace("_", "-"), v) for (k, v) in kwargs.items()) - attrs.update(kwargs) - el = ET.SubElement(parent, "text", attrs) - el.text = text - - if title is not None: - title_el = ET.SubElement(el, "title") - title_el.text = title - - return el - - -def plot_timeline(client_id, data, metrics_rows, baseline_rows): - """ - Make the SVG timeline. - """ - data = sorted(data, key=lambda x: x["start_time_hour"]) - - # Find the date range to determine the size of the plot - min_time = data[0]["start_time_hour"] - max_time = max(ping["end_time_hour"] for ping in data) - width = max_time - min_time - height = (len(metrics_rows) + len(baseline_rows) + 2) * ROW_HEIGHT - - svg = ET.Element( - "svg", - { - "version": "1.1", - "width": str(width), - "height": str(height), - "xmlns": "http://www.w3.org/2000/svg", - }, - ) - ET.SubElement( - svg, - "rect", - { - "x": "0", - "y": "0", - "width": str(width), - "height": str(height), - "fill": "white", - }, - ) - - # Draw vertical lines at midnight and 4am, with the date indicated - dt = data[0]["start_time_local"].replace(hour=0, minute=0, second=0) - while get_fractional_hour(dt) < max_time: - x = get_fractional_hour(dt) - min_time - draw_line(svg, x, x, 0, height, stroke="#cccccc") - draw_text(svg, x + 2, height - 2, dt.strftime("%m-%d")) - - four = dt.replace(hour=4) - x = get_fractional_hour(four) - min_time - draw_line(svg, x, x, 0, height, stroke="#cccccc", stroke_dasharray="2,1") - - dt += datetime.timedelta(days=1) - - # Draw markers for the first time key "FIX" versions were seen in the ping metadata - fixes = list(enumerate(FIXES)) - for ping in sorted(data, key=lambda x: x["end_time_local"]): - if ping["version_date"] is not None and ping["version_date"] >= fixes[0][1][1]: - x = ping["end_time_hour"] - min_time - draw_line(svg, x, x, 0, height, stroke="#33aa33") - draw_text(svg, x + 2, 12, str(fixes[0][0] + 1), title=fixes[0][1][0]) - fixes.pop(0) - - if len(fixes) == 0: - break - - # Draw the actual pings in the timeline - y = ROW_HEIGHT - for (rows, color) in ((baseline_rows, "#000088"), (metrics_rows, "#880000")): - for row in rows[::-1]: - for ping in row: - draw_line( - svg, - ping["start_time_hour"] - min_time, - ping["end_time_hour"] - min_time, - y, - y, - stroke=color, - stroke_width="0.5", - ) - - if ping["ping_type"] == "baseline" and ping["duration"]: - session_start = ( - get_fractional_hour( - ping["end_time_local"] - - datetime.timedelta(seconds=int(ping["duration"])) - ) - - min_time - ) - draw_line( - svg, - session_start, - ping["end_time_hour"] - min_time, - y, - y, - stroke=color, - stroke_width="3", - ) - - if ping["notes"]: - x = 0 - for note in sorted(list(ping["notes"])): - draw_text( - svg, - ping["end_time_hour"] - min_time + 2 + x, - y + 3, - str(note), - font_size="6px", - title=NOTE_SUMMARIES[note], - ) - x += 8 - - y += ROW_HEIGHT - - draw_text(svg, 2, 12, f"Android SDK: {data[0]['sdk']}") - - tree = ET.ElementTree(svg) - - with open(f"{client_id}.svg", "wb") as fd: - tree.write(fd) - - -def find_issues(client_data, stats): - """ - Find and notate issues for a client's data. - """ - client_data = sorted(client_data, key=lambda x: (x["end_time_local"], x["seq"])) - last_ping = None - last_by_type = {} - client_stats = {} - for ping in client_data: - # Find zero-length pings - if ( - ping["ping_type"] == "metrics" - and ping["start_time_local"] == ping["end_time_local"] - ): - ping["notes"].add(ZERO_LENGTH) - - # Find multiple pings with the same end_time - if last_ping is not None: - if ping["ping_type"] == "metrics" and last_ping["ping_type"] == "metrics": - if ping["end_time_local"] == last_ping["end_time_local"]: - ping["notes"].add(DUPLICATE_TIME) - last_ping["notes"].add(DUPLICATE_TIME) - else: - ping["notes"].add(NO_BASELINE) - - # Find missing or duplicate seq numbers - last_of_same_type = last_by_type.get(ping["ping_type"]) - if last_of_same_type is not None: - if int(last_of_same_type["seq"]) + 1 != int(ping["seq"]): - ping["notes"].add(MISSING_SEQ) - elif int(last_of_same_type["seq"]) == int(ping["seq"]): - ping["notes"].add(DUPLICATE_SEQ) - last_of_same_type["notes"].add(DUPLICATE_SEQ) - - if ping["ping_type"] == "metrics": - # Find metrics pings that are more than 24 hours after the last baseline ping - last_baseline = last_by_type.get("baseline") - if last_baseline is not None and ping["end_time_local"] > last_baseline[ - "end_time_local" - ] + datetime.timedelta(days=1): - ping["notes"].add(TOO_LATE) - - # Find metrics pings that are more than +/-1 hour from 4am - if abs( - ping["end_time_local"] - - ping["end_time_local"].replace(hour=4, minute=0, second=0) - ) > datetime.timedelta(hours=1): - ping["notes"].add(FAR_FROM_4AM) - - # Add notes to the overall client stats - for note in ping["notes"]: - client_stats.setdefault(note, 0) - client_stats[note] += 1 - - last_ping = ping - last_by_type[ping["ping_type"]] = ping - - # Add client stats to the overall stats - for note in client_stats.keys(): - stats.setdefault(note, 0) - stats[note] += 1 - - return client_stats - - -def process_single_client(client_id, client_data, stats): - """ - Process a single client, performing the analysis and writing out a plot. - """ - if has_timezone_change(client_data): - stats["changed_timezones"] += 1 - return {"changed_timezones": True} - - client_stats = find_issues(client_data, stats) - - client_data.sort(key=lambda x: x["start_time_local"]) - metrics_rows = organize_plot(x for x in client_data if x["ping_type"] == "metrics") - baseline_rows = organize_plot( - x for x in client_data if x["ping_type"] == "baseline" - ) - - plot_timeline(client_id, client_data, metrics_rows, baseline_rows) - - return client_stats - - -def analyse_by_day(data): - """ - Find the "issues" notated in the `notes` field on each ping and generate - a graph of their frequencies over time. - """ - data_by_day = {} - for ping in data: - if ping["ping_type"] == "metrics": - day = ping["end_time_local"].replace(hour=0, minute=0, second=0) - data_by_day.setdefault(day, {}) - day_data = data_by_day[day] - day_data.setdefault("total", 0) - day_data["total"] += 1 - for note in ping["notes"]: - day_data.setdefault(note, 0) - day_data[note] += 1 - for i, fix in enumerate(FIXES): - if ping["version_date"] is not None and ping["version_date"] >= fix[1]: - fix_id = f"fix{i}" - day_data.setdefault(fix_id, 0) - day_data[fix_id] += 1 - - # Trim the first and last couple of days, since they aren't meaningful - data_by_day = sorted(list(data_by_day.items()))[2:-2] - - return data_by_day - - -def plot_summary(data_by_day, output_filename="summary.svg"): - """ - Plot the summary of issues by day. - """ - dates = [x[0] for x in data_by_day] - - plt.figure(figsize=(20, 20)) - plt.subplot(211) - plt.title("Frequency of notes by day") - for note in range(1, MAX_NOTES): - note_values = [x[1].get(note, 0) / float(x[1]["total"]) for x in data_by_day] - plt.plot(dates, note_values, label=NOTE_SUMMARIES[note]) - plt.legend() - plt.grid() - - plt.subplot(212) - plt.title("Uptake of fixes by day") - for i, fix in enumerate(FIXES): - fix_values = [ - x[1].get(f"fix{i}", 0) / float(x[1]["total"]) for x in data_by_day - ] - plt.plot(dates, fix_values, label=fix[0]) - plt.legend() - plt.grid() - - plt.savefig(output_filename) - - -def main(input, output): - data = load_data(input) - data = filter_data(data) - annotate_data(data) - data_by_client_id = sort_data_by_client_id(data) - - if not os.path.isdir(output): - os.makedirs(output) - os.chdir(output) - - stats = { - "total_clients": len(data_by_client_id), - "changed_timezones": 0, - } - client_stats = {} - for i, (client_id, client_data) in enumerate(data_by_client_id.items()): - print(f"Analysing client: {i}/{len(data_by_client_id)}", end="\r") - client_stats[client_id] = process_single_client(client_id, client_data, stats) - - plot_summary(analyse_by_day(data)) - - print(stats) - - -if __name__ == "__main__": - # Parse commandline arguments - parser = argparse.ArgumentParser("Analyse patterns in baseline and metrics pings") - parser.add_argument("input", nargs=1, help="The input dataset (in csv)") - parser.add_argument("output", nargs=1, help="The output directory") - args = parser.parse_args() - input = args.input[0] - output = args.output[0] - main(input, output)