Add 2 more AB Test audience spaces

ab-testing/config/lib/constants.ts

@@ -13,10 +13,4 @@ const MVT_COUNT = 1000;
  */
 const MAX_SERVER_SIDE_TESTS = 20;
 
-/**
- * The spaces used for test groups, each space covers the entire mvt space allowing for concurrent overlapping tests where necessary.
- * If this is increased, the fastly VCL configuration will need to be updated to match.
- */
-const AUDIENCE_SPACES = ["A", "B", "C"];
-
-export { MVT_COUNT, MAX_SERVER_SIDE_TESTS, AUDIENCE_SPACES };
+export { MVT_COUNT, MAX_SERVER_SIDE_TESTS };

ab-testing/config/lib/fastly-subfield.test.ts

@@ -1,6 +1,6 @@
 import { deepEqual, equal, throws } from "node:assert";
 import test from "node:test";
-import { AUDIENCE_SPACES } from "./constants.ts";
+import { AudienceSpaces } from "../types.ts";
 import {
 	parseFastlySubfield,
 	parseMVTValue,
@@ -136,7 +136,7 @@ test("parseMVTValue", async (t) => {
 			"group:0=test1,group:0:type=control,group:0:exp=50,group:1=test2,group:1:type=variant,group:1:exp=25,group:2=test3,group:2:type=variant,group:2:exp=75";
 		const result = parseMVTValue(subfield);
 
-		equal(result.length, AUDIENCE_SPACES.length);
+		equal(result.length, AudienceSpaces.length);
 		deepEqual(result[0], { name: "test1", type: "control", exp: 50 });
 		deepEqual(result[1], { name: "test2", type: "variant", exp: 25 });
 		deepEqual(result[2], { name: "test3", type: "variant", exp: 75 });
@@ -148,7 +148,7 @@ test("parseMVTValue", async (t) => {
 			const subfield = "group:0=test1,group:1:type=variant";
 			const result = parseMVTValue(subfield);
 
-			equal(result.length, AUDIENCE_SPACES.length);
+			equal(result.length, AudienceSpaces.length);
 			deepEqual(result[0], {
 				name: "test1",
 				type: "undefined",
@@ -171,7 +171,7 @@ test("parseMVTValue", async (t) => {
 		const subfield = "";
 		const result = parseMVTValue(subfield);
 
-		equal(result.length, AUDIENCE_SPACES.length);
+		equal(result.length, AudienceSpaces.length);
 		result.forEach((item) => {
 			deepEqual(item, {
 				name: "undefined",
@@ -277,6 +277,8 @@ test("round-trip compatibility", async (t) => {
 				{ name: "control", type: "control", exp: 50 },
 				{ name: "variant1", type: "variant", exp: 25 },
 				{ name: "variant2", type: "variant", exp: 25 },
+				{ name: "variant3", type: "variant", exp: 25 },
+				{ name: "variant4", type: "variant", exp: 25 },
 			];
 			const stringified = stringifyMVTValue(original);
 			const parsed = parseMVTValue(stringified);

ab-testing/config/lib/fastly-subfield.ts

@@ -1,4 +1,4 @@
-import { AUDIENCE_SPACES } from "./constants.ts";
+import { AudienceSpaces } from "../types.ts";
 import type { FastlyTestParams } from "./types.ts";
 
 const validateValue = (value: string | number, allowedColons: number): void => {
@@ -59,7 +59,7 @@ const parseFastlySubfield = (str: string): Record<string, string | number> => {
  */
 const parseMVTValue = (subfield: string): FastlyTestParams[] => {
 	const value = parseFastlySubfield(subfield);
-	return AUDIENCE_SPACES.map((_, i) => {
+	return AudienceSpaces.map((_, i) => {
 		return {
 			name: String(value[`group:${i}`]),
 			type: String(value[`group:${i}:type`]),

ab-testing/config/scripts/build/calculate-mvt-updates.ts

@@ -1,10 +1,10 @@
-import { AUDIENCE_SPACES, MVT_COUNT } from "../../lib/constants.ts";
+import { MVT_COUNT } from "../../lib/constants.ts";
 import type {
 	AllSpace,
 	AudienceSpace,
 	FastlyTestParams,
 } from "../../lib/types.ts";
-import type { ABTest } from "../../types.ts";
+import { type ABTest, AudienceSpaces } from "../../types.ts";
 import { TestGroupMVTManager } from "./test-group-mvt-manager.ts";
 
 const getTestGroupName = (
@@ -104,27 +104,25 @@ const calculateAllSpaceUpdates = (
 	mvtGroups: AllSpace,
 	tests: ABTest[],
 ): AllSpace => {
-	const updatedTestSpace: AudienceSpace[] = AUDIENCE_SPACES.map(
-		(space, i) => {
-			console.log(`Calculating updates for space: ${space}`);
-			const spaceTests = tests.filter(
-				(test) => (test.audienceSpace ?? "A") === space, // 'A' is the default space
-			);
+	const updatedTestSpace: AudienceSpace[] = AudienceSpaces.map((space, i) => {
+		console.log(`Calculating updates for space: ${space}`);
+		const spaceTests = tests.filter(
+			(test) => (test.audienceSpace ?? "A") === space, // 'A' is the default space
+		);
 
-			if (spaceTests.length === 0) {
-				console.log(`No tests for space: ${space}`);
-				return new Map<string, FastlyTestParams>();
-			}
+		if (spaceTests.length === 0) {
+			console.log(`No tests for space: ${space}`);
+			return new Map<string, FastlyTestParams>();
+		}
 
-			const spaceMVTGroups = new Map(
-				mvtGroups
-					.entries()
-					.map(([key, value]) => [key, value[i] as FastlyTestParams]),
-			);
+		const spaceMVTGroups = new Map(
+			mvtGroups
+				.entries()
+				.map(([key, value]) => [key, value[i] as FastlyTestParams]),
+		);
 
-			return calculateSpaceUpdates(spaceMVTGroups, spaceTests);
-		},
-	);
+		return calculateSpaceUpdates(spaceMVTGroups, spaceTests);
+	});
 
 	return updatedTestSpace.reduce((acc, curr) => {
 		curr.forEach((value, key) => {

ab-testing/config/scripts/validation/enoughSpace.test.ts

@@ -1,13 +1,13 @@
 import { throws } from "node:assert";
 import test from "node:test";
-import type { ABTest } from "../../types.ts";
+import type { ABTest, AudienceSpaceId } from "../../types.ts";
 import { enoughSpace } from "./enoughSpace.ts";
 
 // Helper function to create a test AB test object
 function createABTest(
 	name: string,
 	audienceSize: number,
-	audienceSpace?: "A" | "B" | "C",
+	audienceSpace?: AudienceSpaceId,
 ): ABTest {
 	return {
 		name: `commercial-${name}` as const,

ab-testing/config/types.ts

@@ -24,6 +24,10 @@ type Year = `${number}${number}${number}${number}`;
 type Month = `${number}${number}`;
 type Day = `${number}${number}`;
 
+const AudienceSpaces = ["A", "B", "C", "D", "E"] as const;
+
+type AudienceSpaceId = (typeof AudienceSpaces)[number];
+
 type ABTest = {
 	/** Name of the AB test */
 	name: TestName;
@@ -47,7 +51,7 @@ type ABTest = {
 	 * Having multiple test spaces allows deliberate overlapping of test audiences
 	 * Defaults to A
 	 */
-	audienceSpace?: "A" | "B" | "C";
+	audienceSpace?: AudienceSpaceId;
 	/** Test group definition */
 	groups: string[];
 	/**
@@ -69,4 +73,12 @@ type ABTest = {
 	shouldReportToOphan?: () => boolean;
 };
 
-export type { ABTest, FastlyTestParams, AudienceSpace, AllSpace };
+export { AudienceSpaces };
+
+export type {
+	ABTest,
+	FastlyTestParams,
+	AudienceSpace,
+	AllSpace,
+	AudienceSpaceId,
+};

ab-testing/frontend/src/lib/components/AudienceBreakdown.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import type { ABTest } from '../../../../types.js';
+	import { AudienceSpaces, type ABTest } from "../../../../config/types";
 
 	interface Props {
 		tests: ABTest[];
@@ -20,12 +20,10 @@
 	const BAR_MARGIN_X = 0.1;
 	const BAR_MARGIN_Y = 2;
 
-	const testSpaces = ['A', 'B', 'C'];
+	const chartHeight = AudienceSpaces.length * BAR_HEIGHT + BAR_HEIGHT + 16;
 
-	const chartHeight = testSpaces.length * BAR_HEIGHT + BAR_HEIGHT + 16;
-
-	const testsBySpace = testSpaces.map((space) => {
-		if (space === 'A') {
+	const testsBySpace = AudienceSpaces.map((space) => {
+		if (space === "A") {
 			return tests.filter(
 				(test) => test.audienceSpace === space || !test.audienceSpace,
 			);

dotcom-rendering/docs/development/ab-testing-in-dcr.md

@@ -41,7 +41,7 @@ When your PR is merged, the A/B test will be automatically deployed to Fastly an
 | expirationDate               | false    | A/B tests should have an expiration date set in the future. This is to ensure that tests do not run indefinitely. The duration of a test should be determined by how long we believe it will need to run in order to reach a statistically significant result - if you're unsure how long this is we'd recommend reaching out to our Data Analysts. Adding a buffer of an additional +2 weeks beyond the expected completion date will ensure tests do not end unexpectedly before we have a statistically significant result. <br><br>Expired tests will cause the A/B testing validation to fail, and will not be deployed. <br><br>Tests that expire while they are are in-flight will not be served by Fastly, and should be removed from the `abTest.ts` file as soon as possible. |
 | type                         | false    | All requests are processed by Fastly at the edge, however, A/B testing of server-side logic in Frontend or DCR will need to be cached separately. Client side tests do not need to be cached separately, as they are applied in the browser after the response is delivered. <br><br>Ensure that the `type` field is set to either `client` or `server` to indicate the type of test so that server side tests can be cached correctly, and client side tests are not splitting the cache unnecessarily. <br><br>There's a limit of the number of concurrent server-side tests that can be run, enforced by the validation script, so it's important to use client-side tests where possible.                                                                                           |
 | audienceSize                 | false    | The `audienceSize` is the size of the whole test and is divided between the test groups that you specify. For example, consider an audience size of 10% (`10 / 100`) with two test groups: control and variant. The control and variant test groups will each be allocated 5%. The "resolution" of sizing is down to 0.1%, so groups will be rounded to the nearest 0.1%. <br><br> **Caution:** If you ever need to run a 100% server-side test AKA the entire audience, start with a smaller test first e.g. 30%. This is because going from 0 -> 100% will invalidate the cache for all page views pretty much all at once. Starting with a smaller test will build up test group pages in the cache first, so there will be a less drastic spike in load on origin.                  |
-| audienceSpace                | true     | Ideally A/B tests would never overlap (users being in multiple tests), but sometimes this is unavoidable, for example when running a very large 50+% test without interrupting existing tests. <br><br>To add a test where there is not enough space in the default audience space (`A`), you can specify a different `audienceSpace` in the test definition. <br><br>For example if there are already 3 25% tests in space `A` totalling 75%, and you want to run a 50% test, you can set the `audienceSpace` to `B` to allow this test to overlap with the existing tests. Currently there are [3 spaces configured](https://github.com/guardian/dotcom-rendering/blob/main/ab-testing/config/lib/constants.ts) [`A`,`B`,`C`].                                                        |
+| audienceSpace                | true     | Ideally A/B tests would never overlap (users being in multiple tests), but sometimes this is unavoidable, for example when running a very large 50+% test without interrupting existing tests. <br><br>To add a test where there is not enough space in the default audience space (`A`), you can specify a different `audienceSpace` in the test definition. <br><br>For example if there are already 3 25% tests in space `A` totalling 75%, and you want to run a 50% test, you can set the `audienceSpace` to `B` to allow this test to overlap with the existing tests. Currently there are [5 spaces configured](https://github.com/guardian/dotcom-rendering/blob/main/ab-testing/config/lib/constants.ts) [`A`,`B`,`C`,`D`,`E`].                                                |
 | groups                       | false    | Test group definitions, eg. `['control', 'variant']`. Convention is to have groups named control and variant, but you can name them as you wish. <br><br>A single group is also possible, for example if you're rolling out a new feature and don't need a control.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
 | shouldForceMetricsCollection | true     | `true` or `false`. Bypasses sampling to force metrics collection for this test. See DCR Metrics component for end usage.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |