diff --git a/ab-testing/config/abTests.ts b/ab-testing/config/abTests.ts index 0f76f699b45..808572e4e14 100644 --- a/ab-testing/config/abTests.ts +++ b/ab-testing/config/abTests.ts @@ -209,7 +209,6 @@ const ABTests: ABTest[] = [ groups: ["a", "b"], shouldForceMetricsCollection: false, }, - { name: "commercial-prebid-failsafe-timeout", description: @@ -223,6 +222,30 @@ const ABTests: ABTest[] = [ groups: ["control", "variant"], shouldForceMetricsCollection: true, }, + { + name: "webx-test-test", + description: "Test for webx test", + owners: ["dotcom.platform@theguardian.com"], + status: "ON", + expirationDate: "2026-08-31", + type: "client", + audienceSize: 10 / 100, + audienceSpace: "D", + groups: ["control", "variant"], + shouldForceMetricsCollection: false, + }, + { + name: "webx-test-test-2", + description: "Test for webx test", + owners: ["dotcom.platform@theguardian.com"], + status: "ON", + expirationDate: "2026-08-31", + type: "client", + audienceSize: 50 / 100, + audienceSpace: "E", + groups: ["control", "variant"], + shouldForceMetricsCollection: false, + }, ]; const activeABtests = ABTests.filter((test) => test.status === "ON"); diff --git a/ab-testing/config/lib/fastly-subfield.test.ts b/ab-testing/config/lib/fastly-subfield.test.ts index 60908282c85..3485e589360 100644 --- a/ab-testing/config/lib/fastly-subfield.test.ts +++ b/ab-testing/config/lib/fastly-subfield.test.ts @@ -1,5 +1,6 @@ import { deepEqual, equal, throws } from "node:assert"; import test from "node:test"; +import { AudienceSpaces } from "../types.ts"; import { parseFastlySubfield, parseMVTValue, @@ -7,7 +8,6 @@ import { stringifyMVTValue, } from "./fastly-subfield.ts"; import type { FastlyTestParams } from "./types.ts"; -import { AudienceSpaces } from "../types.ts"; test("stringifyFastlySubfield", async (t) => { await t.test("should stringify object with string values", () => { diff --git a/ab-testing/config/lib/shuffled-space.ts b/ab-testing/config/lib/shuffled-space.ts new file mode 100644 index 00000000000..d90ec8f0bd6 --- /dev/null +++ b/ab-testing/config/lib/shuffled-space.ts @@ -0,0 +1,170 @@ +import { type AudienceSpaceId } from "../types.ts"; +import { MVT_COUNT } from "./constants.ts"; + +/** + * A shuffled space of MVTs for each audience space, storing the MVTs in a random order here is easier than trying to deterministically calculate a pseudorandom order each time we need to allocate MVTs for a test group. + * + * This ensures that tests are assigned to "random" mvts rather than contiguous blocks, so tests do not overlap in the MVT space. And should ensure that large blocks of tests do not overlap each other in the MVT space, which could lead to skewed results. + * + * It is only partial because not all spaces have been shuffled yet due to ongoing tests that we don't want to invalidate by changing their MVTs. Once those tests have expired, we can shuffle the remaining spaces. + */ +const shuffledSpace: Partial> = { + D: [ + 821, 194, 197, 159, 951, 229, 324, 880, 277, 945, 705, 243, 806, 326, + 688, 734, 773, 971, 209, 814, 915, 759, 115, 490, 899, 225, 804, 679, + 485, 80, 117, 153, 870, 913, 248, 604, 819, 605, 358, 471, 626, 523, 79, + 820, 536, 757, 662, 694, 638, 714, 444, 600, 622, 30, 498, 802, 579, + 632, 633, 357, 139, 217, 585, 14, 43, 763, 82, 762, 152, 842, 172, 950, + 177, 858, 427, 867, 582, 13, 418, 375, 918, 282, 113, 669, 943, 298, + 251, 936, 96, 24, 165, 979, 28, 390, 609, 98, 380, 134, 457, 823, 31, + 836, 331, 15, 149, 75, 798, 238, 89, 26, 687, 347, 186, 161, 571, 297, + 847, 542, 645, 772, 612, 690, 493, 668, 467, 167, 3, 144, 720, 366, 204, + 214, 160, 748, 755, 682, 816, 247, 766, 392, 39, 395, 374, 904, 97, 676, + 399, 572, 977, 985, 354, 460, 141, 987, 787, 501, 482, 512, 46, 737, + 741, 652, 599, 451, 587, 195, 265, 552, 305, 539, 598, 142, 958, 486, + 568, 616, 363, 615, 433, 989, 511, 954, 614, 56, 752, 887, 470, 611, + 650, 312, 608, 839, 792, 4, 381, 746, 208, 466, 179, 329, 594, 751, 469, + 406, 567, 854, 436, 240, 505, 437, 280, 868, 516, 825, 995, 309, 66, + 550, 584, 190, 118, 91, 344, 647, 507, 352, 67, 838, 201, 898, 231, 783, + 665, 253, 481, 933, 264, 827, 866, 128, 158, 454, 578, 174, 780, 245, 0, + 465, 659, 878, 909, 127, 929, 403, 685, 727, 447, 765, 740, 946, 111, + 566, 982, 796, 671, 829, 387, 831, 187, 199, 981, 233, 314, 641, 475, + 495, 736, 569, 332, 19, 636, 938, 808, 817, 33, 60, 596, 222, 6, 573, + 586, 728, 64, 284, 175, 407, 508, 703, 327, 712, 719, 383, 700, 304, + 619, 148, 213, 940, 565, 675, 824, 166, 244, 917, 555, 957, 570, 803, + 83, 10, 695, 525, 341, 397, 393, 234, 972, 906, 948, 708, 968, 318, 157, + 29, 301, 564, 294, 924, 372, 263, 150, 409, 241, 964, 716, 429, 131, + 499, 843, 9, 992, 404, 355, 998, 249, 303, 562, 826, 592, 634, 970, 593, + 325, 239, 212, 371, 476, 472, 147, 22, 903, 439, 178, 528, 768, 343, + 184, 367, 377, 491, 699, 610, 953, 385, 336, 966, 653, 36, 32, 360, 271, + 588, 975, 373, 236, 785, 841, 994, 978, 877, 729, 955, 281, 189, 45, + 129, 738, 334, 378, 935, 416, 590, 200, 745, 670, 580, 767, 342, 307, + 338, 944, 216, 319, 753, 308, 797, 346, 855, 532, 260, 68, 857, 537, + 889, 660, 846, 563, 885, 223, 74, 856, 438, 37, 425, 440, 595, 901, 629, + 627, 521, 637, 769, 623, 269, 533, 807, 865, 835, 443, 861, 317, 893, + 394, 293, 313, 908, 488, 446, 546, 947, 920, 94, 997, 474, 790, 121, 88, + 458, 793, 718, 625, 408, 230, 25, 419, 450, 410, 35, 252, 420, 76, 348, + 833, 242, 87, 999, 362, 707, 353, 706, 869, 548, 560, 237, 554, 227, + 435, 691, 405, 356, 299, 426, 680, 218, 402, 607, 556, 365, 202, 726, + 926, 123, 370, 993, 449, 986, 845, 120, 62, 262, 941, 84, 900, 812, 503, + 478, 613, 267, 361, 967, 104, 323, 859, 549, 873, 140, 369, 789, 923, + 268, 23, 125, 487, 73, 927, 72, 530, 391, 651, 794, 54, 310, 432, 136, + 112, 545, 207, 364, 163, 686, 684, 8, 852, 276, 296, 963, 198, 146, 774, + 764, 860, 132, 603, 674, 12, 937, 984, 85, 754, 514, 292, 44, 928, 81, + 965, 333, 181, 100, 110, 206, 424, 811, 698, 646, 914, 837, 905, 256, + 49, 884, 547, 368, 895, 538, 509, 692, 119, 583, 640, 784, 725, 2, 92, + 78, 527, 704, 890, 290, 255, 500, 351, 830, 388, 283, 962, 27, 520, 534, + 320, 442, 162, 973, 801, 180, 266, 874, 664, 911, 916, 883, 50, 666, + 721, 618, 463, 853, 606, 52, 744, 448, 991, 922, 275, 960, 559, 558, + 220, 788, 988, 849, 735, 99, 672, 621, 254, 739, 710, 894, 654, 756, + 510, 574, 732, 980, 434, 863, 934, 340, 673, 524, 235, 155, 103, 459, + 919, 48, 114, 492, 775, 55, 431, 693, 107, 910, 557, 620, 246, 818, 543, + 602, 330, 850, 430, 513, 939, 291, 412, 93, 170, 228, 5, 65, 834, 749, + 758, 143, 791, 597, 630, 295, 77, 809, 164, 108, 34, 661, 156, 138, 192, + 71, 742, 930, 379, 872, 642, 270, 191, 828, 188, 400, 776, 961, 183, + 479, 20, 221, 577, 779, 896, 715, 761, 541, 389, 345, 717, 881, 777, + 137, 713, 169, 59, 502, 494, 126, 639, 504, 38, 337, 805, 90, 484, 601, + 258, 413, 384, 58, 53, 840, 862, 382, 949, 151, 7, 497, 921, 875, 591, + 279, 42, 428, 529, 453, 519, 925, 781, 47, 786, 215, 349, 813, 61, 540, + 983, 689, 130, 102, 696, 287, 959, 441, 496, 452, 316, 300, 205, 778, + 272, 631, 278, 657, 69, 730, 990, 974, 800, 882, 359, 288, 799, 224, + 576, 648, 339, 16, 445, 462, 575, 697, 411, 848, 624, 643, 892, 464, + 879, 724, 635, 63, 259, 306, 864, 145, 95, 473, 506, 551, 286, 722, 553, + 891, 40, 273, 203, 750, 219, 321, 480, 386, 182, 515, 398, 350, 711, + 667, 628, 815, 733, 976, 396, 912, 18, 456, 154, 644, 544, 851, 731, + 168, 414, 876, 232, 173, 289, 101, 969, 561, 1, 17, 810, 315, 683, 956, + 517, 489, 261, 649, 477, 931, 897, 193, 782, 422, 250, 401, 795, 105, + 723, 106, 531, 322, 116, 417, 709, 86, 171, 952, 211, 581, 902, 844, + 124, 589, 522, 942, 335, 677, 51, 468, 133, 617, 747, 770, 760, 871, + 302, 185, 535, 743, 122, 455, 415, 376, 907, 655, 109, 658, 257, 274, + 70, 678, 886, 663, 483, 41, 423, 421, 932, 996, 888, 311, 226, 832, 196, + 702, 771, 518, 210, 135, 11, 701, 656, 822, 328, 21, 461, 285, 526, 681, + 57, 176, + ], + E: [ + 906, 3, 35, 985, 263, 97, 198, 158, 119, 795, 118, 926, 442, 162, 667, + 390, 553, 569, 585, 312, 995, 253, 928, 216, 190, 342, 334, 124, 76, + 652, 437, 470, 380, 903, 945, 315, 881, 615, 245, 687, 810, 319, 582, + 356, 548, 347, 834, 161, 889, 351, 101, 787, 598, 89, 744, 539, 469, + 576, 181, 990, 398, 491, 480, 608, 246, 980, 991, 759, 15, 732, 175, + 479, 734, 500, 134, 625, 796, 357, 24, 174, 340, 142, 714, 867, 973, + 619, 694, 369, 711, 910, 83, 122, 371, 728, 167, 512, 445, 949, 247, + 415, 674, 268, 201, 912, 717, 828, 893, 622, 164, 852, 956, 499, 349, + 95, 758, 574, 251, 74, 202, 623, 633, 752, 41, 49, 63, 641, 270, 680, + 918, 705, 756, 950, 679, 117, 22, 341, 790, 239, 131, 338, 311, 78, 153, + 318, 452, 517, 502, 866, 274, 565, 80, 0, 406, 141, 859, 726, 663, 854, + 837, 100, 364, 389, 572, 730, 529, 951, 546, 566, 229, 930, 70, 58, 257, + 317, 568, 195, 525, 81, 286, 737, 125, 212, 477, 696, 157, 423, 682, + 865, 768, 587, 722, 778, 999, 861, 747, 154, 557, 526, 111, 353, 620, + 843, 420, 886, 106, 646, 782, 178, 299, 948, 400, 733, 26, 272, 784, + 562, 381, 138, 297, 626, 704, 888, 846, 993, 28, 954, 660, 740, 29, 176, + 214, 825, 230, 136, 911, 322, 144, 509, 835, 697, 305, 743, 804, 120, + 978, 775, 764, 291, 558, 135, 361, 840, 46, 501, 228, 900, 11, 688, 542, + 670, 147, 636, 219, 672, 218, 399, 428, 456, 820, 240, 112, 987, 424, + 266, 409, 478, 515, 701, 333, 232, 600, 870, 47, 269, 60, 607, 750, 407, + 121, 146, 848, 137, 282, 632, 51, 673, 878, 684, 414, 363, 963, 326, + 320, 440, 544, 278, 199, 729, 506, 362, 602, 488, 686, 352, 207, 710, + 289, 992, 17, 767, 39, 150, 919, 772, 335, 391, 581, 727, 617, 507, 343, + 955, 231, 92, 388, 535, 658, 302, 25, 200, 630, 316, 271, 360, 595, 612, + 225, 669, 922, 988, 895, 324, 458, 981, 69, 751, 551, 459, 244, 862, + 187, 763, 489, 929, 648, 933, 441, 613, 605, 596, 915, 967, 603, 104, + 226, 379, 8, 102, 868, 413, 969, 510, 210, 869, 592, 761, 405, 508, 894, + 13, 323, 287, 48, 563, 977, 651, 275, 590, 79, 875, 591, 455, 627, 702, + 463, 372, 936, 461, 182, 126, 781, 308, 932, 947, 731, 907, 114, 794, + 82, 807, 325, 511, 536, 821, 609, 151, 451, 573, 346, 203, 989, 54, 466, + 528, 98, 185, 475, 156, 276, 450, 564, 109, 637, 527, 439, 538, 769, + 416, 38, 503, 14, 73, 723, 179, 631, 920, 891, 890, 467, 215, 31, 290, + 690, 378, 497, 766, 370, 589, 998, 601, 145, 975, 62, 250, 762, 974, + 264, 431, 191, 584, 757, 685, 860, 71, 496, 422, 552, 171, 172, 785, + 968, 721, 642, 382, 217, 946, 699, 309, 855, 116, 331, 304, 19, 64, 914, + 498, 554, 958, 545, 65, 724, 777, 683, 196, 2, 819, 653, 447, 402, 36, + 86, 72, 832, 238, 850, 735, 864, 18, 375, 148, 189, 885, 514, 149, 559, + 186, 656, 249, 127, 923, 678, 429, 845, 337, 344, 294, 677, 394, 718, + 635, 448, 454, 30, 523, 105, 300, 831, 123, 283, 236, 348, 354, 313, + 863, 307, 638, 412, 824, 927, 21, 401, 482, 113, 492, 578, 516, 267, + 255, 938, 188, 292, 421, 52, 518, 719, 376, 329, 908, 773, 75, 833, 243, + 996, 328, 383, 537, 952, 805, 800, 957, 700, 160, 849, 224, 783, 753, + 44, 817, 556, 404, 720, 811, 396, 806, 960, 140, 746, 808, 664, 213, + 611, 465, 962, 77, 449, 659, 12, 468, 88, 830, 487, 579, 327, 66, 997, + 50, 385, 209, 921, 917, 543, 839, 816, 513, 411, 892, 5, 827, 580, 233, + 662, 964, 180, 853, 395, 57, 143, 298, 814, 706, 681, 306, 242, 770, + 464, 983, 797, 793, 943, 812, 692, 223, 614, 177, 460, 350, 262, 534, + 183, 904, 880, 103, 220, 939, 128, 707, 616, 384, 872, 798, 32, 279, + 367, 654, 273, 169, 709, 40, 675, 6, 882, 427, 786, 976, 897, 780, 321, + 386, 237, 665, 639, 374, 597, 802, 549, 426, 803, 67, 280, 844, 252, 87, + 586, 37, 671, 155, 288, 387, 624, 588, 285, 836, 355, 593, 594, 748, + 446, 93, 495, 879, 765, 643, 901, 296, 359, 792, 235, 657, 668, 165, + 443, 368, 373, 16, 791, 419, 644, 494, 628, 610, 883, 330, 634, 129, + 159, 206, 899, 227, 45, 397, 703, 4, 483, 716, 986, 432, 715, 485, 708, + 567, 462, 691, 403, 33, 741, 173, 524, 942, 170, 666, 540, 647, 935, 10, + 924, 857, 96, 693, 481, 959, 676, 211, 577, 953, 505, 94, 944, 575, 982, + 583, 471, 826, 877, 655, 132, 453, 774, 254, 194, 284, 754, 59, 689, + 377, 541, 713, 645, 898, 725, 1, 555, 484, 570, 438, 163, 410, 417, 618, + 222, 760, 295, 994, 115, 521, 184, 649, 472, 941, 133, 736, 813, 314, + 533, 874, 530, 90, 493, 139, 99, 408, 265, 661, 571, 192, 979, 815, 847, + 966, 940, 208, 221, 433, 345, 606, 303, 205, 43, 392, 91, 749, 909, 550, + 838, 23, 152, 818, 42, 436, 629, 53, 970, 896, 108, 261, 822, 85, 871, + 712, 68, 739, 931, 248, 738, 393, 168, 522, 520, 241, 984, 473, 301, + 905, 884, 425, 476, 532, 842, 640, 27, 444, 281, 130, 256, 55, 695, 358, + 823, 434, 560, 418, 9, 873, 490, 197, 504, 110, 61, 258, 84, 20, 547, + 745, 902, 259, 332, 339, 851, 519, 34, 755, 809, 841, 430, 204, 937, + 277, 876, 829, 858, 604, 474, 336, 971, 789, 435, 310, 925, 788, 457, + 961, 965, 234, 193, 531, 166, 801, 486, 776, 698, 561, 799, 742, 107, + 916, 913, 260, 934, 7, 856, 771, 366, 293, 887, 650, 599, 56, 621, 779, + 365, 972, + ], +}; + +export const getShuffledSpace = ( + audienceSpaceId: AudienceSpaceId, +): number[] => { + const space = shuffledSpace[audienceSpaceId]; + if (!space || space.length < MVT_COUNT) { + console.warn( + `Shuffled space for audience space ${audienceSpaceId} is not fully populated. Returning a default ordered space of ${MVT_COUNT} MVTs.`, + ); + return Array.from({ length: MVT_COUNT }, (_, i) => i); + } + + return space; +}; diff --git a/ab-testing/config/lib/types.ts b/ab-testing/config/lib/types.ts index 88eadb1e8a5..558a9b950ab 100644 --- a/ab-testing/config/lib/types.ts +++ b/ab-testing/config/lib/types.ts @@ -1,6 +1,6 @@ type FastlyTestParams = { name: string; type: string; exp: number }; type AudienceSpace = Map; -type AllSpace = Map; +type AllSpaces = Map; -export type { FastlyTestParams, AudienceSpace, AllSpace }; +export type { FastlyTestParams, AudienceSpace, AllSpaces as AllSpace }; diff --git a/ab-testing/config/scripts/build/calculate-mvt-updates.test.ts b/ab-testing/config/scripts/build/calculate-mvt-updates.test.ts index cd38a7f4f86..86a9dc13391 100644 --- a/ab-testing/config/scripts/build/calculate-mvt-updates.test.ts +++ b/ab-testing/config/scripts/build/calculate-mvt-updates.test.ts @@ -82,7 +82,7 @@ test("calculateSpaceUpdates - handles empty audience space and tests", () => { const emptyAudienceSpace = new Map(); const emptyTests: ABTest[] = []; - const result = calculateSpaceUpdates(emptyAudienceSpace, emptyTests); + const result = calculateSpaceUpdates("A", emptyAudienceSpace, emptyTests); equal(result.size, 0); equal(deleteTestGroupSpy.mock.callCount(), 0); @@ -99,7 +99,7 @@ test("calculateSpaceUpdates - adds new test groups correctly", () => { }), ]; - const result = calculateSpaceUpdates(emptyAudienceSpace, tests); + const result = calculateSpaceUpdates("A", emptyAudienceSpace, tests); // Should have 2 MVT entries (one for each group) equal(result.size, 2); @@ -137,7 +137,7 @@ test("calculateSpaceUpdates - removes tests no longer present", () => { }), ]; - const result = calculateSpaceUpdates(existingAudienceSpace, tests); + const result = calculateSpaceUpdates("A", existingAudienceSpace, tests); // Should only have entries for test1 const testNames = new Set( @@ -166,7 +166,7 @@ test("calculateSpaceUpdates - resizes existing test groups", () => { }), ]; - const result = calculateSpaceUpdates(existingAudienceSpace, tests); + const result = calculateSpaceUpdates("A", existingAudienceSpace, tests); // Should have 4 MVT entries total (2 per group) equal(result.size, 4); @@ -189,7 +189,7 @@ test("calculateSpaceUpdates - handles fractional audience sizes correctly", () = }), ]; - const result = calculateSpaceUpdates(emptyAudienceSpace, tests); + const result = calculateSpaceUpdates("A", emptyAudienceSpace, tests); // With 0.004 audience size and 2 groups, each group gets 0.002 * 1000 = 2 MVTs equal(result.size, 4); // 2 * 2 groups @@ -208,7 +208,7 @@ test("calculateSpaceUpdates - handles single group test", () => { }), ]; - const result = calculateSpaceUpdates(emptyAudienceSpace, tests); + const result = calculateSpaceUpdates("A", emptyAudienceSpace, tests); // All audience size goes to the single group equal(result.size, 2); // 0.002 * 1000 = 2 MVTs @@ -234,7 +234,7 @@ test("calculateSpaceUpdates - handles multiple tests", () => { }), ]; - const result = calculateSpaceUpdates(emptyAudienceSpace, tests); + const result = calculateSpaceUpdates("A", emptyAudienceSpace, tests); // test1: 2 MVTs (1 per group), test2: 1 MVT = 3 total equal(result.size, 3); @@ -263,7 +263,7 @@ test("calculateSpaceUpdates - preserves expiration dates", () => { }), ]; - const result = calculateSpaceUpdates(emptyAudienceSpace, tests); + const result = calculateSpaceUpdates("A", emptyAudienceSpace, tests); const entry = result.get("mvt:0"); equal(entry?.exp, Math.floor(new Date(expirationDate).getTime() / 1000)); @@ -283,7 +283,7 @@ test("calculateSpaceUpdates - handles client-side tests", () => { }), ]; - const result = calculateSpaceUpdates(emptyAudienceSpace, tests); + const result = calculateSpaceUpdates("A", emptyAudienceSpace, tests); const entry = result.get("mvt:0"); equal(entry?.type, "client"); @@ -475,7 +475,7 @@ test("calculateSpaceUpdates - resizes middle test with adjacent tests", () => { }), ]; - const result = calculateSpaceUpdates(existingAudienceSpace, tests); + const result = calculateSpaceUpdates("A", existingAudienceSpace, tests); // Should have 14 MVT entries total (4 + 6 + 4) equal(result.size, 14); @@ -559,7 +559,7 @@ test("calculateSpaceUpdates - shrinks middle test with adjacent tests", () => { }), ]; - const result = calculateSpaceUpdates(existingAudienceSpace, tests); + const result = calculateSpaceUpdates("A", existingAudienceSpace, tests); // Should have 10 MVT entries total (4 + 2 + 4) equal(result.size, 10); @@ -611,7 +611,7 @@ test("calculateSpaceUpdates - dividing tests into non integer group sizes should }), ]; - const result = calculateSpaceUpdates(existingAudienceSpace, tests); + const result = calculateSpaceUpdates("A", existingAudienceSpace, tests); // Should have 198 MVT entries total equal(result.size, 198); @@ -686,7 +686,7 @@ test("calculateSpaceUpdates - handles insufficient MVTs when resizing middle tes // This should throw an error because there aren't enough available MVTs // to resize test2 from 4 MVTs to 20 MVTs (needs 16 additional MVTs but only 6 are available) throws( - () => calculateSpaceUpdates(existingAudienceSpace, tests), + () => calculateSpaceUpdates("A", existingAudienceSpace, tests), Error, "Not enough available MVTs for test commercial-test2:control", ); @@ -710,7 +710,7 @@ test("calculateSpaceUpdates - updates expiration date for existing test", () => }), ]; - const result = calculateSpaceUpdates(existingAudienceSpace, tests); + const result = calculateSpaceUpdates("A", existingAudienceSpace, tests); // Check that all entries have the updated expiration date const controlEntry = result.get("mvt:0"); @@ -756,7 +756,7 @@ test("calculateSpaceUpdates - handles status change from ON to OFF by removing t }), ]; - const result = calculateSpaceUpdates(existingAudienceSpace, tests); + const result = calculateSpaceUpdates("A", existingAudienceSpace, tests); // Should only have entries for test1 const testNames = new Set( @@ -793,7 +793,7 @@ test("calculateSpaceUpdates - handles status change from OFF to ON by adding tes }), ]; - const result = calculateSpaceUpdates(existingAudienceSpace, tests); + const result = calculateSpaceUpdates("A", existingAudienceSpace, tests); // Should have entries for both tests const testNames = new Set( @@ -827,7 +827,7 @@ test("calculateSpaceUpdates - updates both expiration and size simultaneously", }), ]; - const result = calculateSpaceUpdates(existingAudienceSpace, tests); + const result = calculateSpaceUpdates("A", existingAudienceSpace, tests); // Should have 4 MVT entries (2 per group) instead of 2 equal(result.size, 4); diff --git a/ab-testing/config/scripts/build/calculate-mvt-updates.ts b/ab-testing/config/scripts/build/calculate-mvt-updates.ts index 5a73af87f95..ff8e6690361 100644 --- a/ab-testing/config/scripts/build/calculate-mvt-updates.ts +++ b/ab-testing/config/scripts/build/calculate-mvt-updates.ts @@ -4,7 +4,11 @@ import type { AudienceSpace, FastlyTestParams, } from "../../lib/types.ts"; -import { AudienceSpaces, type ABTest } from "../../types.ts"; +import { + type ABTest, + type AudienceSpaceId, + AudienceSpaces, +} from "../../types.ts"; import { TestGroupMVTManager } from "./test-group-mvt-manager.ts"; const getTestGroupName = ( @@ -13,10 +17,14 @@ const getTestGroupName = ( ) => `${test.name}:${group ?? test.group}`; const calculateSpaceUpdates = ( + audienceSpaceId: AudienceSpaceId, audienceSpace: AudienceSpace, tests: ABTest[], ) => { - const testGroupMVTs = new TestGroupMVTManager(audienceSpace); + const testGroupMVTs = new TestGroupMVTManager( + audienceSpaceId, + audienceSpace, + ); const updateTestGroups = new Map( tests.flatMap((test) => @@ -104,27 +112,40 @@ const calculateAllSpaceUpdates = ( mvtGroups: AllSpace, tests: ABTest[], ): AllSpace => { - const updatedTestSpace: AudienceSpace[] = AudienceSpaces.map((space, i) => { - console.log(`Calculating updates for space: ${space}`); - const spaceTests = tests.filter( - (test) => (test.audienceSpace ?? "A") === space, // 'A' is the default space - ); + const updatedTestSpaces: AudienceSpace[] = AudienceSpaces.map( + (space, i) => { + console.log(`Calculating updates for space: ${space}`); + const spaceTests = tests.filter( + (test) => (test.audienceSpace ?? "A") === space, // 'A' is the default space + ); - if (spaceTests.length === 0) { - console.log(`No tests for space: ${space}`); - return new Map(); - } + if (spaceTests.length === 0) { + console.log(`No tests for space: ${space}`); + return new Map(); + } - const spaceMVTGroups = new Map( - mvtGroups - .entries() - .map(([key, value]) => [key, value[i] as FastlyTestParams]), - ); + const spaceMVTGroups = new Map( + Array.from(mvtGroups.entries()) + .map(([mvtId, tests]) => [ + mvtId, + spaceTests.find((test) => + tests.find( + (t) => + t.name === test.name && + t.type === test.type, + ), + ), + ]) + .filter(([_, test]) => test !== undefined) as Array< + [string, FastlyTestParams] + >, + ); - return calculateSpaceUpdates(spaceMVTGroups, spaceTests); - }); + return calculateSpaceUpdates(space, spaceMVTGroups, spaceTests); + }, + ); - return updatedTestSpace.reduce((acc, curr) => { + return updatedTestSpaces.reduce((acc, curr) => { curr.forEach((value, key) => { if (!acc.has(key)) { acc.set(key, []); diff --git a/ab-testing/config/scripts/build/test-group-mvt-manager.test.ts b/ab-testing/config/scripts/build/test-group-mvt-manager.test.ts index 6baef85ad7a..b8ef6d15479 100644 --- a/ab-testing/config/scripts/build/test-group-mvt-manager.test.ts +++ b/ab-testing/config/scripts/build/test-group-mvt-manager.test.ts @@ -30,7 +30,7 @@ test("TestGroupMVTManager - constructor initializes with empty audience space", string, { name: string; type: string; exp: number } >(); - const manager = new TestGroupMVTManager(emptyAudienceSpace); + const manager = new TestGroupMVTManager("A", emptyAudienceSpace); equal(manager.testGroups.size, 0); }); @@ -42,7 +42,7 @@ test("TestGroupMVTManager - constructor initializes with existing test groups", "test2:control": [6, 7], }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); equal(manager.testGroups.size, 3); deepEqual(manager.getTestGroup("test1:control"), [0, 1, 2]); @@ -55,7 +55,7 @@ test("TestGroupMVTManager - getTestGroup returns undefined for non-existent grou string, { name: string; type: string; exp: number } >(); - const manager = new TestGroupMVTManager(emptyAudienceSpace); + const manager = new TestGroupMVTManager("A", emptyAudienceSpace); equal(manager.getTestGroup("non-existent"), undefined); }); @@ -65,7 +65,7 @@ test("TestGroupMVTManager - addTestGroup successfully adds new test group", () = string, { name: string; type: string; exp: number } >(); - const manager = new TestGroupMVTManager(emptyAudienceSpace); + const manager = new TestGroupMVTManager("A", emptyAudienceSpace); manager.addTestGroup("test1:control", 3); @@ -80,7 +80,7 @@ test("TestGroupMVTManager - addTestGroup throws error when test group already ex "test1:control": [0, 1, 2], }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); throws( () => manager.addTestGroup("test1:control", 2), @@ -95,7 +95,7 @@ test("TestGroupMVTManager - addTestGroup throws error when not enough available "test1:control": Array.from({ length: 998 }, (_, i) => i), }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); throws( () => manager.addTestGroup("test2:control", 5), @@ -109,7 +109,7 @@ test("TestGroupMVTManager - resizeTestGroup increases group size", () => { "test1:control": [0, 1, 2], }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); manager.resizeTestGroup("test1:control", 5); @@ -124,7 +124,7 @@ test("TestGroupMVTManager - resizeTestGroup decreases group size", () => { "test1:control": [0, 1, 2, 3, 4], }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); manager.resizeTestGroup("test1:control", 3); @@ -138,7 +138,7 @@ test("TestGroupMVTManager - resizeTestGroup with same size does nothing", () => "test1:control": [0, 1, 2], }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); manager.resizeTestGroup("test1:control", 3); @@ -152,7 +152,7 @@ test("TestGroupMVTManager - resizeTestGroup throws error for non-existent group" string, { name: string; type: string; exp: number } >(); - const manager = new TestGroupMVTManager(emptyAudienceSpace); + const manager = new TestGroupMVTManager("A", emptyAudienceSpace); throws( () => manager.resizeTestGroup("non-existent", 5), @@ -167,7 +167,7 @@ test("TestGroupMVTManager - resizeTestGroup throws error when not enough availab "test2:control": Array.from({ length: 995 }, (_, i) => i + 3), // Occupy most MVTs }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); throws( () => manager.resizeTestGroup("test1:control", 10), @@ -182,7 +182,7 @@ test("TestGroupMVTManager - deleteTestGroup removes group and frees MVTs", () => "test2:control": [3, 4, 5], }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); manager.deleteTestGroup("test1:control"); @@ -192,7 +192,7 @@ test("TestGroupMVTManager - deleteTestGroup removes group and frees MVTs", () => // Should be able to create a new group with the freed MVTs manager.addTestGroup("test3:control", 3); const newTestGroup = manager.getTestGroup("test3:control"); - deepEqual(newTestGroup, [0, 1, 2]); + deepEqual(newTestGroup, [6, 7, 8]); }); test("TestGroupMVTManager - deleteTestGroup handles non-existent group gracefully", () => { @@ -200,7 +200,7 @@ test("TestGroupMVTManager - deleteTestGroup handles non-existent group gracefull string, { name: string; type: string; exp: number } >(); - const manager = new TestGroupMVTManager(emptyAudienceSpace); + const manager = new TestGroupMVTManager("A", emptyAudienceSpace); // Should not throw an error manager.deleteTestGroup("non-existent"); @@ -213,7 +213,7 @@ test("TestGroupMVTManager - complex scenario with multiple operations", () => { "test1:variant": [3, 4, 5], }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); // Add a new test group manager.addTestGroup("test2:control", 4); @@ -240,7 +240,7 @@ test("TestGroupMVTManager - MVT allocation preserves low-to-high order", () => { string, { name: string; type: string; exp: number } >(); - const manager = new TestGroupMVTManager(emptyAudienceSpace); + const manager = new TestGroupMVTManager("A", emptyAudienceSpace); // Add first group - should get lowest MVTs manager.addTestGroup("test1:control", 3); @@ -270,7 +270,7 @@ test("TestGroupMVTManager - handles MVT parsing from key correctly", () => { exp: Date.now() + 86400000, }); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); equal(manager.testGroups.size, 1); const testGroup = manager.getTestGroup("test1:control"); @@ -284,7 +284,7 @@ test("TestGroupMVTManager - addTestGroup with size 0", () => { string, { name: string; type: string; exp: number } >(); - const manager = new TestGroupMVTManager(emptyAudienceSpace); + const manager = new TestGroupMVTManager("A", emptyAudienceSpace); manager.addTestGroup("test1:control", 0); @@ -298,7 +298,7 @@ test("TestGroupMVTManager - resizeTestGroup to size 0", () => { "test1:control": [0, 1, 2], }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); manager.resizeTestGroup("test1:control", 0); @@ -312,7 +312,7 @@ test("TestGroupMVTManager - available MVTs maintain sorted order after operation "test1:control": [500, 600, 700], }; const audienceSpace = createMockAudienceSpace(existingGroups); - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); // Delete the group to free MVTs manager.deleteTestGroup("test1:control"); @@ -328,7 +328,7 @@ test("TestGroupMVTManager - stress test with maximum MVTs", () => { string, { name: string; type: string; exp: number } >(); - const manager = new TestGroupMVTManager(emptyAudienceSpace); + const manager = new TestGroupMVTManager("A", emptyAudienceSpace); // Add a group that uses all available MVTs manager.addTestGroup("test1:control", 1000); @@ -366,7 +366,7 @@ test("TestGroupMVTManager - constructor with invalid MVT key format", () => { }); // Constructor should handle invalid keys gracefully - const manager = new TestGroupMVTManager(audienceSpace); + const manager = new TestGroupMVTManager("A", audienceSpace); // The group should exist but only with valid MVTs const testGroup = manager.getTestGroup("test1:control"); diff --git a/ab-testing/config/scripts/build/test-group-mvt-manager.ts b/ab-testing/config/scripts/build/test-group-mvt-manager.ts index 0a5c7e461ec..b7d79f95ce8 100644 --- a/ab-testing/config/scripts/build/test-group-mvt-manager.ts +++ b/ab-testing/config/scripts/build/test-group-mvt-manager.ts @@ -1,5 +1,6 @@ -import { MVT_COUNT } from "../../lib/constants.ts"; +import { getShuffledSpace } from "../../lib/shuffled-space.ts"; import type { AudienceSpace } from "../../lib/types.ts"; +import type { AudienceSpaceId } from "../../types.ts"; /** * A class to manage MVTs for test groups in a test space. @@ -22,7 +23,7 @@ class TestGroupMVTManager { * Create a new TestGroupMVTs instance. * @param mvtGroups - An array of current MVT groups from the Fastly dictionary. */ - constructor(mvtGroups: AudienceSpace) { + constructor(audienceSpaceId: AudienceSpaceId, mvtGroups: AudienceSpace) { this.testGroups = new Map( Object.entries( mvtGroups @@ -47,9 +48,11 @@ class TestGroupMVTManager { Array.from(this.testGroups.values()).flat(), ); - this.availableMVTs = Array.from({ length: MVT_COUNT }, (_, i) => i) - .filter((i) => !this.occupiedMVTs.has(i)) - .sort((a, b) => a - b); + const shuffledMVTs = getShuffledSpace(audienceSpaceId); + + this.availableMVTs = shuffledMVTs.filter( + (i) => !this.occupiedMVTs.has(i), + ); console.log( `Initialized TestGroupMVTs with ${this.availableMVTs.length} available MVTs`, @@ -109,11 +112,14 @@ class TestGroupMVTManager { throw new Error(`Not enough available MVTs for test ${name}`); } for (let i = 0; i < additionalMVTsNeeded; i++) { - const mvtIndex = this.availableMVTs.shift(); - if (mvtIndex !== undefined) { - currentMVTs.push(mvtIndex); - this.occupiedMVTs.add(mvtIndex); + const mvtId = this.availableMVTs.shift(); + if (mvtId === undefined) { + throw new Error( + `No available MVTs left to expand test ${name}`, + ); } + currentMVTs.push(mvtId); + this.occupiedMVTs.add(mvtId); } } else if (newSize < currentSize) { const removedMVTs = currentMVTs.slice(newSize); @@ -122,8 +128,6 @@ class TestGroupMVTManager { this.availableMVTs.push(mvt); }); currentMVTs.length = newSize; - // Keep available MVTs sorted in ascending order - this.availableMVTs.sort((a, b) => a - b); } this.testGroups.set(name, currentMVTs); } @@ -140,8 +144,6 @@ class TestGroupMVTManager { this.availableMVTs.push(mvt); }); this.testGroups.delete(name); - // Keep available MVTs sorted in ascending order - this.availableMVTs.sort((a, b) => a - b); } } }