diff --git a/bricks/test_optimizer/brick.yaml b/bricks/test_optimizer/brick.yaml index 07aa58c3d..3c2b80b62 100644 --- a/bricks/test_optimizer/brick.yaml +++ b/bricks/test_optimizer/brick.yaml @@ -12,3 +12,11 @@ vars: default: "." description: The path to the package root. prompt: Please enter the path to the package root. + shard-index: + type: number + description: The 1-based index of the shard to generate tests for. + prompt: Please enter the shard index. + total-shards: + type: number + description: The total number of shards the test suite is split into. + prompt: Please enter the total number of shards. diff --git a/bricks/test_optimizer/hooks/lib/pre_gen.dart b/bricks/test_optimizer/hooks/lib/pre_gen.dart index c0348ef02..cd6d5dcfd 100644 --- a/bricks/test_optimizer/hooks/lib/pre_gen.dart +++ b/bricks/test_optimizer/hooks/lib/pre_gen.dart @@ -33,35 +33,96 @@ Future run(HookContext context) async { final flutterSdkRegExp = RegExp(r'sdk:\s*flutter$', multiLine: true); final isFlutter = flutterSdkRegExp.hasMatch(pubspecContents); - final identifierGenerator = DartIdentifierGenerator(); - final testIdentifierTable = >[]; + final shardIndex = context.vars['shard-index'] as int?; + final totalShards = context.vars['total-shards'] as int?; + + // The CLI validates these before it gets here, but `mason make` prompts for + // them directly, so guard the round-robin below against values that would + // never terminate or index out of range. + if (shardIndex != null && + totalShards != null && + (totalShards < 1 || shardIndex < 1 || shardIndex > totalShards)) { + context.logger.err( + 'shard-index must be between 1 and total-shards, but got ' + 'shard-index $shardIndex and total-shards $totalShards', + ); + exitFn(1); + } + final tests = testDir .listSync(recursive: true) .where((entity) => entity.isTest); - final notOptimizedTests = await getNotOptimizedTests(tests, testDir.path); - - for (final entity in tests) { - final relativePath = path - .relative(entity.path, from: testDir.path) - .replaceAll(r'\', '/'); - testIdentifierTable.add({ - 'path': relativePath, - 'identifier': identifierGenerator.next(), - }); - } - - final optimizedTestsIdentifierTable = testIdentifierTable - .where((e) => !notOptimizedTests.contains(e['path'])) + final notOptimizedTests = (await getNotOptimizedTests( + tests, + testDir.path, + )).toSet(); + + // Sorting guarantees a deterministic order across machines, which is what + // makes sharding reproducible: `Directory.listSync` order is filesystem + // dependent, so without this two runners could disagree on the partition + // and either skip or duplicate tests. + final testPaths = + tests + .map( + (entity) => path + .relative(entity.path, from: testDir.path) + .replaceAll(r'\', '/'), + ) + .toList() + ..sort(); + + // Non optimized tests run as standalone files alongside the optimizer + // entrypoint, so they are sharded too, and in the same deal as the + // optimized ones: dealing out one list keeps every shard within one file + // of the others, whereas dealing out the two lists separately would hand + // the first shards a file from each. + final shardPaths = _shardOf( + testPaths, + shardIndex: shardIndex, + totalShards: totalShards, + ); + final optimizedTestPaths = shardPaths + .where((p) => !notOptimizedTests.contains(p)) + .toList(); + final shardedNotOptimizedTests = shardPaths + .where(notOptimizedTests.contains) .toList(); + final identifierGenerator = DartIdentifierGenerator(); + final optimizedTestsIdentifierTable = [ + for (final relativePath in optimizedTestPaths) + {'path': relativePath, 'identifier': identifierGenerator.next()}, + ]; + context.vars = { 'tests': optimizedTestsIdentifierTable, 'isFlutter': isFlutter, - 'notOptimizedTests': notOptimizedTests, + 'notOptimizedTests': shardedNotOptimizedTests, }; } +/// Returns the subset of [paths] that belongs to the shard [shardIndex] out of +/// [totalShards]. +/// +/// Returns [paths] unchanged when sharding is not enabled (either value is +/// `null`). +/// +/// Files are dealt out round-robin (index modulo [totalShards]) over the +/// already sorted [paths], which keeps shards balanced in file count and makes +/// the partition stable for a given test suite. +List _shardOf( + List paths, { + required int? shardIndex, + required int? totalShards, +}) { + if (shardIndex == null || totalShards == null) return paths; + + return [ + for (var i = shardIndex - 1; i < paths.length; i += totalShards) paths[i], + ]; +} + extension on FileSystemEntity { bool get isTest { return this is File && path.basename(this.path).endsWith('_test.dart'); @@ -85,9 +146,10 @@ Future> getNotOptimizedTests( } } - /// Format to relative path + /// Format to relative path, normalizing separators so the paths compare + /// equal to the ones built in [run] on Windows too. final relativePaths = testWithVeryGoodTest - .map((e) => path.relative(e, from: testDir)) + .map((e) => path.relative(e, from: testDir).replaceAll(r'\', '/')) .toList(); return relativePaths; diff --git a/bricks/test_optimizer/hooks/test/pre_gen_test.dart b/bricks/test_optimizer/hooks/test/pre_gen_test.dart index 67a2ccf19..1ea0c3f72 100644 --- a/bricks/test_optimizer/hooks/test/pre_gen_test.dart +++ b/bricks/test_optimizer/hooks/test/pre_gen_test.dart @@ -1,4 +1,5 @@ import 'dart:io'; +import 'dart:math'; import 'package:hooks/pre_gen.dart' as pre_gen; import 'package:mason/mason.dart'; @@ -188,6 +189,33 @@ dependencies: expect(context.vars['isFlutter'], isNull); }); + test('when the shard values are out of range', () async { + File(path.join(tempDirectory.path, 'pubspec.yaml')).createSync(); + Directory(path.join(tempDirectory.path, 'test')).createSync(); + + context.vars['package-root'] = tempDirectory.absolute.path; + context.vars['shard-index'] = 1; + context.vars['total-shards'] = 0; + + await expectLater( + () => pre_gen.run(context), + throwsA( + isA().having( + (ex) => ex.arguments.first, + 'error code', + equals('1'), + ), + ), + ); + + verify( + () => context.logger.err( + 'shard-index must be between 1 and total-shards, but got ' + 'shard-index 1 and total-shards 0', + ), + ).called(1); + }); + test('when target dir does not contain a pubspec.yaml', () async { final testDir = Directory(path.join(tempDirectory.path, 'test')) ..createSync(); @@ -278,5 +306,142 @@ dependencies: }, ); }); + group('Sharding', () { + /// Creates a package with [count] optimizable test files, plus any + /// [notOptimized] files carrying the skip optimization tag. + Directory createPackage(int count, {int notOptimized = 0}) { + File(path.join(tempDirectory.path, 'pubspec.yaml')).createSync(); + final testDir = Directory(path.join(tempDirectory.path, 'test')) + ..createSync(); + for (var i = 0; i < count; i++) { + File(path.join(testDir.path, 'test${i}_test.dart')).createSync(); + } + for (var i = 0; i < notOptimized; i++) { + File(path.join(testDir.path, 'skip${i}_test.dart')) + .writeAsStringSync(notOptimizedTestContent); + } + return testDir; + } + + List pathsOf(HookContext context) { + final tests = context.vars['tests'] as List>; + return tests.map((e) => e['path']!).toList(); + } + + Future> runShard(int index, int total) async { + final context = _FakeContext() + ..vars['package-root'] = tempDirectory.absolute.path + ..vars['shard-index'] = index + ..vars['total-shards'] = total; + await pre_gen.run(context); + return [ + ...pathsOf(context), + ...(context.vars['notOptimizedTests']! as List).cast(), + ]; + } + + test('runs every test exactly once across all shards', () async { + createPackage(7, notOptimized: 2); + + final shards = [for (var i = 1; i <= 3; i++) await runShard(i, 3)]; + final union = shards.expand((shard) => shard).toList(); + + expect( + union..sort(), + [ + for (var i = 0; i < 7; i++) 'test${i}_test.dart', + for (var i = 0; i < 2; i++) 'skip${i}_test.dart', + ]..sort(), + reason: 'Shards must be a complete and disjoint partition', + ); + }); + + test('shards non optimized tests as well', () async { + createPackage(0, notOptimized: 4); + + final first = await runShard(1, 2); + final second = await runShard(2, 2); + + expect(first, ['skip0_test.dart', 'skip2_test.dart']); + expect(second, ['skip1_test.dart', 'skip3_test.dart']); + }); + + test('deals optimized and non optimized tests out together', () async { + createPackage(2, notOptimized: 3); + + final sizes = [ + for (var i = 1; i <= 6; i++) (await runShard(i, 6)).length, + ]; + + expect( + sizes, + [1, 1, 1, 1, 1, 0], + reason: + 'Sharding the two lists separately would give the first ' + 'shards a file from each while later shards stay empty', + ); + }); + + test('is deterministic across runs', () async { + createPackage(9); + + expect(await runShard(2, 4), await runShard(2, 4)); + }); + + test('balances shards within one file of each other', () async { + createPackage(10); + + final sizes = [ + for (var i = 1; i <= 4; i++) (await runShard(i, 4)).length, + ]; + + expect(sizes.reduce(max) - sizes.reduce(min), lessThanOrEqualTo(1)); + }); + + test( + 'yields an empty shard when there are more shards than tests', + () async { + createPackage(2); + + expect(await runShard(3, 3), isEmpty); + }, + ); + + test( + 'excludes nested non optimized tests from the optimized set', + () async { + File(path.join(tempDirectory.path, 'pubspec.yaml')).createSync(); + final testDir = Directory(path.join(tempDirectory.path, 'test')) + ..createSync(); + final nested = Directory(path.join(testDir.path, 'sub')) + ..createSync(); + File(path.join(nested.path, 'skip_test.dart')) + .writeAsStringSync(notOptimizedTestContent); + + final context = _FakeContext() + ..vars['package-root'] = tempDirectory.absolute.path; + await pre_gen.run(context); + + expect( + pathsOf(context), + isEmpty, + reason: + 'A tagged test in a subdirectory must not be optimized, ' + 'otherwise it runs both inlined and standalone', + ); + expect(context.vars['notOptimizedTests'], ['sub/skip_test.dart']); + }, + ); + + test('includes every test when sharding is not requested', () async { + createPackage(3); + + final context = _FakeContext() + ..vars['package-root'] = tempDirectory.absolute.path; + await pre_gen.run(context); + + expect(pathsOf(context), hasLength(3)); + }); + }); }); } diff --git a/lib/src/cli/dart_cli.dart b/lib/src/cli/dart_cli.dart index 95f0bbba1..1b1f68d01 100644 --- a/lib/src/cli/dart_cli.dart +++ b/lib/src/cli/dart_cli.dart @@ -136,6 +136,8 @@ class Dart { void Function(String)? stderr, GeneratorBuilder buildGenerator = MasonGenerator.fromBundle, List? reportOn, + int? shardIndex, + int? totalShards, }) { return TestCLIRunner.test( logger: logger, @@ -157,6 +159,8 @@ class Dart { stderr: stderr, reportOn: reportOn, buildGenerator: buildGenerator, + shardIndex: shardIndex, + totalShards: totalShards, ); } } diff --git a/lib/src/cli/flutter_cli.dart b/lib/src/cli/flutter_cli.dart index 26b9c93b0..79b940b74 100644 --- a/lib/src/cli/flutter_cli.dart +++ b/lib/src/cli/flutter_cli.dart @@ -215,6 +215,8 @@ class Flutter { void Function(String)? stderr, GeneratorBuilder buildGenerator = MasonGenerator.fromBundle, List? reportOn, + int? shardIndex, + int? totalShards, }) { return TestCLIRunner.test( logger: logger, @@ -235,6 +237,8 @@ class Flutter { stderr: stderr, buildGenerator: buildGenerator, reportOn: reportOn, + shardIndex: shardIndex, + totalShards: totalShards, ); } } diff --git a/lib/src/cli/templates/test_optimizer_bundle.dart b/lib/src/cli/templates/test_optimizer_bundle.dart index 22cc57024..c32fbce59 100644 --- a/lib/src/cli/templates/test_optimizer_bundle.dart +++ b/lib/src/cli/templates/test_optimizer_bundle.dart @@ -20,7 +20,7 @@ final testOptimizerBundle = MasonBundle.fromJson({ }, { "path": "lib/pre_gen.dart", - "data": "aW1wb3J0ICdkYXJ0OmlvJzsKCmltcG9ydCAncGFja2FnZTpob29rcy9kYXJ0X2lkZW50aWZpZXJfZ2VuZXJhdG9yLmRhcnQnOwppbXBvcnQgJ3BhY2thZ2U6bWFzb24vbWFzb24uZGFydCc7CmltcG9ydCAncGFja2FnZTpwYXRoL3BhdGguZGFydCcgYXMgcGF0aDsKCnR5cGVkZWYgRXhpdEZuID0gTmV2ZXIgRnVuY3Rpb24oaW50IGNvZGUpOwoKRXhpdEZuIGV4aXRGbiA9IGV4aXQ7CgpTdHJpbmcgc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uVGFnID0gJ3NraXBfdmVyeV9nb29kX29wdGltaXphdGlvbic7ClJlZ0V4cCBza2lwVmVyeUdvb2RPcHRpbWl6YXRpb25SZWdFeHAgPSBSZWdFeHAoCiAgIkBUYWdzXFxzKlxcKFxccypcXFtbXFxzXFxTXSo/W1wiJ10kc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uVGFnW1wiJ11bXFxzXFxTXSo/XFxdXFxzKlxcKSIsCiAgbXVsdGlMaW5lOiB0cnVlLAopOwoKRnV0dXJlPHZvaWQ+IHJ1bihIb29rQ29udGV4dCBjb250ZXh0KSBhc3luYyB7CiAgZmluYWwgcGFja2FnZVJvb3QgPSBjb250ZXh0LnZhcnNbJ3BhY2thZ2Utcm9vdCddIGFzIFN0cmluZzsKICBmaW5hbCB0ZXN0RGlyID0gRGlyZWN0b3J5KHBhdGguam9pbihwYWNrYWdlUm9vdCwgJ3Rlc3QnKSk7CgogIGlmICghdGVzdERpci5leGlzdHNTeW5jKCkpIHsKICAgIGNvbnRleHQubG9nZ2VyLmVycignQ291bGQgbm90IGZpbmQgZGlyZWN0b3J5ICR7dGVzdERpci5wYXRofScpOwogICAgZXhpdEZuKDEpOwogIH0KCiAgZmluYWwgcHVic3BlYyA9IEZpbGUocGF0aC5qb2luKHBhY2thZ2VSb290LCAncHVic3BlYy55YW1sJykpOwogIGlmICghcHVic3BlYy5leGlzdHNTeW5jKCkpIHsKICAgIGNvbnRleHQubG9nZ2VyLmVycignQ291bGQgbm90IGZpbmQgcHVic3BlYy55YW1sIGF0ICR7dGVzdERpci5wYXRofScpOwogICAgZXhpdEZuKDEpOwogIH0KCiAgZmluYWwgcHVic3BlY0NvbnRlbnRzID0gYXdhaXQgcHVic3BlYy5yZWFkQXNTdHJpbmcoKTsKICBmaW5hbCBmbHV0dGVyU2RrUmVnRXhwID0gUmVnRXhwKHInc2RrOlxzKmZsdXR0ZXIkJywgbXVsdGlMaW5lOiB0cnVlKTsKICBmaW5hbCBpc0ZsdXR0ZXIgPSBmbHV0dGVyU2RrUmVnRXhwLmhhc01hdGNoKHB1YnNwZWNDb250ZW50cyk7CgogIGZpbmFsIGlkZW50aWZpZXJHZW5lcmF0b3IgPSBEYXJ0SWRlbnRpZmllckdlbmVyYXRvcigpOwogIGZpbmFsIHRlc3RJZGVudGlmaWVyVGFibGUgPSA8TWFwPFN0cmluZywgU3RyaW5nPj5bXTsKICBmaW5hbCB0ZXN0cyA9IHRlc3REaXIKICAgICAgLmxpc3RTeW5jKHJlY3Vyc2l2ZTogdHJ1ZSkKICAgICAgLndoZXJlKChlbnRpdHkpID0+IGVudGl0eS5pc1Rlc3QpOwoKICBmaW5hbCBub3RPcHRpbWl6ZWRUZXN0cyA9IGF3YWl0IGdldE5vdE9wdGltaXplZFRlc3RzKHRlc3RzLCB0ZXN0RGlyLnBhdGgpOwoKICBmb3IgKGZpbmFsIGVudGl0eSBpbiB0ZXN0cykgewogICAgZmluYWwgcmVsYXRpdmVQYXRoID0gcGF0aAogICAgICAgIC5yZWxhdGl2ZShlbnRpdHkucGF0aCwgZnJvbTogdGVzdERpci5wYXRoKQogICAgICAgIC5yZXBsYWNlQWxsKHInXCcsICcvJyk7CiAgICB0ZXN0SWRlbnRpZmllclRhYmxlLmFkZCh7CiAgICAgICdwYXRoJzogcmVsYXRpdmVQYXRoLAogICAgICAnaWRlbnRpZmllcic6IGlkZW50aWZpZXJHZW5lcmF0b3IubmV4dCgpLAogICAgfSk7CiAgfQoKICBmaW5hbCBvcHRpbWl6ZWRUZXN0c0lkZW50aWZpZXJUYWJsZSA9IHRlc3RJZGVudGlmaWVyVGFibGUKICAgICAgLndoZXJlKChlKSA9PiAhbm90T3B0aW1pemVkVGVzdHMuY29udGFpbnMoZVsncGF0aCddKSkKICAgICAgLnRvTGlzdCgpOwoKICBjb250ZXh0LnZhcnMgPSB7CiAgICAndGVzdHMnOiBvcHRpbWl6ZWRUZXN0c0lkZW50aWZpZXJUYWJsZSwKICAgICdpc0ZsdXR0ZXInOiBpc0ZsdXR0ZXIsCiAgICAnbm90T3B0aW1pemVkVGVzdHMnOiBub3RPcHRpbWl6ZWRUZXN0cywKICB9Owp9CgpleHRlbnNpb24gb24gRmlsZVN5c3RlbUVudGl0eSB7CiAgYm9vbCBnZXQgaXNUZXN0IHsKICAgIHJldHVybiB0aGlzIGlzIEZpbGUgJiYgcGF0aC5iYXNlbmFtZSh0aGlzLnBhdGgpLmVuZHNXaXRoKCdfdGVzdC5kYXJ0Jyk7CiAgfQp9CgpGdXR1cmU8TGlzdDxTdHJpbmc+PiBnZXROb3RPcHRpbWl6ZWRUZXN0cygKICBJdGVyYWJsZTxGaWxlU3lzdGVtRW50aXR5PiB0ZXN0cywKICBTdHJpbmcgdGVzdERpciwKKSBhc3luYyB7CiAgZmluYWwgcGF0aHMgPSB0ZXN0cy5tYXAoKGUpID0+IGUucGF0aCkudG9MaXN0KCk7CiAgZmluYWwgZm9ybWF0dGVkUGF0aHMgPSBwYXRocy5tYXAoKGUpID0+IGUucmVwbGFjZUFsbCgnLy4vJywgJy8nKSkudG9MaXN0KCk7CgogIGZpbmFsIGZpbGVGdXR1cmVzID0gZm9ybWF0dGVkUGF0aHMubWFwKF9jaGVja0ZpbGVGb3JTa2lwVmVyeUdvb2RPcHRpbWl6YXRpb24pOwogIGZpbmFsIGZpbGVSZXN1bHRzID0gYXdhaXQgRnV0dXJlLndhaXQoZmlsZUZ1dHVyZXMpOwoKICBmaW5hbCB0ZXN0V2l0aFZlcnlHb29kVGVzdCA9IDxTdHJpbmc+W107CiAgZm9yICh2YXIgaSA9IDA7IGkgPCBmb3JtYXR0ZWRQYXRocy5sZW5ndGg7IGkrKykgewogICAgaWYgKGZpbGVSZXN1bHRzW2ldKSB7CiAgICAgIHRlc3RXaXRoVmVyeUdvb2RUZXN0LmFkZChmb3JtYXR0ZWRQYXRoc1tpXSk7CiAgICB9CiAgfQoKICAvLy8gRm9ybWF0IHRvIHJlbGF0aXZlIHBhdGgKICBmaW5hbCByZWxhdGl2ZVBhdGhzID0gdGVzdFdpdGhWZXJ5R29vZFRlc3QKICAgICAgLm1hcCgoZSkgPT4gcGF0aC5yZWxhdGl2ZShlLCBmcm9tOiB0ZXN0RGlyKSkKICAgICAgLnRvTGlzdCgpOwoKICByZXR1cm4gcmVsYXRpdmVQYXRoczsKfQoKLy8vIENoZWNrIGlmIGEgc2luZ2xlIGZpbGUgY29udGFpbnMgc2tpcF92ZXJ5X2dvb2Rfb3B0aW1pemF0aW9uIHRhZwpGdXR1cmU8Ym9vbD4gX2NoZWNrRmlsZUZvclNraXBWZXJ5R29vZE9wdGltaXphdGlvbihTdHJpbmcgcGF0aCkgYXN5bmMgewogIGZpbmFsIGZpbGUgPSBGaWxlKHBhdGgpOwogIGlmICghZmlsZS5leGlzdHNTeW5jKCkpIHJldHVybiBmYWxzZTsKICBmaW5hbCBjb250ZW50ID0gYXdhaXQgZmlsZS5yZWFkQXNTdHJpbmcoKTsKICByZXR1cm4gc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uUmVnRXhwLmhhc01hdGNoKGNvbnRlbnQpOwp9Cg==", + "data": "aW1wb3J0ICdkYXJ0OmlvJzsKCmltcG9ydCAncGFja2FnZTpob29rcy9kYXJ0X2lkZW50aWZpZXJfZ2VuZXJhdG9yLmRhcnQnOwppbXBvcnQgJ3BhY2thZ2U6bWFzb24vbWFzb24uZGFydCc7CmltcG9ydCAncGFja2FnZTpwYXRoL3BhdGguZGFydCcgYXMgcGF0aDsKCnR5cGVkZWYgRXhpdEZuID0gTmV2ZXIgRnVuY3Rpb24oaW50IGNvZGUpOwoKRXhpdEZuIGV4aXRGbiA9IGV4aXQ7CgpTdHJpbmcgc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uVGFnID0gJ3NraXBfdmVyeV9nb29kX29wdGltaXphdGlvbic7ClJlZ0V4cCBza2lwVmVyeUdvb2RPcHRpbWl6YXRpb25SZWdFeHAgPSBSZWdFeHAoCiAgIkBUYWdzXFxzKlxcKFxccypcXFtbXFxzXFxTXSo/W1wiJ10kc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uVGFnW1wiJ11bXFxzXFxTXSo/XFxdXFxzKlxcKSIsCiAgbXVsdGlMaW5lOiB0cnVlLAopOwoKRnV0dXJlPHZvaWQ+IHJ1bihIb29rQ29udGV4dCBjb250ZXh0KSBhc3luYyB7CiAgZmluYWwgcGFja2FnZVJvb3QgPSBjb250ZXh0LnZhcnNbJ3BhY2thZ2Utcm9vdCddIGFzIFN0cmluZzsKICBmaW5hbCB0ZXN0RGlyID0gRGlyZWN0b3J5KHBhdGguam9pbihwYWNrYWdlUm9vdCwgJ3Rlc3QnKSk7CgogIGlmICghdGVzdERpci5leGlzdHNTeW5jKCkpIHsKICAgIGNvbnRleHQubG9nZ2VyLmVycignQ291bGQgbm90IGZpbmQgZGlyZWN0b3J5ICR7dGVzdERpci5wYXRofScpOwogICAgZXhpdEZuKDEpOwogIH0KCiAgZmluYWwgcHVic3BlYyA9IEZpbGUocGF0aC5qb2luKHBhY2thZ2VSb290LCAncHVic3BlYy55YW1sJykpOwogIGlmICghcHVic3BlYy5leGlzdHNTeW5jKCkpIHsKICAgIGNvbnRleHQubG9nZ2VyLmVycignQ291bGQgbm90IGZpbmQgcHVic3BlYy55YW1sIGF0ICR7dGVzdERpci5wYXRofScpOwogICAgZXhpdEZuKDEpOwogIH0KCiAgZmluYWwgcHVic3BlY0NvbnRlbnRzID0gYXdhaXQgcHVic3BlYy5yZWFkQXNTdHJpbmcoKTsKICBmaW5hbCBmbHV0dGVyU2RrUmVnRXhwID0gUmVnRXhwKHInc2RrOlxzKmZsdXR0ZXIkJywgbXVsdGlMaW5lOiB0cnVlKTsKICBmaW5hbCBpc0ZsdXR0ZXIgPSBmbHV0dGVyU2RrUmVnRXhwLmhhc01hdGNoKHB1YnNwZWNDb250ZW50cyk7CgogIGZpbmFsIHNoYXJkSW5kZXggPSBjb250ZXh0LnZhcnNbJ3NoYXJkLWluZGV4J10gYXMgaW50PzsKICBmaW5hbCB0b3RhbFNoYXJkcyA9IGNvbnRleHQudmFyc1sndG90YWwtc2hhcmRzJ10gYXMgaW50PzsKCiAgLy8gVGhlIENMSSB2YWxpZGF0ZXMgdGhlc2UgYmVmb3JlIGl0IGdldHMgaGVyZSwgYnV0IGBtYXNvbiBtYWtlYCBwcm9tcHRzIGZvcgogIC8vIHRoZW0gZGlyZWN0bHksIHNvIGd1YXJkIHRoZSByb3VuZC1yb2JpbiBiZWxvdyBhZ2FpbnN0IHZhbHVlcyB0aGF0IHdvdWxkCiAgLy8gbmV2ZXIgdGVybWluYXRlIG9yIGluZGV4IG91dCBvZiByYW5nZS4KICBpZiAoc2hhcmRJbmRleCAhPSBudWxsICYmCiAgICAgIHRvdGFsU2hhcmRzICE9IG51bGwgJiYKICAgICAgKHRvdGFsU2hhcmRzIDwgMSB8fCBzaGFyZEluZGV4IDwgMSB8fCBzaGFyZEluZGV4ID4gdG90YWxTaGFyZHMpKSB7CiAgICBjb250ZXh0LmxvZ2dlci5lcnIoCiAgICAgICdzaGFyZC1pbmRleCBtdXN0IGJlIGJldHdlZW4gMSBhbmQgdG90YWwtc2hhcmRzLCBidXQgZ290ICcKICAgICAgJ3NoYXJkLWluZGV4ICRzaGFyZEluZGV4IGFuZCB0b3RhbC1zaGFyZHMgJHRvdGFsU2hhcmRzJywKICAgICk7CiAgICBleGl0Rm4oMSk7CiAgfQoKICBmaW5hbCB0ZXN0cyA9IHRlc3REaXIKICAgICAgLmxpc3RTeW5jKHJlY3Vyc2l2ZTogdHJ1ZSkKICAgICAgLndoZXJlKChlbnRpdHkpID0+IGVudGl0eS5pc1Rlc3QpOwoKICBmaW5hbCBub3RPcHRpbWl6ZWRUZXN0cyA9IChhd2FpdCBnZXROb3RPcHRpbWl6ZWRUZXN0cygKICAgIHRlc3RzLAogICAgdGVzdERpci5wYXRoLAogICkpLnRvU2V0KCk7CgogIC8vIFNvcnRpbmcgZ3VhcmFudGVlcyBhIGRldGVybWluaXN0aWMgb3JkZXIgYWNyb3NzIG1hY2hpbmVzLCB3aGljaCBpcyB3aGF0CiAgLy8gbWFrZXMgc2hhcmRpbmcgcmVwcm9kdWNpYmxlOiBgRGlyZWN0b3J5Lmxpc3RTeW5jYCBvcmRlciBpcyBmaWxlc3lzdGVtCiAgLy8gZGVwZW5kZW50LCBzbyB3aXRob3V0IHRoaXMgdHdvIHJ1bm5lcnMgY291bGQgZGlzYWdyZWUgb24gdGhlIHBhcnRpdGlvbgogIC8vIGFuZCBlaXRoZXIgc2tpcCBvciBkdXBsaWNhdGUgdGVzdHMuCiAgZmluYWwgdGVzdFBhdGhzID0KICAgICAgdGVzdHMKICAgICAgICAgIC5tYXAoCiAgICAgICAgICAgIChlbnRpdHkpID0+IHBhdGgKICAgICAgICAgICAgICAgIC5yZWxhdGl2ZShlbnRpdHkucGF0aCwgZnJvbTogdGVzdERpci5wYXRoKQogICAgICAgICAgICAgICAgLnJlcGxhY2VBbGwocidcJywgJy8nKSwKICAgICAgICAgICkKICAgICAgICAgIC50b0xpc3QoKQogICAgICAgIC4uc29ydCgpOwoKICAvLyBOb24gb3B0aW1pemVkIHRlc3RzIHJ1biBhcyBzdGFuZGFsb25lIGZpbGVzIGFsb25nc2lkZSB0aGUgb3B0aW1pemVyCiAgLy8gZW50cnlwb2ludCwgc28gdGhleSBhcmUgc2hhcmRlZCB0b28sIGFuZCBpbiB0aGUgc2FtZSBkZWFsIGFzIHRoZQogIC8vIG9wdGltaXplZCBvbmVzOiBkZWFsaW5nIG91dCBvbmUgbGlzdCBrZWVwcyBldmVyeSBzaGFyZCB3aXRoaW4gb25lIGZpbGUKICAvLyBvZiB0aGUgb3RoZXJzLCB3aGVyZWFzIGRlYWxpbmcgb3V0IHRoZSB0d28gbGlzdHMgc2VwYXJhdGVseSB3b3VsZCBoYW5kCiAgLy8gdGhlIGZpcnN0IHNoYXJkcyBhIGZpbGUgZnJvbSBlYWNoLgogIGZpbmFsIHNoYXJkUGF0aHMgPSBfc2hhcmRPZigKICAgIHRlc3RQYXRocywKICAgIHNoYXJkSW5kZXg6IHNoYXJkSW5kZXgsCiAgICB0b3RhbFNoYXJkczogdG90YWxTaGFyZHMsCiAgKTsKICBmaW5hbCBvcHRpbWl6ZWRUZXN0UGF0aHMgPSBzaGFyZFBhdGhzCiAgICAgIC53aGVyZSgocCkgPT4gIW5vdE9wdGltaXplZFRlc3RzLmNvbnRhaW5zKHApKQogICAgICAudG9MaXN0KCk7CiAgZmluYWwgc2hhcmRlZE5vdE9wdGltaXplZFRlc3RzID0gc2hhcmRQYXRocwogICAgICAud2hlcmUobm90T3B0aW1pemVkVGVzdHMuY29udGFpbnMpCiAgICAgIC50b0xpc3QoKTsKCiAgZmluYWwgaWRlbnRpZmllckdlbmVyYXRvciA9IERhcnRJZGVudGlmaWVyR2VuZXJhdG9yKCk7CiAgZmluYWwgb3B0aW1pemVkVGVzdHNJZGVudGlmaWVyVGFibGUgPSBbCiAgICBmb3IgKGZpbmFsIHJlbGF0aXZlUGF0aCBpbiBvcHRpbWl6ZWRUZXN0UGF0aHMpCiAgICAgIHsncGF0aCc6IHJlbGF0aXZlUGF0aCwgJ2lkZW50aWZpZXInOiBpZGVudGlmaWVyR2VuZXJhdG9yLm5leHQoKX0sCiAgXTsKCiAgY29udGV4dC52YXJzID0gewogICAgJ3Rlc3RzJzogb3B0aW1pemVkVGVzdHNJZGVudGlmaWVyVGFibGUsCiAgICAnaXNGbHV0dGVyJzogaXNGbHV0dGVyLAogICAgJ25vdE9wdGltaXplZFRlc3RzJzogc2hhcmRlZE5vdE9wdGltaXplZFRlc3RzLAogIH07Cn0KCi8vLyBSZXR1cm5zIHRoZSBzdWJzZXQgb2YgW3BhdGhzXSB0aGF0IGJlbG9uZ3MgdG8gdGhlIHNoYXJkIFtzaGFyZEluZGV4XSBvdXQgb2YKLy8vIFt0b3RhbFNoYXJkc10uCi8vLwovLy8gUmV0dXJucyBbcGF0aHNdIHVuY2hhbmdlZCB3aGVuIHNoYXJkaW5nIGlzIG5vdCBlbmFibGVkIChlaXRoZXIgdmFsdWUgaXMKLy8vIGBudWxsYCkuCi8vLwovLy8gRmlsZXMgYXJlIGRlYWx0IG91dCByb3VuZC1yb2JpbiAoaW5kZXggbW9kdWxvIFt0b3RhbFNoYXJkc10pIG92ZXIgdGhlCi8vLyBhbHJlYWR5IHNvcnRlZCBbcGF0aHNdLCB3aGljaCBrZWVwcyBzaGFyZHMgYmFsYW5jZWQgaW4gZmlsZSBjb3VudCBhbmQgbWFrZXMKLy8vIHRoZSBwYXJ0aXRpb24gc3RhYmxlIGZvciBhIGdpdmVuIHRlc3Qgc3VpdGUuCkxpc3Q8U3RyaW5nPiBfc2hhcmRPZigKICBMaXN0PFN0cmluZz4gcGF0aHMsIHsKICByZXF1aXJlZCBpbnQ/IHNoYXJkSW5kZXgsCiAgcmVxdWlyZWQgaW50PyB0b3RhbFNoYXJkcywKfSkgewogIGlmIChzaGFyZEluZGV4ID09IG51bGwgfHwgdG90YWxTaGFyZHMgPT0gbnVsbCkgcmV0dXJuIHBhdGhzOwoKICByZXR1cm4gWwogICAgZm9yICh2YXIgaSA9IHNoYXJkSW5kZXggLSAxOyBpIDwgcGF0aHMubGVuZ3RoOyBpICs9IHRvdGFsU2hhcmRzKSBwYXRoc1tpXSwKICBdOwp9CgpleHRlbnNpb24gb24gRmlsZVN5c3RlbUVudGl0eSB7CiAgYm9vbCBnZXQgaXNUZXN0IHsKICAgIHJldHVybiB0aGlzIGlzIEZpbGUgJiYgcGF0aC5iYXNlbmFtZSh0aGlzLnBhdGgpLmVuZHNXaXRoKCdfdGVzdC5kYXJ0Jyk7CiAgfQp9CgpGdXR1cmU8TGlzdDxTdHJpbmc+PiBnZXROb3RPcHRpbWl6ZWRUZXN0cygKICBJdGVyYWJsZTxGaWxlU3lzdGVtRW50aXR5PiB0ZXN0cywKICBTdHJpbmcgdGVzdERpciwKKSBhc3luYyB7CiAgZmluYWwgcGF0aHMgPSB0ZXN0cy5tYXAoKGUpID0+IGUucGF0aCkudG9MaXN0KCk7CiAgZmluYWwgZm9ybWF0dGVkUGF0aHMgPSBwYXRocy5tYXAoKGUpID0+IGUucmVwbGFjZUFsbCgnLy4vJywgJy8nKSkudG9MaXN0KCk7CgogIGZpbmFsIGZpbGVGdXR1cmVzID0gZm9ybWF0dGVkUGF0aHMubWFwKF9jaGVja0ZpbGVGb3JTa2lwVmVyeUdvb2RPcHRpbWl6YXRpb24pOwogIGZpbmFsIGZpbGVSZXN1bHRzID0gYXdhaXQgRnV0dXJlLndhaXQoZmlsZUZ1dHVyZXMpOwoKICBmaW5hbCB0ZXN0V2l0aFZlcnlHb29kVGVzdCA9IDxTdHJpbmc+W107CiAgZm9yICh2YXIgaSA9IDA7IGkgPCBmb3JtYXR0ZWRQYXRocy5sZW5ndGg7IGkrKykgewogICAgaWYgKGZpbGVSZXN1bHRzW2ldKSB7CiAgICAgIHRlc3RXaXRoVmVyeUdvb2RUZXN0LmFkZChmb3JtYXR0ZWRQYXRoc1tpXSk7CiAgICB9CiAgfQoKICAvLy8gRm9ybWF0IHRvIHJlbGF0aXZlIHBhdGgsIG5vcm1hbGl6aW5nIHNlcGFyYXRvcnMgc28gdGhlIHBhdGhzIGNvbXBhcmUKICAvLy8gZXF1YWwgdG8gdGhlIG9uZXMgYnVpbHQgaW4gW3J1bl0gb24gV2luZG93cyB0b28uCiAgZmluYWwgcmVsYXRpdmVQYXRocyA9IHRlc3RXaXRoVmVyeUdvb2RUZXN0CiAgICAgIC5tYXAoKGUpID0+IHBhdGgucmVsYXRpdmUoZSwgZnJvbTogdGVzdERpcikucmVwbGFjZUFsbChyJ1wnLCAnLycpKQogICAgICAudG9MaXN0KCk7CgogIHJldHVybiByZWxhdGl2ZVBhdGhzOwp9CgovLy8gQ2hlY2sgaWYgYSBzaW5nbGUgZmlsZSBjb250YWlucyBza2lwX3ZlcnlfZ29vZF9vcHRpbWl6YXRpb24gdGFnCkZ1dHVyZTxib29sPiBfY2hlY2tGaWxlRm9yU2tpcFZlcnlHb29kT3B0aW1pemF0aW9uKFN0cmluZyBwYXRoKSBhc3luYyB7CiAgZmluYWwgZmlsZSA9IEZpbGUocGF0aCk7CiAgaWYgKCFmaWxlLmV4aXN0c1N5bmMoKSkgcmV0dXJuIGZhbHNlOwogIGZpbmFsIGNvbnRlbnQgPSBhd2FpdCBmaWxlLnJlYWRBc1N0cmluZygpOwogIHJldHVybiBza2lwVmVyeUdvb2RPcHRpbWl6YXRpb25SZWdFeHAuaGFzTWF0Y2goY29udGVudCk7Cn0K", "type": "text", }, { @@ -40,7 +40,7 @@ final testOptimizerBundle = MasonBundle.fromJson({ }, { "path": "test/pre_gen_test.dart", - "data": "aW1wb3J0ICdkYXJ0OmlvJzsKCmltcG9ydCAncGFja2FnZTpob29rcy9wcmVfZ2VuLmRhcnQnIGFzIHByZV9nZW47CmltcG9ydCAncGFja2FnZTptYXNvbi9tYXNvbi5kYXJ0JzsKaW1wb3J0ICdwYWNrYWdlOm1vY2t0YWlsL21vY2t0YWlsLmRhcnQnOwppbXBvcnQgJ3BhY2thZ2U6cGF0aC9wYXRoLmRhcnQnIGFzIHBhdGg7CmltcG9ydCAncGFja2FnZTp0ZXN0L3Rlc3QuZGFydCc7CgpjbGFzcyBfTW9ja0xvZ2dlciBleHRlbmRzIE1vY2sgaW1wbGVtZW50cyBMb2dnZXI7CgpjbGFzcyBfRmFrZUNvbnRleHQgZXh0ZW5kcyBGYWtlIGltcGxlbWVudHMgSG9va0NvbnRleHQgewogIEBvdmVycmlkZQogIGZpbmFsIGxvZ2dlciA9IF9Nb2NrTG9nZ2VyKCk7CgogIEBvdmVycmlkZQogIE1hcDxTdHJpbmcsIE9iamVjdD8+IHZhcnMgPSB7fTsKfQoKZmluYWwgbm90T3B0aW1pemVkVGVzdENvbnRlbnQgPQogICAgJycnCkBUYWdzKFsnJHtwcmVfZ2VuLnNraXBWZXJ5R29vZE9wdGltaXphdGlvblRhZ30nXSkKdm9pZCBtYWluKCkgewogIHRlc3QoJ3Rlc3QnLCAoKSB7CiAgICBleHBlY3QoMSwgMSk7CiAgfSk7Cn0KJycnOwoKZmluYWwgYW5vdGhlck5vdE9wdGltaXplZFRlc3RDb250ZW50ID0KICAgICcnJwpAVGFncyhbJyR7cHJlX2dlbi5za2lwVmVyeUdvb2RPcHRpbWl6YXRpb25UYWd9JywgJ2Fub3RoZXJfdGFnJ10pCnZvaWQgbWFpbigpIHsKICB0ZXN0KCdhbm90aGVyIHRlc3QnLCAoKSB7CiAgICBleHBlY3QoMSwgMSk7CiAgfSk7Cn0KJycnOwoKdm9pZCBtYWluKCkgewogIGxhdGUgRGlyZWN0b3J5IHRlbXBEaXJlY3Rvcnk7CgogIHNldFVwKCgpIHsKICAgIHRlbXBEaXJlY3RvcnkgPSBEaXJlY3Rvcnkuc3lzdGVtVGVtcC5jcmVhdGVUZW1wU3luYygndGVzdF9vcHRpbWl6ZXInKTsKICB9KTsKCiAgdGVhckRvd24oKCkgewogICAgdGVtcERpcmVjdG9yeS5kZWxldGVTeW5jKHJlY3Vyc2l2ZTogdHJ1ZSk7CiAgfSk7CgogIGdyb3VwKCdQcmUgZ2VuIGhvb2snLCAoKSB7CiAgICBsYXRlIEhvb2tDb250ZXh0IGNvbnRleHQ7CgogICAgc2V0VXAoKCkgewogICAgICBjb250ZXh0ID0gX0Zha2VDb250ZXh0KCk7CiAgICB9KTsKCiAgICBncm91cCgnQ29tcGxldGVzJywgKCkgewogICAgICB0ZXN0KCd3aXRoIHRlc3QgZmlsZXMgbGlzdCcsICgpIGFzeW5jIHsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZW1wRGlyZWN0b3J5LnBhdGgsICdwdWJzcGVjLnlhbWwnKSkuY3JlYXRlU3luYygpOwoKICAgICAgICBmaW5hbCB0ZXN0RGlyID0gRGlyZWN0b3J5KHBhdGguam9pbih0ZW1wRGlyZWN0b3J5LnBhdGgsICd0ZXN0JykpCiAgICAgICAgICAuLmNyZWF0ZVN5bmMoKTsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZXN0RGlyLnBhdGgsICd0ZXN0MV90ZXN0LmRhcnQnKSkuY3JlYXRlU3luYygpOwogICAgICAgIEZpbGUocGF0aC5qb2luKHRlc3REaXIucGF0aCwgJ3Rlc3QyX3Rlc3QuZGFydCcpKS5jcmVhdGVTeW5jKCk7CiAgICAgICAgRmlsZShwYXRoLmpvaW4odGVzdERpci5wYXRoLCAnbm9fdGVzdF9oZXJlLmRhcnQnKSkuY3JlYXRlU3luYygpOwoKICAgICAgICBjb250ZXh0LnZhcnNbJ3BhY2thZ2Utcm9vdCddID0gdGVtcERpcmVjdG9yeS5hYnNvbHV0ZS5wYXRoOwoKICAgICAgICBhd2FpdCBwcmVfZ2VuLnJ1bihjb250ZXh0KTsKCiAgICAgICAgZmluYWwgdGVzdHMgPSBjb250ZXh0LnZhcnNbJ3Rlc3RzJ10gYXMgTGlzdDxNYXA8U3RyaW5nLCBTdHJpbmc+PjsKICAgICAgICBmaW5hbCB0ZXN0c01hcCA9IDxTdHJpbmcsIFN0cmluZz57fTsKICAgICAgICBmb3IgKGZpbmFsIHRlc3QgaW4gdGVzdHMpIHsKICAgICAgICAgIGZpbmFsIHBhdGggPSB0ZXN0WydwYXRoJ10hOwogICAgICAgICAgZmluYWwgaWRlbnRpZmllciA9IHRlc3RbJ2lkZW50aWZpZXInXSE7CiAgICAgICAgICB0ZXN0c01hcFtwYXRoXSA9IGlkZW50aWZpZXI7CiAgICAgICAgfQoKICAgICAgICBmaW5hbCBwYXRocyA9IHRlc3RzTWFwLmtleXM7CiAgICAgICAgZXhwZWN0KHBhdGhzLCBjb250YWlucygndGVzdDFfdGVzdC5kYXJ0JykpOwogICAgICAgIGV4cGVjdChwYXRocywgY29udGFpbnMoJ3Rlc3QyX3Rlc3QuZGFydCcpKTsKICAgICAgICBleHBlY3QocGF0aHMsIGlzTm90KGNvbnRhaW5zKCdub190ZXN0X2hlcmUuZGFydCcpKSk7CgogICAgICAgIGV4cGVjdCgKICAgICAgICAgIHRlc3RzTWFwLnZhbHVlcy50b1NldCgpLmxlbmd0aCwKICAgICAgICAgIGVxdWFscyh0ZXN0cy5sZW5ndGgpLAogICAgICAgICAgcmVhc29uOiAnQWxsIHRlc3RzIGZpbGVzIHNob3VsZCBoYXZlIHVuaXF1ZSBpZGVudGlmaWVycycsCiAgICAgICAgKTsKCiAgICAgICAgZXhwZWN0KGNvbnRleHQudmFyc1snaXNGbHV0dGVyJ10sIGZhbHNlKTsKICAgICAgfSk7CgogICAgICB0ZXN0KCd3aXRoIHByb3BlciBpc0ZsdXR0ZXIgaWRlbnRpZmljYXRpb24nLCAoKSBhc3luYyB7CiAgICAgICAgRmlsZShwYXRoLmpvaW4odGVtcERpcmVjdG9yeS5wYXRoLCAncHVic3BlYy55YW1sJykpCiAgICAgICAgICAuLmNyZWF0ZVN5bmMoKQogICAgICAgICAgLi53cml0ZUFzU3RyaW5nU3luYygnJycKZGVwZW5kZW5jaWVzOgogIGZsdXR0ZXI6CiAgICBzZGs6IGZsdXR0ZXInJycpOwoKICAgICAgICBEaXJlY3RvcnkocGF0aC5qb2luKHRlbXBEaXJlY3RvcnkucGF0aCwgJ3Rlc3QnKSkuY3JlYXRlU3luYygpOwoKICAgICAgICBjb250ZXh0LnZhcnNbJ3BhY2thZ2Utcm9vdCddID0gdGVtcERpcmVjdG9yeS5hYnNvbHV0ZS5wYXRoOwoKICAgICAgICBhd2FpdCBwcmVfZ2VuLnJ1bihjb250ZXh0KTsKCiAgICAgICAgZXhwZWN0KGNvbnRleHQudmFyc1snaXNGbHV0dGVyJ10sIHRydWUpOwogICAgICB9KTsKCiAgICAgIHRlc3QoJ3dpdGggcHJvcGVyIG5vdCBvcHRpbWl6ZWQgdGVzdHMgaWRlbnRpZmljYXRpb24nLCAoKSBhc3luYyB7CiAgICAgICAgRmlsZShwYXRoLmpvaW4odGVtcERpcmVjdG9yeS5wYXRoLCAncHVic3BlYy55YW1sJykpLmNyZWF0ZVN5bmMoKTsKCiAgICAgICAgZmluYWwgdGVzdERpciA9IERpcmVjdG9yeShwYXRoLmpvaW4odGVtcERpcmVjdG9yeS5wYXRoLCAndGVzdCcpKQogICAgICAgICAgLi5jcmVhdGVTeW5jKCk7CiAgICAgICAgRmlsZShwYXRoLmpvaW4odGVzdERpci5wYXRoLCAndGVzdDFfdGVzdC5kYXJ0JykpLmNyZWF0ZVN5bmMoKTsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZXN0RGlyLnBhdGgsICd0ZXN0Ml90ZXN0LmRhcnQnKSkuY3JlYXRlU3luYygpOwogICAgICAgIEZpbGUocGF0aC5qb2luKHRlc3REaXIucGF0aCwgJ25vX3Rlc3RfaGVyZS5kYXJ0JykpLmNyZWF0ZVN5bmMoKTsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZXN0RGlyLnBhdGgsICdub3Rfb3B0aW1pemVkX3Rlc3QuZGFydCcpKQogICAgICAgICAgICAud3JpdGVBc1N0cmluZ1N5bmMobm90T3B0aW1pemVkVGVzdENvbnRlbnQpOwogICAgICAgIEZpbGUocGF0aC5qb2luKHRlc3REaXIucGF0aCwgJ2Fub3RoZXJfbm90X29wdGltaXplZF90ZXN0LmRhcnQnKSkKICAgICAgICAgICAgLndyaXRlQXNTdHJpbmdTeW5jKGFub3RoZXJOb3RPcHRpbWl6ZWRUZXN0Q29udGVudCk7CgogICAgICAgIGNvbnRleHQudmFyc1sncGFja2FnZS1yb290J10gPSB0ZW1wRGlyZWN0b3J5LmFic29sdXRlLnBhdGg7CgogICAgICAgIGF3YWl0IHByZV9nZW4ucnVuKGNvbnRleHQpOwoKICAgICAgICBmaW5hbCB0ZXN0cyA9IGNvbnRleHQudmFyc1sndGVzdHMnXSBhcyBMaXN0PE1hcDxTdHJpbmcsIFN0cmluZz4+OwogICAgICAgIGZpbmFsIHRlc3RzTWFwID0gPFN0cmluZywgU3RyaW5nPnt9OwogICAgICAgIGZvciAoZmluYWwgdGVzdCBpbiB0ZXN0cykgewogICAgICAgICAgZmluYWwgcGF0aCA9IHRlc3RbJ3BhdGgnXSE7CiAgICAgICAgICBmaW5hbCBpZGVudGlmaWVyID0gdGVzdFsnaWRlbnRpZmllciddITsKICAgICAgICAgIHRlc3RzTWFwW3BhdGhdID0gaWRlbnRpZmllcjsKICAgICAgICB9CgogICAgICAgIGZpbmFsIHBhdGhzID0gdGVzdHNNYXAua2V5czsKICAgICAgICBleHBlY3QocGF0aHMsIGNvbnRhaW5zKCd0ZXN0MV90ZXN0LmRhcnQnKSk7CiAgICAgICAgZXhwZWN0KHBhdGhzLCBjb250YWlucygndGVzdDJfdGVzdC5kYXJ0JykpOwogICAgICAgIGV4cGVjdChwYXRocywgaXNOb3QoY29udGFpbnMoJ25vX3Rlc3RfaGVyZS5kYXJ0JykpKTsKICAgICAgICBleHBlY3QocGF0aHMsIGlzTm90KGNvbnRhaW5zKCdub3Rfb3B0aW1pemVkX3Rlc3QuZGFydCcpKSk7CiAgICAgICAgZXhwZWN0KHBhdGhzLCBpc05vdChjb250YWlucygnYW5vdGhlcl9ub3Rfb3B0aW1pemVkX3Rlc3QuZGFydCcpKSk7CgogICAgICAgIGV4cGVjdCgKICAgICAgICAgIHRlc3RzTWFwLnZhbHVlcy50b1NldCgpLmxlbmd0aCwKICAgICAgICAgIGVxdWFscyh0ZXN0cy5sZW5ndGgpLAogICAgICAgICAgcmVhc29uOiAnQWxsIHRlc3RzIGZpbGVzIHNob3VsZCBoYXZlIHVuaXF1ZSBpZGVudGlmaWVycycsCiAgICAgICAgKTsKICAgICAgICBmaW5hbCBub3RPcHRpbWl6ZWRUZXN0cyA9CiAgICAgICAgICAgIGNvbnRleHQudmFyc1snbm90T3B0aW1pemVkVGVzdHMnXSBhcyBMaXN0PFN0cmluZz47CiAgICAgICAgZXhwZWN0KG5vdE9wdGltaXplZFRlc3RzLCBjb250YWlucygnbm90X29wdGltaXplZF90ZXN0LmRhcnQnKSk7CiAgICAgICAgZXhwZWN0KG5vdE9wdGltaXplZFRlc3RzLCBjb250YWlucygnYW5vdGhlcl9ub3Rfb3B0aW1pemVkX3Rlc3QuZGFydCcpKTsKICAgICAgfSk7CiAgICB9KTsKCiAgICBncm91cCgnRmFpbHMnLCAoKSB7CiAgICAgIHNldFVwKCgpIHsKICAgICAgICBwcmVfZ2VuLmV4aXRGbiA9IChjb2RlKSB7CiAgICAgICAgICB0aHJvdyBQcm9jZXNzRXhjZXB0aW9uKCdleGl0JywgW2NvZGUudG9TdHJpbmcoKV0pOwogICAgICAgIH07CiAgICAgIH0pOwoKICAgICAgdGVhckRvd24oKCkgewogICAgICAgIHByZV9nZW4uZXhpdEZuID0gZXhpdDsKICAgICAgfSk7CgogICAgICB0ZXN0KCd3aGVuIHRhcmdldCB0ZXN0IGRpciBkb2VzIG5vdCBleGlzdCcsICgpIGFzeW5jIHsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZW1wRGlyZWN0b3J5LnBhdGgsICdwdWJzcGVjLnlhbWwnKSkuY3JlYXRlU3luYygpOwoKICAgICAgICBmaW5hbCB0ZXN0RGlyID0gRGlyZWN0b3J5KHBhdGguam9pbih0ZW1wRGlyZWN0b3J5LnBhdGgsICd0ZXN0JykpOwoKICAgICAgICBjb250ZXh0LnZhcnNbJ3BhY2thZ2Utcm9vdCddID0gdGVtcERpcmVjdG9yeS5hYnNvbHV0ZS5wYXRoOwoKICAgICAgICBhd2FpdCBleHBlY3RMYXRlcigKICAgICAgICAgICgpID0+IHByZV9nZW4ucnVuKGNvbnRleHQpLAogICAgICAgICAgdGhyb3dzQSgKICAgICAgICAgICAgaXNBPFByb2Nlc3NFeGNlcHRpb24+KCkuaGF2aW5nKAogICAgICAgICAgICAgIChleCkgPT4gZXguYXJndW1lbnRzLmZpcnN0LAogICAgICAgICAgICAgICdlcnJvciBjb2RlJywKICAgICAgICAgICAgICBlcXVhbHMoJzEnKSwKICAgICAgICAgICAgKSwKICAgICAgICAgICksCiAgICAgICAgKTsKCiAgICAgICAgdmVyaWZ5KAogICAgICAgICAgKCkgPT4gY29udGV4dC5sb2dnZXIuZXJyKCdDb3VsZCBub3QgZmluZCBkaXJlY3RvcnkgJHt0ZXN0RGlyLnBhdGh9JyksCiAgICAgICAgKS5jYWxsZWQoMSk7CgogICAgICAgIGV4cGVjdChjb250ZXh0LnZhcnNbJ3Rlc3RzJ10sIGlzTnVsbCk7CiAgICAgICAgZXhwZWN0KGNvbnRleHQudmFyc1snaXNGbHV0dGVyJ10sIGlzTnVsbCk7CiAgICAgIH0pOwoKICAgICAgdGVzdCgnd2hlbiB0YXJnZXQgZGlyIGRvZXMgbm90IGNvbnRhaW4gYSBwdWJzcGVjLnlhbWwnLCAoKSBhc3luYyB7CiAgICAgICAgZmluYWwgdGVzdERpciA9IERpcmVjdG9yeShwYXRoLmpvaW4odGVtcERpcmVjdG9yeS5wYXRoLCAndGVzdCcpKQogICAgICAgICAgLi5jcmVhdGVTeW5jKCk7CiAgICAgICAgRmlsZShwYXRoLmpvaW4odGVzdERpci5wYXRoLCAndGVzdDFfdGVzdC5kYXJ0JykpLmNyZWF0ZVN5bmMoKTsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZXN0RGlyLnBhdGgsICd0ZXN0Ml90ZXN0LmRhcnQnKSkuY3JlYXRlU3luYygpOwogICAgICAgIEZpbGUocGF0aC5qb2luKHRlc3REaXIucGF0aCwgJ25vX3Rlc3RfaGVyZS5kYXJ0JykpLmNyZWF0ZVN5bmMoKTsKCiAgICAgICAgY29udGV4dC52YXJzWydwYWNrYWdlLXJvb3QnXSA9IHRlbXBEaXJlY3RvcnkuYWJzb2x1dGUucGF0aDsKCiAgICAgICAgYXdhaXQgZXhwZWN0TGF0ZXIoCiAgICAgICAgICAoKSA9PiBwcmVfZ2VuLnJ1bihjb250ZXh0KSwKICAgICAgICAgIHRocm93c0EoCiAgICAgICAgICAgIGlzQTxQcm9jZXNzRXhjZXB0aW9uPigpLmhhdmluZygKICAgICAgICAgICAgICAoZXgpID0+IGV4LmFyZ3VtZW50cy5maXJzdCwKICAgICAgICAgICAgICAnZXJyb3IgY29kZScsCiAgICAgICAgICAgICAgZXF1YWxzKCcxJyksCiAgICAgICAgICAgICksCiAgICAgICAgICApLAogICAgICAgICk7CgogICAgICAgIHZlcmlmeSgKICAgICAgICAgICgpID0+IGNvbnRleHQubG9nZ2VyLmVycigKICAgICAgICAgICAgJ0NvdWxkIG5vdCBmaW5kIHB1YnNwZWMueWFtbCBhdCAke3Rlc3REaXIucGF0aH0nLAogICAgICAgICAgKSwKICAgICAgICApLmNhbGxlZCgxKTsKCiAgICAgICAgZXhwZWN0KGNvbnRleHQudmFyc1sndGVzdHMnXSwgaXNOdWxsKTsKICAgICAgICBleHBlY3QoY29udGV4dC52YXJzWydpc0ZsdXR0ZXInXSwgaXNOdWxsKTsKICAgICAgfSk7CiAgICB9KTsKCiAgICBncm91cCgnc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uUmVnRXhwIHJlZ2V4JywgKCkgewogICAgICBmaW5hbCByZWdleCA9IHByZV9nZW4uc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uUmVnRXhwOwogICAgICB0ZXN0KCdtYXRjaGVzIHNpbmdsZS1saW5lIHRhZycsICgpIHsKICAgICAgICBmaW5hbCBjb250ZW50ID0gIkBUYWdzKFsnJHtwcmVfZ2VuLnNraXBWZXJ5R29vZE9wdGltaXphdGlvblRhZ30nXSkiOwogICAgICAgIGV4cGVjdChyZWdleC5oYXNNYXRjaChjb250ZW50KSwgaXNUcnVlKTsKICAgICAgfSk7CgogICAgICB0ZXN0KCdtYXRjaGVzIHNpbmdsZS1saW5lIHdpdGggbXVsdGlwbGUgdGFncycsICgpIHsKICAgICAgICBmaW5hbCBjb250ZW50ID0KICAgICAgICAgICAgIkBUYWdzKFsnJHtwcmVfZ2VuLnNraXBWZXJ5R29vZE9wdGltaXphdGlvblRhZ30nLCAnY2hyb21lJ10pIjsKICAgICAgICBleHBlY3QocmVnZXguaGFzTWF0Y2goY29udGVudCksIGlzVHJ1ZSk7CiAgICAgIH0pOwoKICAgICAgdGVzdCgnbWF0Y2hlcyBtdWx0aS1saW5lIHRhZyBsaXN0JywgKCkgewogICAgICAgIGZpbmFsIGNvbnRlbnQgPQogICAgICAgICAgICAnJycKICAgICAgQFRhZ3MoWwogICAgICAgICcke3ByZV9nZW4uc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uVGFnfScsCiAgICAgICAgJ2Nocm9tZScsCiAgICAgICAgJ3Rlc3QnLAogICAgICBdKQogICAgICAnJyc7CiAgICAgICAgZXhwZWN0KHJlZ2V4Lmhhc01hdGNoKGNvbnRlbnQpLCBpc1RydWUpOwogICAgICB9KTsKCiAgICAgIHRlc3QoJ21hdGNoZXMgbXVsdGktbGluZSB3aGVyZSB0YWcgaXMgbm90IHRoZSBmaXJzdCcsICgpIHsKICAgICAgICBmaW5hbCBjb250ZW50ID0KICAgICAgICAgICAgJycnCiAgICAgIEBUYWdzKFsKICAgICAgICAnY2hyb21lJywKICAgICAgICAnJHtwcmVfZ2VuLnNraXBWZXJ5R29vZE9wdGltaXphdGlvblRhZ30nLAogICAgICAgICd0ZXN0JywKICAgICAgXSkKICAgICAgJycnOwogICAgICAgIGV4cGVjdChyZWdleC5oYXNNYXRjaChjb250ZW50KSwgaXNUcnVlKTsKICAgICAgfSk7CgogICAgICB0ZXN0KCdkb2VzIG5vdCBtYXRjaCB3aGVuIHRhZyBtaXNzaW5nJywgKCkgewogICAgICAgIGNvbnN0IGNvbnRlbnQgPSAiQFRhZ3MoWydjaHJvbWUnLCAndGVzdCddKSI7CiAgICAgICAgZXhwZWN0KHJlZ2V4Lmhhc01hdGNoKGNvbnRlbnQpLCBpc0ZhbHNlKTsKICAgICAgfSk7CgogICAgICB0ZXN0KAogICAgICAgICdkb2VzIG5vdCBtYXRjaCBzdWJzdHJpbmcgb25seSAoZS5nLiBza2lwX3ZlcnlfZ29vZF9vcHRpbWl6YXRpb24sdGVzdCknLAogICAgICAgICgpIHsKICAgICAgICAgIGZpbmFsIGNvbnRlbnQgPQogICAgICAgICAgICAgICcnJwogICAgICBAVGFncyhbCiAgICAgICAgJyR7cHJlX2dlbi5za2lwVmVyeUdvb2RPcHRpbWl6YXRpb25UYWd9LHRlc3QnLAogICAgICAgICdjaHJvbWUnLAogICAgICBdKQogICAgICAnJyc7CiAgICAgICAgICBleHBlY3QoCiAgICAgICAgICAgIHJlZ2V4Lmhhc01hdGNoKGNvbnRlbnQpLAogICAgICAgICAgICBpc0ZhbHNlLAogICAgICAgICAgKTsgLy8gb25seSBleGFjdCB0YWcgc2hvdWxkIG1hdGNoCiAgICAgICAgfSwKICAgICAgKTsKICAgIH0pOwogIH0pOwp9Cg==", + "data": "aW1wb3J0ICdkYXJ0OmlvJzsKaW1wb3J0ICdkYXJ0Om1hdGgnOwoKaW1wb3J0ICdwYWNrYWdlOmhvb2tzL3ByZV9nZW4uZGFydCcgYXMgcHJlX2dlbjsKaW1wb3J0ICdwYWNrYWdlOm1hc29uL21hc29uLmRhcnQnOwppbXBvcnQgJ3BhY2thZ2U6bW9ja3RhaWwvbW9ja3RhaWwuZGFydCc7CmltcG9ydCAncGFja2FnZTpwYXRoL3BhdGguZGFydCcgYXMgcGF0aDsKaW1wb3J0ICdwYWNrYWdlOnRlc3QvdGVzdC5kYXJ0JzsKCmNsYXNzIF9Nb2NrTG9nZ2VyIGV4dGVuZHMgTW9jayBpbXBsZW1lbnRzIExvZ2dlcjsKCmNsYXNzIF9GYWtlQ29udGV4dCBleHRlbmRzIEZha2UgaW1wbGVtZW50cyBIb29rQ29udGV4dCB7CiAgQG92ZXJyaWRlCiAgZmluYWwgbG9nZ2VyID0gX01vY2tMb2dnZXIoKTsKCiAgQG92ZXJyaWRlCiAgTWFwPFN0cmluZywgT2JqZWN0Pz4gdmFycyA9IHt9Owp9CgpmaW5hbCBub3RPcHRpbWl6ZWRUZXN0Q29udGVudCA9CiAgICAnJycKQFRhZ3MoWycke3ByZV9nZW4uc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uVGFnfSddKQp2b2lkIG1haW4oKSB7CiAgdGVzdCgndGVzdCcsICgpIHsKICAgIGV4cGVjdCgxLCAxKTsKICB9KTsKfQonJyc7CgpmaW5hbCBhbm90aGVyTm90T3B0aW1pemVkVGVzdENvbnRlbnQgPQogICAgJycnCkBUYWdzKFsnJHtwcmVfZ2VuLnNraXBWZXJ5R29vZE9wdGltaXphdGlvblRhZ30nLCAnYW5vdGhlcl90YWcnXSkKdm9pZCBtYWluKCkgewogIHRlc3QoJ2Fub3RoZXIgdGVzdCcsICgpIHsKICAgIGV4cGVjdCgxLCAxKTsKICB9KTsKfQonJyc7Cgp2b2lkIG1haW4oKSB7CiAgbGF0ZSBEaXJlY3RvcnkgdGVtcERpcmVjdG9yeTsKCiAgc2V0VXAoKCkgewogICAgdGVtcERpcmVjdG9yeSA9IERpcmVjdG9yeS5zeXN0ZW1UZW1wLmNyZWF0ZVRlbXBTeW5jKCd0ZXN0X29wdGltaXplcicpOwogIH0pOwoKICB0ZWFyRG93bigoKSB7CiAgICB0ZW1wRGlyZWN0b3J5LmRlbGV0ZVN5bmMocmVjdXJzaXZlOiB0cnVlKTsKICB9KTsKCiAgZ3JvdXAoJ1ByZSBnZW4gaG9vaycsICgpIHsKICAgIGxhdGUgSG9va0NvbnRleHQgY29udGV4dDsKCiAgICBzZXRVcCgoKSB7CiAgICAgIGNvbnRleHQgPSBfRmFrZUNvbnRleHQoKTsKICAgIH0pOwoKICAgIGdyb3VwKCdDb21wbGV0ZXMnLCAoKSB7CiAgICAgIHRlc3QoJ3dpdGggdGVzdCBmaWxlcyBsaXN0JywgKCkgYXN5bmMgewogICAgICAgIEZpbGUocGF0aC5qb2luKHRlbXBEaXJlY3RvcnkucGF0aCwgJ3B1YnNwZWMueWFtbCcpKS5jcmVhdGVTeW5jKCk7CgogICAgICAgIGZpbmFsIHRlc3REaXIgPSBEaXJlY3RvcnkocGF0aC5qb2luKHRlbXBEaXJlY3RvcnkucGF0aCwgJ3Rlc3QnKSkKICAgICAgICAgIC4uY3JlYXRlU3luYygpOwogICAgICAgIEZpbGUocGF0aC5qb2luKHRlc3REaXIucGF0aCwgJ3Rlc3QxX3Rlc3QuZGFydCcpKS5jcmVhdGVTeW5jKCk7CiAgICAgICAgRmlsZShwYXRoLmpvaW4odGVzdERpci5wYXRoLCAndGVzdDJfdGVzdC5kYXJ0JykpLmNyZWF0ZVN5bmMoKTsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZXN0RGlyLnBhdGgsICdub190ZXN0X2hlcmUuZGFydCcpKS5jcmVhdGVTeW5jKCk7CgogICAgICAgIGNvbnRleHQudmFyc1sncGFja2FnZS1yb290J10gPSB0ZW1wRGlyZWN0b3J5LmFic29sdXRlLnBhdGg7CgogICAgICAgIGF3YWl0IHByZV9nZW4ucnVuKGNvbnRleHQpOwoKICAgICAgICBmaW5hbCB0ZXN0cyA9IGNvbnRleHQudmFyc1sndGVzdHMnXSBhcyBMaXN0PE1hcDxTdHJpbmcsIFN0cmluZz4+OwogICAgICAgIGZpbmFsIHRlc3RzTWFwID0gPFN0cmluZywgU3RyaW5nPnt9OwogICAgICAgIGZvciAoZmluYWwgdGVzdCBpbiB0ZXN0cykgewogICAgICAgICAgZmluYWwgcGF0aCA9IHRlc3RbJ3BhdGgnXSE7CiAgICAgICAgICBmaW5hbCBpZGVudGlmaWVyID0gdGVzdFsnaWRlbnRpZmllciddITsKICAgICAgICAgIHRlc3RzTWFwW3BhdGhdID0gaWRlbnRpZmllcjsKICAgICAgICB9CgogICAgICAgIGZpbmFsIHBhdGhzID0gdGVzdHNNYXAua2V5czsKICAgICAgICBleHBlY3QocGF0aHMsIGNvbnRhaW5zKCd0ZXN0MV90ZXN0LmRhcnQnKSk7CiAgICAgICAgZXhwZWN0KHBhdGhzLCBjb250YWlucygndGVzdDJfdGVzdC5kYXJ0JykpOwogICAgICAgIGV4cGVjdChwYXRocywgaXNOb3QoY29udGFpbnMoJ25vX3Rlc3RfaGVyZS5kYXJ0JykpKTsKCiAgICAgICAgZXhwZWN0KAogICAgICAgICAgdGVzdHNNYXAudmFsdWVzLnRvU2V0KCkubGVuZ3RoLAogICAgICAgICAgZXF1YWxzKHRlc3RzLmxlbmd0aCksCiAgICAgICAgICByZWFzb246ICdBbGwgdGVzdHMgZmlsZXMgc2hvdWxkIGhhdmUgdW5pcXVlIGlkZW50aWZpZXJzJywKICAgICAgICApOwoKICAgICAgICBleHBlY3QoY29udGV4dC52YXJzWydpc0ZsdXR0ZXInXSwgZmFsc2UpOwogICAgICB9KTsKCiAgICAgIHRlc3QoJ3dpdGggcHJvcGVyIGlzRmx1dHRlciBpZGVudGlmaWNhdGlvbicsICgpIGFzeW5jIHsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZW1wRGlyZWN0b3J5LnBhdGgsICdwdWJzcGVjLnlhbWwnKSkKICAgICAgICAgIC4uY3JlYXRlU3luYygpCiAgICAgICAgICAuLndyaXRlQXNTdHJpbmdTeW5jKCcnJwpkZXBlbmRlbmNpZXM6CiAgZmx1dHRlcjoKICAgIHNkazogZmx1dHRlcicnJyk7CgogICAgICAgIERpcmVjdG9yeShwYXRoLmpvaW4odGVtcERpcmVjdG9yeS5wYXRoLCAndGVzdCcpKS5jcmVhdGVTeW5jKCk7CgogICAgICAgIGNvbnRleHQudmFyc1sncGFja2FnZS1yb290J10gPSB0ZW1wRGlyZWN0b3J5LmFic29sdXRlLnBhdGg7CgogICAgICAgIGF3YWl0IHByZV9nZW4ucnVuKGNvbnRleHQpOwoKICAgICAgICBleHBlY3QoY29udGV4dC52YXJzWydpc0ZsdXR0ZXInXSwgdHJ1ZSk7CiAgICAgIH0pOwoKICAgICAgdGVzdCgnd2l0aCBwcm9wZXIgbm90IG9wdGltaXplZCB0ZXN0cyBpZGVudGlmaWNhdGlvbicsICgpIGFzeW5jIHsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZW1wRGlyZWN0b3J5LnBhdGgsICdwdWJzcGVjLnlhbWwnKSkuY3JlYXRlU3luYygpOwoKICAgICAgICBmaW5hbCB0ZXN0RGlyID0gRGlyZWN0b3J5KHBhdGguam9pbih0ZW1wRGlyZWN0b3J5LnBhdGgsICd0ZXN0JykpCiAgICAgICAgICAuLmNyZWF0ZVN5bmMoKTsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZXN0RGlyLnBhdGgsICd0ZXN0MV90ZXN0LmRhcnQnKSkuY3JlYXRlU3luYygpOwogICAgICAgIEZpbGUocGF0aC5qb2luKHRlc3REaXIucGF0aCwgJ3Rlc3QyX3Rlc3QuZGFydCcpKS5jcmVhdGVTeW5jKCk7CiAgICAgICAgRmlsZShwYXRoLmpvaW4odGVzdERpci5wYXRoLCAnbm9fdGVzdF9oZXJlLmRhcnQnKSkuY3JlYXRlU3luYygpOwogICAgICAgIEZpbGUocGF0aC5qb2luKHRlc3REaXIucGF0aCwgJ25vdF9vcHRpbWl6ZWRfdGVzdC5kYXJ0JykpCiAgICAgICAgICAgIC53cml0ZUFzU3RyaW5nU3luYyhub3RPcHRpbWl6ZWRUZXN0Q29udGVudCk7CiAgICAgICAgRmlsZShwYXRoLmpvaW4odGVzdERpci5wYXRoLCAnYW5vdGhlcl9ub3Rfb3B0aW1pemVkX3Rlc3QuZGFydCcpKQogICAgICAgICAgICAud3JpdGVBc1N0cmluZ1N5bmMoYW5vdGhlck5vdE9wdGltaXplZFRlc3RDb250ZW50KTsKCiAgICAgICAgY29udGV4dC52YXJzWydwYWNrYWdlLXJvb3QnXSA9IHRlbXBEaXJlY3RvcnkuYWJzb2x1dGUucGF0aDsKCiAgICAgICAgYXdhaXQgcHJlX2dlbi5ydW4oY29udGV4dCk7CgogICAgICAgIGZpbmFsIHRlc3RzID0gY29udGV4dC52YXJzWyd0ZXN0cyddIGFzIExpc3Q8TWFwPFN0cmluZywgU3RyaW5nPj47CiAgICAgICAgZmluYWwgdGVzdHNNYXAgPSA8U3RyaW5nLCBTdHJpbmc+e307CiAgICAgICAgZm9yIChmaW5hbCB0ZXN0IGluIHRlc3RzKSB7CiAgICAgICAgICBmaW5hbCBwYXRoID0gdGVzdFsncGF0aCddITsKICAgICAgICAgIGZpbmFsIGlkZW50aWZpZXIgPSB0ZXN0WydpZGVudGlmaWVyJ10hOwogICAgICAgICAgdGVzdHNNYXBbcGF0aF0gPSBpZGVudGlmaWVyOwogICAgICAgIH0KCiAgICAgICAgZmluYWwgcGF0aHMgPSB0ZXN0c01hcC5rZXlzOwogICAgICAgIGV4cGVjdChwYXRocywgY29udGFpbnMoJ3Rlc3QxX3Rlc3QuZGFydCcpKTsKICAgICAgICBleHBlY3QocGF0aHMsIGNvbnRhaW5zKCd0ZXN0Ml90ZXN0LmRhcnQnKSk7CiAgICAgICAgZXhwZWN0KHBhdGhzLCBpc05vdChjb250YWlucygnbm9fdGVzdF9oZXJlLmRhcnQnKSkpOwogICAgICAgIGV4cGVjdChwYXRocywgaXNOb3QoY29udGFpbnMoJ25vdF9vcHRpbWl6ZWRfdGVzdC5kYXJ0JykpKTsKICAgICAgICBleHBlY3QocGF0aHMsIGlzTm90KGNvbnRhaW5zKCdhbm90aGVyX25vdF9vcHRpbWl6ZWRfdGVzdC5kYXJ0JykpKTsKCiAgICAgICAgZXhwZWN0KAogICAgICAgICAgdGVzdHNNYXAudmFsdWVzLnRvU2V0KCkubGVuZ3RoLAogICAgICAgICAgZXF1YWxzKHRlc3RzLmxlbmd0aCksCiAgICAgICAgICByZWFzb246ICdBbGwgdGVzdHMgZmlsZXMgc2hvdWxkIGhhdmUgdW5pcXVlIGlkZW50aWZpZXJzJywKICAgICAgICApOwogICAgICAgIGZpbmFsIG5vdE9wdGltaXplZFRlc3RzID0KICAgICAgICAgICAgY29udGV4dC52YXJzWydub3RPcHRpbWl6ZWRUZXN0cyddIGFzIExpc3Q8U3RyaW5nPjsKICAgICAgICBleHBlY3Qobm90T3B0aW1pemVkVGVzdHMsIGNvbnRhaW5zKCdub3Rfb3B0aW1pemVkX3Rlc3QuZGFydCcpKTsKICAgICAgICBleHBlY3Qobm90T3B0aW1pemVkVGVzdHMsIGNvbnRhaW5zKCdhbm90aGVyX25vdF9vcHRpbWl6ZWRfdGVzdC5kYXJ0JykpOwogICAgICB9KTsKICAgIH0pOwoKICAgIGdyb3VwKCdGYWlscycsICgpIHsKICAgICAgc2V0VXAoKCkgewogICAgICAgIHByZV9nZW4uZXhpdEZuID0gKGNvZGUpIHsKICAgICAgICAgIHRocm93IFByb2Nlc3NFeGNlcHRpb24oJ2V4aXQnLCBbY29kZS50b1N0cmluZygpXSk7CiAgICAgICAgfTsKICAgICAgfSk7CgogICAgICB0ZWFyRG93bigoKSB7CiAgICAgICAgcHJlX2dlbi5leGl0Rm4gPSBleGl0OwogICAgICB9KTsKCiAgICAgIHRlc3QoJ3doZW4gdGFyZ2V0IHRlc3QgZGlyIGRvZXMgbm90IGV4aXN0JywgKCkgYXN5bmMgewogICAgICAgIEZpbGUocGF0aC5qb2luKHRlbXBEaXJlY3RvcnkucGF0aCwgJ3B1YnNwZWMueWFtbCcpKS5jcmVhdGVTeW5jKCk7CgogICAgICAgIGZpbmFsIHRlc3REaXIgPSBEaXJlY3RvcnkocGF0aC5qb2luKHRlbXBEaXJlY3RvcnkucGF0aCwgJ3Rlc3QnKSk7CgogICAgICAgIGNvbnRleHQudmFyc1sncGFja2FnZS1yb290J10gPSB0ZW1wRGlyZWN0b3J5LmFic29sdXRlLnBhdGg7CgogICAgICAgIGF3YWl0IGV4cGVjdExhdGVyKAogICAgICAgICAgKCkgPT4gcHJlX2dlbi5ydW4oY29udGV4dCksCiAgICAgICAgICB0aHJvd3NBKAogICAgICAgICAgICBpc0E8UHJvY2Vzc0V4Y2VwdGlvbj4oKS5oYXZpbmcoCiAgICAgICAgICAgICAgKGV4KSA9PiBleC5hcmd1bWVudHMuZmlyc3QsCiAgICAgICAgICAgICAgJ2Vycm9yIGNvZGUnLAogICAgICAgICAgICAgIGVxdWFscygnMScpLAogICAgICAgICAgICApLAogICAgICAgICAgKSwKICAgICAgICApOwoKICAgICAgICB2ZXJpZnkoCiAgICAgICAgICAoKSA9PiBjb250ZXh0LmxvZ2dlci5lcnIoJ0NvdWxkIG5vdCBmaW5kIGRpcmVjdG9yeSAke3Rlc3REaXIucGF0aH0nKSwKICAgICAgICApLmNhbGxlZCgxKTsKCiAgICAgICAgZXhwZWN0KGNvbnRleHQudmFyc1sndGVzdHMnXSwgaXNOdWxsKTsKICAgICAgICBleHBlY3QoY29udGV4dC52YXJzWydpc0ZsdXR0ZXInXSwgaXNOdWxsKTsKICAgICAgfSk7CgogICAgICB0ZXN0KCd3aGVuIHRoZSBzaGFyZCB2YWx1ZXMgYXJlIG91dCBvZiByYW5nZScsICgpIGFzeW5jIHsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZW1wRGlyZWN0b3J5LnBhdGgsICdwdWJzcGVjLnlhbWwnKSkuY3JlYXRlU3luYygpOwogICAgICAgIERpcmVjdG9yeShwYXRoLmpvaW4odGVtcERpcmVjdG9yeS5wYXRoLCAndGVzdCcpKS5jcmVhdGVTeW5jKCk7CgogICAgICAgIGNvbnRleHQudmFyc1sncGFja2FnZS1yb290J10gPSB0ZW1wRGlyZWN0b3J5LmFic29sdXRlLnBhdGg7CiAgICAgICAgY29udGV4dC52YXJzWydzaGFyZC1pbmRleCddID0gMTsKICAgICAgICBjb250ZXh0LnZhcnNbJ3RvdGFsLXNoYXJkcyddID0gMDsKCiAgICAgICAgYXdhaXQgZXhwZWN0TGF0ZXIoCiAgICAgICAgICAoKSA9PiBwcmVfZ2VuLnJ1bihjb250ZXh0KSwKICAgICAgICAgIHRocm93c0EoCiAgICAgICAgICAgIGlzQTxQcm9jZXNzRXhjZXB0aW9uPigpLmhhdmluZygKICAgICAgICAgICAgICAoZXgpID0+IGV4LmFyZ3VtZW50cy5maXJzdCwKICAgICAgICAgICAgICAnZXJyb3IgY29kZScsCiAgICAgICAgICAgICAgZXF1YWxzKCcxJyksCiAgICAgICAgICAgICksCiAgICAgICAgICApLAogICAgICAgICk7CgogICAgICAgIHZlcmlmeSgKICAgICAgICAgICgpID0+IGNvbnRleHQubG9nZ2VyLmVycigKICAgICAgICAgICAgJ3NoYXJkLWluZGV4IG11c3QgYmUgYmV0d2VlbiAxIGFuZCB0b3RhbC1zaGFyZHMsIGJ1dCBnb3QgJwogICAgICAgICAgICAnc2hhcmQtaW5kZXggMSBhbmQgdG90YWwtc2hhcmRzIDAnLAogICAgICAgICAgKSwKICAgICAgICApLmNhbGxlZCgxKTsKICAgICAgfSk7CgogICAgICB0ZXN0KCd3aGVuIHRhcmdldCBkaXIgZG9lcyBub3QgY29udGFpbiBhIHB1YnNwZWMueWFtbCcsICgpIGFzeW5jIHsKICAgICAgICBmaW5hbCB0ZXN0RGlyID0gRGlyZWN0b3J5KHBhdGguam9pbih0ZW1wRGlyZWN0b3J5LnBhdGgsICd0ZXN0JykpCiAgICAgICAgICAuLmNyZWF0ZVN5bmMoKTsKICAgICAgICBGaWxlKHBhdGguam9pbih0ZXN0RGlyLnBhdGgsICd0ZXN0MV90ZXN0LmRhcnQnKSkuY3JlYXRlU3luYygpOwogICAgICAgIEZpbGUocGF0aC5qb2luKHRlc3REaXIucGF0aCwgJ3Rlc3QyX3Rlc3QuZGFydCcpKS5jcmVhdGVTeW5jKCk7CiAgICAgICAgRmlsZShwYXRoLmpvaW4odGVzdERpci5wYXRoLCAnbm9fdGVzdF9oZXJlLmRhcnQnKSkuY3JlYXRlU3luYygpOwoKICAgICAgICBjb250ZXh0LnZhcnNbJ3BhY2thZ2Utcm9vdCddID0gdGVtcERpcmVjdG9yeS5hYnNvbHV0ZS5wYXRoOwoKICAgICAgICBhd2FpdCBleHBlY3RMYXRlcigKICAgICAgICAgICgpID0+IHByZV9nZW4ucnVuKGNvbnRleHQpLAogICAgICAgICAgdGhyb3dzQSgKICAgICAgICAgICAgaXNBPFByb2Nlc3NFeGNlcHRpb24+KCkuaGF2aW5nKAogICAgICAgICAgICAgIChleCkgPT4gZXguYXJndW1lbnRzLmZpcnN0LAogICAgICAgICAgICAgICdlcnJvciBjb2RlJywKICAgICAgICAgICAgICBlcXVhbHMoJzEnKSwKICAgICAgICAgICAgKSwKICAgICAgICAgICksCiAgICAgICAgKTsKCiAgICAgICAgdmVyaWZ5KAogICAgICAgICAgKCkgPT4gY29udGV4dC5sb2dnZXIuZXJyKAogICAgICAgICAgICAnQ291bGQgbm90IGZpbmQgcHVic3BlYy55YW1sIGF0ICR7dGVzdERpci5wYXRofScsCiAgICAgICAgICApLAogICAgICAgICkuY2FsbGVkKDEpOwoKICAgICAgICBleHBlY3QoY29udGV4dC52YXJzWyd0ZXN0cyddLCBpc051bGwpOwogICAgICAgIGV4cGVjdChjb250ZXh0LnZhcnNbJ2lzRmx1dHRlciddLCBpc051bGwpOwogICAgICB9KTsKICAgIH0pOwoKICAgIGdyb3VwKCdza2lwVmVyeUdvb2RPcHRpbWl6YXRpb25SZWdFeHAgcmVnZXgnLCAoKSB7CiAgICAgIGZpbmFsIHJlZ2V4ID0gcHJlX2dlbi5za2lwVmVyeUdvb2RPcHRpbWl6YXRpb25SZWdFeHA7CiAgICAgIHRlc3QoJ21hdGNoZXMgc2luZ2xlLWxpbmUgdGFnJywgKCkgewogICAgICAgIGZpbmFsIGNvbnRlbnQgPSAiQFRhZ3MoWycke3ByZV9nZW4uc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uVGFnfSddKSI7CiAgICAgICAgZXhwZWN0KHJlZ2V4Lmhhc01hdGNoKGNvbnRlbnQpLCBpc1RydWUpOwogICAgICB9KTsKCiAgICAgIHRlc3QoJ21hdGNoZXMgc2luZ2xlLWxpbmUgd2l0aCBtdWx0aXBsZSB0YWdzJywgKCkgewogICAgICAgIGZpbmFsIGNvbnRlbnQgPQogICAgICAgICAgICAiQFRhZ3MoWycke3ByZV9nZW4uc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uVGFnfScsICdjaHJvbWUnXSkiOwogICAgICAgIGV4cGVjdChyZWdleC5oYXNNYXRjaChjb250ZW50KSwgaXNUcnVlKTsKICAgICAgfSk7CgogICAgICB0ZXN0KCdtYXRjaGVzIG11bHRpLWxpbmUgdGFnIGxpc3QnLCAoKSB7CiAgICAgICAgZmluYWwgY29udGVudCA9CiAgICAgICAgICAgICcnJwogICAgICBAVGFncyhbCiAgICAgICAgJyR7cHJlX2dlbi5za2lwVmVyeUdvb2RPcHRpbWl6YXRpb25UYWd9JywKICAgICAgICAnY2hyb21lJywKICAgICAgICAndGVzdCcsCiAgICAgIF0pCiAgICAgICcnJzsKICAgICAgICBleHBlY3QocmVnZXguaGFzTWF0Y2goY29udGVudCksIGlzVHJ1ZSk7CiAgICAgIH0pOwoKICAgICAgdGVzdCgnbWF0Y2hlcyBtdWx0aS1saW5lIHdoZXJlIHRhZyBpcyBub3QgdGhlIGZpcnN0JywgKCkgewogICAgICAgIGZpbmFsIGNvbnRlbnQgPQogICAgICAgICAgICAnJycKICAgICAgQFRhZ3MoWwogICAgICAgICdjaHJvbWUnLAogICAgICAgICcke3ByZV9nZW4uc2tpcFZlcnlHb29kT3B0aW1pemF0aW9uVGFnfScsCiAgICAgICAgJ3Rlc3QnLAogICAgICBdKQogICAgICAnJyc7CiAgICAgICAgZXhwZWN0KHJlZ2V4Lmhhc01hdGNoKGNvbnRlbnQpLCBpc1RydWUpOwogICAgICB9KTsKCiAgICAgIHRlc3QoJ2RvZXMgbm90IG1hdGNoIHdoZW4gdGFnIG1pc3NpbmcnLCAoKSB7CiAgICAgICAgY29uc3QgY29udGVudCA9ICJAVGFncyhbJ2Nocm9tZScsICd0ZXN0J10pIjsKICAgICAgICBleHBlY3QocmVnZXguaGFzTWF0Y2goY29udGVudCksIGlzRmFsc2UpOwogICAgICB9KTsKCiAgICAgIHRlc3QoCiAgICAgICAgJ2RvZXMgbm90IG1hdGNoIHN1YnN0cmluZyBvbmx5IChlLmcuIHNraXBfdmVyeV9nb29kX29wdGltaXphdGlvbix0ZXN0KScsCiAgICAgICAgKCkgewogICAgICAgICAgZmluYWwgY29udGVudCA9CiAgICAgICAgICAgICAgJycnCiAgICAgIEBUYWdzKFsKICAgICAgICAnJHtwcmVfZ2VuLnNraXBWZXJ5R29vZE9wdGltaXphdGlvblRhZ30sdGVzdCcsCiAgICAgICAgJ2Nocm9tZScsCiAgICAgIF0pCiAgICAgICcnJzsKICAgICAgICAgIGV4cGVjdCgKICAgICAgICAgICAgcmVnZXguaGFzTWF0Y2goY29udGVudCksCiAgICAgICAgICAgIGlzRmFsc2UsCiAgICAgICAgICApOyAvLyBvbmx5IGV4YWN0IHRhZyBzaG91bGQgbWF0Y2gKICAgICAgICB9LAogICAgICApOwogICAgfSk7CiAgICBncm91cCgnU2hhcmRpbmcnLCAoKSB7CiAgICAgIC8vLyBDcmVhdGVzIGEgcGFja2FnZSB3aXRoIFtjb3VudF0gb3B0aW1pemFibGUgdGVzdCBmaWxlcywgcGx1cyBhbnkKICAgICAgLy8vIFtub3RPcHRpbWl6ZWRdIGZpbGVzIGNhcnJ5aW5nIHRoZSBza2lwIG9wdGltaXphdGlvbiB0YWcuCiAgICAgIERpcmVjdG9yeSBjcmVhdGVQYWNrYWdlKGludCBjb3VudCwge2ludCBub3RPcHRpbWl6ZWQgPSAwfSkgewogICAgICAgIEZpbGUocGF0aC5qb2luKHRlbXBEaXJlY3RvcnkucGF0aCwgJ3B1YnNwZWMueWFtbCcpKS5jcmVhdGVTeW5jKCk7CiAgICAgICAgZmluYWwgdGVzdERpciA9IERpcmVjdG9yeShwYXRoLmpvaW4odGVtcERpcmVjdG9yeS5wYXRoLCAndGVzdCcpKQogICAgICAgICAgLi5jcmVhdGVTeW5jKCk7CiAgICAgICAgZm9yICh2YXIgaSA9IDA7IGkgPCBjb3VudDsgaSsrKSB7CiAgICAgICAgICBGaWxlKHBhdGguam9pbih0ZXN0RGlyLnBhdGgsICd0ZXN0JHtpfV90ZXN0LmRhcnQnKSkuY3JlYXRlU3luYygpOwogICAgICAgIH0KICAgICAgICBmb3IgKHZhciBpID0gMDsgaSA8IG5vdE9wdGltaXplZDsgaSsrKSB7CiAgICAgICAgICBGaWxlKHBhdGguam9pbih0ZXN0RGlyLnBhdGgsICdza2lwJHtpfV90ZXN0LmRhcnQnKSkKICAgICAgICAgICAgICAud3JpdGVBc1N0cmluZ1N5bmMobm90T3B0aW1pemVkVGVzdENvbnRlbnQpOwogICAgICAgIH0KICAgICAgICByZXR1cm4gdGVzdERpcjsKICAgICAgfQoKICAgICAgTGlzdDxTdHJpbmc+IHBhdGhzT2YoSG9va0NvbnRleHQgY29udGV4dCkgewogICAgICAgIGZpbmFsIHRlc3RzID0gY29udGV4dC52YXJzWyd0ZXN0cyddIGFzIExpc3Q8TWFwPFN0cmluZywgU3RyaW5nPj47CiAgICAgICAgcmV0dXJuIHRlc3RzLm1hcCgoZSkgPT4gZVsncGF0aCddISkudG9MaXN0KCk7CiAgICAgIH0KCiAgICAgIEZ1dHVyZTxMaXN0PFN0cmluZz4+IHJ1blNoYXJkKGludCBpbmRleCwgaW50IHRvdGFsKSBhc3luYyB7CiAgICAgICAgZmluYWwgY29udGV4dCA9IF9GYWtlQ29udGV4dCgpCiAgICAgICAgICAuLnZhcnNbJ3BhY2thZ2Utcm9vdCddID0gdGVtcERpcmVjdG9yeS5hYnNvbHV0ZS5wYXRoCiAgICAgICAgICAuLnZhcnNbJ3NoYXJkLWluZGV4J10gPSBpbmRleAogICAgICAgICAgLi52YXJzWyd0b3RhbC1zaGFyZHMnXSA9IHRvdGFsOwogICAgICAgIGF3YWl0IHByZV9nZW4ucnVuKGNvbnRleHQpOwogICAgICAgIHJldHVybiBbCiAgICAgICAgICAuLi5wYXRoc09mKGNvbnRleHQpLAogICAgICAgICAgLi4uKGNvbnRleHQudmFyc1snbm90T3B0aW1pemVkVGVzdHMnXSEgYXMgTGlzdCkuY2FzdDxTdHJpbmc+KCksCiAgICAgICAgXTsKICAgICAgfQoKICAgICAgdGVzdCgncnVucyBldmVyeSB0ZXN0IGV4YWN0bHkgb25jZSBhY3Jvc3MgYWxsIHNoYXJkcycsICgpIGFzeW5jIHsKICAgICAgICBjcmVhdGVQYWNrYWdlKDcsIG5vdE9wdGltaXplZDogMik7CgogICAgICAgIGZpbmFsIHNoYXJkcyA9IFtmb3IgKHZhciBpID0gMTsgaSA8PSAzOyBpKyspIGF3YWl0IHJ1blNoYXJkKGksIDMpXTsKICAgICAgICBmaW5hbCB1bmlvbiA9IHNoYXJkcy5leHBhbmQoKHNoYXJkKSA9PiBzaGFyZCkudG9MaXN0KCk7CgogICAgICAgIGV4cGVjdCgKICAgICAgICAgIHVuaW9uLi5zb3J0KCksCiAgICAgICAgICBbCiAgICAgICAgICAgIGZvciAodmFyIGkgPSAwOyBpIDwgNzsgaSsrKSAndGVzdCR7aX1fdGVzdC5kYXJ0JywKICAgICAgICAgICAgZm9yICh2YXIgaSA9IDA7IGkgPCAyOyBpKyspICdza2lwJHtpfV90ZXN0LmRhcnQnLAogICAgICAgICAgXS4uc29ydCgpLAogICAgICAgICAgcmVhc29uOiAnU2hhcmRzIG11c3QgYmUgYSBjb21wbGV0ZSBhbmQgZGlzam9pbnQgcGFydGl0aW9uJywKICAgICAgICApOwogICAgICB9KTsKCiAgICAgIHRlc3QoJ3NoYXJkcyBub24gb3B0aW1pemVkIHRlc3RzIGFzIHdlbGwnLCAoKSBhc3luYyB7CiAgICAgICAgY3JlYXRlUGFja2FnZSgwLCBub3RPcHRpbWl6ZWQ6IDQpOwoKICAgICAgICBmaW5hbCBmaXJzdCA9IGF3YWl0IHJ1blNoYXJkKDEsIDIpOwogICAgICAgIGZpbmFsIHNlY29uZCA9IGF3YWl0IHJ1blNoYXJkKDIsIDIpOwoKICAgICAgICBleHBlY3QoZmlyc3QsIFsnc2tpcDBfdGVzdC5kYXJ0JywgJ3NraXAyX3Rlc3QuZGFydCddKTsKICAgICAgICBleHBlY3Qoc2Vjb25kLCBbJ3NraXAxX3Rlc3QuZGFydCcsICdza2lwM190ZXN0LmRhcnQnXSk7CiAgICAgIH0pOwoKICAgICAgdGVzdCgnZGVhbHMgb3B0aW1pemVkIGFuZCBub24gb3B0aW1pemVkIHRlc3RzIG91dCB0b2dldGhlcicsICgpIGFzeW5jIHsKICAgICAgICBjcmVhdGVQYWNrYWdlKDIsIG5vdE9wdGltaXplZDogMyk7CgogICAgICAgIGZpbmFsIHNpemVzID0gWwogICAgICAgICAgZm9yICh2YXIgaSA9IDE7IGkgPD0gNjsgaSsrKSAoYXdhaXQgcnVuU2hhcmQoaSwgNikpLmxlbmd0aCwKICAgICAgICBdOwoKICAgICAgICBleHBlY3QoCiAgICAgICAgICBzaXplcywKICAgICAgICAgIFsxLCAxLCAxLCAxLCAxLCAwXSwKICAgICAgICAgIHJlYXNvbjoKICAgICAgICAgICAgICAnU2hhcmRpbmcgdGhlIHR3byBsaXN0cyBzZXBhcmF0ZWx5IHdvdWxkIGdpdmUgdGhlIGZpcnN0ICcKICAgICAgICAgICAgICAnc2hhcmRzIGEgZmlsZSBmcm9tIGVhY2ggd2hpbGUgbGF0ZXIgc2hhcmRzIHN0YXkgZW1wdHknLAogICAgICAgICk7CiAgICAgIH0pOwoKICAgICAgdGVzdCgnaXMgZGV0ZXJtaW5pc3RpYyBhY3Jvc3MgcnVucycsICgpIGFzeW5jIHsKICAgICAgICBjcmVhdGVQYWNrYWdlKDkpOwoKICAgICAgICBleHBlY3QoYXdhaXQgcnVuU2hhcmQoMiwgNCksIGF3YWl0IHJ1blNoYXJkKDIsIDQpKTsKICAgICAgfSk7CgogICAgICB0ZXN0KCdiYWxhbmNlcyBzaGFyZHMgd2l0aGluIG9uZSBmaWxlIG9mIGVhY2ggb3RoZXInLCAoKSBhc3luYyB7CiAgICAgICAgY3JlYXRlUGFja2FnZSgxMCk7CgogICAgICAgIGZpbmFsIHNpemVzID0gWwogICAgICAgICAgZm9yICh2YXIgaSA9IDE7IGkgPD0gNDsgaSsrKSAoYXdhaXQgcnVuU2hhcmQoaSwgNCkpLmxlbmd0aCwKICAgICAgICBdOwoKICAgICAgICBleHBlY3Qoc2l6ZXMucmVkdWNlKG1heCkgLSBzaXplcy5yZWR1Y2UobWluKSwgbGVzc1RoYW5PckVxdWFsVG8oMSkpOwogICAgICB9KTsKCiAgICAgIHRlc3QoCiAgICAgICAgJ3lpZWxkcyBhbiBlbXB0eSBzaGFyZCB3aGVuIHRoZXJlIGFyZSBtb3JlIHNoYXJkcyB0aGFuIHRlc3RzJywKICAgICAgICAoKSBhc3luYyB7CiAgICAgICAgICBjcmVhdGVQYWNrYWdlKDIpOwoKICAgICAgICAgIGV4cGVjdChhd2FpdCBydW5TaGFyZCgzLCAzKSwgaXNFbXB0eSk7CiAgICAgICAgfSwKICAgICAgKTsKCiAgICAgIHRlc3QoCiAgICAgICAgJ2V4Y2x1ZGVzIG5lc3RlZCBub24gb3B0aW1pemVkIHRlc3RzIGZyb20gdGhlIG9wdGltaXplZCBzZXQnLAogICAgICAgICgpIGFzeW5jIHsKICAgICAgICAgIEZpbGUocGF0aC5qb2luKHRlbXBEaXJlY3RvcnkucGF0aCwgJ3B1YnNwZWMueWFtbCcpKS5jcmVhdGVTeW5jKCk7CiAgICAgICAgICBmaW5hbCB0ZXN0RGlyID0gRGlyZWN0b3J5KHBhdGguam9pbih0ZW1wRGlyZWN0b3J5LnBhdGgsICd0ZXN0JykpCiAgICAgICAgICAgIC4uY3JlYXRlU3luYygpOwogICAgICAgICAgZmluYWwgbmVzdGVkID0gRGlyZWN0b3J5KHBhdGguam9pbih0ZXN0RGlyLnBhdGgsICdzdWInKSkKICAgICAgICAgICAgLi5jcmVhdGVTeW5jKCk7CiAgICAgICAgICBGaWxlKHBhdGguam9pbihuZXN0ZWQucGF0aCwgJ3NraXBfdGVzdC5kYXJ0JykpCiAgICAgICAgICAgICAgLndyaXRlQXNTdHJpbmdTeW5jKG5vdE9wdGltaXplZFRlc3RDb250ZW50KTsKCiAgICAgICAgICBmaW5hbCBjb250ZXh0ID0gX0Zha2VDb250ZXh0KCkKICAgICAgICAgICAgLi52YXJzWydwYWNrYWdlLXJvb3QnXSA9IHRlbXBEaXJlY3RvcnkuYWJzb2x1dGUucGF0aDsKICAgICAgICAgIGF3YWl0IHByZV9nZW4ucnVuKGNvbnRleHQpOwoKICAgICAgICAgIGV4cGVjdCgKICAgICAgICAgICAgcGF0aHNPZihjb250ZXh0KSwKICAgICAgICAgICAgaXNFbXB0eSwKICAgICAgICAgICAgcmVhc29uOgogICAgICAgICAgICAgICAgJ0EgdGFnZ2VkIHRlc3QgaW4gYSBzdWJkaXJlY3RvcnkgbXVzdCBub3QgYmUgb3B0aW1pemVkLCAnCiAgICAgICAgICAgICAgICAnb3RoZXJ3aXNlIGl0IHJ1bnMgYm90aCBpbmxpbmVkIGFuZCBzdGFuZGFsb25lJywKICAgICAgICAgICk7CiAgICAgICAgICBleHBlY3QoY29udGV4dC52YXJzWydub3RPcHRpbWl6ZWRUZXN0cyddLCBbJ3N1Yi9za2lwX3Rlc3QuZGFydCddKTsKICAgICAgICB9LAogICAgICApOwoKICAgICAgdGVzdCgnaW5jbHVkZXMgZXZlcnkgdGVzdCB3aGVuIHNoYXJkaW5nIGlzIG5vdCByZXF1ZXN0ZWQnLCAoKSBhc3luYyB7CiAgICAgICAgY3JlYXRlUGFja2FnZSgzKTsKCiAgICAgICAgZmluYWwgY29udGV4dCA9IF9GYWtlQ29udGV4dCgpCiAgICAgICAgICAuLnZhcnNbJ3BhY2thZ2Utcm9vdCddID0gdGVtcERpcmVjdG9yeS5hYnNvbHV0ZS5wYXRoOwogICAgICAgIGF3YWl0IHByZV9nZW4ucnVuKGNvbnRleHQpOwoKICAgICAgICBleHBlY3QocGF0aHNPZihjb250ZXh0KSwgaGFzTGVuZ3RoKDMpKTsKICAgICAgfSk7CiAgICB9KTsKICB9KTsKfQo=", "type": "text", }, ], @@ -61,5 +61,15 @@ final testOptimizerBundle = MasonBundle.fromJson({ "default": ".", "prompt": "Please enter the path to the package root.", }, + "shard-index": { + "type": "number", + "description": "The 1-based index of the shard to generate tests for.", + "prompt": "Please enter the shard index.", + }, + "total-shards": { + "type": "number", + "description": "The total number of shards the test suite is split into.", + "prompt": "Please enter the total number of shards.", + }, }, }); diff --git a/lib/src/cli/test_cli_runner.dart b/lib/src/cli/test_cli_runner.dart index e78aeb786..5e6a8bbc4 100644 --- a/lib/src/cli/test_cli_runner.dart +++ b/lib/src/cli/test_cli_runner.dart @@ -86,6 +86,65 @@ class TestCLIRunner { return rest.where((arg) => !arg.startsWith('-')).isNotEmpty; } + /// Validates the `--shard-index` / `--total-shards` combination. + /// + /// Returns an error message describing the problem, or `null` when the + /// configuration is valid. + /// + /// [rawShardIndex], [rawTotalShards] and [rawMinCoverage] are the unparsed + /// command line values, so that "not provided" can be told apart from + /// "provided but not a number", and an explicit `--min-coverage` from one + /// inherited from `very_good.yaml`. [optimizePerformance] is the effective + /// value, after every option that disables the optimizer is accounted for. + static String? validateSharding({ + required String? rawShardIndex, + required String? rawTotalShards, + required String? rawMinCoverage, + required bool optimizePerformance, + }) { + if (rawShardIndex == null && rawTotalShards == null) return null; + + if (rawShardIndex == null || rawTotalShards == null) { + return '--shard-index and --total-shards must be used together.'; + } + + final totalShards = int.tryParse(rawTotalShards); + if (totalShards == null || totalShards < 1) { + return '--total-shards must be a positive integer, ' + 'but got "$rawTotalShards".'; + } + + final shardIndex = int.tryParse(rawShardIndex); + if (shardIndex == null || shardIndex < 1) { + return '--shard-index must be a positive integer, ' + 'but got "$rawShardIndex".'; + } + + if (shardIndex > totalShards) { + return '--shard-index ($shardIndex) must be less than or equal to ' + '--total-shards ($totalShards).'; + } + + // Sharding partitions the test files consolidated by the optimizer, so it + // cannot work when the optimization is turned off. + if (!optimizePerformance) { + return 'Sharding requires the test optimizer, which is disabled by ' + '--no-optimization, --platform, --update-goldens or by targeting ' + 'specific test files.'; + } + + // Each shard only exercises a fraction of the codebase, so its coverage is + // not representative of the whole suite. Merge the lcov files from every + // shard and enforce the threshold in a separate job instead. + if (rawMinCoverage != null) { + return '--min-coverage cannot be combined with sharding. Collect ' + 'coverage per shard with --coverage, merge the lcov reports, then ' + 'check the threshold in a separate job.'; + } + + return null; + } + /// Run tests (`flutter test`). /// Returns a list of exit codes for each test process. static Future> test({ @@ -108,6 +167,8 @@ class TestCLIRunner { GeneratorBuilder buildGenerator = MasonGenerator.fromBundle, List? reportOn, bool checkIgnore = false, + int? shardIndex, + int? totalShards, @visibleForTesting VeryGoodTestRunner? overrideTestRunner, }) { final initialCwd = cwd; @@ -145,7 +206,11 @@ class TestCLIRunner { '''Shuffling test order with --test-randomize-ordering-seed=$randomSeed\n''', ); } - var vars = {'package-root': workingDirectory}; + var vars = { + 'package-root': workingDirectory, + 'shard-index': ?shardIndex, + 'total-shards': ?totalShards, + }; if (optimizePerformance) { final optimizationProgress = logger.progress('Optimizing tests'); try { @@ -167,6 +232,23 @@ class TestCLIRunner { final notOptimizedTests = vars['notOptimizedTests'] as List? ?? []; + + // A shard can legitimately end up empty when there are more shards + // than test files. That is not a failure: the shard has nothing to do, + // so report success instead of letting the test runner exit with + // "No tests were found", which would fail the CI job. + if (shardIndex != null && + optimizePerformance && + (vars['tests'] as List? ?? []).isEmpty && + notOptimizedTests.isEmpty) { + stdout?.call('No tests found for shard $shardIndex in $path\n'); + await _cleanupOptimizerFile(cwd); + // The merge step downstream still expects a report from every + // shard, so leave an empty one behind. + if (collectCoverage) await lcovFile.create(recursive: true); + return ExitCode.success.code; + } + return await _overrideAnsiOutput( forceAnsi, () => @@ -185,8 +267,10 @@ class TestCLIRunner { p.join('test', _testOptimizerFileName), // Include non-optimized tests that require separate execution if (notOptimizedTests.isNotEmpty && optimizePerformance) + // The hook emits forward slashes; keep the argument native. ...notOptimizedTests.map( - (e) => p.join('test', e.toString()), + (e) => + p.joinAll(['test', ...p.posix.split(e.toString())]), ), ], stdout: stdout ?? noop, diff --git a/lib/src/commands/dart/commands/dart_test_command.dart b/lib/src/commands/dart/commands/dart_test_command.dart index 08896162c..0aedef8b6 100644 --- a/lib/src/commands/dart/commands/dart_test_command.dart +++ b/lib/src/commands/dart/commands/dart_test_command.dart @@ -30,6 +30,9 @@ class DartTestOptions { required this.runSkipped, required this.checkIgnore, required this.fileReporter, + required this.shardIndex, + required this.totalShards, + required this.rawMinCoverage, }); /// Parses [ArgResults] into a [DartTestOptions] instance. @@ -102,6 +105,9 @@ class DartTestOptions { 'file-reporter', testConfig.fileReporter, ); + final shardIndex = argResults['shard-index'] as String?; + final totalShards = argResults['total-shards'] as String?; + final rawMinCoverage = argResults['min-coverage'] as String?; final rest = argResults.rest; return DartTestOptions._( @@ -122,6 +128,9 @@ class DartTestOptions { runSkipped: runSkipped, checkIgnore: checkIgnore, fileReporter: fileReporter, + shardIndex: shardIndex, + totalShards: totalShards, + rawMinCoverage: rawMinCoverage, rest: rest, ); } @@ -179,6 +188,19 @@ class DartTestOptions { /// `:` (e.g. `json:reports/tests.json`). final String? fileReporter; + /// The raw `--shard-index` value, validated by + /// [TestCLIRunner.validateSharding]. + final String? shardIndex; + + /// The raw `--total-shards` value, validated by + /// [TestCLIRunner.validateSharding]. + final String? totalShards; + + /// The raw `--min-coverage` value, without the `very_good.yaml` fallback + /// that [minCoverage] applies, so sharding can reject an explicit threshold + /// while ignoring an inherited one. + final String? rawMinCoverage; + /// The remaining arguments passed to the `dart test` command. final List rest; } @@ -204,6 +226,8 @@ typedef DartTestCommandCall = Future> Function({ void Function(String)? stderr, List? reportOn, bool checkIgnore, + int? shardIndex, + int? totalShards, }); /// {@template dart_test_command} @@ -335,6 +359,23 @@ class DartTestCommand extends Command { 'Enable an additional reporter writing test results to a file. ' 'Should be in the form : (e.g. "json:reports/tests.json").', valueHelp: 'name:path', + ) + ..addOption( + 'shard-index', + help: + 'The 1-based index of the shard to run. ' + 'Must be used together with --total-shards. ' + 'Requires optimization to be enabled. ' + 'When omitted, no sharding is applied.', + valueHelp: 'index', + ) + ..addOption( + 'total-shards', + help: + 'Split the test suite into this many shards and run only the one ' + 'selected by --shard-index. Useful to parallelize tests across ' + 'multiple CI runners. When omitted, no sharding is applied.', + valueHelp: 'count', ); } @@ -383,24 +424,43 @@ This command should be run from the root of your Dart project.'''); final options = DartTestOptions.parse(_argResults, config: config); + final optimizePerformance = + options.optimizePerformance && + !TestCLIRunner.isTargettingTestFiles(options.rest) && + // Disabled optimization when platform is specified + // https://github.com/VeryGoodOpenSource/very_good_cli/issues/1363 + options.platform == null; + + final shardingError = TestCLIRunner.validateSharding( + rawShardIndex: options.shardIndex, + rawTotalShards: options.totalShards, + rawMinCoverage: options.rawMinCoverage, + optimizePerformance: optimizePerformance, + ); + if (shardingError != null) { + _logger.err(shardingError); + return ExitCode.usage.code; + } + + // A threshold inherited from very_good.yaml only applies to un-sharded + // runs: a single shard covers a fraction of the code and would fail it. + final minCoverage = options.totalShards == null + ? options.minCoverage + : null; + if (isDartInstalled) { try { final results = await _dartTest( - optimizePerformance: - options.optimizePerformance && - !TestCLIRunner.isTargettingTestFiles(options.rest) && - // Disabled optimization when platform is specified - // https://github.com/VeryGoodOpenSource/very_good_cli/issues/1363 - options.platform == null, + optimizePerformance: optimizePerformance, recursive: recursive, logger: _logger, stdout: _logger.write, stderr: _logger.err, collectCoverage: options.collectCoverage || - options.minCoverage != null || + minCoverage != null || options.showUncovered, - minCoverage: options.minCoverage, + minCoverage: minCoverage, showUncovered: options.showUncovered, excludeFromCoverage: options.excludeFromCoverage, collectCoverageFrom: options.collectCoverageFrom, @@ -419,6 +479,8 @@ This command should be run from the root of your Dart project.'''); ], reportOn: options.reportOn.isEmpty ? null : options.reportOn, checkIgnore: options.checkIgnore, + shardIndex: int.tryParse(options.shardIndex ?? ''), + totalShards: int.tryParse(options.totalShards ?? ''), ); if (results.any((code) => code != ExitCode.success.code)) { return ExitCode.unavailable.code; @@ -426,7 +488,7 @@ This command should be run from the root of your Dart project.'''); } on MinCoverageNotMet catch (e) { TestCLIRunner.handleMinCoverageNotMet( logger: _logger, - minCoverage: options.minCoverage, + minCoverage: minCoverage, e: e, ); return ExitCode.unavailable.code; diff --git a/lib/src/commands/test/test.dart b/lib/src/commands/test/test.dart index 698dce96b..c3e666877 100644 --- a/lib/src/commands/test/test.dart +++ b/lib/src/commands/test/test.dart @@ -33,6 +33,9 @@ class FlutterTestOptions { required this.flavor, required this.timeout, required this.fileReporter, + required this.shardIndex, + required this.totalShards, + required this.rawMinCoverage, required this.rest, }); @@ -124,6 +127,9 @@ class FlutterTestOptions { 'file-reporter', testConfig.fileReporter, ); + final shardIndex = argResults['shard-index'] as String?; + final totalShards = argResults['total-shards'] as String?; + final rawMinCoverage = argResults['min-coverage'] as String?; final rest = argResults.rest; return FlutterTestOptions._( @@ -148,6 +154,9 @@ class FlutterTestOptions { flavor: flavor, timeout: effectiveTimeout, fileReporter: fileReporter, + shardIndex: shardIndex, + totalShards: totalShards, + rawMinCoverage: rawMinCoverage, rest: rest, ); } @@ -218,6 +227,19 @@ class FlutterTestOptions { /// `:` (e.g. `json:reports/tests.json`). final String? fileReporter; + /// The raw `--shard-index` value, validated by + /// [TestCLIRunner.validateSharding]. + final String? shardIndex; + + /// The raw `--total-shards` value, validated by + /// [TestCLIRunner.validateSharding]. + final String? totalShards; + + /// The raw `--min-coverage` value, without the `very_good.yaml` fallback + /// that [minCoverage] applies, so sharding can reject an explicit threshold + /// while ignoring an inherited one. + final String? rawMinCoverage; + /// The remaining arguments passed to the test command. final List rest; } @@ -244,6 +266,8 @@ typedef FlutterTestCommand = Future> Function({ void Function(String)? stdout, void Function(String)? stderr, List? reportOn, + int? shardIndex, + int? totalShards, }); /// {@template test_command} @@ -411,6 +435,23 @@ class TestCommand extends Command { 'Enable an additional reporter writing test results to a file. ' 'Should be in the form : (e.g. "json:reports/tests.json").', valueHelp: 'name:path', + ) + ..addOption( + 'shard-index', + help: + 'The 1-based index of the shard to run. ' + 'Must be used together with --total-shards. ' + 'Requires optimization to be enabled. ' + 'When omitted, no sharding is applied.', + valueHelp: 'index', + ) + ..addOption( + 'total-shards', + help: + 'Split the test suite into this many shards and run only the one ' + 'selected by --shard-index. Useful to parallelize tests across ' + 'multiple CI runners. When omitted, no sharding is applied.', + valueHelp: 'count', ); } @@ -461,31 +502,52 @@ This command should be run from the root of your Flutter project.'''); final options = FlutterTestOptions.parse(_argResults, config: config); + final optimizePerformance = + options.optimizePerformance && + !TestCLIRunner.isTargettingTestFiles(options.rest) && + !options.updateGoldens && + // Disabled optimization when platform is specified + // https://github.com/VeryGoodOpenSource/very_good_cli/issues/1363 + options.platform == null; + + final shardingError = TestCLIRunner.validateSharding( + rawShardIndex: options.shardIndex, + rawTotalShards: options.totalShards, + rawMinCoverage: options.rawMinCoverage, + optimizePerformance: optimizePerformance, + ); + if (shardingError != null) { + _logger.err(shardingError); + return ExitCode.usage.code; + } + + // A threshold inherited from very_good.yaml only applies to un-sharded + // runs: a single shard covers a fraction of the code and would fail it. + final minCoverage = options.totalShards == null + ? options.minCoverage + : null; + if (isFlutterInstalled) { try { final results = await _flutterTest( - optimizePerformance: - options.optimizePerformance && - !TestCLIRunner.isTargettingTestFiles(options.rest) && - !options.updateGoldens && - // Disabled optimization when platform is specified - // https://github.com/VeryGoodOpenSource/very_good_cli/issues/1363 - options.platform == null, + optimizePerformance: optimizePerformance, recursive: recursive, logger: _logger, stdout: _logger.write, stderr: _logger.err, collectCoverage: options.collectCoverage || - options.minCoverage != null || + minCoverage != null || options.showUncovered, - minCoverage: options.minCoverage, + minCoverage: minCoverage, showUncovered: options.showUncovered, excludeFromCoverage: options.excludeFromCoverage, collectCoverageFrom: options.collectCoverageFrom, randomSeed: options.randomSeed, forceAnsi: options.forceAnsi, reportOn: options.reportOn.isEmpty ? null : options.reportOn, + shardIndex: int.tryParse(options.shardIndex ?? ''), + totalShards: int.tryParse(options.totalShards ?? ''), arguments: [ if (options.excludeTags != null) ...['-x', options.excludeTags!], if (options.tags != null) ...['-t', options.tags!], @@ -514,7 +576,7 @@ This command should be run from the root of your Flutter project.'''); } on MinCoverageNotMet catch (e) { TestCLIRunner.handleMinCoverageNotMet( logger: _logger, - minCoverage: options.minCoverage, + minCoverage: minCoverage, e: e, ); return ExitCode.unavailable.code; diff --git a/site/docs/commands/test.md b/site/docs/commands/test.md index 505fc3d8d..f3f555b76 100644 --- a/site/docs/commands/test.md +++ b/site/docs/commands/test.md @@ -51,6 +51,14 @@ very_good test [arguments] Useful when tests hang due to an unbounded pumpAndSettle() call. --file-reporter= Enable an additional reporter writing test results to a file. Should be in the form : (e.g. "json:reports/tests.json"). + --shard-index= The 1-based index of the shard to run. + Must be used together with --total-shards. + Requires optimization to be enabled. + When omitted, no sharding is applied. + --total-shards= Split the test suite into this many shards and run only the one + selected by --shard-index. + Useful to parallelize tests across multiple CI runners. + When omitted, no sharding is applied. Run "very_good help" to see global options. ``` @@ -74,6 +82,52 @@ For **Dart** projects, use **`very_good dart test`** instead. All test parameters and options work identically with this command. ::: +### Sharding tests across CI runners + +When the test phase is your CI bottleneck, split the suite across several +runners with `--shard-index` and `--total-shards`. Each runner generates its own +optimized entrypoint containing only its slice of the test files, so the shards +together run every test exactly once. + +```yaml +jobs: + test: + strategy: + matrix: + shard: [1, 2, 3] + steps: + - run: very_good test --shard-index ${{ matrix.shard }} --total-shards 3 +``` + +Test files are sorted and then dealt out round-robin, which keeps the partition +deterministic: the same suite always produces the same shards, on every machine. +Files tagged with `skip_very_good_optimization` are sharded as well, so they do +not run on every runner. + +A shard with no test files (more shards than test files) succeeds without +running anything, so an oversized matrix will not fail your build. + +When combined with `--recursive`, each package is sharded independently, so a +given runner executes a slice of every package. Balance therefore degrades when +a workspace contains many packages with few tests each. + +:::caution +Sharding cannot be combined with `--min-coverage`, and a `min_coverage` set in +`very_good.yaml` is ignored while sharding. Each shard only exercises a subset +of the codebase, so its coverage is not representative of the whole suite. +Collect coverage per shard with `--coverage`, merge the resulting lcov reports +once every shard has finished, and enforce the threshold on the merged report +in a separate job. A shard without tests still writes an empty +`coverage/lcov.info`; pass `--ignore-errors empty` to `lcov` when merging so it +is accepted. +::: + +:::info +Sharding requires the test optimizer, so it is rejected whenever the optimizer +is off: with `--no-optimization`, `--platform`, `--update-goldens`, or when +targeting specific test files. +::: + ### Passing Flutter specific arguments The `dart test` and `flutter test` commands expose more arguments than those available through `very_good dart test` or `very_good test`. However, you can use the argument terminator `--` to pass additional arguments directly to the underlying test commands. Any arguments after `--` will be forwarded to `dart test` or `flutter test`, making all their options available. diff --git a/test/src/cli/test_cli_runner_test.dart b/test/src/cli/test_cli_runner_test.dart index f2d454832..3cd08c2b5 100644 --- a/test/src/cli/test_cli_runner_test.dart +++ b/test/src/cli/test_cli_runner_test.dart @@ -1135,6 +1135,210 @@ void main() { expect(testRunnerArgs, equals(['--coverage', '--coverage'])); }); + test('passes shard vars to the generator', () async { + final tempDirectory = Directory.systemTemp.createTempSync(); + addTearDown(() => tempDirectory.deleteSync(recursive: true)); + + File(p.join(tempDirectory.path, 'pubspec.yaml')).createSync(); + Directory(p.join(tempDirectory.path, 'test')).createSync(); + when( + () => hooks.preGen( + vars: any(named: 'vars'), + onVarsChanged: any(named: 'onVarsChanged'), + workingDirectory: any(named: 'workingDirectory'), + ), + ).thenAnswer((invocation) async { + (invocation.namedArguments[#onVarsChanged] + as void Function(Map vars)) + .call({ + 'package-root': tempDirectory.path, + 'tests': [ + {'path': 'a_test.dart', 'identifier': 'a'}, + ], + }); + }); + + await expectLater( + TestCLIRunner.test( + testType: TestRunType.flutter, + cwd: tempDirectory.path, + logger: logger, + stdout: stdoutLogs.add, + stderr: stderrLogs.add, + buildGenerator: generatorBuilder(), + optimizePerformance: true, + shardIndex: 2, + totalShards: 3, + overrideTestRunner: testRunner( + Stream.fromIterable([ + const DoneTestEvent(success: true, time: 0), + const ExitTestEvent(exitCode: 0, time: 0), + ]), + ), + ), + completion(equals([ExitCode.success.code])), + ); + + verify( + () => hooks.preGen( + vars: { + 'package-root': tempDirectory.path, + 'shard-index': 2, + 'total-shards': 3, + }, + onVarsChanged: any(named: 'onVarsChanged'), + workingDirectory: tempDirectory.path, + ), + ).called(1); + }); + + test( + 'writes an empty lcov file for an empty shard with coverage', + () async { + final tempDirectory = Directory.systemTemp.createTempSync(); + addTearDown(() => tempDirectory.deleteSync(recursive: true)); + + File(p.join(tempDirectory.path, 'pubspec.yaml')).createSync(); + Directory(p.join(tempDirectory.path, 'test')).createSync(); + when( + () => hooks.preGen( + vars: any(named: 'vars'), + onVarsChanged: any(named: 'onVarsChanged'), + workingDirectory: any(named: 'workingDirectory'), + ), + ).thenAnswer((invocation) async { + (invocation.namedArguments[#onVarsChanged] + as void Function(Map vars)) + .call({ + 'package-root': tempDirectory.path, + 'tests': >[], + 'notOptimizedTests': [], + }); + }); + + await expectLater( + TestCLIRunner.test( + testType: TestRunType.flutter, + cwd: tempDirectory.path, + logger: logger, + stdout: stdoutLogs.add, + stderr: stderrLogs.add, + buildGenerator: generatorBuilder(), + optimizePerformance: true, + collectCoverage: true, + shardIndex: 3, + totalShards: 3, + overrideTestRunner: testRunner(const Stream.empty()), + ), + completion(equals([ExitCode.success.code])), + ); + + final lcov = File( + p.join(tempDirectory.path, 'coverage', 'lcov.info'), + ); + expect(lcov.existsSync(), isTrue); + expect(lcov.lengthSync(), 0); + expect(testRunnerArgs, isEmpty); + }, + ); + + test('succeeds without running tests when the shard is empty', () async { + final tempDirectory = Directory.systemTemp.createTempSync(); + addTearDown(() => tempDirectory.deleteSync(recursive: true)); + + File(p.join(tempDirectory.path, 'pubspec.yaml')).createSync(); + Directory(p.join(tempDirectory.path, 'test')).createSync(); + when( + () => hooks.preGen( + vars: any(named: 'vars'), + onVarsChanged: any(named: 'onVarsChanged'), + workingDirectory: any(named: 'workingDirectory'), + ), + ).thenAnswer((invocation) async { + (invocation.namedArguments[#onVarsChanged] + as void Function(Map vars)) + .call({ + 'package-root': tempDirectory.path, + 'tests': >[], + 'notOptimizedTests': [], + }); + }); + + await expectLater( + TestCLIRunner.test( + testType: TestRunType.flutter, + cwd: tempDirectory.path, + logger: logger, + stdout: stdoutLogs.add, + stderr: stderrLogs.add, + buildGenerator: generatorBuilder(), + optimizePerformance: true, + shardIndex: 9, + totalShards: 9, + overrideTestRunner: testRunner(const Stream.empty()), + ), + completion(equals([ExitCode.success.code])), + ); + + expect( + stdoutLogs, + equals([ + 'Running "flutter test" in . ...\n', + 'No tests found for shard 9 in .\n', + ]), + ); + expect(testRunnerArgs, isEmpty); + }); + + test( + 'runs the shard when it only contains non optimized tests', + () async { + final tempDirectory = Directory.systemTemp.createTempSync(); + addTearDown(() => tempDirectory.deleteSync(recursive: true)); + + File(p.join(tempDirectory.path, 'pubspec.yaml')).createSync(); + Directory(p.join(tempDirectory.path, 'test')).createSync(); + when( + () => hooks.preGen( + vars: any(named: 'vars'), + onVarsChanged: any(named: 'onVarsChanged'), + workingDirectory: any(named: 'workingDirectory'), + ), + ).thenAnswer((invocation) async { + (invocation.namedArguments[#onVarsChanged] + as void Function(Map vars)) + .call({ + 'package-root': tempDirectory.path, + 'tests': >[], + 'notOptimizedTests': ['skip0_test.dart'], + }); + }); + + await expectLater( + TestCLIRunner.test( + testType: TestRunType.flutter, + cwd: tempDirectory.path, + logger: logger, + stdout: stdoutLogs.add, + stderr: stderrLogs.add, + buildGenerator: generatorBuilder(), + optimizePerformance: true, + shardIndex: 2, + totalShards: 2, + overrideTestRunner: testRunner( + Stream.fromIterable([ + const DoneTestEvent(success: true, time: 0), + const ExitTestEvent(exitCode: 0, time: 0), + ]), + ), + ), + completion(equals([ExitCode.success.code])), + ); + + expect(testRunnerArgs, contains(p.join('test', 'skip0_test.dart'))); + }, + ); + test('runs tests w/optimizations (passing)', () async { final tempDirectory = Directory.systemTemp.createTempSync(); addTearDown(() => tempDirectory.deleteSync(recursive: true)); diff --git a/test/src/commands/dart/commands/dart_test_test.dart b/test/src/commands/dart/commands/dart_test_test.dart index ea9f483e6..9a957da1c 100644 --- a/test/src/commands/dart/commands/dart_test_test.dart +++ b/test/src/commands/dart/commands/dart_test_test.dart @@ -50,6 +50,8 @@ const expectedTestUsage = [ ' --[no-]check-ignore Whether to check for and respect coverage ignore comments (e.g. // coverage:ignore-line).\n' ' (defaults to on)\n' ' --file-reporter= Enable an additional reporter writing test results to a file. Should be in the form : (e.g. "json:reports/tests.json").\n' + ' --shard-index= The 1-based index of the shard to run. Must be used together with --total-shards. Requires optimization to be enabled. When omitted, no sharding is applied.\n' + ' --total-shards= Split the test suite into this many shards and run only the one selected by --shard-index. Useful to parallelize tests across multiple CI runners. When omitted, no sharding is applied.\n' '\n' 'Run "very_good help" to see global options.', ]; @@ -74,6 +76,8 @@ abstract class DartTestCommandCall { void Function(String)? stderr, bool? forceAnsi, List? reportOn, + int? shardIndex, + int? totalShards, }); } @@ -121,6 +125,8 @@ void main() { stderr: any(named: 'stderr'), forceAnsi: any(named: 'forceAnsi'), reportOn: any(named: 'reportOn'), + shardIndex: any(named: 'shardIndex'), + totalShards: any(named: 'totalShards'), ), ).thenAnswer((_) async => [0]); when(() => argResults['concurrency']).thenReturn(concurrency); @@ -132,6 +138,8 @@ void main() { when(() => argResults['run-skipped']).thenReturn(false); when(() => argResults['optimization']).thenReturn(true); when(() => argResults['platform']).thenReturn(null); + when(() => argResults['shard-index']).thenReturn(null); + when(() => argResults['total-shards']).thenReturn(null); when(() => argResults['collect-coverage-from']) .thenReturn('imports'); when(() => argResults['report-on']).thenReturn([]); @@ -853,6 +861,143 @@ void main() { ).called(1); }); + group('sharding', () { + void withShards(String? index, String? total) { + when(() => argResults['shard-index']).thenReturn(index); + when(() => argResults['total-shards']).thenReturn(total); + } + + test('forwards shard values to the test runner', () async { + withShards('2', '3'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.success.code)); + verify( + () => dartTest( + shardIndex: 2, + totalShards: 3, + optimizePerformance: true, + arguments: defaultArguments, + logger: logger, + stdout: logger.write, + stderr: logger.err, + ), + ).called(1); + }); + + test('fails when only one shard option is provided', () async { + withShards('1', null); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify( + () => logger.err( + '--shard-index and --total-shards must be used together.', + ), + ).called(1); + }); + + test('fails when --shard-index exceeds --total-shards', () async { + withShards('4', '3'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify( + () => logger.err( + '--shard-index (4) must be less than or equal to ' + '--total-shards (3).', + ), + ).called(1); + }); + + const optimizerDisabledError = + 'Sharding requires the test optimizer, which is disabled by ' + '--no-optimization, --platform, --update-goldens or by targeting ' + 'specific test files.'; + + test('fails when sharding with optimization disabled', () async { + withShards('1', '3'); + when(() => argResults['optimization']).thenReturn(false); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify(() => logger.err(optimizerDisabledError)).called(1); + verifyNever(() => dartTest(shardIndex: 1, totalShards: 3)); + }); + + test('fails when sharding with --platform', () async { + withShards('1', '3'); + when(() => argResults['platform']).thenReturn('chrome'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify(() => logger.err(optimizerDisabledError)).called(1); + }); + + test('fails when sharding while targeting test files', () async { + withShards('1', '3'); + when(() => argResults.rest).thenReturn(['test/foo_test.dart']); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify(() => logger.err(optimizerDisabledError)).called(1); + }); + + test( + 'ignores a min_coverage from very_good.yaml when sharding', + () async { + final tempDirectory = Directory.systemTemp.createTempSync(); + addTearDown(() { + Directory.current = cwd; + tempDirectory.deleteSync(recursive: true); + }); + Directory.current = tempDirectory.path; + File(path.join(tempDirectory.path, 'pubspec.yaml')).createSync(); + File(path.join(tempDirectory.path, 'very_good.yaml')) + .writeAsStringSync('dart:\n test:\n min_coverage: 90\n'); + when(() => argResults.wasParsed(any())).thenReturn(false); + withShards('1', '3'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.success.code)); + verify( + () => dartTest( + shardIndex: 1, + totalShards: 3, + optimizePerformance: true, + arguments: defaultArguments, + logger: logger, + stdout: logger.write, + stderr: logger.err, + ), + ).called(1); + }, + ); + + test('fails when sharding is combined with --min-coverage', () async { + withShards('1', '3'); + when(() => argResults['min-coverage']).thenReturn('100'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify( + () => logger.err( + '--min-coverage cannot be combined with sharding. Collect ' + 'coverage per shard with --coverage, merge the lcov reports, ' + 'then check the threshold in a separate job.', + ), + ).called(1); + }); + }); + group('very_good.yaml configuration', () { test( 'fails with exit code ${ExitCode.config.code} ' @@ -953,6 +1098,7 @@ void main() { ), ); expect(options.minCoverage, equals(50)); + expect(options.rawMinCoverage, equals('50')); }); test('CLI --file-reporter takes precedence over config value', () { diff --git a/test/src/commands/test/test_test.dart b/test/src/commands/test/test_test.dart index df58d189c..e780449e4 100644 --- a/test/src/commands/test/test_test.dart +++ b/test/src/commands/test/test_test.dart @@ -53,6 +53,8 @@ const expectedTestUsage = [ ' --flavor Build a custom app flavor as defined by platform-specific build setup. Supports the use of product flavors in Android Gradle scripts, and the use of custom Xcode schemes.\n' ' --timeout= Maximum seconds to let tests run before killing the process. Useful when tests hang due to an unbounded pumpAndSettle() call.\n' ' --file-reporter= Enable an additional reporter writing test results to a file. Should be in the form : (e.g. "json:reports/tests.json").\n' + ' --shard-index= The 1-based index of the shard to run. Must be used together with --total-shards. Requires optimization to be enabled. When omitted, no sharding is applied.\n' + ' --total-shards= Split the test suite into this many shards and run only the one selected by --shard-index. Useful to parallelize tests across multiple CI runners. When omitted, no sharding is applied.\n' '\n' 'Run "very_good help" to see global options.', ]; @@ -76,6 +78,8 @@ abstract class FlutterTestCommand { void Function(String)? stderr, bool? forceAnsi, List? reportOn, + int? shardIndex, + int? totalShards, }); } @@ -122,6 +126,8 @@ void main() { stderr: any(named: 'stderr'), forceAnsi: any(named: 'forceAnsi'), reportOn: any(named: 'reportOn'), + shardIndex: any(named: 'shardIndex'), + totalShards: any(named: 'totalShards'), ), ).thenAnswer((_) async => [0]); when(() => argResults['concurrency']).thenReturn(concurrency); @@ -132,6 +138,8 @@ void main() { when(() => argResults['fail-fast']).thenReturn(false); when(() => argResults['optimization']).thenReturn(true); when(() => argResults['platform']).thenReturn(null); + when(() => argResults['shard-index']).thenReturn(null); + when(() => argResults['total-shards']).thenReturn(null); when(() => argResults['report-on']).thenReturn([]); when(() => argResults['run-skipped']).thenReturn(false); when(() => argResults['flavor']).thenReturn(null); @@ -957,6 +965,216 @@ void main() { ); }); + group('sharding', () { + void withShards(String? index, String? total) { + when(() => argResults['shard-index']).thenReturn(index); + when(() => argResults['total-shards']).thenReturn(total); + } + + test('forwards shard values to the test runner', () async { + withShards('2', '3'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.success.code)); + verify( + () => flutterTest( + shardIndex: 2, + totalShards: 3, + optimizePerformance: true, + arguments: defaultArguments, + logger: logger, + stdout: logger.write, + stderr: logger.err, + ), + ).called(1); + }); + + test('does not forward shard values when not sharding', () async { + final result = await testCommand.run(); + + expect(result, equals(ExitCode.success.code)); + verify( + () => flutterTest( + optimizePerformance: true, + arguments: defaultArguments, + logger: logger, + stdout: logger.write, + stderr: logger.err, + ), + ).called(1); + }); + + test('fails when --shard-index is used without --total-shards', () async { + withShards('1', null); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify( + () => logger.err( + '--shard-index and --total-shards must be used together.', + ), + ).called(1); + verifyNever( + () => flutterTest( + shardIndex: any(named: 'shardIndex'), + totalShards: any(named: 'totalShards'), + ), + ); + }); + + test('fails when --total-shards is used without --shard-index', () async { + withShards(null, '3'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify( + () => logger.err( + '--shard-index and --total-shards must be used together.', + ), + ).called(1); + }); + + test('fails when --shard-index exceeds --total-shards', () async { + withShards('4', '3'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify( + () => logger.err( + '--shard-index (4) must be less than or equal to ' + '--total-shards (3).', + ), + ).called(1); + }); + + test('fails when --shard-index is not a positive integer', () async { + withShards('0', '3'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify( + () => logger.err( + '--shard-index must be a positive integer, but got "0".', + ), + ).called(1); + }); + + test('fails when --shard-index is not a number', () async { + withShards('abc', '3'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify( + () => logger.err( + '--shard-index must be a positive integer, but got "abc".', + ), + ).called(1); + }); + + test('fails when --total-shards is not a positive integer', () async { + withShards('1', '0'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify( + () => logger.err( + '--total-shards must be a positive integer, but got "0".', + ), + ).called(1); + }); + + const optimizerDisabledError = + 'Sharding requires the test optimizer, which is disabled by ' + '--no-optimization, --platform, --update-goldens or by targeting ' + 'specific test files.'; + + test('fails when sharding with optimization disabled', () async { + withShards('1', '3'); + when(() => argResults['optimization']).thenReturn(false); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify(() => logger.err(optimizerDisabledError)).called(1); + verifyNever(() => flutterTest(shardIndex: 1, totalShards: 3)); + }); + + test('fails when sharding with --platform', () async { + withShards('1', '3'); + when(() => argResults['platform']).thenReturn('chrome'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify(() => logger.err(optimizerDisabledError)).called(1); + }); + + test('fails when sharding with --update-goldens', () async { + withShards('1', '3'); + when(() => argResults['update-goldens']).thenReturn(true); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify(() => logger.err(optimizerDisabledError)).called(1); + }); + + test( + 'ignores a min_coverage from very_good.yaml when sharding', + () async { + final tempDirectory = Directory.systemTemp.createTempSync(); + addTearDown(() { + Directory.current = cwd; + tempDirectory.deleteSync(recursive: true); + }); + Directory.current = tempDirectory.path; + File(path.join(tempDirectory.path, 'pubspec.yaml')).createSync(); + File(path.join(tempDirectory.path, 'very_good.yaml')) + .writeAsStringSync('test:\n min_coverage: 90\n'); + when(() => argResults.wasParsed(any())).thenReturn(false); + withShards('1', '3'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.success.code)); + verify( + () => flutterTest( + shardIndex: 1, + totalShards: 3, + optimizePerformance: true, + arguments: defaultArguments, + logger: logger, + stdout: logger.write, + stderr: logger.err, + ), + ).called(1); + }, + ); + + test('fails when sharding is combined with --min-coverage', () async { + withShards('1', '3'); + when(() => argResults['min-coverage']).thenReturn('100'); + + final result = await testCommand.run(); + + expect(result, equals(ExitCode.usage.code)); + verify( + () => logger.err( + '--min-coverage cannot be combined with sharding. Collect ' + 'coverage per shard with --coverage, merge the lcov reports, ' + 'then check the threshold in a separate job.', + ), + ).called(1); + }); + }); + group('very_good.yaml configuration', () { test( 'fails with exit code ${ExitCode.config.code} ' @@ -997,6 +1215,7 @@ void main() { ), ); expect(options.minCoverage, equals(90)); + expect(options.rawMinCoverage, isNull); expect(options.excludeFromCoverage, equals('**/*.g.dart')); expect(options.reportOn, equals(['lib/'])); expect(options.fileReporter, equals('json:reports/tests.json')); @@ -1014,6 +1233,7 @@ void main() { ), ); expect(options.minCoverage, equals(50)); + expect(options.rawMinCoverage, equals('50')); }); test('CLI --file-reporter takes precedence over config value', () {