From bde6211c2b9c74cf3305aed26328ce133adade16 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Thu, 9 Jul 2026 18:49:32 +0500 Subject: [PATCH 01/29] feat(test_runner): support live Firebase and HTTP under flutter test Bridge Cloud Functions, Auth, and Firestore pigeon calls to HTTPS so YAML app tests can exercise real backends without native plugins. Preserve route settings for navigation waits, register startTimer delay timers for lifecycle cleanup, and keep unit tests free of real credentials. --- .../framework/apiproviders/api_provider.dart | 2 +- .../lib/layout/ensemble_page_route.dart | 6 +- modules/ensemble/lib/screen_controller.dart | 25 +- .../lib/actions/test_step_executor.dart | 42 +- .../lib/cli/yaml_test_app_patcher.dart | 53 +- .../lib/entry/ensemble_test_entry.dart | 65 ++- .../lib/mocks/adobe_test_setup.dart | 53 ++ .../lib/mocks/firebase_auth_test_setup.dart | 164 ++++++ .../mocks/firebase_firestore_test_setup.dart | 539 ++++++++++++++++++ .../mocks/firebase_functions_test_setup.dart | 148 +++++ .../lib/mocks/firebase_test_setup.dart | 69 +++ .../lib/mocks/live_firebase_auth_http.dart | 98 ++++ .../mocks/live_sign_in_with_custom_token.dart | 182 ++++++ .../lib/mocks/mock_api_provider.dart | 166 +++++- .../lib/runner/ensemble_test_context.dart | 4 + .../lib/runner/ensemble_test_harness.dart | 220 ++++++- .../lib/runner/ensemble_test_runner.dart | 24 +- .../lib/runner/live_async_call.dart | 70 +++ .../lib/runner/yaml_test_session.dart | 28 +- tools/ensemble_test_runner/pubspec.yaml | 8 + .../test/ensemble_test_harness_test.dart | 25 + .../test/ensemble_test_parser_test.dart | 25 + .../test/firebase_auth_test_setup_test.dart | 16 + .../firebase_functions_test_setup_test.dart | 29 + .../test/mock_api_provider_test.dart | 146 +++++ .../test/navigation_flow_recorder_test.dart | 25 +- .../test/yaml_test_app_patcher_test.dart | 62 +- 27 files changed, 2239 insertions(+), 55 deletions(-) create mode 100644 tools/ensemble_test_runner/lib/mocks/adobe_test_setup.dart create mode 100644 tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart create mode 100644 tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart create mode 100644 tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart create mode 100644 tools/ensemble_test_runner/lib/mocks/firebase_test_setup.dart create mode 100644 tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart create mode 100644 tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart create mode 100644 tools/ensemble_test_runner/lib/runner/live_async_call.dart create mode 100644 tools/ensemble_test_runner/test/ensemble_test_harness_test.dart create mode 100644 tools/ensemble_test_runner/test/firebase_auth_test_setup_test.dart create mode 100644 tools/ensemble_test_runner/test/firebase_functions_test_setup_test.dart create mode 100644 tools/ensemble_test_runner/test/mock_api_provider_test.dart diff --git a/modules/ensemble/lib/framework/apiproviders/api_provider.dart b/modules/ensemble/lib/framework/apiproviders/api_provider.dart index 2dd658927..234078b31 100644 --- a/modules/ensemble/lib/framework/apiproviders/api_provider.dart +++ b/modules/ensemble/lib/framework/apiproviders/api_provider.dart @@ -29,7 +29,7 @@ class APIProviders extends InheritedWidget { if (provider == null) { return httpProvider; } else if (provider == 'firebaseFunction') { - return FirebaseFunctionsAPIProvider(); + return providers[provider] ?? FirebaseFunctionsAPIProvider(); } else if (provider == 'sse') { return providers[provider] ?? SSEAPIProvider(); } else { diff --git a/modules/ensemble/lib/layout/ensemble_page_route.dart b/modules/ensemble/lib/layout/ensemble_page_route.dart index 199f8eb75..d5d9a9b93 100644 --- a/modules/ensemble/lib/layout/ensemble_page_route.dart +++ b/modules/ensemble/lib/layout/ensemble_page_route.dart @@ -78,8 +78,12 @@ extension PageTransitionTypeX on PageTransitionType { /// Page route builder that presents a screen without transition animation. class EnsemblePageRouteNoTransitionBuilder extends PageRouteBuilder { /// Creates a [EnsemblePageRouteNoTransitionBuilder] object. - EnsemblePageRouteNoTransitionBuilder({required Widget screenWidget}) + EnsemblePageRouteNoTransitionBuilder({ + required Widget screenWidget, + RouteSettings? settings, + }) : super( + settings: settings, pageBuilder: (context, animation, secondaryAnimation) => screenWidget, ); } diff --git a/modules/ensemble/lib/screen_controller.dart b/modules/ensemble/lib/screen_controller.dart index cd7f02fd1..5cb63a9c9 100644 --- a/modules/ensemble/lib/screen_controller.dart +++ b/modules/ensemble/lib/screen_controller.dart @@ -279,7 +279,7 @@ class ScreenController { (isRepeat ? repeatInterval! : 0); // we always execute at least once, delayed by startAfter and fallback to repeatInterval (or immediate if startAfter is 0) - Timer(Duration(seconds: delay), () { + final timer = Timer(Duration(seconds: delay), () { // execute the action executeActionWithScope(context, scopeManager, action.onTimer); @@ -298,7 +298,7 @@ class ScreenController { int? repeatCount = maxTimes != null ? maxTimes - 1 : null; if (repeatCount != 0) { int counter = 0; - final timer = + final periodicTimer = Timer.periodic(Duration(seconds: repeatInterval), (timer) { // execute the action executeActionWithScope(context, scopeManager, action.onTimer); @@ -317,10 +317,11 @@ class ScreenController { // save our timer to our PageData since user may want to cancel at anytime // and also when we navigate away from the page - scopeManager.addTimer(action, timer); + scopeManager.addTimer(action, periodicTimer); } } }); + scopeManager.addTimer(action, timer); } else if (action is StopTimerAction) { try { scopeManager.removeTimer(action.id); @@ -584,12 +585,24 @@ class ScreenController { defaultTransitionOptions[_pageType]?['duration'], fallback: 250); + final routeSettings = RouteSettings( + name: screenName ?? screenId, + arguments: ScreenPayload( + screenId: screenId, + screenName: screenName, + pageType: pageType, + arguments: pageArgs, + isExternal: isExternal, + ), + ); + PageRouteBuilder route = getScreenBuilder( screenWidget, pageType: pageType, transitionType: transitionType, alignment: alignment, duration: duration, + settings: routeSettings, ); // push the new route and remove all existing screens. This is suitable for logging out. if (routeOption == RouteOption.clearAllScreens) { @@ -651,12 +664,14 @@ class ScreenController { PageTransitionType? transitionType, Alignment? alignment, int? duration, + RouteSettings? settings, }) { const enableTransition = bool.fromEnvironment('transitions', defaultValue: true); if (!enableTransition) { - return EnsemblePageRouteNoTransitionBuilder(screenWidget: screenWidget); + return EnsemblePageRouteNoTransitionBuilder( + screenWidget: screenWidget, settings: settings); } if (pageType == PageType.modal) { @@ -669,6 +684,7 @@ class ScreenController { duration: Duration(milliseconds: duration ?? 250), barrierDismissible: true, barrierColor: Colors.black54, + settings: settings, ); } else { return EnsemblePageRouteBuilder( @@ -676,6 +692,7 @@ class ScreenController { transitionType: transitionType ?? PageTransitionType.fade, alignment: alignment ?? Alignment.center, duration: Duration(milliseconds: duration ?? 250), + settings: settings, ); } } diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 371a4631e..d361173ef 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -1,3 +1,5 @@ +import 'dart:async'; + import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; @@ -434,6 +436,29 @@ class TestStepExecutor { EnginePhase.sendSemanticsUpdate, timeout ?? config.settleTimeout, ); + await _yieldToLiveApiWork(); + } + + /// Lets in-flight live HTTP (wrapped in [WidgetTester.runAsync]) finish and + /// pumps a frame so Ensemble can apply API state. Uses [Duration.zero] so + /// timers from departed screens are not advanced while draining live HTTP. + Future _yieldToLiveApiWork() async { + for (var i = 0; i < 200; i++) { + final hadPending = context.mockApiProvider.hasPendingLiveCalls; + if (hadPending) { + try { + await context.mockApiProvider + .waitForLiveCalls() + .timeout(config.waitPollInterval); + } on TimeoutException { + // Keep polling; HTTP may still be in flight inside runAsync. + } + } + await tester.pump(); + if (!hadPending) { + return; + } + } } Future _tap(String id) async { @@ -572,8 +597,10 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { + await _yieldToLiveApiWork(); await tester.pump(config.waitPollInterval); if (context.mockApiProvider.callCount(name) >= times) { + await _yieldToLiveApiWork(); return; } } @@ -589,12 +616,21 @@ class TestStepExecutor { }) async { final stopwatch = Stopwatch()..start(); final tracker = ScreenTracker(); + bool isVisible() => + tracker.isScreenVisible(screenName: screen) || + tracker.isScreenVisible(screenId: screen); + while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); - if (tracker.isScreenVisible(screenName: screen) || - tracker.isScreenVisible(screenId: screen)) { + if (isVisible()) { return; } + await _yieldToLiveApiWork(); + await tester.pump(config.waitPollInterval); + } + await _yieldToLiveApiWork(); + await tester.pump(); + if (isVisible()) { + return; } throw EnsembleTestFailure( 'Timed out after ${timeoutMs}ms waiting for navigation to "$screen"', diff --git a/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart b/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart index 1d4b09e2d..7e1c43c8a 100644 --- a/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart +++ b/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart @@ -13,13 +13,30 @@ class YamlTestAppPatcher { static String testsAssetLineFor(String testsDirRelative) => ' - ${_withTrailingSlash(testsDirRelative)}'; - static const testEntryContents = ''' + /// Legacy CLI stub without module bootstrap. + static const legacyTestEntryContents = ''' // Generated by ensemble_test_runner — run `dart run ensemble_test_runner:ensemble_test`. import 'package:ensemble_test_runner/entry/ensemble_test_entry.dart'; void main() => runEnsembleYamlTests(); '''; + static String testEntryContentsFor(String packageName) => ''' +// Generated by ensemble_test_runner — customize bootstrap here. +import 'package:ensemble_test_runner/entry/ensemble_test_entry.dart'; +import 'package:$packageName/generated/ensemble_modules.dart'; + +Future main() async { + await runEnsembleYamlTests( + bootstrap: () => EnsembleModules().init(), + ); +} +'''; + + /// @deprecated Use [testEntryContentsFor]. + static String get testEntryContents => + testEntryContentsFor('your_app_package'); + final Map _backups = {}; bool _enabled = false; bool _removeTestEntryOnRestore = false; @@ -52,10 +69,6 @@ void main() => runEnsembleYamlTests(); _backup(_pubspecPath); _backup(_testEntryPath, optional: true); - final priorTestEntry = _backups[_testEntryPath] ?? ''; - _removeTestEntryOnRestore = - priorTestEntry.isEmpty || priorTestEntry == testEntryContents; - final pubspec = File(_pubspecPath).readAsStringSync(); final activated = _activatePubspec(pubspec); _pubspecChanged = activated != pubspec; @@ -63,11 +76,23 @@ void main() => runEnsembleYamlTests(); File(_pubspecPath).writeAsStringSync(activated); } - Directory(_testDirPath).createSync(recursive: true); final testEntry = File(_testEntryPath); - if (!testEntry.existsSync() || - testEntry.readAsStringSync() != testEntryContents) { - testEntry.writeAsStringSync(testEntryContents); + final testEntryExisted = testEntry.existsSync(); + _removeTestEntryOnRestore = !testEntryExisted; + + if (!testEntryExisted) { + Directory(_testDirPath).createSync(recursive: true); + testEntry.writeAsStringSync( + testEntryContentsFor(_readPackageName()), + ); + } else { + final content = testEntry.readAsStringSync(); + if (content.trim() == legacyTestEntryContents.trim()) { + final upgraded = testEntryContentsFor(_readPackageName()); + testEntry.writeAsStringSync(upgraded); + _backups[_testEntryPath] = upgraded; + _removeTestEntryOnRestore = false; + } } _enabled = true; @@ -107,6 +132,16 @@ void main() => runEnsembleYamlTests(); File(path).writeAsStringSync(backup); } + String _readPackageName() { + final content = File(_pubspecPath).readAsStringSync(); + final match = + RegExp(r'^name:\s*(\S+)', multiLine: true).firstMatch(content); + if (match == null) { + throw StateError('Could not read package name from pubspec.yaml'); + } + return match.group(1)!; + } + void _deleteTestEntry() { final file = File(_testEntryPath); if (file.existsSync()) { diff --git a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart index ff7327502..81c00f7cb 100644 --- a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart +++ b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart @@ -6,6 +6,7 @@ import 'dart:io'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; import 'package:ensemble_test_runner/ensemble_test_runner.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -16,8 +17,52 @@ const _timeoutSeconds = int.fromEnvironment( defaultValue: _defaultTimeoutSeconds, ); +/// Options for [runEnsembleYamlTests], typically set from `test/ensemble_tests.dart`. +class EnsembleYamlTestOptions { + /// App bootstrap, typically `() => EnsembleModules().init()` from + /// `lib/generated/ensemble_modules.dart` (same as `main.dart`). + final Future Function()? bootstrap; + + /// Host-app methods passed to [EnsembleApp], e.g. `captureCertificateForHost`. + final Map? externalMethods; + + const EnsembleYamlTestOptions({ + this.bootstrap, + this.externalMethods, + }); +} + /// Flutter test entry: discovers app-local `tests/*.test.yaml` and runs them. -void runEnsembleYamlTests() { +/// +/// Mirror `main.dart` from `test/ensemble_tests.dart`: +/// ```dart +/// import 'package:my_app/generated/ensemble_modules.dart'; +/// +/// Future main() async { +/// await runEnsembleYamlTests( +/// bootstrap: () => EnsembleModules().init(), +/// externalMethods: { +/// 'captureCertificateForHost': captureCertificateForHost, +/// }, +/// ); +/// } +/// ``` +Future runEnsembleYamlTests({ + Future Function()? bootstrap, + Map? externalMethods, +}) { + return runEnsembleYamlTestsWithOptions( + EnsembleYamlTestOptions( + bootstrap: bootstrap, + externalMethods: externalMethods, + ), + ); +} + +/// Same as [runEnsembleYamlTests] with an explicit options object. +Future runEnsembleYamlTestsWithOptions( + EnsembleYamlTestOptions options, +) async { EnsembleTestHarness.ensureTestPlugins(); tearDown(() { TestErrorTracker.reset(); @@ -28,6 +73,21 @@ void runEnsembleYamlTests() { testWidgets( 'Ensemble app *.test.yaml', (tester) async { + if (options.bootstrap == null) { + fail( + 'Ensemble YAML tests require module bootstrap. ' + 'In test/ensemble_tests.dart call runEnsembleYamlTests with ' + 'bootstrap: () => EnsembleModules().init() ' + '(see ensemble_test_runner README).', + ); + } + await tester.runAsync(() async { + await options.bootstrap!(); + ensureLiveAuthActionsForTest(); + // Module constructors may schedule follow-up async init work. + await Future.delayed(Duration.zero); + }); + final target = await EnsembleTestDiscovery.loadAppTarget(); final plan = await EnsembleTestExecutionPlanner.build( target: target, @@ -37,10 +97,13 @@ void runEnsembleYamlTests() { appPath: target.appPath, appHome: target.appHome, i18nPath: target.i18nPath, + externalMethods: options.externalMethods, ); final runner = EnsembleTestRunner(harness: harness); final resultsById = await runner.runPlan(plan, tester); + await YamlTestSession.navigationFlow.flushPending(); + await tester.pump(); final failures = []; final orderedResults = []; diff --git a/tools/ensemble_test_runner/lib/mocks/adobe_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/adobe_test_setup.dart new file mode 100644 index 000000000..02b39f25c --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/adobe_test_setup.dart @@ -0,0 +1,53 @@ +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _adobeMocksInstalled = false; + +/// Installs no-op Adobe Experience Platform method channel handlers so +/// [AdobeAnalyticsImpl] can initialize under [flutter test] without native SDKs. +void ensureAdobeAnalyticsMocksForTest() { + if (_adobeMocksInstalled) return; + + const channelNames = [ + 'flutter_aepcore', + 'flutter_aepedge', + 'flutter_aepidentity', + 'flutter_aeplifecycle', + 'flutter_aepsignal', + 'flutter_aepedgeidentity', + 'flutter_aepedgeconsent', + 'flutter_aepassurance', + 'flutter_aepuserprofile', + ]; + + for (final name in channelNames) { + _installAdobeChannelMock(name); + } + + _adobeMocksInstalled = true; +} + +void _installAdobeChannelMock(String channelName) { + final channel = MethodChannel(channelName); + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(channel, (call) async { + switch (call.method) { + case 'sendEvent': + return []; + case 'extensionVersion': + return 'ensemble-test'; + case 'getExperienceCloudId': + case 'getLocationHint': + case 'getUrlVariables': + return null; + case 'getIdentities': + return {}; + case 'getUserAttributes': + return {}; + case 'getConsents': + return {}; + default: + return null; + } + }); +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart new file mode 100644 index 000000000..af110c68d --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart @@ -0,0 +1,164 @@ +import 'package:firebase_auth_platform_interface/src/pigeon/messages.pigeon.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _firebaseAuthBridgeInstalled = false; + +class _LiveAuthSession { + _LiveAuthSession({ + required this.idToken, + required this.refreshToken, + required this.uid, + required this.expiresAtMs, + this.email, + this.isEmailVerified = false, + }); + + String idToken; + final String refreshToken; + final String uid; + int expiresAtMs; + final String? email; + final bool isEmailVerified; +} + +final Map _sessionsByApp = {}; + +void recordLiveAuthSession({ + required String appName, + required String idToken, + required String refreshToken, + required String uid, + required int expiresAtMs, + String? email, + bool isEmailVerified = false, +}) { + _sessionsByApp[appName] = _LiveAuthSession( + idToken: idToken, + refreshToken: refreshToken, + uid: uid, + expiresAtMs: expiresAtMs, + email: email, + isEmailVerified: isEmailVerified, + ); +} + +bool hasLiveAuthSession(String appName) => _sessionsByApp.containsKey(appName); + +_LiveAuthSession? liveAuthSessionForApp(String appName) => + _sessionsByApp[appName]; + +/// Bridges Firebase Auth user token pigeon calls for code paths that still use +/// [FirebaseAuth] after [LiveSignInWithCustomToken] has established a session. +void ensureLiveFirebaseAuthForTest() { + if (_firebaseAuthBridgeInstalled) return; + TestWidgetsFlutterBinding.ensureInitialized(); + + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + final codec = FirebaseAuthHostApi.pigeonChannelCodec; + + void registerHandler( + String channelName, + Future> Function(List args) handler, + ) { + messenger.setMockDecodedMessageHandler( + BasicMessageChannel(channelName, codec), + (Object? message) async { + final args = (message! as List); + try { + return await handler(args); + } on PlatformException catch (error) { + return [error.code, error.message, error.details]; + } catch (error) { + return ['error', error.toString(), null]; + } + }, + ); + } + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthHostApi.registerIdTokenListener', + (args) async { + final app = _decodeApp(args[0]); + final channel = 'ensemble_test_runner/auth/id-token/${app.appName}'; + _mockEventChannel(channel); + return [channel]; + }, + ); + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthHostApi.registerAuthStateListener', + (args) async { + final app = _decodeApp(args[0]); + final channel = 'ensemble_test_runner/auth/auth-state/${app.appName}'; + _mockEventChannel(channel); + return [channel]; + }, + ); + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthHostApi.signOut', + (args) async { + final app = _decodeApp(args[0]); + _sessionsByApp.remove(app.appName); + return []; + }, + ); + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthUserHostApi.getIdToken', + (args) async { + final app = _decodeApp(args[0]); + final session = _sessionForApp(app); + return [ + InternalIdTokenResult( + token: session.idToken, + expirationTimestamp: session.expiresAtMs, + authTimestamp: DateTime.now().millisecondsSinceEpoch, + issuedAtTimestamp: DateTime.now().millisecondsSinceEpoch, + signInProvider: 'custom', + ), + ]; + }, + ); + + _firebaseAuthBridgeInstalled = true; +} + +void _mockEventChannel(String channelName) { + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + messenger.setMockMethodCallHandler(MethodChannel(channelName), (call) async { + switch (call.method) { + case 'listen': + return 0; + case 'cancel': + return null; + default: + return null; + } + }); +} + +AuthPigeonFirebaseApp _decodeApp(Object? value) { + if (value is AuthPigeonFirebaseApp) { + return value; + } + if (value is List) { + return AuthPigeonFirebaseApp.decode(value); + } + throw ArgumentError('Unexpected Firebase app argument: $value'); +} + +_LiveAuthSession _sessionForApp(AuthPigeonFirebaseApp app) { + final session = _sessionsByApp[app.appName]; + if (session == null) { + throw PlatformException( + code: 'no-current-user', + message: 'No signed-in Firebase user for app ${app.appName}', + ); + } + return session; +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart new file mode 100644 index 000000000..46291f34c --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart @@ -0,0 +1,539 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:cloud_firestore_platform_interface/cloud_firestore_platform_interface.dart'; +import 'package:ensemble_test_runner/mocks/firebase_auth_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _firestoreBridgeInstalled = false; +int _snapshotListenerCounter = 0; + +/// Bridges Firestore pigeon calls to the real Firestore REST API during +/// [flutter test]. Native plugins are unavailable in the VM test binding. +void ensureLiveFirestoreForTest() { + if (_firestoreBridgeInstalled) return; + TestWidgetsFlutterBinding.ensureInitialized(); + + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + final codec = FirebaseFirestoreHostApi.pigeonChannelCodec; + + void register( + String method, + Future> Function(List args) handler, + ) { + messenger.setMockDecodedMessageHandler( + BasicMessageChannel( + 'dev.flutter.pigeon.cloud_firestore_platform_interface.FirebaseFirestoreHostApi.$method', + codec, + ), + (Object? message) async { + final args = (message! as List); + try { + return await handler(args); + } on PlatformException catch (error) { + return [error.code, error.message, error.details]; + } catch (error) { + return ['error', error.toString(), null]; + } + }, + ); + } + + Future> noop(List args) async => []; + + register('setLoggingEnabled', (_) async => []); + register('enableNetwork', noop); + register('disableNetwork', noop); + register('clearPersistence', noop); + register('terminate', noop); + register('waitForPendingWrites', noop); + register('snapshotsInSyncSetup', (_) async => [ + 'ensemble_test_runner/firestore/snapshots-in-sync', + ]); + register('persistenceCacheIndexManagerRequest', noop); + + register('documentReferenceSet', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + await _LiveFirestoreRestClient().documentSet(app, request); + return []; + }); + + register('documentReferenceUpdate', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + await _LiveFirestoreRestClient().documentUpdate(app, request); + return []; + }); + + register('documentReferenceDelete', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + await _LiveFirestoreRestClient().documentDelete(app, request); + return []; + }); + + register('documentReferenceGet', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + final snapshot = await _LiveFirestoreRestClient().documentGet(app, request); + return [snapshot]; + }); + + register('queryGet', (args) async { + final app = _decodeApp(args[0]); + final path = args[1]! as String; + final isCollectionGroup = args[2]! as bool; + final parameters = args[3]! as InternalQueryParameters; + final options = args[4]! as InternalGetOptions; + final snapshot = await _LiveFirestoreRestClient().queryGet( + app, + path: path, + isCollectionGroup: isCollectionGroup, + parameters: parameters, + options: options, + ); + return [snapshot]; + }); + + register('documentReferenceSnapshot', (args) async { + final request = _decodeDocumentRequest(args[1]); + final listenerId = + 'ensemble_test_runner/firestore/document/${_snapshotListenerCounter++}/${request.path}'; + _mockFirestoreEventChannel('document', listenerId); + return [listenerId]; + }); + + register('querySnapshot', (args) async { + final path = args[1]! as String; + final listenerId = + 'ensemble_test_runner/firestore/query/${_snapshotListenerCounter++}/$path'; + _mockFirestoreEventChannel('query', listenerId); + return [listenerId]; + }); + + _firestoreBridgeInstalled = true; +} + +void _mockFirestoreEventChannel(String kind, String listenerId) { + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + messenger.setMockMethodCallHandler( + MethodChannel('plugins.flutter.io/firebase_firestore/$kind/$listenerId'), + (call) async { + switch (call.method) { + case 'listen': + return 0; + case 'cancel': + return null; + default: + return null; + } + }, + ); +} + +FirestorePigeonFirebaseApp _decodeApp(Object? value) { + if (value is FirestorePigeonFirebaseApp) return value; + if (value is List) return FirestorePigeonFirebaseApp.decode(value); + throw ArgumentError('Unexpected Firestore app argument: $value'); +} + +DocumentReferenceRequest _decodeDocumentRequest(Object? value) { + if (value is DocumentReferenceRequest) return value; + if (value is List) return DocumentReferenceRequest.decode(value); + throw ArgumentError('Unexpected document request argument: $value'); +} + +class _LiveFirestoreRestClient { + Future documentSet( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + final encoded = _encodeDocumentFields(request.data ?? {}); + final transforms = encoded.transforms; + final body = { + if (encoded.fields.isNotEmpty) 'fields': encoded.fields, + if (transforms.isNotEmpty) 'updateTransforms': transforms, + }; + + final option = request.option; + final updateMaskPaths = []; + if (option?.merge == true) { + updateMaskPaths.addAll(encoded.fieldPaths); + } else if (option?.mergeFields != null && option!.mergeFields!.isNotEmpty) { + updateMaskPaths.addAll( + option.mergeFields! + .map((components) => components?.whereType().join('.')) + .whereType() + .where((path) => path.isNotEmpty), + ); + } + + await _request( + app, + method: 'PATCH', + path: request.path, + updateMaskPaths: updateMaskPaths, + body: body, + ); + } + + Future documentUpdate( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + final encoded = _encodeDocumentFields(request.data ?? {}); + final body = { + if (encoded.fields.isNotEmpty) 'fields': encoded.fields, + if (encoded.transforms.isNotEmpty) + 'updateTransforms': encoded.transforms, + }; + await _request( + app, + method: 'PATCH', + path: request.path, + updateMaskPaths: encoded.fieldPaths, + body: body, + ); + } + + Future documentDelete( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + await _request(app, method: 'DELETE', path: request.path); + } + + Future documentGet( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + final decoded = await _request(app, method: 'GET', path: request.path); + if (decoded == null) { + return InternalDocumentSnapshot( + path: request.path, + data: null, + metadata: InternalSnapshotMetadata( + hasPendingWrites: false, + isFromCache: false, + ), + ); + } + return _toPigeonDocumentSnapshot(request.path, decoded); + } + + Future queryGet( + FirestorePigeonFirebaseApp app, { + required String path, + required bool isCollectionGroup, + required InternalQueryParameters parameters, + required InternalGetOptions options, + }) async { + final structuredQuery = { + 'from': [ + if (isCollectionGroup) + {'collectionId': path, 'allDescendants': true} + else + {'collectionId': path.split('/').last}, + ], + if (parameters.limit != null) 'limit': parameters.limit, + }; + + final body = { + 'structuredQuery': structuredQuery, + }; + if (path.contains('/') && !isCollectionGroup) { + final parentPath = path.split('/')..removeLast(); + body['parent'] = '${_documentsRoot(app)}/${parentPath.join('/')}'; + } else { + body['parent'] = _documentsRoot(app); + } + + final decoded = await _request( + app, + method: 'POST', + suffix: ':runQuery', + body: body, + ); + + final documents = []; + if (decoded is List) { + for (final entry in decoded) { + if (entry is! Map) continue; + final document = entry['document']; + if (document is! Map) continue; + final name = document['name']?.toString() ?? ''; + final docPath = _pathFromDocumentName(name); + documents.add(_toPigeonDocumentSnapshot(docPath, document)); + } + } + + return InternalQuerySnapshot( + documents: documents, + documentChanges: documents + .whereType() + .map( + (document) => InternalDocumentChange( + type: DocumentChangeType.added, + document: document, + oldIndex: -1, + newIndex: documents.indexOf(document), + ), + ) + .toList(), + metadata: InternalSnapshotMetadata( + hasPendingWrites: false, + isFromCache: options.source == Source.cache, + ), + ); + } + + Future _request( + FirestorePigeonFirebaseApp app, { + required String method, + String? path, + String suffix = '', + List updateMaskPaths = const [], + Map? body, + }) async { + final projectId = _projectIdForApp(app.appName); + final idToken = _idTokenForApp(app.appName); + final encodedPath = path == null + ? '' + : path.split('/').map(Uri.encodeComponent).join('/'); + final queryParts = [ + for (final fieldPath in updateMaskPaths) + 'updateMask.fieldPaths=${Uri.encodeQueryComponent(fieldPath)}', + ]; + final uri = Uri.parse( + 'https://firestore.googleapis.com/v1/projects/$projectId/databases/(default)/documents' + '${encodedPath.isEmpty ? '' : '/$encodedPath'}$suffix' + '${queryParts.isEmpty ? '' : '?${queryParts.join('&')}'}', + ); + + final savedOverrides = HttpOverrides.current; + HttpOverrides.global = null; + final client = HttpClient(); + try { + late final HttpClientRequest request; + switch (method) { + case 'GET': + request = await client.getUrl(uri); + case 'DELETE': + request = await client.deleteUrl(uri); + case 'PATCH': + request = await client.patchUrl(uri); + case 'POST': + request = await client.postUrl(uri); + default: + throw UnsupportedError('Unsupported HTTP method: $method'); + } + request.headers.set(HttpHeaders.contentTypeHeader, 'application/json'); + request.headers.set(HttpHeaders.authorizationHeader, 'Bearer $idToken'); + if (body != null) { + request.write(jsonEncode(body)); + } + + final response = await request.close(); + final responseBody = await response.transform(utf8.decoder).join(); + + if (response.statusCode == 404 && method == 'GET') { + return null; + } + if (response.statusCode < 200 || response.statusCode >= 300) { + throw PlatformException( + code: 'firestore', + message: 'HTTP ${response.statusCode} $method $uri: $responseBody', + ); + } + if (responseBody.isEmpty) { + return null; + } + return jsonDecode(responseBody); + } finally { + client.close(); + HttpOverrides.global = savedOverrides; + } + } + + String _documentsRoot(FirestorePigeonFirebaseApp app) { + final projectId = _projectIdForApp(app.appName); + return 'projects/$projectId/databases/(default)/documents'; + } +} + +class _EncodedDocument { + _EncodedDocument({ + required this.fields, + required this.transforms, + required this.fieldPaths, + }); + + final Map fields; + final List> transforms; + final List fieldPaths; +} + +_EncodedDocument _encodeDocumentFields(Map data) { + final fields = {}; + final fieldPaths = []; + + data.forEach((key, value) { + final fieldPath = key?.toString(); + if (fieldPath == null || fieldPath.isEmpty) return; + + fields[fieldPath] = _encodeValue(value); + fieldPaths.add(fieldPath); + }); + + return _EncodedDocument( + fields: fields, + transforms: const [], + fieldPaths: fieldPaths, + ); +} + +Map _encodeValue(Object? value) { + if (value == null) return {'nullValue': null}; + if (value is bool) return {'booleanValue': value}; + if (value is int) return {'integerValue': value.toString()}; + if (value is double) return {'doubleValue': value}; + if (value is String) return {'stringValue': value}; + if (value is Timestamp) { + return {'timestampValue': value.toDate().toUtc().toIso8601String()}; + } + if (value is GeoPoint) { + return { + 'geoPointValue': { + 'latitude': value.latitude, + 'longitude': value.longitude, + }, + }; + } + if (value is Map) { + final nested = {}; + value.forEach((key, nestedValue) { + final nestedKey = key?.toString(); + if (nestedKey == null || nestedKey.isEmpty) return; + nested[nestedKey] = _encodeValue(nestedValue); + }); + return {'mapValue': {'fields': nested}}; + } + if (value is Iterable) { + return { + 'arrayValue': { + 'values': value.map(_encodeValue).toList(), + }, + }; + } + return {'stringValue': value.toString()}; +} + +InternalDocumentSnapshot _toPigeonDocumentSnapshot( + String path, + Map document, +) { + final fields = document['fields']; + return InternalDocumentSnapshot( + path: path, + data: fields is Map + ? _decodeFields(fields.cast()) + : {}, + metadata: InternalSnapshotMetadata( + hasPendingWrites: false, + isFromCache: false, + ), + ); +} + +Map _decodeFields(Map fields) { + final decoded = {}; + fields.forEach((key, value) { + decoded[key?.toString()] = _decodeValue(value); + }); + return decoded; +} + +Object? _decodeValue(Object? value) { + if (value is! Map) return value; + final map = value.cast(); + if (map.containsKey('stringValue')) return map['stringValue']; + if (map.containsKey('booleanValue')) return map['booleanValue']; + if (map.containsKey('integerValue')) { + return int.tryParse(map['integerValue'].toString()) ?? + map['integerValue']; + } + if (map.containsKey('doubleValue')) { + final raw = map['doubleValue']; + return raw is num ? raw.toDouble() : double.tryParse(raw.toString()); + } + if (map.containsKey('nullValue')) return null; + if (map.containsKey('timestampValue')) { + return Timestamp.fromDate(DateTime.parse(map['timestampValue'].toString())); + } + if (map.containsKey('geoPointValue')) { + final geo = map['geoPointValue'] as Map; + return GeoPoint( + (geo['latitude'] as num).toDouble(), + (geo['longitude'] as num).toDouble(), + ); + } + if (map.containsKey('mapValue')) { + final nested = (map['mapValue'] as Map?)?['fields']; + if (nested is Map) { + return _decodeFields(nested.cast()); + } + return {}; + } + if (map.containsKey('arrayValue')) { + final values = (map['arrayValue'] as Map?)?['values']; + if (values is List) { + return values.map(_decodeValue).toList(); + } + return []; + } + return map; +} + +String _pathFromDocumentName(String name) { + const marker = '/documents/'; + final index = name.indexOf(marker); + if (index < 0) return name; + return name.substring(index + marker.length); +} + +String _projectIdForApp(String appName) { + final fromInit = firebaseProjectIdsByApp[appName]; + if (fromInit != null && fromInit.isNotEmpty) { + return fromInit; + } + try { + final app = appName.isEmpty || appName == defaultFirebaseAppName + ? Firebase.app() + : Firebase.app(appName); + return app.options.projectId; + } catch (_) { + throw StateError( + 'Could not resolve Firebase projectId for Firestore call (app: $appName).', + ); + } +} + +String _idTokenForApp(String appName) { + final session = liveAuthSessionForApp(appName) ?? + liveAuthSessionForApp(defaultFirebaseAppName); + final token = session?.idToken; + if (token == null || token.isEmpty) { + throw PlatformException( + code: 'firestore', + message: + 'No signed-in Firebase user for Firestore REST call (app: $appName).', + ); + } + return token; +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart new file mode 100644 index 000000000..8d35b13ff --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart @@ -0,0 +1,148 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:cloud_functions_platform_interface/src/pigeon/messages.pigeon.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _cloudFunctionsBridgeInstalled = false; + +/// Bridges Firebase Cloud Functions pigeon calls to real HTTPS callable endpoints +/// during [flutter test]. Native macOS/iOS/Android plugins are unavailable in the +/// VM test binding, so without this handler [httpsCallable] fails immediately. +void ensureLiveCloudFunctionsForTest() { + if (_cloudFunctionsBridgeInstalled) return; + TestWidgetsFlutterBinding.ensureInitialized(); + + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + + messenger.setMockDecodedMessageHandler( + BasicMessageChannel( + 'dev.flutter.pigeon.cloud_functions_platform_interface.CloudFunctionsHostApi.call', + CloudFunctionsHostApi.pigeonChannelCodec, + ), + (Object? message) async { + final args = (message! as List)[0]! as Map; + final arguments = args.cast(); + try { + final result = await _invokeCallableOverHttp(arguments); + return [result]; + } on PlatformException catch (error) { + return [error.code, error.message, error.details]; + } catch (error) { + return ['error', error.toString(), null]; + } + }, + ); + + messenger.setMockDecodedMessageHandler( + BasicMessageChannel( + 'dev.flutter.pigeon.cloud_functions_platform_interface.CloudFunctionsHostApi.registerEventChannel', + CloudFunctionsHostApi.pigeonChannelCodec, + ), + (_) async => [], + ); + + _cloudFunctionsBridgeInstalled = true; +} + +Future _invokeCallableOverHttp(Map arguments) async { + final uri = _resolveCallableUri(arguments); + final parameters = arguments['parameters']; + final timeoutMs = arguments['timeout'] as int? ?? 60000; + + final savedOverrides = HttpOverrides.current; + HttpOverrides.global = null; + final client = HttpClient(); + client.connectionTimeout = Duration(milliseconds: timeoutMs); + + try { + final request = await client.postUrl(uri); + request.headers.set(HttpHeaders.contentTypeHeader, 'application/json'); + request.write(jsonEncode({'data': parameters})); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + + if (response.statusCode < 200 || response.statusCode >= 300) { + throw PlatformException( + code: 'firebase_functions', + message: 'HTTP ${response.statusCode} calling $uri: $body', + ); + } + + final decoded = jsonDecode(body); + if (decoded is! Map) { + throw PlatformException( + code: 'firebase_functions', + message: 'Unexpected callable response from $uri: $body', + ); + } + + final map = Map.from(decoded); + if (map.containsKey('error')) { + throw PlatformException( + code: 'firebase_functions', + message: map['error'].toString(), + ); + } + + final result = map['result']; + if (result is Map) { + return Map.from(result); + } + return result; + } finally { + client.close(); + HttpOverrides.global = savedOverrides; + } +} + +Uri _resolveCallableUri(Map arguments) { + final functionUri = arguments['functionUri'] as String?; + if (functionUri != null && functionUri.isNotEmpty) { + return Uri.parse(functionUri); + } + + final origin = arguments['origin'] as String?; + if (origin != null && origin.isNotEmpty) { + return Uri.parse(origin); + } + + final functionName = arguments['functionName'] as String?; + if (functionName == null || functionName.isEmpty) { + throw ArgumentError('Cloud Functions call missing functionName'); + } + + final region = (arguments['region'] as String?) ?? 'us-central1'; + final projectId = _projectIdForApp(arguments['appName'] as String?); + if (projectId == null || projectId.isEmpty) { + throw StateError( + 'Could not resolve Firebase projectId for Cloud Functions call. ' + 'Ensure firebase_config is loaded before invoking callable APIs.', + ); + } + + return Uri.parse('https://$region-$projectId.cloudfunctions.net/$functionName'); +} + +String? _projectIdForApp(String? appName) { + if (appName != null && appName.isNotEmpty) { + final fromInit = firebaseProjectIdsByApp[appName]; + if (fromInit != null && fromInit.isNotEmpty) { + return fromInit; + } + } + try { + final app = (appName == null || + appName.isEmpty || + appName == defaultFirebaseAppName) + ? Firebase.app() + : Firebase.app(appName); + return app.options.projectId; + } catch (_) { + return null; + } +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_test_setup.dart new file mode 100644 index 000000000..d866bb89a --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_test_setup.dart @@ -0,0 +1,69 @@ +import 'package:ensemble_test_runner/mocks/firebase_auth_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_firestore_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_functions_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/live_sign_in_with_custom_token.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:firebase_core_platform_interface/test.dart'; + +bool _firebaseCoreMocksInstalled = false; + +final Map firebaseProjectIdsByApp = {}; + +/// Preserves [CoreFirebaseOptions] from [Firebase.initializeApp] instead of +/// the stock mock that always returns projectId `123`. +class _PreservingFirebaseCoreHostApi implements TestFirebaseCoreHostApi { + @override + Future initializeApp( + String appName, + CoreFirebaseOptions initializeAppRequest, + ) async { + firebaseProjectIdsByApp[appName] = initializeAppRequest.projectId; + return CoreInitializeResponse( + name: appName, + options: initializeAppRequest, + pluginConstants: {}, + ); + } + + @override + Future> initializeCore() async { + return [ + CoreInitializeResponse( + name: defaultFirebaseAppName, + options: CoreFirebaseOptions( + apiKey: 'test-api-key', + projectId: 'test-project', + appId: 'test-app-id', + messagingSenderId: 'test-sender', + ), + pluginConstants: {}, + ), + ]; + } + + @override + Future optionsFromResource() async { + return CoreFirebaseOptions( + apiKey: 'test-api-key', + projectId: 'test-project', + appId: 'test-app-id', + messagingSenderId: 'test-sender', + ); + } +} + +/// Installs Firebase Core platform mocks so [Firebase.initializeApp] works under +/// [flutter test] (no native Firebase host). Required before app bootstrap. +void ensureFirebaseCoreMocksForTest() { + if (_firebaseCoreMocksInstalled) return; + TestFirebaseCoreHostApi.setUp(_PreservingFirebaseCoreHostApi()); + ensureLiveCloudFunctionsForTest(); + ensureLiveFirebaseAuthForTest(); + ensureLiveFirestoreForTest(); + _firebaseCoreMocksInstalled = true; +} + +/// Call after app module bootstrap so [signInWithCustomToken] uses live Firebase. +void ensureLiveAuthActionsForTest() { + ensureLiveSignInWithCustomTokenForTest(); +} diff --git a/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart b/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart new file mode 100644 index 000000000..fca53319c --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart @@ -0,0 +1,98 @@ +import 'dart:convert'; +import 'dart:io'; + +/// Signs in with a Firebase custom token via the Identity Toolkit REST API. +Future> postIdentityToolkitSignInWithCustomToken({ + required String customToken, + required String apiKey, + int maxAttempts = 3, +}) async { + Object? lastError; + for (var attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await _postIdentityToolkitSignInOnce( + customToken: customToken, + apiKey: apiKey, + ); + } catch (error) { + lastError = error; + final retryable = _isRetryableAuthError(error); + if (!retryable || attempt >= maxAttempts) { + rethrow; + } + await Future.delayed(Duration(milliseconds: 250 * attempt)); + } + } + throw lastError ?? StateError('Identity Toolkit sign-in failed.'); +} + +bool _isRetryableAuthError(Object error) { + final message = error.toString().toLowerCase(); + return message.contains('connection reset') || + message.contains('connection closed') || + message.contains('timed out') || + message.contains('socketexception'); +} + +Future> _postIdentityToolkitSignInOnce({ + required String customToken, + required String apiKey, +}) async { + final uri = Uri.parse( + 'https://identitytoolkit.googleapis.com/v1/accounts:signInWithCustomToken?key=$apiKey', + ); + final savedOverrides = HttpOverrides.current; + HttpOverrides.global = null; + final client = HttpClient(); + client.connectionTimeout = const Duration(seconds: 60); + try { + final request = await client.postUrl(uri); + request.headers.set(HttpHeaders.contentTypeHeader, 'application/json'); + request.write(jsonEncode({ + 'token': customToken, + 'returnSecureToken': true, + })); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + if (response.statusCode < 200 || response.statusCode >= 300) { + throw HttpException( + 'HTTP ${response.statusCode} calling $uri: $body', + uri: uri, + ); + } + final decoded = jsonDecode(body); + if (decoded is! Map) { + throw HttpException('Unexpected auth response from $uri', uri: uri); + } + return Map.from(decoded); + } finally { + client.close(); + HttpOverrides.global = savedOverrides; + } +} + +/// Firebase ID tokens use `user_id`; custom tokens may expose `localId`. +String? uidFromFirebaseAuthResponse(Map authBody) { + final localId = authBody['localId']?.toString(); + if (localId != null && localId.isNotEmpty) { + return localId; + } + return uidFromFirebaseIdToken(authBody['idToken']?.toString() ?? ''); +} + +String? uidFromFirebaseIdToken(String idToken) { + final parts = idToken.split('.'); + if (parts.length < 2) { + return null; + } + try { + final normalized = base64Url.normalize(parts[1]); + final payload = jsonDecode(utf8.decode(base64Url.decode(normalized))); + if (payload is Map) { + return payload['user_id']?.toString() ?? payload['sub']?.toString(); + } + } catch (_) { + return null; + } + return null; +} diff --git a/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart b/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart new file mode 100644 index 000000000..51ce6bfd4 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart @@ -0,0 +1,182 @@ +import 'package:ensemble/ensemble.dart'; +import 'package:ensemble/framework/action.dart'; +import 'package:ensemble/framework/apiproviders/firebase_functions/firebase_functions_api_provider.dart'; +import 'package:ensemble/framework/error_handling.dart'; +import 'package:ensemble/framework/event.dart'; +import 'package:ensemble/framework/scope.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble/framework/stub/auth_context_manager.dart'; +import 'package:ensemble/screen_controller.dart'; +import 'package:ensemble/widget/stub_widgets.dart'; +import 'package:ensemble_test_runner/mocks/firebase_auth_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/live_firebase_auth_http.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter/material.dart'; +import 'package:get_it/get_it.dart'; + +/// Signs in with a Firebase custom token via the real Identity Toolkit REST API. +/// +/// HTTP runs through [LiveAsyncCallSupport] (same [WidgetTester.runAsync] queue +/// as live [invokeAPI] calls) because direct sockets from action callbacks fail +/// under [flutter test]. +class LiveSignInWithCustomToken implements SignInWithCustomToken { + @override + Future signInWithCustomToken( + BuildContext context, { + required SignInWithCustomTokenAction action, + }) async { + try { + final token = _evaluatedToken(context, action.jwtToken); + if (token.isEmpty) { + throw LanguageError( + "signInWithCustomToken requires jwtToken as 'token' parameter.", + recovery: + "Fix: pass valid jwtToken as 'token' under signInWithCustomToken", + ); + } + + final appName = _resolveAppName(); + final apiKey = _resolveApiKey(); + final authBody = await _signInOverHttp(token: token, apiKey: apiKey); + + final idToken = authBody['idToken']?.toString(); + final refreshToken = authBody['refreshToken']?.toString(); + final localId = uidFromFirebaseAuthResponse(authBody); + if (idToken == null || localId == null || refreshToken == null) { + throw StateError( + 'Firebase signInWithCustomToken response missing tokens ' + '(keys: ${authBody.keys.toList()}).', + ); + } + + final expiresInSec = + int.tryParse(authBody['expiresIn']?.toString() ?? '') ?? 3600; + recordLiveAuthSession( + appName: appName, + idToken: idToken, + refreshToken: refreshToken, + uid: localId, + expiresAtMs: DateTime.now() + .add(Duration(seconds: expiresInSec)) + .millisecondsSinceEpoch, + email: authBody['email']?.toString(), + isEmailVerified: authBody['emailVerified'] == true, + ); + + await StorageManager().writeToSystemStorage('user.id', localId); + await StorageManager().writeToSystemStorage('user.isAnonymous', false); + + if (action.onAuthenticated != null) { + final user = AuthenticatedUser( + provider: SignInProvider.firebase, + id: localId, + ); + ScreenController().executeAction( + context, + action.onAuthenticated!, + event: EnsembleEvent( + null, + data: {'user': user, 'idToken': idToken}, + ), + ); + } + } catch (error) { + if (action.onError != null) { + ScreenController().executeAction( + context, + action.onError!, + event: EnsembleEvent( + null, + error: {'error': _redactSecrets(error.toString())}, + ), + ); + } + } + } + + /// Avoid dumping JWTs / refresh tokens into test logs via onError. + static String _redactSecrets(String message) { + return message + .replaceAllMapped( + RegExp(r'eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+'), + (_) => '[redacted-jwt]', + ) + .replaceAllMapped( + RegExp(r'(refreshToken|idToken|accessToken|token)["\s:=]+[^\s,"}]+', + caseSensitive: false), + (match) => '${match.group(1)}=[redacted]', + ); + } + + Future> _signInOverHttp({ + required String token, + required String apiKey, + }) async { + final result = await LiveAsyncCallSupport.run( + () => postIdentityToolkitSignInWithCustomToken( + customToken: token, + apiKey: apiKey, + ), + ); + if (result == null) { + throw StateError( + 'Firebase signInWithCustomToken did not complete under runAsync.', + ); + } + return result; + } + + String _evaluatedToken(BuildContext context, String? jwtToken) { + var token = jwtToken ?? ''; + final scopeManager = ScreenController().getScopeManager(context); + if (scopeManager != null) { + token = scopeManager.dataContext.eval(token)?.toString() ?? token; + } + return token; + } + + String _resolveApiKey() { + final apiKey = _resolveFirebaseApp().options.apiKey; + if (apiKey.isEmpty) { + throw ConfigError('Firebase apiKey is missing for live auth sign-in.'); + } + return apiKey; + } + + String _resolveAppName() { + try { + return _resolveFirebaseApp().name; + } catch (_) { + return defaultFirebaseAppName; + } + } + + FirebaseApp _resolveFirebaseApp() { + final functionsApp = FirebaseFunctionsAPIProvider.getFirebaseAppContext(); + if (functionsApp != null) { + return functionsApp; + } + if (Firebase.apps.isNotEmpty) { + return Firebase.apps.first; + } + throw ConfigError( + 'No Firebase app is initialized for live auth sign-in.', + ); + } +} + +/// Registers [LiveSignInWithCustomToken] so YAML `signInWithCustomToken` +/// actions use the real Identity Toolkit API during tests. +void ensureLiveSignInWithCustomTokenForTest() { + if (!GetIt.I.isRegistered()) { + GetIt.I.registerFactory( + () => LiveSignInWithCustomToken(), + ); + return; + } + GetIt.I.unregister(); + GetIt.I.registerFactory( + () => LiveSignInWithCustomToken(), + ); +} diff --git a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart b/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart index 5e5407851..8a572e9fd 100644 --- a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart +++ b/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart @@ -1,9 +1,12 @@ +import 'dart:async'; import 'dart:convert'; +import 'dart:io'; import 'package:ensemble/framework/apiproviders/api_provider.dart'; import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; import 'package:ensemble/framework/data_context.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; import 'package:flutter/widgets.dart'; import 'package:yaml/yaml.dart'; @@ -25,7 +28,7 @@ class APICallRecord { }); } -/// Records API calls and returns YAML-configured mock responses by API name. +/// Records API calls, returns YAML mocks when configured, otherwise delegates. class MockAPIProvider extends HTTPAPIProvider { MockAPIProvider({ required Map mocks, @@ -34,15 +37,19 @@ class MockAPIProvider extends HTTPAPIProvider { _delegate = delegate ?? HTTPAPIProvider(); final Map _mocks; - final HTTPAPIProvider _delegate; + HTTPAPIProvider _delegate; final List calls = []; + Future Function(Future Function())? liveAsyncRunner; - int callCount(String apiName) => - calls.where((c) => c.name == apiName).length; + int callCount(String apiName) => calls.where((c) => c.name == apiName).length; List callsFor(String apiName) => calls.where((c) => c.name == apiName).toList(); + void bindHttpDelegate(HTTPAPIProvider delegate) { + _delegate = delegate; + } + void setMock(String apiName, MockAPIResponse response) { _mocks[apiName] = response; } @@ -59,6 +66,10 @@ class MockAPIProvider extends HTTPAPIProvider { void clearApiExceptions() => _forcedExceptions.clear(); + bool get hasPendingLiveCalls => LiveAsyncCallSupport.hasPendingLiveCalls; + + Future waitForLiveCalls() => LiveAsyncCallSupport.waitForLiveCalls(); + @override Future init(String appId, Map config) => _delegate.init(appId, config); @@ -72,6 +83,32 @@ class MockAPIProvider extends HTTPAPIProvider { YamlMap api, DataContext eContext, String apiName, + ) async { + final response = await invokeApiWithDelegate( + _delegate, + context, + api, + eContext, + apiName, + ); + if (response is HttpResponse) { + return response; + } + return HttpResponse.fromBody( + response.body, + response.headers?.map((k, v) => MapEntry(k, v.toString())), + response.statusCode, + response.reasonPhrase, + response.apiState, + ); + } + + Future invokeApiWithDelegate( + APIProvider delegate, + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, ) async { if (simulateNetworkOffline) { calls.add(APICallRecord( @@ -110,7 +147,13 @@ class MockAPIProvider extends HTTPAPIProvider { final mock = _mocks[apiName]; if (mock == null) { - return _delegate.invokeApi(context, api, eContext, apiName); + return _invokeLiveDelegate( + delegate, + context, + api, + eContext, + apiName, + ); } if (mock.delayMs != null && mock.delayMs! > 0) { @@ -126,6 +169,81 @@ class MockAPIProvider extends HTTPAPIProvider { ); } + Future _invokeLiveDelegate( + APIProvider delegate, + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + final runner = liveAsyncRunner; + Future call() async { + final savedOverrides = _clearHttpOverridesForLiveHttps(api, eContext); + try { + return await delegate.invokeApi(context, api, eContext, apiName); + } finally { + _restoreHttpOverrides(savedOverrides); + } + } + + if (runner != null) { + try { + final response = await _runLiveApiCall(call); + if (response != null) { + return response; + } + return HttpResponse.fromBody( + 'Live API call did not return a response.', + {'Content-Type': 'text/plain'}, + 500, + 'Internal Server Error', + APIState.error, + ); + } catch (error) { + return HttpResponse.fromBody( + 'Unexpected error: $error.', + {'Content-Type': 'text/plain'}, + 500, + 'Internal Server Error', + APIState.error, + ); + } + } + + return call(); + } + + Future _runLiveApiCall(Future Function() call) { + LiveAsyncCallSupport.runner ??= liveAsyncRunner; + return LiveAsyncCallSupport.run(call); + } + + /// Remote HTTPS endpoints (e.g. api.acc.kpn.com) fail TLS under the test + /// harness [HttpOverrides]. Local HTTP gateways still need those overrides. + HttpOverrides? _clearHttpOverridesForLiveHttps( + YamlMap api, + DataContext eContext, + ) { + final rawUrl = (api['url'] ?? api['uri'] ?? '').toString().trim(); + if (rawUrl.isEmpty) { + return null; + } + final url = HTTPAPIProvider.resolveUrl(eContext, rawUrl); + final uri = Uri.tryParse(url); + if (uri?.scheme != 'https') { + return null; + } + final saved = HttpOverrides.current; + HttpOverrides.global = null; + return saved; + } + + void _restoreHttpOverrides(HttpOverrides? saved) { + if (saved != null) { + HttpOverrides.global = saved; + } + } + @override Future invokeMockAPI(DataContext eContext, dynamic mock) => _delegate.invokeMockAPI(eContext, mock); @@ -138,6 +256,44 @@ class MockAPIProvider extends HTTPAPIProvider { void dispose() => _delegate.dispose(); } +/// Delegates to a real provider unless [host] has a mock for the API name. +class ApiMockOverlay implements APIProvider { + ApiMockOverlay(this._host, this._delegate); + + final MockAPIProvider _host; + final APIProvider _delegate; + + @override + Future init(String appId, Map config) => + _delegate.init(appId, config); + + @override + Future invokeApi( + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) { + return _host.invokeApiWithDelegate( + _delegate, + context, + api, + eContext, + apiName, + ); + } + + @override + Future invokeMockAPI(DataContext eContext, dynamic mock) => + _delegate.invokeMockAPI(eContext, mock); + + @override + ApiMockOverlay clone() => ApiMockOverlay(_host, _delegate.clone()); + + @override + void dispose() => _delegate.dispose(); +} + class _CapturedRequest { final dynamic body; final Map? query; diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart index 046bdcbbb..8e5a6be0a 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart @@ -31,6 +31,7 @@ class EnsembleTestContext { ); final storage = testCase.initialState['storage']; + final keychain = testCase.initialState['keychain']; final env = testCase.initialState['env']; final envMap = env is Map @@ -42,6 +43,9 @@ class EnsembleTestContext { initialPublicStorage: storage is Map ? Map.from(storage) : null, + initialKeychain: keychain is Map + ? Map.from(keychain) + : null, ); final ctx = EnsembleTestContext( diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index 05d1cde70..19fdddf84 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -2,11 +2,14 @@ import 'dart:io'; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/ensemble_app.dart'; +import 'package:ensemble/framework/apiproviders/api_provider.dart'; import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; import 'package:ensemble/framework/definition_providers/local_provider.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/page_model.dart'; +import 'package:ensemble_test_runner/mocks/adobe_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; @@ -19,21 +22,33 @@ import 'package:flutter_test/flutter_test.dart'; class EnsembleTestSetup { final Map? envOverrides; final Map? initialPublicStorage; + final Map? initialKeychain; const EnsembleTestSetup({ this.envOverrides, this.initialPublicStorage, + this.initialKeychain, }); } /// Applies YAML test environment and storage bootstrap data to [config]. -void applyYamlTestBootstrap(EnsembleConfig config, EnsembleTestSetup setup) { +Future applyYamlTestBootstrap( + EnsembleConfig config, EnsembleTestSetup setup) async { if (setup.envOverrides != null && setup.envOverrides!.isNotEmpty) { config.updateEnvOverrides(setup.envOverrides!); } - setup.initialPublicStorage?.forEach((key, value) { - StorageManager().write(key, value); - }); + await applyYamlTestStorageBootstrap(setup); +} + +Future applyYamlTestStorageBootstrap(EnsembleTestSetup setup) async { + for (final entry in setup.initialPublicStorage?.entries ?? + const Iterable>.empty()) { + await StorageManager().write(entry.key, entry.value); + } + for (final entry in setup.initialKeychain?.entries ?? + const Iterable>.empty()) { + await StorageManager().writeSecurely(key: entry.key, value: entry.value); + } } /// Boots the real Ensemble runtime for widget tests. @@ -43,6 +58,9 @@ class EnsembleTestHarness { static void ensureTestPlugins() { TestWidgetsFlutterBinding.ensureInitialized(); + ensureFirebaseCoreMocksForTest(); + ensureAdobeAnalyticsMocksForTest(); + HttpOverrides.global = _YamlTestHttpOverrides(); const pathProviderChannel = MethodChannel('plugins.flutter.io/path_provider'); TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger @@ -180,11 +198,13 @@ class EnsembleTestHarness { final String appPath; final String appHome; final String? i18nPath; + final Map? externalMethods; EnsembleTestHarness({ required this.appPath, required this.appHome, this.i18nPath, + this.externalMethods, }); static String normalizeAppPath(String path) { @@ -208,34 +228,75 @@ class EnsembleTestHarness { return updated; } - static void installHttpMockProvider( + static Future initializeRealApiProviders(EnsembleConfig config) async { + await Ensemble.initializeAPIProviders(config); + config.apiProviders ??= {}; + config.apiProviders!.putIfAbsent('http', () => HTTPAPIProvider()); + + final firebase = config.apiProviders!['firebase']; + if (firebase != null) { + config.apiProviders!['firebaseFunction'] = firebase; + } + } + + /// Wraps real providers with [mock] for call recording and optional overrides. + static void installApiMockOverrides( EnsembleConfig config, MockAPIProvider mock, ) { - config.apiProviders = { - ...?config.apiProviders, - 'http': mock, - }; + final realProviders = Map.from( + config.apiProviders ?? const {}, + ); + + final installed = {}; + for (final entry in realProviders.entries) { + if (entry.key == 'http') { + mock.bindHttpDelegate(entry.value as HTTPAPIProvider); + installed['http'] = mock; + continue; + } + installed[entry.key] = ApiMockOverlay(mock, entry.value); + } + + if (!installed.containsKey('http')) { + mock.bindHttpDelegate(HTTPAPIProvider()); + installed['http'] = mock; + } + + final firebase = realProviders['firebase']; + if (firebase != null && !installed.containsKey('firebaseFunction')) { + installed['firebaseFunction'] = + installed['firebase'] ?? ApiMockOverlay(mock, firebase); + } + + config.apiProviders = installed; } Future bootstrapRuntime( EnsembleConfig config, EnsembleTestSetup setup, { - MockAPIProvider? httpMock, + MockAPIProvider? apiMockOverlay, }) async { ensureTestPlugins(); - applyYamlTestBootstrap(config, setup); + final env = Map.from(config.envOverrides ?? {}); + env['firebase_app_check'] = 'false'; + if (setup.envOverrides != null && setup.envOverrides!.isNotEmpty) { + env.addAll(setup.envOverrides!); + } + config.updateEnvOverrides(env); Ensemble().setEnsembleConfig(config); - - if (httpMock != null) { - installHttpMockProvider(config, httpMock); + if (externalMethods != null && externalMethods!.isNotEmpty) { + Ensemble().setExternalMethods(externalMethods!); } await Ensemble().initManagers(); + await initializeRealApiProviders(config); - config.apiProviders ??= {'http': HTTPAPIProvider()}; - config.apiProviders!.putIfAbsent('http', () => HTTPAPIProvider()); + if (apiMockOverlay != null) { + installApiMockOverrides(config, apiMockOverlay); + } + await applyYamlTestStorageBootstrap(setup); YamlTestSession.markRuntimeBootstrapped(); return config; @@ -252,12 +313,13 @@ class EnsembleTestHarness { YamlTestSession.navigationFlow.clear(); final ctx = context ?? EnsembleTestContext.fromTestCase(testCase); + await _ensureDefaultViewport(tester, ctx); var config = existingConfig ?? await buildConfig(); final bootstrapped = await tester.runAsync(() async { return bootstrapRuntime( config, ctx.setup, - httpMock: ctx.mockApiProvider, + apiMockOverlay: ctx.mockApiProvider, ); }); config = bootstrapped!; @@ -269,8 +331,8 @@ class EnsembleTestHarness { ); } - // Let EnsembleApp load the bundle through its existing config path, which - // preserves the test-installed API providers and avoids real provider init. + // Skip re-initializing providers in EnsembleApp.initApp; bootstrapRuntime + // already installed real providers and mock overlays. config.appBundle = null; await tester.pumpWidget( EnsembleApp( @@ -283,6 +345,17 @@ class EnsembleTestHarness { return config; } + static Future _ensureDefaultViewport( + WidgetTester tester, + EnsembleTestContext context, + ) async { + if (context.runtime.deviceSize != null) return; + const size = Size(800, 844); + tester.view.physicalSize = size; + tester.view.devicePixelRatio = 1.0; + context.runtime.deviceSize = size; + } + static Future waitForInitialWidgets( WidgetTester tester, { EnsembleTestCase? testCase, @@ -324,17 +397,17 @@ class EnsembleTestHarness { }); } - static void applyInPlaceSetup(EnsembleTestContext ctx) { + static Future applyInPlaceSetup(EnsembleTestContext ctx) async { final config = Ensemble().getConfig(); if (config != null) { - applyYamlTestBootstrap(config, ctx.setup); + await applyYamlTestBootstrap(config, ctx.setup); } ctx.applyRuntimeEnv(); for (final entry in ctx.testCase.mocks.apis.entries) { ctx.mockApiProvider.setMock(entry.key, entry.value); } - if (config != null) { - installHttpMockProvider(config, ctx.mockApiProvider); + if (config != null && config.apiProviders?['http'] is! MockAPIProvider) { + installApiMockOverrides(config, ctx.mockApiProvider); } } @@ -342,3 +415,104 @@ class EnsembleTestHarness { YamlTestSession.reset(); } } + +class _YamlTestHttpOverrides extends HttpOverrides { + @override + HttpClient createHttpClient(SecurityContext? context) { + final client = super.createHttpClient(context); + client.connectionFactory = ( + Uri uri, + String? proxyHost, + int? proxyPort, + ) { + // HTTP to local gateways (e.g. instellen.local) uses filtered DNS so mDNS + // does not prefer loopback. All HTTPS uses the platform connector so TLS + // handshakes (cloud APIs and local gateway cert capture) work normally. + if (uri.scheme == 'https') { + return _platformDefaultConnection(uri, proxyHost, proxyPort); + } + return _connectWithoutStaggeredLookup(uri, proxyHost, proxyPort); + }; + return client; + } + + static Future> _platformDefaultConnection( + Uri uri, + String? proxyHost, + int? proxyPort, + ) async { + final savedOverrides = HttpOverrides.current; + HttpOverrides.global = null; + try { + return Socket.startConnect(proxyHost ?? uri.host, proxyPort ?? uri.port); + } finally { + HttpOverrides.global = savedOverrides; + } + } + + static Future> _connectWithoutStaggeredLookup( + Uri uri, + String? proxyHost, + int? proxyPort, + ) async { + final host = proxyHost ?? uri.host; + final port = proxyPort ?? uri.port; + final addresses = _usableAddresses( + host, + await InternetAddress.lookup(host), + ); + if (addresses.isEmpty) { + throw const SocketException('No addresses resolved'); + } + var cancelled = false; + final socket = _connectToFirstAvailableAddress( + addresses, + port, + isCancelled: () => cancelled, + ); + return ConnectionTask.fromSocket( + socket, + () { + cancelled = true; + }, + ); + } + + /// Dart's [InternetAddress.lookup] can return loopback before LAN IPs for + /// mDNS names like `instellen.local`, causing slow connection-refused retries. + static List _usableAddresses( + String host, + List addresses, + ) { + if (host == 'localhost' || host == '127.0.0.1' || host == '::1') { + return addresses; + } + final filtered = + addresses.where((address) => !address.isLoopback).toList(); + return filtered.isEmpty ? addresses : filtered; + } + + static Future _connectToFirstAvailableAddress( + List addresses, + int port, { + required bool Function() isCancelled, + }) async { + Object? lastError; + StackTrace? lastStackTrace; + for (final address in addresses) { + if (isCancelled()) { + throw const SocketException('Connection attempt cancelled'); + } + try { + return await Socket.connect(address, port); + } catch (error, stackTrace) { + lastError = error; + lastStackTrace = stackTrace; + } + } + Error.throwWithStackTrace( + lastError ?? const SocketException('Connection failed'), + lastStackTrace ?? StackTrace.current, + ); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index 1f6612828..9a2380d27 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -6,6 +6,7 @@ import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/reporters/test_reporter.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -77,6 +78,8 @@ class EnsembleTestRunner { try { final ctx = EnsembleTestContext.fromTestCase(test); + ctx.mockApiProvider.liveAsyncRunner = tester.runAsync; + LiveAsyncCallSupport.runner = tester.runAsync; TestErrorTracker.install(ctx.runtime); late final EnsembleConfig config; @@ -87,7 +90,7 @@ class EnsembleTestRunner { 'runtime is not bootstrapped — ensure the prerequisite test runs first', ); } - EnsembleTestHarness.applyInPlaceSetup(ctx); + await EnsembleTestHarness.applyInPlaceSetup(ctx); config = existingConfig ?? Ensemble().getConfig()!; await EnsembleTestHarness.waitForInitialWidgets(tester, testCase: test); } else { @@ -145,7 +148,10 @@ class EnsembleTestRunner { final step = test.steps[i]; try { await executor.execute(step); + await YamlTestSession.navigationFlow.flushPending(); } catch (error, stackTrace) { + await _settleLiveApiWork(tester, ctx); + await YamlTestSession.navigationFlow.flushPending(); return EnsembleSingleTestResult.failed( testId: test.id, metadata: test.metadataJson, @@ -160,6 +166,8 @@ class EnsembleTestRunner { } } + await YamlTestSession.navigationFlow.flushPending(); + return EnsembleSingleTestResult.passed( testId: test.id, metadata: test.metadataJson, @@ -168,4 +176,18 @@ class EnsembleTestRunner { report: buildTestReportDetails(test), ); } + + Future _settleLiveApiWork( + WidgetTester tester, + EnsembleTestContext ctx, + ) async { + for (var i = 0; i < 20; i++) { + await ctx.mockApiProvider.waitForLiveCalls(); + await tester.pump(); + await YamlTestSession.navigationFlow.flushPending(); + if (!ctx.mockApiProvider.hasPendingLiveCalls) { + return; + } + } + } } diff --git a/tools/ensemble_test_runner/lib/runner/live_async_call.dart b/tools/ensemble_test_runner/lib/runner/live_async_call.dart new file mode 100644 index 000000000..6d45d119b --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/live_async_call.dart @@ -0,0 +1,70 @@ +import 'dart:async'; +import 'dart:io'; + +/// Serializes real async work (HTTP, etc.) through [WidgetTester.runAsync]. +/// +/// Flutter rejects reentrant [WidgetTester.runAsync] calls, so live network +/// requests from the app must be queued when multiple actions run in parallel. +class LiveAsyncCallSupport { + static Future Function(Future Function())? runner; + + static int _pendingLiveCalls = 0; + static final Set> _inFlightLiveCalls = {}; + static Future _liveCallQueue = Future.value(); + + static bool get hasPendingLiveCalls => _pendingLiveCalls > 0; + + static Future waitForLiveCalls() async { + if (_inFlightLiveCalls.isEmpty) { + return; + } + await Future.wait(_inFlightLiveCalls.toList()); + } + + /// Runs [call] on the platform HTTP stack (no test [HttpOverrides]). + static Future runWithPlatformHttp(Future Function() call) { + return run(() async { + final saved = HttpOverrides.current; + HttpOverrides.global = null; + try { + return await call(); + } finally { + HttpOverrides.global = saved; + } + }); + } + + static Future run(Future Function() call) { + final liveRunner = runner; + if (liveRunner == null) { + return call(); + } + + _pendingLiveCalls++; + final completer = Completer(); + final tracked = completer.future; + _inFlightLiveCalls.add(tracked); + + _liveCallQueue = _liveCallQueue.then((_) async { + try { + completer.complete(await liveRunner(call)); + } catch (error, stackTrace) { + if (!completer.isCompleted) { + completer.completeError(error, stackTrace); + } + } finally { + _pendingLiveCalls--; + _inFlightLiveCalls.remove(tracked); + } + }); + + return tracked; + } + + static void reset() { + runner = null; + _pendingLiveCalls = 0; + _inFlightLiveCalls.clear(); + _liveCallQueue = Future.value(); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart b/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart index bf90c5d19..54c5d755d 100644 --- a/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart +++ b/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart @@ -2,11 +2,14 @@ import 'dart:async'; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; /// Records screen visits for YAML tests (including back-navigation revisits). class NavigationFlowRecorder { final List _flow = []; StreamSubscription? _subscription; + VisibleScreen? _pendingScreen; + bool _flushScheduled = false; List get flow => List.unmodifiable(_flow); @@ -15,7 +18,11 @@ class NavigationFlowRecorder { _subscription = ScreenTracker().onScreenChange.listen(_onScreenChange); } - void clear() => _flow.clear(); + void clear() { + _flow.clear(); + _pendingScreen = null; + _flushScheduled = false; + } /// For unit tests only. void seed(Iterable names) { @@ -34,6 +41,24 @@ class NavigationFlowRecorder { void recordScreenChange(VisibleScreen? screen) => _onScreenChange(screen); void _onScreenChange(VisibleScreen? screen) { + if (screen == null) return; + _pendingScreen = screen; + if (_flushScheduled) return; + _flushScheduled = true; + scheduleMicrotask(_flushPendingScreen); + } + + Future flushPending() async { + await Future.microtask(() {}); + if (_flushScheduled) { + _flushPendingScreen(); + } + } + + void _flushPendingScreen() { + _flushScheduled = false; + final screen = _pendingScreen; + _pendingScreen = null; if (screen == null) return; final name = screen.screenName ?? screen.screenId; if (name == null) return; @@ -59,6 +84,7 @@ class YamlTestSession { static void reset() { runtimeBootstrapped = false; navigationFlow.clear(); + LiveAsyncCallSupport.reset(); Ensemble.resetInitManagersForTest(); } diff --git a/tools/ensemble_test_runner/pubspec.yaml b/tools/ensemble_test_runner/pubspec.yaml index a67a02483..6ba2fe8b6 100644 --- a/tools/ensemble_test_runner/pubspec.yaml +++ b/tools/ensemble_test_runner/pubspec.yaml @@ -30,9 +30,17 @@ dependencies: sdk: flutter yaml: ^3.1.2 path: ^1.9.0 + firebase_core_platform_interface: ^7.0.1 + cloud_functions_platform_interface: ^6.0.1 + cloud_firestore_platform_interface: ^8.0.1 + firebase_auth_platform_interface: ^9.0.3 + http: ^1.2.2 + get_it: ^8.0.3 dev_dependencies: flutter_lints: ^2.0.3 + cloud_functions: ^6.0.3 + firebase_auth: ^6.5.4 flutter: uses-material-design: true diff --git a/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart new file mode 100644 index 000000000..701fa8b29 --- /dev/null +++ b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart @@ -0,0 +1,25 @@ +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + testWidgets('initial keychain state is written to secure storage', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + + await applyYamlTestStorageBootstrap( + const EnsembleTestSetup( + initialKeychain: { + 'kpnPsi': 'test-psi', + 'authPayload': {'token': 'abc'}, + }, + ), + ); + + expect(await StorageManager().readSecurely('kpnPsi'), 'test-psi'); + expect( + await StorageManager().readSecurely('authPayload'), + {'token': 'abc'}, + ); + }); +} diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 241018977..648e6ca67 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -47,6 +47,31 @@ steps: ); }); + test('parses initial keychain state', () { + const yaml = ''' +id: resumes_session +startScreen: Login +initialState: + storage: + languageSet: true + keychain: + kpnPsi: test-psi + authPayload: + token: abc +steps: + - expectVisible: + id: login_button +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.initialState['keychain'], isA()); + expect((test.initialState['keychain'] as Map)['kpnPsi'], 'test-psi'); + expect( + ((test.initialState['keychain'] as Map)['authPayload'] as Map)['token'], + 'abc', + ); + }); + test('parses AI-friendly metadata fields', () { const yaml = ''' id: login_valid diff --git a/tools/ensemble_test_runner/test/firebase_auth_test_setup_test.dart b/tools/ensemble_test_runner/test/firebase_auth_test_setup_test.dart new file mode 100644 index 000000000..e16f0c824 --- /dev/null +++ b/tools/ensemble_test_runner/test/firebase_auth_test_setup_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/mocks/live_firebase_auth_http.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('extracts Firebase uid from idToken payload', () { + const uid = 'test-user-uid'; + final payload = base64Url.encode(utf8.encode(jsonEncode({ + 'user_id': uid, + 'sub': uid, + }))); + final idToken = 'header.$payload.signature'; + expect(uidFromFirebaseIdToken(idToken), uid); + }); +} diff --git a/tools/ensemble_test_runner/test/firebase_functions_test_setup_test.dart b/tools/ensemble_test_runner/test/firebase_functions_test_setup_test.dart new file mode 100644 index 000000000..6706c98b2 --- /dev/null +++ b/tools/ensemble_test_runner/test/firebase_functions_test_setup_test.dart @@ -0,0 +1,29 @@ +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + testWidgets( + 'firebase core mock preserves projectId from initializeApp', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + + await tester.runAsync(() async { + const appName = 'firebaseFunctionsTest'; + await Firebase.initializeApp( + name: appName, + options: const FirebaseOptions( + apiKey: 'test-api-key', + appId: 'test-app-id', + messagingSenderId: 'test-sender', + projectId: 'test-project-id', + ), + ); + + expect(firebaseProjectIdsByApp[appName], 'test-project-id'); + expect(Firebase.app(appName).options.projectId, 'test-project-id'); + }); + }, + ); +} diff --git a/tools/ensemble_test_runner/test/mock_api_provider_test.dart b/tools/ensemble_test_runner/test/mock_api_provider_test.dart new file mode 100644 index 000000000..c7f3ab986 --- /dev/null +++ b/tools/ensemble_test_runner/test/mock_api_provider_test.dart @@ -0,0 +1,146 @@ +import 'package:ensemble/framework/apiproviders/api_provider.dart'; +import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; +import 'package:ensemble/framework/data_context.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + testWidgets('unmocked firebase functions delegate to the real provider', + (tester) async { + await tester.pumpWidget(const MaterialApp(home: SizedBox.shrink())); + + final delegate = _RecordingAPIProvider(); + final provider = MockAPIProvider(mocks: {}); + final context = tester.element(find.byType(SizedBox)); + final api = YamlMap.wrap({'type': 'firebaseFunction', 'name': 'exampleCallable'}); + + await provider.invokeApiWithDelegate( + delegate, + context, + api, + DataContext(buildContext: context), + 'exampleCallable', + ); + + expect(delegate.invoked, isTrue); + expect(provider.callCount('exampleCallable'), 1); + }); + + testWidgets('mocked APIs return configured responses without delegating', + (tester) async { + await tester.pumpWidget(const MaterialApp(home: SizedBox.shrink())); + + final delegate = _RecordingAPIProvider(); + final provider = MockAPIProvider( + mocks: { + 'login': MockAPIResponse(statusCode: 200, body: {'token': 'abc'}), + }, + delegate: delegate, + ); + final context = tester.element(find.byType(SizedBox)); + final response = await provider.invokeApi( + context, + YamlMap.wrap({'type': 'http', 'url': 'https://example.test/login'}), + DataContext(buildContext: context), + 'login', + ); + + expect(delegate.invoked, isFalse); + expect(response.body, {'token': 'abc'}); + }); + + test('serializes live runner calls to avoid reentrant runAsync', () async { + var inFlight = 0; + var maxInFlight = 0; + final delegate = _ConcurrentRecordingAPIProvider( + onStart: () { + inFlight++; + if (inFlight > maxInFlight) { + maxInFlight = inFlight; + } + }, + onEnd: () { + inFlight--; + }, + ); + final provider = MockAPIProvider(mocks: {}, delegate: delegate); + provider.liveAsyncRunner = (Future Function() fn) => fn(); + + final buildContext = _FakeBuildContext(); + final first = provider.invokeApiWithDelegate( + delegate, + buildContext, + YamlMap.wrap({'type': 'http', 'url': 'https://example.test/a'}), + DataContext(buildContext: buildContext), + 'first', + ); + final second = provider.invokeApiWithDelegate( + delegate, + buildContext, + YamlMap.wrap({'type': 'http', 'url': 'https://example.test/b'}), + DataContext(buildContext: buildContext), + 'second', + ); + + await Future.wait([first, second]); + expect(maxInFlight, 1); + expect(delegate.invoked, 2); + }); +} + +class _ConcurrentRecordingAPIProvider extends HTTPAPIProvider { + _ConcurrentRecordingAPIProvider({ + required this.onStart, + required this.onEnd, + }); + + final void Function() onStart; + final void Function() onEnd; + int invoked = 0; + + @override + Future invokeApi( + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + onStart(); + invoked++; + await Future.delayed(const Duration(milliseconds: 50)); + onEnd(); + return HttpResponse.fromBody( + {'ok': true}, + {'Content-Type': 'application/json'}, + 200, + null, + APIState.success, + ); + } +} + +class _RecordingAPIProvider extends HTTPAPIProvider { + bool invoked = false; + + @override + Future invokeApi( + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + invoked = true; + return HttpResponse.fromBody( + {'ok': true}, + {'Content-Type': 'application/json'}, + 200, + null, + APIState.success, + ); + } +} + +class _FakeBuildContext extends Fake implements BuildContext {} diff --git a/tools/ensemble_test_runner/test/navigation_flow_recorder_test.dart b/tools/ensemble_test_runner/test/navigation_flow_recorder_test.dart index f337763a0..92619bf9c 100644 --- a/tools/ensemble_test_runner/test/navigation_flow_recorder_test.dart +++ b/tools/ensemble_test_runner/test/navigation_flow_recorder_test.dart @@ -3,25 +3,48 @@ import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { - test('dedupes consecutive screen names', () { + test('dedupes consecutive screen names', () async { final recorder = NavigationFlowRecorder(); recorder.recordScreenChange( VisibleScreen(screenName: 'Hello Home', visibleSince: DateTime.now()), ); + await recorder.flushPending(); recorder.recordScreenChange( VisibleScreen(screenName: 'Goodbye', visibleSince: DateTime.now()), ); + await recorder.flushPending(); recorder.recordScreenChange( VisibleScreen(screenName: 'Hello Home', visibleSince: DateTime.now()), ); + await recorder.flushPending(); recorder.recordScreenChange( VisibleScreen(screenName: 'Hello Home', visibleSince: DateTime.now()), ); + await recorder.flushPending(); expect( recorder.flow, ['Hello Home', 'Goodbye', 'Hello Home'], ); }); + + test('coalesces transient route removal screen changes', () async { + final recorder = NavigationFlowRecorder(); + + recorder.recordScreenChange( + VisibleScreen(screenName: 'InitApp', visibleSince: DateTime.now()), + ); + await recorder.flushPending(); + + recorder.recordScreenChange( + VisibleScreen(screenName: 'Login', visibleSince: DateTime.now()), + ); + recorder.recordScreenChange( + VisibleScreen(screenName: 'AutoSignIn', visibleSince: DateTime.now()), + ); + await recorder.flushPending(); + + expect(recorder.flow, ['InitApp', 'AutoSignIn']); + }); } diff --git a/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart b/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart index ce77c823e..e51bfb9a7 100644 --- a/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart +++ b/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart @@ -45,7 +45,7 @@ steps: [] expect( File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') .readAsStringSync(), - contains('runEnsembleYamlTests'), + contains('EnsembleModules().init()'), ); patcher.restore(); @@ -58,7 +58,7 @@ steps: [] ); }); - test('restore deletes leftover generated test entry from a prior run', () { + test('upgrades legacy stub and keeps it after restore', () { final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); addTearDown(() => dir.deleteSync(recursive: true)); @@ -82,16 +82,68 @@ steps: [] '''); Directory('${dir.path}/test').createSync(recursive: true); File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') - .writeAsStringSync(YamlTestAppPatcher.testEntryContents); + .writeAsStringSync(YamlTestAppPatcher.legacyTestEntryContents); final patcher = YamlTestAppPatcher(dir.path); patcher.enable(); patcher.restore(); + final restored = + File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') + .readAsStringSync(); + expect(restored, contains('EnsembleModules().init()')); + }); + + test('does not overwrite a customized test entry', () { + final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); + addTearDown(() => dir.deleteSync(recursive: true)); + + File('${dir.path}/pubspec.yaml').writeAsStringSync(''' +name: sample_app +dev_dependencies: + flutter_test: + sdk: flutter +flutter: + assets: + - ensemble/ +'''); + _writeConfig(dir); + Directory('${dir.path}/ensemble/apps/helloApp/tests') + .createSync(recursive: true); + File('${dir.path}/ensemble/apps/helloApp/tests/sample.test.yaml') + .writeAsStringSync(''' +id: sample +startScreen: Home +steps: [] +'''); + const customEntry = ''' +import 'package:ensemble_test_runner/entry/ensemble_test_entry.dart'; + +Future main() async { + await runEnsembleYamlTests( + bootstrap: () async {}, + ); +} +'''; + Directory('${dir.path}/test').createSync(recursive: true); + File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') + .writeAsStringSync(customEntry); + + final patcher = YamlTestAppPatcher(dir.path); + patcher.enable(); + expect( File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') - .existsSync(), - isFalse, + .readAsStringSync(), + customEntry, + ); + + patcher.restore(); + + expect( + File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') + .readAsStringSync(), + customEntry, ); }); From 8d6c6ba48b506dbb6ddbf74ac36338194f901fbc Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Thu, 9 Jul 2026 22:47:32 +0500 Subject: [PATCH 02/29] refactor(test_runner): clean up HTTP overrides and improve code readability Removed unused HTTP override logic from various files and simplified the implementation of HTTP client creation. Enhanced code formatting for better readability and maintainability. --- .../mocks/firebase_firestore_test_setup.dart | 26 +++-- .../mocks/firebase_functions_test_setup.dart | 6 +- .../lib/mocks/live_firebase_auth_http.dart | 3 - .../lib/mocks/mock_api_provider.dart | 37 +------ .../lib/runner/ensemble_test_harness.dart | 100 +----------------- .../lib/runner/live_async_call.dart | 16 +-- 6 files changed, 22 insertions(+), 166 deletions(-) diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart index 46291f34c..2e4b8cb24 100644 --- a/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart +++ b/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart @@ -51,9 +51,11 @@ void ensureLiveFirestoreForTest() { register('clearPersistence', noop); register('terminate', noop); register('waitForPendingWrites', noop); - register('snapshotsInSyncSetup', (_) async => [ - 'ensemble_test_runner/firestore/snapshots-in-sync', - ]); + register( + 'snapshotsInSyncSetup', + (_) async => [ + 'ensemble_test_runner/firestore/snapshots-in-sync', + ]); register('persistenceCacheIndexManagerRequest', noop); register('documentReferenceSet', (args) async { @@ -190,8 +192,7 @@ class _LiveFirestoreRestClient { final encoded = _encodeDocumentFields(request.data ?? {}); final body = { if (encoded.fields.isNotEmpty) 'fields': encoded.fields, - if (encoded.transforms.isNotEmpty) - 'updateTransforms': encoded.transforms, + if (encoded.transforms.isNotEmpty) 'updateTransforms': encoded.transforms, }; await _request( app, @@ -303,9 +304,8 @@ class _LiveFirestoreRestClient { }) async { final projectId = _projectIdForApp(app.appName); final idToken = _idTokenForApp(app.appName); - final encodedPath = path == null - ? '' - : path.split('/').map(Uri.encodeComponent).join('/'); + final encodedPath = + path == null ? '' : path.split('/').map(Uri.encodeComponent).join('/'); final queryParts = [ for (final fieldPath in updateMaskPaths) 'updateMask.fieldPaths=${Uri.encodeQueryComponent(fieldPath)}', @@ -316,8 +316,6 @@ class _LiveFirestoreRestClient { '${queryParts.isEmpty ? '' : '?${queryParts.join('&')}'}', ); - final savedOverrides = HttpOverrides.current; - HttpOverrides.global = null; final client = HttpClient(); try { late final HttpClientRequest request; @@ -357,7 +355,6 @@ class _LiveFirestoreRestClient { return jsonDecode(responseBody); } finally { client.close(); - HttpOverrides.global = savedOverrides; } } @@ -422,7 +419,9 @@ Map _encodeValue(Object? value) { if (nestedKey == null || nestedKey.isEmpty) return; nested[nestedKey] = _encodeValue(nestedValue); }); - return {'mapValue': {'fields': nested}}; + return { + 'mapValue': {'fields': nested} + }; } if (value is Iterable) { return { @@ -465,8 +464,7 @@ Object? _decodeValue(Object? value) { if (map.containsKey('stringValue')) return map['stringValue']; if (map.containsKey('booleanValue')) return map['booleanValue']; if (map.containsKey('integerValue')) { - return int.tryParse(map['integerValue'].toString()) ?? - map['integerValue']; + return int.tryParse(map['integerValue'].toString()) ?? map['integerValue']; } if (map.containsKey('doubleValue')) { final raw = map['doubleValue']; diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart index 8d35b13ff..849395363 100644 --- a/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart +++ b/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart @@ -54,8 +54,6 @@ Future _invokeCallableOverHttp(Map arguments) async { final parameters = arguments['parameters']; final timeoutMs = arguments['timeout'] as int? ?? 60000; - final savedOverrides = HttpOverrides.current; - HttpOverrides.global = null; final client = HttpClient(); client.connectionTimeout = Duration(milliseconds: timeoutMs); @@ -96,7 +94,6 @@ Future _invokeCallableOverHttp(Map arguments) async { return result; } finally { client.close(); - HttpOverrides.global = savedOverrides; } } @@ -125,7 +122,8 @@ Uri _resolveCallableUri(Map arguments) { ); } - return Uri.parse('https://$region-$projectId.cloudfunctions.net/$functionName'); + return Uri.parse( + 'https://$region-$projectId.cloudfunctions.net/$functionName'); } String? _projectIdForApp(String? appName) { diff --git a/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart b/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart index fca53319c..75105497b 100644 --- a/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart +++ b/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart @@ -41,8 +41,6 @@ Future> _postIdentityToolkitSignInOnce({ final uri = Uri.parse( 'https://identitytoolkit.googleapis.com/v1/accounts:signInWithCustomToken?key=$apiKey', ); - final savedOverrides = HttpOverrides.current; - HttpOverrides.global = null; final client = HttpClient(); client.connectionTimeout = const Duration(seconds: 60); try { @@ -67,7 +65,6 @@ Future> _postIdentityToolkitSignInOnce({ return Map.from(decoded); } finally { client.close(); - HttpOverrides.global = savedOverrides; } } diff --git a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart b/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart index 8a572e9fd..b92742d39 100644 --- a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart +++ b/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart @@ -1,6 +1,5 @@ import 'dart:async'; import 'dart:convert'; -import 'dart:io'; import 'package:ensemble/framework/apiproviders/api_provider.dart'; import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; @@ -177,14 +176,8 @@ class MockAPIProvider extends HTTPAPIProvider { String apiName, ) async { final runner = liveAsyncRunner; - Future call() async { - final savedOverrides = _clearHttpOverridesForLiveHttps(api, eContext); - try { - return await delegate.invokeApi(context, api, eContext, apiName); - } finally { - _restoreHttpOverrides(savedOverrides); - } - } + Future call() => + delegate.invokeApi(context, api, eContext, apiName); if (runner != null) { try { @@ -218,32 +211,6 @@ class MockAPIProvider extends HTTPAPIProvider { return LiveAsyncCallSupport.run(call); } - /// Remote HTTPS endpoints (e.g. api.acc.kpn.com) fail TLS under the test - /// harness [HttpOverrides]. Local HTTP gateways still need those overrides. - HttpOverrides? _clearHttpOverridesForLiveHttps( - YamlMap api, - DataContext eContext, - ) { - final rawUrl = (api['url'] ?? api['uri'] ?? '').toString().trim(); - if (rawUrl.isEmpty) { - return null; - } - final url = HTTPAPIProvider.resolveUrl(eContext, rawUrl); - final uri = Uri.tryParse(url); - if (uri?.scheme != 'https') { - return null; - } - final saved = HttpOverrides.current; - HttpOverrides.global = null; - return saved; - } - - void _restoreHttpOverrides(HttpOverrides? saved) { - if (saved != null) { - HttpOverrides.global = saved; - } - } - @override Future invokeMockAPI(DataContext eContext, dynamic mock) => _delegate.invokeMockAPI(eContext, mock); diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index 19fdddf84..b8a331bfc 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -60,7 +60,7 @@ class EnsembleTestHarness { TestWidgetsFlutterBinding.ensureInitialized(); ensureFirebaseCoreMocksForTest(); ensureAdobeAnalyticsMocksForTest(); - HttpOverrides.global = _YamlTestHttpOverrides(); + HttpOverrides.global = _RealNetworkHttpOverrides(); const pathProviderChannel = MethodChannel('plugins.flutter.io/path_provider'); TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger @@ -416,103 +416,9 @@ class EnsembleTestHarness { } } -class _YamlTestHttpOverrides extends HttpOverrides { +class _RealNetworkHttpOverrides extends HttpOverrides { @override HttpClient createHttpClient(SecurityContext? context) { - final client = super.createHttpClient(context); - client.connectionFactory = ( - Uri uri, - String? proxyHost, - int? proxyPort, - ) { - // HTTP to local gateways (e.g. instellen.local) uses filtered DNS so mDNS - // does not prefer loopback. All HTTPS uses the platform connector so TLS - // handshakes (cloud APIs and local gateway cert capture) work normally. - if (uri.scheme == 'https') { - return _platformDefaultConnection(uri, proxyHost, proxyPort); - } - return _connectWithoutStaggeredLookup(uri, proxyHost, proxyPort); - }; - return client; - } - - static Future> _platformDefaultConnection( - Uri uri, - String? proxyHost, - int? proxyPort, - ) async { - final savedOverrides = HttpOverrides.current; - HttpOverrides.global = null; - try { - return Socket.startConnect(proxyHost ?? uri.host, proxyPort ?? uri.port); - } finally { - HttpOverrides.global = savedOverrides; - } - } - - static Future> _connectWithoutStaggeredLookup( - Uri uri, - String? proxyHost, - int? proxyPort, - ) async { - final host = proxyHost ?? uri.host; - final port = proxyPort ?? uri.port; - final addresses = _usableAddresses( - host, - await InternetAddress.lookup(host), - ); - if (addresses.isEmpty) { - throw const SocketException('No addresses resolved'); - } - var cancelled = false; - final socket = _connectToFirstAvailableAddress( - addresses, - port, - isCancelled: () => cancelled, - ); - return ConnectionTask.fromSocket( - socket, - () { - cancelled = true; - }, - ); - } - - /// Dart's [InternetAddress.lookup] can return loopback before LAN IPs for - /// mDNS names like `instellen.local`, causing slow connection-refused retries. - static List _usableAddresses( - String host, - List addresses, - ) { - if (host == 'localhost' || host == '127.0.0.1' || host == '::1') { - return addresses; - } - final filtered = - addresses.where((address) => !address.isLoopback).toList(); - return filtered.isEmpty ? addresses : filtered; - } - - static Future _connectToFirstAvailableAddress( - List addresses, - int port, { - required bool Function() isCancelled, - }) async { - Object? lastError; - StackTrace? lastStackTrace; - for (final address in addresses) { - if (isCancelled()) { - throw const SocketException('Connection attempt cancelled'); - } - try { - return await Socket.connect(address, port); - } catch (error, stackTrace) { - lastError = error; - lastStackTrace = stackTrace; - } - } - Error.throwWithStackTrace( - lastError ?? const SocketException('Connection failed'), - lastStackTrace ?? StackTrace.current, - ); + return super.createHttpClient(context); } } diff --git a/tools/ensemble_test_runner/lib/runner/live_async_call.dart b/tools/ensemble_test_runner/lib/runner/live_async_call.dart index 6d45d119b..4e2d0fccf 100644 --- a/tools/ensemble_test_runner/lib/runner/live_async_call.dart +++ b/tools/ensemble_test_runner/lib/runner/live_async_call.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:io'; /// Serializes real async work (HTTP, etc.) through [WidgetTester.runAsync]. /// @@ -21,18 +20,9 @@ class LiveAsyncCallSupport { await Future.wait(_inFlightLiveCalls.toList()); } - /// Runs [call] on the platform HTTP stack (no test [HttpOverrides]). - static Future runWithPlatformHttp(Future Function() call) { - return run(() async { - final saved = HttpOverrides.current; - HttpOverrides.global = null; - try { - return await call(); - } finally { - HttpOverrides.global = saved; - } - }); - } + /// Backwards-compatible alias for callers that need real async work. + static Future runWithPlatformHttp(Future Function() call) => + run(call); static Future run(Future Function() call) { final liveRunner = runner; From 927f9bbc40f04b88af39f98821b67ce41468b32c Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Thu, 9 Jul 2026 23:19:41 +0500 Subject: [PATCH 03/29] refactor(mock_api_provider): streamline mock API response handling Replaced direct construction of HttpResponse with a dedicated method to convert MockAPIResponse to a runtime-compatible format. This change enhances code clarity and maintainability by centralizing response formatting logic. --- .../lib/mocks/mock_api_provider.dart | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart b/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart index b92742d39..891258df1 100644 --- a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart +++ b/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart @@ -159,13 +159,7 @@ class MockAPIProvider extends HTTPAPIProvider { await Future.delayed(Duration(milliseconds: mock.delayMs!)); } - return HttpResponse.fromBody( - mock.body, - mock.headers?.map((k, v) => MapEntry(k, v.toString())), - mock.statusCode, - null, - APIState.success, - ); + return delegate.invokeMockAPI(eContext, _toRuntimeMockResponse(mock)); } Future _invokeLiveDelegate( @@ -261,6 +255,14 @@ class ApiMockOverlay implements APIProvider { void dispose() => _delegate.dispose(); } +Map _toRuntimeMockResponse(MockAPIResponse mock) { + return { + 'body': mock.body, + if (mock.headers != null) 'headers': mock.headers, + 'statusCode': mock.statusCode, + }; +} + class _CapturedRequest { final dynamic body; final Map? query; From ffb9905fc9034d0278f49498a88394f174e80a1d Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Thu, 9 Jul 2026 23:29:05 +0500 Subject: [PATCH 04/29] refactor(api_provider): replace mock API provider with test API provider overlay Updated the ensemble test runner to utilize a new TestApiProviderOverlay, enhancing the handling of API mocks and improving the clarity of API interactions. This change includes renaming and refactoring related components to streamline the testing process and ensure better maintainability. --- .../lib/actions/extended_step_handlers.dart | 14 ++--- .../lib/actions/test_step_executor.dart | 14 ++--- .../lib/assertions/assertion_engine.dart | 14 ++--- .../lib/ensemble_test_runner.dart | 2 +- ...er.dart => test_api_provider_overlay.dart} | 52 ++++++++++++------- .../lib/runner/ensemble_test_context.dart | 25 ++++----- .../lib/runner/ensemble_test_harness.dart | 25 ++++----- .../lib/runner/ensemble_test_runner.dart | 6 +-- ...rt => test_api_provider_overlay_test.dart} | 11 ++-- 9 files changed, 89 insertions(+), 74 deletions(-) rename tools/ensemble_test_runner/lib/mocks/{mock_api_provider.dart => test_api_provider_overlay.dart} (86%) rename tools/ensemble_test_runner/test/{mock_api_provider_test.dart => test_api_provider_overlay_test.dart} (91%) diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 594200688..ccc8ea723 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -174,7 +174,7 @@ class ExtendedStepHandlers { await _mockApiFromFixture(executor, step); return true; case 'clearApiMocks': - executor.context.mockApiProvider.clearMocks(); + executor.context.apiOverlay.clearMocks(); return true; case 'mockApiException': await _mockApiException(executor, step); @@ -333,7 +333,7 @@ class ExtendedStepHandlers { static Future _restartApp(TestStepExecutor e) async { EnsembleTestHarness.resetTestRuntime(); e.context.runtime.clear(); - e.context.mockApiProvider.resetCalls(); + e.context.apiOverlay.resetCalls(); final screen = e.context.testCase.startScreen ?? ScreenTracker().getCurrentScreenIdentifier(); if (screen == null || screen.isEmpty) { @@ -346,7 +346,7 @@ class ExtendedStepHandlers { static Future _resetAppState(TestStepExecutor e) async { ScreenTracker().clearAll(); - e.context.mockApiProvider.resetCalls(); + e.context.apiOverlay.resetCalls(); e.context.runtime.clear(); await StorageManager().clearPublicStorage(); } @@ -486,7 +486,7 @@ class ExtendedStepHandlers { 'mockApiFromFixture requires "name" and "fixture"'); } final body = await _readFixture(e, fixture); - e.context.mockApiProvider.setMock( + e.context.apiOverlay.setMock( name, MockAPIResponse( statusCode: step.args['statusCode'] as int? ?? 200, @@ -500,7 +500,7 @@ class ExtendedStepHandlers { final name = step.args['name']?.toString(); if (name == null) throw EnsembleTestFailure('mockApiException requires "name"'); - e.context.mockApiProvider.setApiException( + e.context.apiOverlay.setApiException( name, Exception(step.args['message']?.toString() ?? 'API exception (test)'), ); @@ -509,7 +509,7 @@ class ExtendedStepHandlers { static Future _mockTimeout(TestStepExecutor e, TestStep step) async { final name = step.args['name']?.toString(); if (name == null) throw EnsembleTestFailure('mockTimeout requires "name"'); - e.context.mockApiProvider.setMock( + e.context.apiOverlay.setMock( name, MockAPIResponse( statusCode: 200, @@ -520,7 +520,7 @@ class ExtendedStepHandlers { } static void _setNetworkOffline(TestStepExecutor e, bool offline) { - e.context.mockApiProvider.simulateNetworkOffline = offline; + e.context.apiOverlay.simulateNetworkOffline = offline; ConnectivityState().isOnline = !offline; e.context.runtime.networkOffline = offline; } diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index d361173ef..2e08fc023 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -338,7 +338,7 @@ class TestStepExecutor { if (name == null) { throw EnsembleTestFailure('mockApi requires "name"'); } - context.mockApiProvider.setMock( + context.apiOverlay.setMock( name, context.mockFromStepArgs(step.args), ); @@ -348,7 +348,7 @@ class TestStepExecutor { if (name == null) { throw EnsembleTestFailure('mockApiError requires "name"'); } - context.mockApiProvider.setMock( + context.apiOverlay.setMock( name, MockAPIResponse( statusCode: step.args['statusCode'] as int? ?? 500, @@ -358,10 +358,10 @@ class TestStepExecutor { ); break; case 'resetApiCalls': - context.mockApiProvider.resetCalls(); + context.apiOverlay.resetCalls(); break; case 'logApiCalls': - for (final call in context.mockApiProvider.calls) { + for (final call in context.apiOverlay.calls) { context.logger.log( 'API ${call.name} body=${call.body} query=${call.query}', ); @@ -444,10 +444,10 @@ class TestStepExecutor { /// timers from departed screens are not advanced while draining live HTTP. Future _yieldToLiveApiWork() async { for (var i = 0; i < 200; i++) { - final hadPending = context.mockApiProvider.hasPendingLiveCalls; + final hadPending = context.apiOverlay.hasPendingLiveCalls; if (hadPending) { try { - await context.mockApiProvider + await context.apiOverlay .waitForLiveCalls() .timeout(config.waitPollInterval); } on TimeoutException { @@ -599,7 +599,7 @@ class TestStepExecutor { while (stopwatch.elapsedMilliseconds < timeoutMs) { await _yieldToLiveApiWork(); await tester.pump(config.waitPollInterval); - if (context.mockApiProvider.callCount(name) >= times) { + if (context.apiOverlay.callCount(name) >= times) { await _yieldToLiveApiWork(); return; } diff --git a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart index 1bf4dbfa7..b4ff29420 100644 --- a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart +++ b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart @@ -6,7 +6,7 @@ import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/framework/view/data_scope_widget.dart'; import 'package:ensemble/framework/view/page_group.dart'; -import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; @@ -82,7 +82,7 @@ class AssertionEngine { } void expectApiNotCalled(String apiName) { - final count = context.mockApiProvider.callCount(apiName); + final count = context.apiOverlay.callCount(apiName); if (count != 0) { throw EnsembleTestFailure( 'Expected API "$apiName" not to be called, but it was called $count times.', @@ -134,7 +134,7 @@ class AssertionEngine { } void expectApiCalled(String apiName, int times) { - final actual = context.mockApiProvider.callCount(apiName); + final actual = context.apiOverlay.callCount(apiName); if (actual != times) { throw EnsembleTestFailure( 'Expected API "$apiName" to be called $times times, but it was called $actual times. ' @@ -358,7 +358,7 @@ class AssertionEngine { } void expectApiCallOrder(List names) { - final actual = context.mockApiProvider.calls.map((c) => c.name).toList(); + final actual = context.apiOverlay.calls.map((c) => c.name).toList(); var index = 0; for (final name in names) { while (index < actual.length && actual[index] != name) { @@ -374,7 +374,7 @@ class AssertionEngine { } void expectLastApiCall(String apiName) { - final calls = context.mockApiProvider.calls; + final calls = context.apiOverlay.calls; if (calls.isEmpty || calls.last.name != apiName) { throw EnsembleTestFailure( 'Expected last API call to be "$apiName", ' @@ -481,7 +481,7 @@ class AssertionEngine { } APICallRecord _selectApiCall(String apiName, {int? times}) { - final calls = context.mockApiProvider.callsFor(apiName); + final calls = context.apiOverlay.callsFor(apiName); if (calls.isEmpty) { throw EnsembleTestFailure('Expected API "$apiName" to be called.'); } @@ -595,7 +595,7 @@ class AssertionEngine { } String apiCallSummary({int limit = 10}) { - final calls = context.mockApiProvider.calls; + final calls = context.apiOverlay.calls; if (calls.isEmpty) return 'No API calls were recorded.'; final names = calls.take(limit).map((call) => call.name).join(', '); final suffix = calls.length > limit ? ', ... (${calls.length} total)' : ''; diff --git a/tools/ensemble_test_runner/lib/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/ensemble_test_runner.dart index defbc1215..59e182835 100644 --- a/tools/ensemble_test_runner/lib/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/ensemble_test_runner.dart @@ -8,7 +8,7 @@ export 'actions/test_execution_config.dart'; export 'actions/test_step_executor.dart'; export 'assertions/assertion_engine.dart'; export 'models/ensemble_test_models.dart'; -export 'mocks/mock_api_provider.dart'; +export 'mocks/test_api_provider_overlay.dart'; export 'mocks/test_logger.dart'; export 'parser/ensemble_test_parser.dart'; export 'schema/ensemble_test_schema_builder.dart'; diff --git a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart b/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart similarity index 86% rename from tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart rename to tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart index 891258df1..2704e8286 100644 --- a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart +++ b/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart @@ -27,23 +27,39 @@ class APICallRecord { }); } -/// Records API calls, returns YAML mocks when configured, otherwise delegates. -class MockAPIProvider extends HTTPAPIProvider { - MockAPIProvider({ +class ApiCallRecorder { + final List calls = []; + + int callCount(String apiName) => calls.where((c) => c.name == apiName).length; + + List callsFor(String apiName) => + calls.where((c) => c.name == apiName).toList(); + + void record(APICallRecord call) => calls.add(call); + + void reset() => calls.clear(); +} + +/// Test provider overlay: observes API calls, applies test overrides, otherwise delegates. +class TestApiProviderOverlay extends HTTPAPIProvider { + TestApiProviderOverlay({ required Map mocks, HTTPAPIProvider? delegate, + ApiCallRecorder? recorder, }) : _mocks = mocks, - _delegate = delegate ?? HTTPAPIProvider(); + _delegate = delegate ?? HTTPAPIProvider(), + recorder = recorder ?? ApiCallRecorder(); final Map _mocks; HTTPAPIProvider _delegate; - final List calls = []; + final ApiCallRecorder recorder; Future Function(Future Function())? liveAsyncRunner; - int callCount(String apiName) => calls.where((c) => c.name == apiName).length; + List get calls => recorder.calls; - List callsFor(String apiName) => - calls.where((c) => c.name == apiName).toList(); + int callCount(String apiName) => recorder.callCount(apiName); + + List callsFor(String apiName) => recorder.callsFor(apiName); void bindHttpDelegate(HTTPAPIProvider delegate) { _delegate = delegate; @@ -53,7 +69,7 @@ class MockAPIProvider extends HTTPAPIProvider { _mocks[apiName] = response; } - void resetCalls() => calls.clear(); + void resetCalls() => recorder.reset(); void clearMocks() => _mocks.clear(); @@ -110,7 +126,7 @@ class MockAPIProvider extends HTTPAPIProvider { String apiName, ) async { if (simulateNetworkOffline) { - calls.add(APICallRecord( + recorder.record(APICallRecord( name: apiName, apiDefinition: api, timestamp: DateTime.now(), @@ -126,7 +142,7 @@ class MockAPIProvider extends HTTPAPIProvider { final forced = _forcedExceptions[apiName]; if (forced != null) { - calls.add(APICallRecord( + recorder.record(APICallRecord( name: apiName, apiDefinition: api, timestamp: DateTime.now(), @@ -135,7 +151,7 @@ class MockAPIProvider extends HTTPAPIProvider { } final captured = _captureRequest(api, eContext); - calls.add(APICallRecord( + recorder.record(APICallRecord( name: apiName, apiDefinition: api, timestamp: DateTime.now(), @@ -211,17 +227,17 @@ class MockAPIProvider extends HTTPAPIProvider { /// Same instance as config — keeps call recording aligned with [EnsembleTestContext]. @override - MockAPIProvider clone() => this; + TestApiProviderOverlay clone() => this; @override void dispose() => _delegate.dispose(); } -/// Delegates to a real provider unless [host] has a mock for the API name. -class ApiMockOverlay implements APIProvider { - ApiMockOverlay(this._host, this._delegate); +/// Delegates to a real provider unless [host] has a test override for the API name. +class TestApiOverlay implements APIProvider { + TestApiOverlay(this._host, this._delegate); - final MockAPIProvider _host; + final TestApiProviderOverlay _host; final APIProvider _delegate; @override @@ -249,7 +265,7 @@ class ApiMockOverlay implements APIProvider { _delegate.invokeMockAPI(eContext, mock); @override - ApiMockOverlay clone() => ApiMockOverlay(_host, _delegate.clone()); + TestApiOverlay clone() => TestApiOverlay(_host, _delegate.clone()); @override void dispose() => _delegate.dispose(); diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart index 8e5a6be0a..63ad7a41b 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart @@ -1,6 +1,6 @@ import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/storage_manager.dart'; -import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/mocks/test_logger.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; @@ -8,7 +8,7 @@ import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; class EnsembleTestContext { final EnsembleTestCase testCase; - final MockAPIProvider mockApiProvider; + final TestApiProviderOverlay apiOverlay; final TestLogger logger; final EnsembleTestSetup setup; @@ -19,14 +19,14 @@ class EnsembleTestContext { EnsembleTestContext({ required this.testCase, - required this.mockApiProvider, + required this.apiOverlay, required this.logger, required this.setup, }); factory EnsembleTestContext.fromTestCase(EnsembleTestCase testCase) { final logger = TestLogger(); - final mockApi = MockAPIProvider( + final mockApi = TestApiProviderOverlay( mocks: Map.from(testCase.mocks.apis), ); @@ -34,23 +34,20 @@ class EnsembleTestContext { final keychain = testCase.initialState['keychain']; final env = testCase.initialState['env']; - final envMap = env is Map - ? Map.from(env) - : {}; + final envMap = + env is Map ? Map.from(env) : {}; final setup = EnsembleTestSetup( envOverrides: envMap.isEmpty ? null : envMap, - initialPublicStorage: storage is Map - ? Map.from(storage) - : null, - initialKeychain: keychain is Map - ? Map.from(keychain) - : null, + initialPublicStorage: + storage is Map ? Map.from(storage) : null, + initialKeychain: + keychain is Map ? Map.from(keychain) : null, ); final ctx = EnsembleTestContext( testCase: testCase, - mockApiProvider: mockApi, + apiOverlay: mockApi, logger: logger, setup: setup, ); diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index b8a331bfc..de0df32e6 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -10,7 +10,7 @@ import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/page_model.dart'; import 'package:ensemble_test_runner/mocks/adobe_test_setup.dart'; import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; -import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; @@ -240,9 +240,9 @@ class EnsembleTestHarness { } /// Wraps real providers with [mock] for call recording and optional overrides. - static void installApiMockOverrides( + static void installTestApiOverlay( EnsembleConfig config, - MockAPIProvider mock, + TestApiProviderOverlay mock, ) { final realProviders = Map.from( config.apiProviders ?? const {}, @@ -255,7 +255,7 @@ class EnsembleTestHarness { installed['http'] = mock; continue; } - installed[entry.key] = ApiMockOverlay(mock, entry.value); + installed[entry.key] = TestApiOverlay(mock, entry.value); } if (!installed.containsKey('http')) { @@ -266,7 +266,7 @@ class EnsembleTestHarness { final firebase = realProviders['firebase']; if (firebase != null && !installed.containsKey('firebaseFunction')) { installed['firebaseFunction'] = - installed['firebase'] ?? ApiMockOverlay(mock, firebase); + installed['firebase'] ?? TestApiOverlay(mock, firebase); } config.apiProviders = installed; @@ -275,7 +275,7 @@ class EnsembleTestHarness { Future bootstrapRuntime( EnsembleConfig config, EnsembleTestSetup setup, { - MockAPIProvider? apiMockOverlay, + TestApiProviderOverlay? apiOverlay, }) async { ensureTestPlugins(); @@ -293,8 +293,8 @@ class EnsembleTestHarness { await Ensemble().initManagers(); await initializeRealApiProviders(config); - if (apiMockOverlay != null) { - installApiMockOverrides(config, apiMockOverlay); + if (apiOverlay != null) { + installTestApiOverlay(config, apiOverlay); } await applyYamlTestStorageBootstrap(setup); @@ -319,7 +319,7 @@ class EnsembleTestHarness { return bootstrapRuntime( config, ctx.setup, - apiMockOverlay: ctx.mockApiProvider, + apiOverlay: ctx.apiOverlay, ); }); config = bootstrapped!; @@ -404,10 +404,11 @@ class EnsembleTestHarness { } ctx.applyRuntimeEnv(); for (final entry in ctx.testCase.mocks.apis.entries) { - ctx.mockApiProvider.setMock(entry.key, entry.value); + ctx.apiOverlay.setMock(entry.key, entry.value); } - if (config != null && config.apiProviders?['http'] is! MockAPIProvider) { - installApiMockOverrides(config, ctx.mockApiProvider); + if (config != null && + config.apiProviders?['http'] is! TestApiProviderOverlay) { + installTestApiOverlay(config, ctx.apiOverlay); } } diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index 9a2380d27..a73fb8ac7 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -78,7 +78,7 @@ class EnsembleTestRunner { try { final ctx = EnsembleTestContext.fromTestCase(test); - ctx.mockApiProvider.liveAsyncRunner = tester.runAsync; + ctx.apiOverlay.liveAsyncRunner = tester.runAsync; LiveAsyncCallSupport.runner = tester.runAsync; TestErrorTracker.install(ctx.runtime); @@ -182,10 +182,10 @@ class EnsembleTestRunner { EnsembleTestContext ctx, ) async { for (var i = 0; i < 20; i++) { - await ctx.mockApiProvider.waitForLiveCalls(); + await ctx.apiOverlay.waitForLiveCalls(); await tester.pump(); await YamlTestSession.navigationFlow.flushPending(); - if (!ctx.mockApiProvider.hasPendingLiveCalls) { + if (!ctx.apiOverlay.hasPendingLiveCalls) { return; } } diff --git a/tools/ensemble_test_runner/test/mock_api_provider_test.dart b/tools/ensemble_test_runner/test/test_api_provider_overlay_test.dart similarity index 91% rename from tools/ensemble_test_runner/test/mock_api_provider_test.dart rename to tools/ensemble_test_runner/test/test_api_provider_overlay_test.dart index c7f3ab986..aebf8f0e5 100644 --- a/tools/ensemble_test_runner/test/mock_api_provider_test.dart +++ b/tools/ensemble_test_runner/test/test_api_provider_overlay_test.dart @@ -2,7 +2,7 @@ import 'package:ensemble/framework/apiproviders/api_provider.dart'; import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; import 'package:ensemble/framework/data_context.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:yaml/yaml.dart'; @@ -13,9 +13,10 @@ void main() { await tester.pumpWidget(const MaterialApp(home: SizedBox.shrink())); final delegate = _RecordingAPIProvider(); - final provider = MockAPIProvider(mocks: {}); + final provider = TestApiProviderOverlay(mocks: {}); final context = tester.element(find.byType(SizedBox)); - final api = YamlMap.wrap({'type': 'firebaseFunction', 'name': 'exampleCallable'}); + final api = + YamlMap.wrap({'type': 'firebaseFunction', 'name': 'exampleCallable'}); await provider.invokeApiWithDelegate( delegate, @@ -34,7 +35,7 @@ void main() { await tester.pumpWidget(const MaterialApp(home: SizedBox.shrink())); final delegate = _RecordingAPIProvider(); - final provider = MockAPIProvider( + final provider = TestApiProviderOverlay( mocks: { 'login': MockAPIResponse(statusCode: 200, body: {'token': 'abc'}), }, @@ -66,7 +67,7 @@ void main() { inFlight--; }, ); - final provider = MockAPIProvider(mocks: {}, delegate: delegate); + final provider = TestApiProviderOverlay(mocks: {}, delegate: delegate); provider.liveAsyncRunner = (Future Function() fn) => fn(); final buildContext = _FakeBuildContext(); From 37058ed58959cea1f6c41ee8bffa7c2a63f22fd7 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Fri, 10 Jul 2026 00:31:49 +0500 Subject: [PATCH 05/29] refactor(api_tests): remove deprecated API mock steps and streamline test vocabulary Eliminated unused API mock steps such as `mockNetworkOffline`, `mockNetworkOnline`, `expectApiRequest`, `expectApiRequestContains`, and `expectApiHeader` from the test vocabulary and related files. This refactor enhances clarity and maintainability of the test runner by focusing on essential functionalities. --- tools/ensemble_test_runner/STEP_VOCABULARY.md | 2 +- .../assets/schema/ensemble_tests_schema.json | 263 ------------------ .../lib/actions/extended_step_handlers.dart | 20 -- .../lib/actions/test_step_executor.dart | 48 +--- .../lib/assertions/assertion_engine.dart | 68 ----- .../lib/entry/ensemble_test_entry.dart | 1 + .../lib/mocks/test_api_provider_overlay.dart | 76 ----- .../lib/reporters/test_reporter.dart | 7 + .../lib/runner/test_runtime_state.dart | 1 - .../lib/vocabulary/test_step_arg_kind.dart | 28 +- .../lib/vocabulary/test_step_registry.dart | 45 --- .../test/test_reporter_test.dart | 17 ++ .../tool/generate_step_registry.dart | 41 --- 13 files changed, 45 insertions(+), 572 deletions(-) diff --git a/tools/ensemble_test_runner/STEP_VOCABULARY.md b/tools/ensemble_test_runner/STEP_VOCABULARY.md index 5c3e1a910..f454da6eb 100644 --- a/tools/ensemble_test_runner/STEP_VOCABULARY.md +++ b/tools/ensemble_test_runner/STEP_VOCABULARY.md @@ -34,7 +34,7 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense `expectScreen` (alias), `expectNavigateTo`, `expectVisited`, `expectNotVisited`, `expectBackStack`, `expectCanGoBack`, `goBack` ### API mock / assert -`mockApi`, `mockApiError`, `mockApiFromFixture`, `mockApiException`, `mockTimeout`, `mockNetworkOffline`, `mockNetworkOnline`, `resetApiCalls`, `clearApiMocks`, `expectApiCalled`, `expectApiNotCalled`, `expectApiRequest`, `expectApiRequestContains`, `expectApiHeader`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` +`mockApi`, `mockApiError`, `mockApiFromFixture`, `mockApiException`, `mockTimeout`, `resetApiCalls`, `clearApiMocks`, `expectApiCalled`, `expectApiNotCalled`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` ### State / storage / runtime `setState`, `expectState`, `expectStateContains`, `expectStateExists`, `expectStateNotExists`, `resetState`, `setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 04dff0fd4..675c1b010 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -1568,24 +1568,6 @@ } ] }, - "args_mockNetworkOffline": { - "type": "object", - "additionalProperties": false, - "title": "mockNetworkOffline", - "description": "Simulate offline network for API calls", - "examples": [ - {} - ] - }, - "args_mockNetworkOnline": { - "type": "object", - "additionalProperties": false, - "title": "mockNetworkOnline", - "description": "Restore online network for API calls", - "examples": [ - {} - ] - }, "args_resetApiCalls": { "type": "object", "additionalProperties": false, @@ -1650,94 +1632,6 @@ } ] }, - "args_expectApiRequest": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "body": true, - "query": true, - "headers": true, - "times": { - "type": "integer" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "expectApiRequest", - "description": "Assert last API request body/query/headers match", - "examples": [ - { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - ] - }, - "args_expectApiRequestContains": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "body": true, - "query": true, - "headers": true, - "times": { - "type": "integer" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "expectApiRequestContains", - "description": "Assert API request contains partial body/query", - "examples": [ - { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - ] - }, - "args_expectApiHeader": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "header": { - "type": "string" - }, - "equals": true, - "times": { - "type": "integer" - } - }, - "required": [ - "name", - "header", - "equals" - ], - "additionalProperties": false, - "title": "expectApiHeader", - "description": "Assert an API request header equals expected", - "examples": [ - { - "name": "login", - "header": "Authorization", - "equals": "Bearer test-token" - } - ] - }, "args_expectApiCallOrder": { "type": "object", "properties": { @@ -4626,56 +4520,6 @@ "mockTimeout" ] }, - { - "type": "object", - "title": "mockNetworkOffline", - "description": "Simulate offline network for API calls", - "examples": [ - { - "mockNetworkOffline": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockNetworkOffline": { - "$ref": "#/$defs/args_mockNetworkOffline", - "description": "Simulate offline network for API calls", - "examples": [ - {} - ] - } - }, - "required": [ - "mockNetworkOffline" - ] - }, - { - "type": "object", - "title": "mockNetworkOnline", - "description": "Restore online network for API calls", - "examples": [ - { - "mockNetworkOnline": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockNetworkOnline": { - "$ref": "#/$defs/args_mockNetworkOnline", - "description": "Restore online network for API calls", - "examples": [ - {} - ] - } - }, - "required": [ - "mockNetworkOnline" - ] - }, { "type": "object", "title": "resetApiCalls", @@ -4788,113 +4632,6 @@ "expectApiNotCalled" ] }, - { - "type": "object", - "title": "expectApiRequest", - "description": "Assert last API request body/query/headers match", - "examples": [ - { - "expectApiRequest": { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectApiRequest": { - "$ref": "#/$defs/args_expectApiRequest", - "description": "Assert last API request body/query/headers match", - "examples": [ - { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - ] - } - }, - "required": [ - "expectApiRequest" - ] - }, - { - "type": "object", - "title": "expectApiRequestContains", - "description": "Assert API request contains partial body/query", - "examples": [ - { - "expectApiRequestContains": { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectApiRequestContains": { - "$ref": "#/$defs/args_expectApiRequestContains", - "description": "Assert API request contains partial body/query", - "examples": [ - { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } - } - ] - } - }, - "required": [ - "expectApiRequestContains" - ] - }, - { - "type": "object", - "title": "expectApiHeader", - "description": "Assert an API request header equals expected", - "examples": [ - { - "expectApiHeader": { - "name": "login", - "header": "Authorization", - "equals": "Bearer test-token" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectApiHeader": { - "$ref": "#/$defs/args_expectApiHeader", - "description": "Assert an API request header equals expected", - "examples": [ - { - "name": "login", - "header": "Authorization", - "equals": "Bearer test-token" - } - ] - } - }, - "required": [ - "expectApiHeader" - ] - }, { "type": "object", "title": "expectApiCallOrder", diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index ccc8ea723..3bf0c75d0 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -182,20 +182,6 @@ class ExtendedStepHandlers { case 'mockTimeout': await _mockTimeout(executor, step); return true; - case 'mockNetworkOffline': - _setNetworkOffline(executor, true); - return true; - case 'mockNetworkOnline': - _setNetworkOffline(executor, false); - return true; - case 'expectApiHeader': - executor.assertions.expectApiHeader( - step.args['name']?.toString() ?? '', - step.args['header']?.toString() ?? '', - step.args['equals'], - times: step.args['times'] as int?, - ); - return true; case 'expectApiCallOrder': executor.assertions.expectApiCallOrder( (step.args['names'] as List?)?.map((e) => e.toString()).toList() ?? @@ -519,12 +505,6 @@ class ExtendedStepHandlers { ); } - static void _setNetworkOffline(TestStepExecutor e, bool offline) { - e.context.apiOverlay.simulateNetworkOffline = offline; - ConnectivityState().isOnline = !offline; - e.context.runtime.networkOffline = offline; - } - static void _setAuth(TestStepExecutor e, TestStep step) { final user = step.args['user']; if (user is Map) { diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 2e08fc023..7179333bf 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -9,6 +9,7 @@ import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -245,32 +246,6 @@ class TestStepExecutor { } assertions.expectApiNotCalled(name); break; - case 'expectApiRequest': - final name = step.args['name']?.toString(); - if (name == null) { - throw EnsembleTestFailure('expectApiRequest requires "name"'); - } - assertions.expectApiRequest( - name, - body: step.args['body'], - query: step.args['query'], - headers: step.args['headers'], - times: step.args['times'] as int?, - ); - break; - case 'expectApiRequestContains': - final containsName = step.args['name']?.toString(); - if (containsName == null) { - throw EnsembleTestFailure('expectApiRequestContains requires "name"'); - } - assertions.expectApiRequestContains( - containsName, - body: step.args['body'], - query: step.args['query'], - headers: step.args['headers'], - times: step.args['times'] as int?, - ); - break; case 'expectCount': final expected = step.args['equals'] as int?; if (expected == null) { @@ -361,10 +336,12 @@ class TestStepExecutor { context.apiOverlay.resetCalls(); break; case 'logApiCalls': + final counts = {}; for (final call in context.apiOverlay.calls) { - context.logger.log( - 'API ${call.name} body=${call.body} query=${call.query}', - ); + counts.update(call.name, (count) => count + 1, ifAbsent: () => 1); + } + for (final entry in counts.entries) { + context.logger.log('API ${entry.key} x${entry.value}'); } break; default: @@ -470,6 +447,8 @@ class TestStepExecutor { ); } _expectSingleWidget(finder, id, 'tap'); + await tester.ensureVisible(finder); + await tester.pump(); await tester.tap(finder); await _settle(); } @@ -616,12 +595,14 @@ class TestStepExecutor { }) async { final stopwatch = Stopwatch()..start(); final tracker = ScreenTracker(); - bool isVisible() => + bool hasNavigated() => tracker.isScreenVisible(screenName: screen) || - tracker.isScreenVisible(screenId: screen); + tracker.isScreenVisible(screenId: screen) || + YamlTestSession.navigationFlow.flow.contains(screen); while (stopwatch.elapsedMilliseconds < timeoutMs) { - if (isVisible()) { + await YamlTestSession.navigationFlow.flushPending(); + if (hasNavigated()) { return; } await _yieldToLiveApiWork(); @@ -629,7 +610,8 @@ class TestStepExecutor { } await _yieldToLiveApiWork(); await tester.pump(); - if (isVisible()) { + await YamlTestSession.navigationFlow.flushPending(); + if (hasNavigated()) { return; } throw EnsembleTestFailure( diff --git a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart index b4ff29420..7c09776bb 100644 --- a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart +++ b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart @@ -143,58 +143,6 @@ class AssertionEngine { } } - void expectApiRequest( - String apiName, { - dynamic body, - dynamic query, - dynamic headers, - int? times, - }) { - final record = _selectApiCall(apiName, times: times); - - if (body != null && !_deepEquals(record.body, body)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" body $body, but got ${record.body}.', - ); - } - if (query != null && !_deepEquals(record.query, query)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" query $query, but got ${record.query}.', - ); - } - if (headers != null && !_deepEquals(record.headers, headers)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" headers $headers, but got ${record.headers}.', - ); - } - } - - void expectApiRequestContains( - String apiName, { - dynamic body, - dynamic query, - dynamic headers, - int? times, - }) { - final record = _selectApiCall(apiName, times: times); - - if (body != null && !_deepContains(record.body, body)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" body to contain $body, but got ${record.body}.', - ); - } - if (query != null && !_deepContains(record.query, query)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" query to contain $query, but got ${record.query}.', - ); - } - if (headers != null && !_deepContains(record.headers, headers)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" headers to contain $headers, but got ${record.headers}.', - ); - } - } - void expectCount(String id, int expected) { final count = finderForId(id).evaluate().length; if (count != expected) { @@ -341,22 +289,6 @@ class AssertionEngine { } } - void expectApiHeader( - String apiName, - String header, - dynamic expected, { - int? times, - }) { - final record = _selectApiCall(apiName, times: times); - final headers = record.headers ?? {}; - final actual = headers[header.toLowerCase()]; - if (!_deepEquals(actual, expected)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" header "$header" = "$expected", got "$actual".', - ); - } - } - void expectApiCallOrder(List names) { final actual = context.apiOverlay.calls.map((c) => c.name).toList(); var index = 0; diff --git a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart index 81c00f7cb..3f42cb488 100644 --- a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart +++ b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart @@ -63,6 +63,7 @@ Future runEnsembleYamlTests({ Future runEnsembleYamlTestsWithOptions( EnsembleYamlTestOptions options, ) async { + LiveTestWidgetsFlutterBinding.ensureInitialized(); EnsembleTestHarness.ensureTestPlugins(); tearDown(() { TestErrorTracker.reset(); diff --git a/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart b/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart index 2704e8286..cdfc3dcd7 100644 --- a/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart +++ b/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:convert'; import 'package:ensemble/framework/apiproviders/api_provider.dart'; import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; @@ -13,17 +12,11 @@ class APICallRecord { final String name; final YamlMap apiDefinition; final DateTime timestamp; - final dynamic body; - final Map? query; - final Map? headers; APICallRecord({ required this.name, required this.apiDefinition, required this.timestamp, - this.body, - this.query, - this.headers, }); } @@ -89,9 +82,6 @@ class TestApiProviderOverlay extends HTTPAPIProvider { Future init(String appId, Map config) => _delegate.init(appId, config); - /// When true, [invokeApi] returns an offline error without calling the delegate. - bool simulateNetworkOffline = false; - @override Future invokeApi( BuildContext context, @@ -125,21 +115,6 @@ class TestApiProviderOverlay extends HTTPAPIProvider { DataContext eContext, String apiName, ) async { - if (simulateNetworkOffline) { - recorder.record(APICallRecord( - name: apiName, - apiDefinition: api, - timestamp: DateTime.now(), - )); - return HttpResponse.fromBody( - {'message': 'Network offline (test)'}, - null, - 503, - null, - APIState.error, - ); - } - final forced = _forcedExceptions[apiName]; if (forced != null) { recorder.record(APICallRecord( @@ -150,14 +125,10 @@ class TestApiProviderOverlay extends HTTPAPIProvider { throw forced; } - final captured = _captureRequest(api, eContext); recorder.record(APICallRecord( name: apiName, apiDefinition: api, timestamp: DateTime.now(), - body: captured.body, - query: captured.query, - headers: captured.headers, )); final mock = _mocks[apiName]; @@ -278,50 +249,3 @@ Map _toRuntimeMockResponse(MockAPIResponse mock) { 'statusCode': mock.statusCode, }; } - -class _CapturedRequest { - final dynamic body; - final Map? query; - final Map? headers; - - const _CapturedRequest({this.body, this.query, this.headers}); -} - -_CapturedRequest _captureRequest(YamlMap api, DataContext eContext) { - Map? headers; - if (api['headers'] is YamlMap) { - headers = {}; - (api['headers'] as YamlMap).forEach((key, value) { - if (value != null) { - headers![key.toString().toLowerCase()] = - eContext.eval(value)?.toString() ?? ''; - } - }); - } - - dynamic body; - if (api['body'] != null) { - final evaluated = eContext.eval(api['body']); - if (evaluated is Map || evaluated is List) { - body = evaluated; - } else if (evaluated is String) { - try { - body = json.decode(evaluated); - } catch (_) { - body = evaluated; - } - } else { - body = evaluated; - } - } - - Map? query; - if (api['parameters'] is YamlMap) { - query = {}; - (api['parameters'] as YamlMap).forEach((key, value) { - query![key.toString()] = eContext.eval(value)?.toString() ?? ''; - }); - } - - return _CapturedRequest(body: body, query: query, headers: headers); -} diff --git a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart index 89a5c0cc3..89c5e4038 100644 --- a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart +++ b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart @@ -143,5 +143,12 @@ class TestReporter { buffer.writeln('│ at step: ${r.failedStepIndex! + 1}'); } } + + if (r.logs.isNotEmpty) { + buffer.writeln('│ logs:'); + for (final log in r.logs) { + buffer.writeln('│ $log'); + } + } } } diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index a87b17712..8b61a885e 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -34,7 +34,6 @@ class TestErrorTracker { _previousHandler = FlutterError.onError; FlutterError.onError = (FlutterErrorDetails details) { runtime.flutterErrors.add(details.exceptionAsString()); - _previousHandler?.call(details); }; } diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart index d9250cfa9..8b5d4102b 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -33,8 +33,6 @@ enum TestStepArgKind { mockApiException, mockTimeout, apiName, - apiRequest, - expectApiHeader, setState, expectState, storageKey, @@ -121,7 +119,10 @@ extension TestStepArgKindSchema on TestStepArgKind { ); case TestStepArgKind.setSlider: return _object( - properties: {'id': _string, 'value': {'type': 'number'}}, + properties: { + 'id': _string, + 'value': {'type': 'number'} + }, required: ['id'], ); case TestStepArgKind.chooseValue: @@ -259,27 +260,6 @@ extension TestStepArgKindSchema on TestStepArgKind { properties: {'name': _string, 'times': _integer}, required: ['name'], ); - case TestStepArgKind.apiRequest: - return _object( - properties: { - 'name': _string, - 'body': _any, - 'query': _any, - 'headers': _any, - 'times': _integer, - }, - required: ['name'], - ); - case TestStepArgKind.expectApiHeader: - return _object( - properties: { - 'name': _string, - 'header': _string, - 'equals': _any, - 'times': _integer, - }, - required: ['name', 'header', 'equals'], - ); case TestStepArgKind.setState: return _object( properties: {'path': _string, 'value': _any}, diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index d57562d26..2b4e1c18a 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -533,20 +533,6 @@ abstract final class TestStepRegistry { description: 'Mock an API with a long delay (simulate timeout)', example: const {'name': 'slow_api', 'delayMs': 60000}, ), - 'mockNetworkOffline': TestStepRegistryEntry( - category: TestStepCategory.network, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Simulate offline network for API calls', - example: const {}, - ), - 'mockNetworkOnline': TestStepRegistryEntry( - category: TestStepCategory.network, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Restore online network for API calls', - example: const {}, - ), 'resetApiCalls': TestStepRegistryEntry( category: TestStepCategory.apiMock, tier: TestStepTier.core, @@ -575,37 +561,6 @@ abstract final class TestStepRegistry { description: 'Assert an API was never called', example: const {'name': 'login', 'times': 1}, ), - 'expectApiRequest': TestStepRegistryEntry( - category: TestStepCategory.apiAssertion, - tier: TestStepTier.core, - argKind: TestStepArgKind.apiRequest, - description: 'Assert last API request body/query/headers match', - example: const { - 'name': 'login', - 'body': const {'email': 'user@test.com', 'password': 'secret'} - }, - ), - 'expectApiRequestContains': TestStepRegistryEntry( - category: TestStepCategory.apiAssertion, - tier: TestStepTier.core, - argKind: TestStepArgKind.apiRequest, - description: 'Assert API request contains partial body/query', - example: const { - 'name': 'login', - 'body': const {'email': 'user@test.com', 'password': 'secret'} - }, - ), - 'expectApiHeader': TestStepRegistryEntry( - category: TestStepCategory.apiAssertion, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectApiHeader, - description: 'Assert an API request header equals expected', - example: const { - 'name': 'login', - 'header': 'Authorization', - 'equals': 'Bearer test-token' - }, - ), 'expectApiCallOrder': TestStepRegistryEntry( category: TestStepCategory.apiAssertion, tier: TestStepTier.core, diff --git a/tools/ensemble_test_runner/test/test_reporter_test.dart b/tools/ensemble_test_runner/test/test_reporter_test.dart index b7cb93205..b19bc20c1 100644 --- a/tools/ensemble_test_runner/test/test_reporter_test.dart +++ b/tools/ensemble_test_runner/test/test_reporter_test.dart @@ -87,5 +87,22 @@ void main() { expect(output, contains('>> 2. tap(submit)')); expect(output, contains('error: not found')); }); + + test('prints step logs', () { + final output = TestReporter().formatSummary( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.passed( + testId: 'api_log_test', + durationMs: 42, + logs: const ['API getCompleteSchedules x1'], + ), + ], + ), + ); + + expect(output, contains('logs:')); + expect(output, contains('API getCompleteSchedules x1')); + }); }); } diff --git a/tools/ensemble_test_runner/tool/generate_step_registry.dart b/tools/ensemble_test_runner/tool/generate_step_registry.dart index e0ba1fe9f..2527512b1 100644 --- a/tools/ensemble_test_runner/tool/generate_step_registry.dart +++ b/tools/ensemble_test_runner/tool/generate_step_registry.dart @@ -435,18 +435,6 @@ void main() { 'mockTimeout', 'Mock an API with a long delay (simulate timeout)', ), - 'mockNetworkOffline': step( - 'network', - 'core', - 'empty', - 'Simulate offline network for API calls', - ), - 'mockNetworkOnline': step( - 'network', - 'core', - 'empty', - 'Restore online network for API calls', - ), 'resetApiCalls': step( 'apiMock', 'core', @@ -471,24 +459,6 @@ void main() { 'apiName', 'Assert an API was never called', ), - 'expectApiRequest': step( - 'apiAssertion', - 'core', - 'apiRequest', - 'Assert last API request body/query/headers match', - ), - 'expectApiRequestContains': step( - 'apiAssertion', - 'core', - 'apiRequest', - 'Assert API request contains partial body/query', - ), - 'expectApiHeader': step( - 'apiAssertion', - 'core', - 'expectApiHeader', - 'Assert an API request header equals expected', - ), 'expectApiCallOrder': step( 'apiAssertion', 'core', @@ -893,17 +863,6 @@ Map defaultExampleForArg(String arg) { return {'name': 'slow_api', 'delayMs': 60000}; case 'apiName': return {'name': 'login', 'times': 1}; - case 'apiRequest': - return { - 'name': 'login', - 'body': {'email': 'user@test.com', 'password': 'secret'}, - }; - case 'expectApiHeader': - return { - 'name': 'login', - 'header': 'Authorization', - 'equals': 'Bearer test-token', - }; case 'setState': return {'path': 'user.name', 'value': 'Jane'}; case 'expectState': From cb4d4adfd034e0333baefd6b01af3dcfec413a7f Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Fri, 10 Jul 2026 01:28:42 +0500 Subject: [PATCH 06/29] refactor(test_runner): update logStorage handling and schema for improved clarity Modified the logStorage functionality to log all public storage values when no key is provided, enhancing its usability. Updated the schema and examples in the ensemble_tests_schema.json to reflect changes in fixture paths and descriptions, ensuring consistency across the test runner. Introduced an optionalStorageKey argument kind to streamline the handling of storage keys. --- .../assets/schema/ensemble_tests_schema.json | 46 ++++++++----------- .../lib/actions/extended_step_handlers.dart | 19 ++++++-- .../lib/vocabulary/test_step_arg_kind.dart | 5 ++ .../lib/vocabulary/test_step_registry.dart | 15 +++--- .../test/test_step_executor_test.dart | 40 +++++++++++++++- .../tool/generate_step_registry.dart | 6 ++- 6 files changed, 90 insertions(+), 41 deletions(-) diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 675c1b010..cb840bfa9 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -1518,7 +1518,7 @@ "examples": [ { "name": "users", - "fixture": "users.json" + "fixture": "fixtures/users.json" } ] }, @@ -2258,20 +2258,14 @@ "properties": { "key": { "type": "string" - }, - "equals": true, - "value": true + } }, - "required": [ - "key" - ], "additionalProperties": false, "title": "logStorage", - "description": "Log public storage value for key", + "description": "Log one public storage value, or all public storage when key is omitted", "examples": [ { - "key": "onboarding_done", - "value": true + "key": "onboarding_done" } ] }, @@ -2401,7 +2395,7 @@ "description": "Load a JSON fixture into the test fixture map", "examples": [ { - "fixture": "user.json" + "fixture": "fixtures/user.json" } ] }, @@ -2426,7 +2420,7 @@ "description": "Apply all keys from a JSON fixture to state", "examples": [ { - "fixture": "user.json" + "fixture": "fixtures/user.json" } ] }, @@ -2451,7 +2445,7 @@ "description": "Assert state or path matches a JSON fixture", "examples": [ { - "fixture": "user.json" + "fixture": "fixtures/user.json" } ] }, @@ -4435,7 +4429,7 @@ { "mockApiFromFixture": { "name": "users", - "fixture": "users.json" + "fixture": "fixtures/users.json" } } ], @@ -4449,7 +4443,7 @@ "examples": [ { "name": "users", - "fixture": "users.json" + "fixture": "fixtures/users.json" } ] } @@ -5579,12 +5573,11 @@ { "type": "object", "title": "logStorage", - "description": "Log public storage value for key", + "description": "Log one public storage value, or all public storage when key is omitted", "examples": [ { "logStorage": { - "key": "onboarding_done", - "value": true + "key": "onboarding_done" } } ], @@ -5594,11 +5587,10 @@ "properties": { "logStorage": { "$ref": "#/$defs/args_logStorage", - "description": "Log public storage value for key", + "description": "Log one public storage value, or all public storage when key is omitted", "examples": [ { - "key": "onboarding_done", - "value": true + "key": "onboarding_done" } ] } @@ -5807,7 +5799,7 @@ "examples": [ { "loadFixture": { - "fixture": "user.json" + "fixture": "fixtures/user.json" } } ], @@ -5820,7 +5812,7 @@ "description": "Load a JSON fixture into the test fixture map", "examples": [ { - "fixture": "user.json" + "fixture": "fixtures/user.json" } ] } @@ -5836,7 +5828,7 @@ "examples": [ { "setStateFromFixture": { - "fixture": "user.json" + "fixture": "fixtures/user.json" } } ], @@ -5849,7 +5841,7 @@ "description": "Apply all keys from a JSON fixture to state", "examples": [ { - "fixture": "user.json" + "fixture": "fixtures/user.json" } ] } @@ -5865,7 +5857,7 @@ "examples": [ { "expectMatchesFixture": { - "fixture": "user.json" + "fixture": "fixtures/user.json" } } ], @@ -5878,7 +5870,7 @@ "description": "Assert state or path matches a JSON fixture", "examples": [ { - "fixture": "user.json" + "fixture": "fixtures/user.json" } ] } diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 3bf0c75d0..7fb4d167c 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -264,9 +264,22 @@ class ExtendedStepHandlers { return true; case 'logStorage': final key = step.args['key']?.toString(); - executor.context.logger.log( - 'storage[$key]=${StorageManager().read(key ?? '')}', - ); + if (key == null || key.isEmpty) { + final storage = StorageManager(); + final entries = storage.getKeys().map( + (key) => MapEntry(key, storage.read(key)), + ); + executor.context.logger.log( + 'storage=${jsonEncode( + Map.fromEntries(entries), + toEncodable: (value) => value.toString(), + )}', + ); + } else { + executor.context.logger.log( + 'storage[$key]=${StorageManager().read(key)}', + ); + } return true; case 'expectAccessible': executor.assertions.expectAccessible(executor.requireId(step)); diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart index 8b5d4102b..59ed975c4 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -36,6 +36,7 @@ enum TestStepArgKind { setState, expectState, storageKey, + optionalStorageKey, group, repeat, optional, @@ -275,6 +276,10 @@ extension TestStepArgKindSchema on TestStepArgKind { properties: {'key': _string, 'equals': _any, 'value': _any}, required: ['key'], ); + case TestStepArgKind.optionalStorageKey: + return _object( + properties: {'key': _string}, + ); case TestStepArgKind.group: return _object( properties: { diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index 2b4e1c18a..5bbd94bd1 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -517,7 +517,7 @@ abstract final class TestStepRegistry { tier: TestStepTier.core, argKind: TestStepArgKind.mockApiFromFixture, description: 'Load mock response body from a JSON fixture asset', - example: const {'name': 'users', 'fixture': 'users.json'}, + example: const {'name': 'users', 'fixture': 'fixtures/users.json'}, ), 'mockApiException': TestStepRegistryEntry( category: TestStepCategory.apiMock, @@ -805,9 +805,10 @@ abstract final class TestStepRegistry { 'logStorage': TestStepRegistryEntry( category: TestStepCategory.debug, tier: TestStepTier.core, - argKind: TestStepArgKind.storageKey, - description: 'Log public storage value for key', - example: const {'key': 'onboarding_done', 'value': true}, + argKind: TestStepArgKind.optionalStorageKey, + description: + 'Log one public storage value, or all public storage when key is omitted', + example: const {'key': 'onboarding_done'}, ), 'expectNoConsoleErrors': TestStepRegistryEntry( category: TestStepCategory.quality, @@ -863,21 +864,21 @@ abstract final class TestStepRegistry { tier: TestStepTier.core, argKind: TestStepArgKind.fixturePath, description: 'Load a JSON fixture into the test fixture map', - example: const {'fixture': 'user.json'}, + example: const {'fixture': 'fixtures/user.json'}, ), 'setStateFromFixture': TestStepRegistryEntry( category: TestStepCategory.fixture, tier: TestStepTier.core, argKind: TestStepArgKind.fixturePath, description: 'Apply all keys from a JSON fixture to state', - example: const {'fixture': 'user.json'}, + example: const {'fixture': 'fixtures/user.json'}, ), 'expectMatchesFixture': TestStepRegistryEntry( category: TestStepCategory.fixture, tier: TestStepTier.core, argKind: TestStepArgKind.fixturePath, description: 'Assert state or path matches a JSON fixture', - example: const {'fixture': 'user.json'}, + example: const {'fixture': 'fixtures/user.json'}, ), }; } diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index 8e128a273..9675d85cc 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -1,8 +1,9 @@ import 'package:ensemble_test_runner/actions/test_step_executor.dart'; -import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble/framework/storage_manager.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { @@ -14,7 +15,8 @@ void main() { steps: [], ), ); - final harness = EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); final executor = TestStepExecutor( tester: tester, context: context, @@ -33,4 +35,38 @@ void main() { ), ); }); + + testWidgets('logStorage logs all public storage when key is omitted', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + await tester.runAsync(() async { + await StorageManager().init(); + await StorageManager().clearPublicStorage(); + await StorageManager().write('first', 'one'); + await StorageManager().write('second', {'nested': true}); + }); + + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 't', + startScreen: 'Home', + steps: [], + ), + ); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: harness, + ); + + await executor.execute(const TestStep(type: 'logStorage', args: {})); + + expect(context.logger.logs, hasLength(1)); + expect(context.logger.logs.single, startsWith('storage=')); + expect(context.logger.logs.single, contains('"first":"one"')); + expect(context.logger.logs.single, contains('"second":{"nested":true}')); + }); } diff --git a/tools/ensemble_test_runner/tool/generate_step_registry.dart b/tools/ensemble_test_runner/tool/generate_step_registry.dart index 2527512b1..cfd9690a9 100644 --- a/tools/ensemble_test_runner/tool/generate_step_registry.dart +++ b/tools/ensemble_test_runner/tool/generate_step_registry.dart @@ -642,8 +642,8 @@ void main() { 'logStorage': step( 'debug', 'core', - 'storageKey', - 'Log public storage value for key', + 'optionalStorageKey', + 'Log one public storage value, or all public storage when key is omitted', ), 'expectNoConsoleErrors': step( 'quality', @@ -869,6 +869,8 @@ Map defaultExampleForArg(String arg) { return {'path': 'user.name', 'equals': 'Jane'}; case 'storageKey': return {'key': 'onboarding_done', 'value': true}; + case 'optionalStorageKey': + return {'key': 'onboarding_done'}; case 'group': return { 'name': 'login_flow', From 049c928019b1b52ebc7694b376e6fed69317cc79 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Fri, 10 Jul 2026 01:42:00 +0500 Subject: [PATCH 07/29] refactor(test_runner): remove state-related steps and streamline test vocabulary Eliminated deprecated state handling steps such as `setState`, `expectState`, and related assertions from the test runner. This refactor enhances clarity and maintainability by focusing on essential functionalities, while also updating the associated documentation and schema to reflect these changes. --- tools/ensemble_test_runner/STEP_VOCABULARY.md | 8 +- .../assets/schema/ensemble_tests_schema.json | 571 ------------------ .../lib/actions/extended_step_handlers.dart | 103 ---- .../lib/actions/state_helper.dart | 34 -- .../lib/actions/test_step_executor.dart | 58 -- .../lib/assertions/assertion_engine.dart | 121 ---- .../lib/mocks/firebase_auth_test_setup.dart | 1 - .../mocks/live_sign_in_with_custom_token.dart | 2 - .../lib/runner/test_runtime_state.dart | 2 - .../lib/vocabulary/test_step_arg_kind.dart | 40 -- .../lib/vocabulary/test_step_registry.dart | 77 --- .../lib/vocabulary/test_step_vocabulary.dart | 1 - .../tool/generate_step_registry.dart | 78 --- 13 files changed, 4 insertions(+), 1092 deletions(-) delete mode 100644 tools/ensemble_test_runner/lib/actions/state_helper.dart diff --git a/tools/ensemble_test_runner/STEP_VOCABULARY.md b/tools/ensemble_test_runner/STEP_VOCABULARY.md index f454da6eb..fa3dbf076 100644 --- a/tools/ensemble_test_runner/STEP_VOCABULARY.md +++ b/tools/ensemble_test_runner/STEP_VOCABULARY.md @@ -22,7 +22,7 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense `scroll`, `scrollUntilVisible`, `swipe`, `drag`, `pullToRefresh` ### Wait / sync -`wait` (alias `pump`), `pump`, `settle`, `waitFor`, `waitForText`, `waitForGone`, `waitForApi`, `waitForNavigation`, `waitUntil` +`wait` (alias `pump`), `pump`, `settle`, `waitFor`, `waitForText`, `waitForGone`, `waitForApi`, `waitForNavigation` ### UI assertions `expectVisible`, `expectNotVisible`, `expectExists`, `expectNotExists`, `expectText`, `expectNoText`, `expectTextContains`, `expectEnabled`, `expectDisabled` @@ -36,11 +36,11 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense ### API mock / assert `mockApi`, `mockApiError`, `mockApiFromFixture`, `mockApiException`, `mockTimeout`, `resetApiCalls`, `clearApiMocks`, `expectApiCalled`, `expectApiNotCalled`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` -### State / storage / runtime -`setState`, `expectState`, `expectStateContains`, `expectStateExists`, `expectStateNotExists`, `resetState`, `setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` +### Storage / runtime +`setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` ### Scripts / fixtures / debug / quality -`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `loadFixture`, `setStateFromFixture`, `expectMatchesFixture`, `logState`, `logStorage`, `screenshot`, `dumpTree`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` +`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `logStorage`, `screenshot`, `dumpTree`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` ### Control flow `group`, `repeat`, `optional`, `ifVisible` diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index cb840bfa9..986e3bdad 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -847,37 +847,6 @@ } ] }, - "args_waitUntil": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "equals": true, - "state": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "equals": true - }, - "additionalProperties": false - }, - "timeoutMs": { - "type": "integer" - } - }, - "additionalProperties": false, - "title": "waitUntil", - "description": "Poll until app state at path equals expected value", - "examples": [ - { - "path": "user.name", - "equals": "Jane" - } - ] - }, "args_expectVisible": { "type": "object", "properties": { @@ -1681,125 +1650,6 @@ } ] }, - "args_setState": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "value": true - }, - "required": [ - "path" - ], - "additionalProperties": false, - "title": "setState", - "description": "Set app data-context state at path to value", - "examples": [ - { - "path": "user.name", - "value": "Jane" - } - ] - }, - "args_expectState": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "equals": true, - "contains": true - }, - "required": [ - "path" - ], - "additionalProperties": false, - "title": "expectState", - "description": "Assert app state at path equals expected", - "examples": [ - { - "path": "user.name", - "equals": "Jane" - } - ] - }, - "args_expectStateContains": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "equals": true, - "contains": true - }, - "required": [ - "path" - ], - "additionalProperties": false, - "title": "expectStateContains", - "description": "Assert app state at path contains subset", - "examples": [ - { - "path": "user.name", - "equals": "Jane" - } - ] - }, - "args_expectStateExists": { - "type": "object", - "properties": { - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "additionalProperties": false, - "title": "expectStateExists", - "description": "Assert state path resolves without error", - "examples": [ - { - "path": "user.id" - } - ] - }, - "args_expectStateNotExists": { - "type": "object", - "properties": { - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "additionalProperties": false, - "title": "expectStateNotExists", - "description": "Assert state path is null or absent", - "examples": [ - { - "path": "user.id" - } - ] - }, - "args_resetState": { - "type": "object", - "properties": { - "path": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "resetState", - "description": "Clear state at path (set to null)", - "examples": [ - { - "path": "cart" - } - ] - }, "args_setStorage": { "type": "object", "properties": { @@ -2234,25 +2084,6 @@ {} ] }, - "args_logState": { - "type": "object", - "properties": { - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "additionalProperties": false, - "title": "logState", - "description": "Log resolved state at path", - "examples": [ - { - "path": "user.id" - } - ] - }, "args_logStorage": { "type": "object", "properties": { @@ -2374,81 +2205,6 @@ } ] }, - "args_loadFixture": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "path": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statePath": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "loadFixture", - "description": "Load a JSON fixture into the test fixture map", - "examples": [ - { - "fixture": "fixtures/user.json" - } - ] - }, - "args_setStateFromFixture": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "path": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statePath": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "setStateFromFixture", - "description": "Apply all keys from a JSON fixture to state", - "examples": [ - { - "fixture": "fixtures/user.json" - } - ] - }, - "args_expectMatchesFixture": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "path": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statePath": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "expectMatchesFixture", - "description": "Assert state or path matches a JSON fixture", - "examples": [ - { - "fixture": "fixtures/user.json" - } - ] - }, "step": { "oneOf": [ { @@ -3507,37 +3263,6 @@ "waitForNavigation" ] }, - { - "type": "object", - "title": "waitUntil", - "description": "Poll until app state at path equals expected value", - "examples": [ - { - "waitUntil": { - "path": "user.name", - "equals": "Jane" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "waitUntil": { - "$ref": "#/$defs/args_waitUntil", - "description": "Poll until app state at path equals expected value", - "examples": [ - { - "path": "user.name", - "equals": "Jane" - } - ] - } - }, - "required": [ - "waitUntil" - ] - }, { "type": "object", "title": "expectVisible", @@ -4692,186 +4417,6 @@ "expectLastApiCall" ] }, - { - "type": "object", - "title": "setState", - "description": "Set app data-context state at path to value", - "examples": [ - { - "setState": { - "path": "user.name", - "value": "Jane" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "setState": { - "$ref": "#/$defs/args_setState", - "description": "Set app data-context state at path to value", - "examples": [ - { - "path": "user.name", - "value": "Jane" - } - ] - } - }, - "required": [ - "setState" - ] - }, - { - "type": "object", - "title": "expectState", - "description": "Assert app state at path equals expected", - "examples": [ - { - "expectState": { - "path": "user.name", - "equals": "Jane" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectState": { - "$ref": "#/$defs/args_expectState", - "description": "Assert app state at path equals expected", - "examples": [ - { - "path": "user.name", - "equals": "Jane" - } - ] - } - }, - "required": [ - "expectState" - ] - }, - { - "type": "object", - "title": "expectStateContains", - "description": "Assert app state at path contains subset", - "examples": [ - { - "expectStateContains": { - "path": "user.name", - "equals": "Jane" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectStateContains": { - "$ref": "#/$defs/args_expectStateContains", - "description": "Assert app state at path contains subset", - "examples": [ - { - "path": "user.name", - "equals": "Jane" - } - ] - } - }, - "required": [ - "expectStateContains" - ] - }, - { - "type": "object", - "title": "expectStateExists", - "description": "Assert state path resolves without error", - "examples": [ - { - "expectStateExists": { - "path": "user.id" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectStateExists": { - "$ref": "#/$defs/args_expectStateExists", - "description": "Assert state path resolves without error", - "examples": [ - { - "path": "user.id" - } - ] - } - }, - "required": [ - "expectStateExists" - ] - }, - { - "type": "object", - "title": "expectStateNotExists", - "description": "Assert state path is null or absent", - "examples": [ - { - "expectStateNotExists": { - "path": "user.id" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectStateNotExists": { - "$ref": "#/$defs/args_expectStateNotExists", - "description": "Assert state path is null or absent", - "examples": [ - { - "path": "user.id" - } - ] - } - }, - "required": [ - "expectStateNotExists" - ] - }, - { - "type": "object", - "title": "resetState", - "description": "Clear state at path (set to null)", - "examples": [ - { - "resetState": { - "path": "cart" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "resetState": { - "$ref": "#/$defs/args_resetState", - "description": "Clear state at path (set to null)", - "examples": [ - { - "path": "cart" - } - ] - } - }, - "required": [ - "resetState" - ] - }, { "type": "object", "title": "setStorage", @@ -5541,35 +5086,6 @@ "dumpTree" ] }, - { - "type": "object", - "title": "logState", - "description": "Log resolved state at path", - "examples": [ - { - "logState": { - "path": "user.id" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "logState": { - "$ref": "#/$defs/args_logState", - "description": "Log resolved state at path", - "examples": [ - { - "path": "user.id" - } - ] - } - }, - "required": [ - "logState" - ] - }, { "type": "object", "title": "logStorage", @@ -5791,93 +5307,6 @@ "required": [ "expectNoOverflow" ] - }, - { - "type": "object", - "title": "loadFixture", - "description": "Load a JSON fixture into the test fixture map", - "examples": [ - { - "loadFixture": { - "fixture": "fixtures/user.json" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "loadFixture": { - "$ref": "#/$defs/args_loadFixture", - "description": "Load a JSON fixture into the test fixture map", - "examples": [ - { - "fixture": "fixtures/user.json" - } - ] - } - }, - "required": [ - "loadFixture" - ] - }, - { - "type": "object", - "title": "setStateFromFixture", - "description": "Apply all keys from a JSON fixture to state", - "examples": [ - { - "setStateFromFixture": { - "fixture": "fixtures/user.json" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "setStateFromFixture": { - "$ref": "#/$defs/args_setStateFromFixture", - "description": "Apply all keys from a JSON fixture to state", - "examples": [ - { - "fixture": "fixtures/user.json" - } - ] - } - }, - "required": [ - "setStateFromFixture" - ] - }, - { - "type": "object", - "title": "expectMatchesFixture", - "description": "Assert state or path matches a JSON fixture", - "examples": [ - { - "expectMatchesFixture": { - "fixture": "fixtures/user.json" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectMatchesFixture": { - "$ref": "#/$defs/args_expectMatchesFixture", - "description": "Assert state or path matches a JSON fixture", - "examples": [ - { - "fixture": "fixtures/user.json" - } - ] - } - }, - "required": [ - "expectMatchesFixture" - ] } ] } diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 7fb4d167c..09b29d692 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -1,11 +1,9 @@ import 'dart:convert'; import 'package:ensemble/action/navigation_action.dart'; -import 'package:ensemble/framework/bindings.dart'; import 'package:ensemble/screen_controller.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; -import 'package:ensemble_test_runner/actions/state_helper.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; @@ -201,24 +199,6 @@ class ExtendedStepHandlers { case 'clearStorage': await executor.context.clearStorage(); return true; - case 'expectStateContains': - executor.assertions.expectStateContains( - step.args['path']?.toString() ?? '', - step.args['contains'], - ); - return true; - case 'expectStateExists': - executor.assertions - .expectStateExists(step.args['path']?.toString() ?? ''); - return true; - case 'expectStateNotExists': - executor.assertions.expectStateNotExists( - step.args['path']?.toString() ?? '', - ); - return true; - case 'resetState': - await _resetState(executor, step); - return true; case 'setAuth': _setAuth(executor, step); return true; @@ -248,20 +228,6 @@ class ExtendedStepHandlers { executor.assertions .expectConsoleLog(step.args['contains']?.toString() ?? ''); return true; - case 'loadFixture': - await _loadFixture(executor, step); - return true; - case 'setStateFromFixture': - await _setStateFromFixture(executor, step); - return true; - case 'expectMatchesFixture': - await _expectMatchesFixture(executor, step); - return true; - case 'logState': - executor.context.logger.log( - 'state: ${executor.assertions.readState(step.args['path']?.toString() ?? '')}', - ); - return true; case 'logStorage': final key = step.args['key']?.toString(); if (key == null || key.isEmpty) { @@ -622,75 +588,6 @@ class ExtendedStepHandlers { return candidates.toSet().toList(); } - static Future _loadFixture(TestStepExecutor e, TestStep step) async { - final key = step.args['key']?.toString(); - final path = - step.args['path']?.toString() ?? step.args['fixture']?.toString(); - if (key == null || path == null) { - throw EnsembleTestFailure('loadFixture requires "key" and "path"'); - } - e.context.runtime.fixtures[key] = await _readFixture(e, path); - } - - static Future _setStateFromFixture( - TestStepExecutor e, TestStep step) async { - final path = - step.args['fixture']?.toString() ?? step.args['path']?.toString(); - if (path == null) { - throw EnsembleTestFailure( - 'setStateFromFixture requires "fixture" or "path"'); - } - final data = await _readFixture(e, path); - if (data is! Map) { - throw EnsembleTestFailure('Fixture must be a JSON object'); - } - final scope = e.assertions.activeScope(); - if (scope == null) { - throw EnsembleTestFailure('setStateFromFixture requires active screen'); - } - for (final entry in data.entries) { - setStatePath(scope, entry.key.toString(), entry.value); - } - await e.settle(); - } - - static Future _expectMatchesFixture( - TestStepExecutor e, - TestStep step, - ) async { - final path = - step.args['fixture']?.toString() ?? step.args['path']?.toString(); - final statePath = step.args['statePath']?.toString(); - if (path == null) { - throw EnsembleTestFailure( - 'expectMatchesFixture requires "fixture" or "path"'); - } - final expected = await _readFixture(e, path); - if (statePath != null) { - e.assertions.expectState(statePath, expected); - } else { - final actual = - e.assertions.readState(step.args['path']?.toString() ?? ''); - if (!_deepEquals(actual, expected)) { - throw EnsembleTestFailure( - 'State does not match fixture $path: expected $expected, got $actual', - ); - } - } - } - - static Future _resetState(TestStepExecutor e, TestStep step) async { - final path = step.args['path']?.toString(); - final scope = e.assertions.activeScope(); - if (scope == null) { - throw EnsembleTestFailure('resetState requires active screen'); - } - if (path != null) { - setStatePath(scope, path, null); - } - await e.settle(); - } - static Future _screenshot(TestStepExecutor e, TestStep step) async { final name = step.args['name']?.toString(); if (name != null && name.isNotEmpty) { diff --git a/tools/ensemble_test_runner/lib/actions/state_helper.dart b/tools/ensemble_test_runner/lib/actions/state_helper.dart deleted file mode 100644 index ed99e5a7e..000000000 --- a/tools/ensemble_test_runner/lib/actions/state_helper.dart +++ /dev/null @@ -1,34 +0,0 @@ -import 'package:ensemble/framework/scope.dart'; -import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; - -/// Writes a dot-path value into the active [ScopeManager] data context. -void setStatePath(ScopeManager scope, String path, dynamic value) { - final parts = path.split('.').where((p) => p.isNotEmpty).toList(); - if (parts.isEmpty) { - throw EnsembleTestFailure('setState path cannot be empty'); - } - if (parts.length == 1) { - scope.dataContext.addDataContextById(parts.first, value); - return; - } - - final root = parts.first; - final existing = scope.dataContext.getContextMap()[root]; - final Map rootMap = existing is Map - ? Map.from(existing) - : {}; - - var cursor = rootMap; - for (var i = 1; i < parts.length - 1; i++) { - final key = parts[i]; - final next = cursor[key]; - if (next is Map) { - cursor[key] = Map.from(next); - } else { - cursor[key] = {}; - } - cursor = cursor[key] as Map; - } - cursor[parts.last] = value; - scope.dataContext.addDataContextById(root, rootMap); -} diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 7179333bf..389efcfcd 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -3,7 +3,6 @@ import 'dart:async'; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; -import 'package:ensemble_test_runner/actions/state_helper.dart'; import 'package:ensemble_test_runner/actions/test_execution_config.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; @@ -113,21 +112,6 @@ class TestStepExecutor { config.defaultWaitTimeout.inMilliseconds, ); return; - case 'waitUntil': - String? statePath = step.args['path']?.toString(); - dynamic expected = step.args['equals']; - final stateNode = step.args['state']; - if (stateNode is Map) { - statePath ??= stateNode['path']?.toString(); - expected ??= stateNode['equals']; - } - await _waitUntil( - path: statePath, - expected: expected, - timeoutMs: step.args['timeoutMs'] as int? ?? - config.defaultWaitTimeout.inMilliseconds, - ); - return; case 'expectScreen': final screen = step.args['name']?.toString() ?? step.args['screen']?.toString(); @@ -274,26 +258,6 @@ class TestStepExecutor { } assertions.expectStorage(key, step.args['equals']); break; - case 'expectState': - final path = step.args['path']?.toString(); - if (path == null) { - throw EnsembleTestFailure('expectState requires "path"'); - } - assertions.expectState(path, step.args['equals']); - break; - case 'setState': - final path = step.args['path']?.toString(); - if (path == null) { - throw EnsembleTestFailure('setState requires "path"'); - } - final scope = assertions.activeScope(); - if (scope == null) { - throw EnsembleTestFailure( - 'setState requires an active Ensemble screen.', - ); - } - setStatePath(scope, path, step.args['value']); - break; case 'setStorage': final key = step.args['key']?.toString(); if (key == null) { @@ -619,28 +583,6 @@ class TestStepExecutor { ); } - Future _waitUntil({ - String? path, - dynamic expected, - required int timeoutMs, - }) async { - if (path == null || path.isEmpty) { - throw EnsembleTestFailure('waitUntil requires "path"'); - } - - final stopwatch = Stopwatch()..start(); - while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); - if (assertions.matchesState(path, expected)) { - return; - } - } - throw EnsembleTestFailure( - 'Timed out after ${timeoutMs}ms waiting for state "$path" ' - 'to equal "$expected"', - ); - } - Future _waitForGone({ String? id, required int timeoutMs, diff --git a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart index 7c09776bb..6418903fd 100644 --- a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart +++ b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart @@ -6,7 +6,6 @@ import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/framework/view/data_scope_widget.dart'; import 'package:ensemble/framework/view/page_group.dart'; -import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; @@ -315,34 +314,6 @@ class AssertionEngine { } } - void expectStateContains(String path, dynamic contains) { - final actual = readState(path); - if (!_deepContains(actual, contains)) { - throw EnsembleTestFailure( - 'Expected state "$path" to contain $contains, got $actual.', - ); - } - } - - void expectStateExists(String path) { - try { - readState(path); - } catch (_) { - throw EnsembleTestFailure('Expected state path "$path" to exist.'); - } - } - - void expectStateNotExists(String path) { - try { - final value = readState(path); - if (value != null) { - throw EnsembleTestFailure('Expected state path "$path" to be absent.'); - } - } on EnsembleTestFailure { - // expected - } - } - void expectConsoleLog(String contains) { final logs = context.runtime.consoleLogs; if (!logs.any((l) => l.contains(contains))) { @@ -412,45 +383,6 @@ class AssertionEngine { } } - APICallRecord _selectApiCall(String apiName, {int? times}) { - final calls = context.apiOverlay.callsFor(apiName); - if (calls.isEmpty) { - throw EnsembleTestFailure('Expected API "$apiName" to be called.'); - } - if (times != null) { - if (times < 1 || times > calls.length) { - throw EnsembleTestFailure( - 'Expected API "$apiName" call #$times, but it was called ${calls.length} times.', - ); - } - return calls[times - 1]; - } - return calls.last; - } - - /// Reads a data-context path without asserting. - dynamic readState(String path) { - final scope = _activeScope(); - if (scope == null) { - throw EnsembleTestFailure( - 'readState requires an active Ensemble screen (no ScopeManager found).', - ); - } - if (path.contains(r'${') || path.contains(r'$(')) { - return scope.dataContext.eval(path); - } - return scope.dataContext.eval('\${$path}'); - } - - bool matchesState(String path, dynamic expected) { - try { - expectState(path, expected); - return true; - } on EnsembleTestFailure { - return false; - } - } - void expectStorage(String key, dynamic expected) { final actual = StorageManager().read(key); if (actual != expected) { @@ -462,28 +394,6 @@ class AssertionEngine { ScopeManager? activeScope() => _activeScope(); - void expectState(String path, dynamic expected) { - final scope = _activeScope(); - if (scope == null) { - throw EnsembleTestFailure( - 'expectState requires an active Ensemble screen (no ScopeManager found).', - ); - } - - dynamic actual; - if (path.contains(r'${') || path.contains(r'$(')) { - actual = scope.dataContext.eval(path); - } else { - actual = scope.dataContext.eval('\${$path}'); - } - - if (!_deepEquals(actual, expected)) { - throw EnsembleTestFailure( - 'Expected state "$path" to equal "$expected", but got "$actual".', - ); - } - } - void expectNavigateTo(String screenName) { final tracker = ScreenTracker(); if (!tracker.isScreenVisible(screenName: screenName) && @@ -546,37 +456,6 @@ class AssertionEngine { return null; } - /// True when [subset] is contained in [value] (maps/lists recurse). - bool _deepContains(dynamic value, dynamic subset) { - if (subset == null) return true; - if (value == null) return false; - - final normalizedValue = _normalizeForCompare(value); - final normalizedSubset = _normalizeForCompare(subset); - - if (normalizedSubset is Map) { - if (normalizedValue is! Map) return false; - for (final entry in normalizedSubset.entries) { - if (!normalizedValue.containsKey(entry.key)) return false; - if (!_deepContains(normalizedValue[entry.key], entry.value)) { - return false; - } - } - return true; - } - - if (normalizedSubset is List) { - if (normalizedValue is! List) return false; - for (final item in normalizedSubset) { - final found = normalizedValue.any((v) => _deepContains(v, item)); - if (!found) return false; - } - return true; - } - - return normalizedValue == normalizedSubset; - } - bool _deepEquals(dynamic a, dynamic b) { if (a == b) return true; diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart index af110c68d..2b932e5f2 100644 --- a/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart +++ b/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart @@ -1,5 +1,4 @@ import 'package:firebase_auth_platform_interface/src/pigeon/messages.pigeon.dart'; -import 'package:firebase_core/firebase_core.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart b/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart index 51ce6bfd4..c94bfa491 100644 --- a/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart +++ b/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart @@ -1,9 +1,7 @@ -import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/action.dart'; import 'package:ensemble/framework/apiproviders/firebase_functions/firebase_functions_api_provider.dart'; import 'package:ensemble/framework/error_handling.dart'; import 'package:ensemble/framework/event.dart'; -import 'package:ensemble/framework/scope.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/framework/stub/auth_context_manager.dart'; import 'package:ensemble/screen_controller.dart'; diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index 8b61a885e..57194fd94 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -11,7 +11,6 @@ class TestRuntimeState { Size? deviceSize; Locale? locale; String? themeMode; - final Map fixtures = {}; void clear() { networkOffline = false; @@ -22,7 +21,6 @@ class TestRuntimeState { deviceSize = null; locale = null; themeMode = null; - fixtures.clear(); } } diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart index 59ed975c4..70e4b84fa 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -18,7 +18,6 @@ enum TestStepArgKind { waitFor, waitForGone, waitForNavigation, - waitUntil, textRequired, expectEquals, expectChecked, @@ -33,8 +32,6 @@ enum TestStepArgKind { mockApiException, mockTimeout, apiName, - setState, - expectState, storageKey, optionalStorageKey, group, @@ -42,8 +39,6 @@ enum TestStepArgKind { optional, ifVisible, screenshot, - expectStatePath, - resetStatePath, setAuth, setPermission, setDevice, @@ -52,7 +47,6 @@ enum TestStepArgKind { runScript, expectConsoleLog, expectErrorContains, - fixturePath, expectApiCallOrder, expectListContains, expectListItem, @@ -170,17 +164,6 @@ extension TestStepArgKindSchema on TestStepArgKind { properties: {'screen': _string, 'timeoutMs': _integer}, required: ['screen'], ); - case TestStepArgKind.waitUntil: - return _object( - properties: { - 'path': _string, - 'equals': _any, - 'state': _object( - properties: {'path': _string, 'equals': _any}, - ), - 'timeoutMs': _integer, - }, - ); case TestStepArgKind.textRequired: return _object(properties: {'text': _string}, required: ['text']); case TestStepArgKind.expectEquals: @@ -261,16 +244,6 @@ extension TestStepArgKindSchema on TestStepArgKind { properties: {'name': _string, 'times': _integer}, required: ['name'], ); - case TestStepArgKind.setState: - return _object( - properties: {'path': _string, 'value': _any}, - required: ['path'], - ); - case TestStepArgKind.expectState: - return _object( - properties: {'path': _string, 'equals': _any, 'contains': _any}, - required: ['path'], - ); case TestStepArgKind.storageKey: return _object( properties: {'key': _string, 'equals': _any, 'value': _any}, @@ -314,10 +287,6 @@ extension TestStepArgKindSchema on TestStepArgKind { ); case TestStepArgKind.screenshot: return _object(properties: {'name': _string}); - case TestStepArgKind.expectStatePath: - return _object(properties: {'path': _string}, required: ['path']); - case TestStepArgKind.resetStatePath: - return _object(properties: {'path': _string}); case TestStepArgKind.setAuth: return _object( properties: { @@ -352,15 +321,6 @@ extension TestStepArgKindSchema on TestStepArgKind { ); case TestStepArgKind.expectErrorContains: return _object(properties: {'contains': _string}); - case TestStepArgKind.fixturePath: - return _object( - properties: { - 'key': _string, - 'path': _string, - 'fixture': _string, - 'statePath': _string, - }, - ); case TestStepArgKind.expectApiCallOrder: return _object( properties: { diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index 5bbd94bd1..3cdae8878 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -288,13 +288,6 @@ abstract final class TestStepRegistry { description: 'Poll until the given screen is visible', example: const {'screen': 'Home', 'timeoutMs': 5000}, ), - 'waitUntil': TestStepRegistryEntry( - category: TestStepCategory.wait, - tier: TestStepTier.core, - argKind: TestStepArgKind.waitUntil, - description: 'Poll until app state at path equals expected value', - example: const {'path': 'user.name', 'equals': 'Jane'}, - ), 'expectVisible': TestStepRegistryEntry( category: TestStepCategory.uiAssertion, tier: TestStepTier.core, @@ -577,48 +570,6 @@ abstract final class TestStepRegistry { description: 'Assert the most recent API call name', example: const {'name': 'login', 'times': 1}, ), - 'setState': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.setState, - description: 'Set app data-context state at path to value', - example: const {'path': 'user.name', 'value': 'Jane'}, - ), - 'expectState': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectState, - description: 'Assert app state at path equals expected', - example: const {'path': 'user.name', 'equals': 'Jane'}, - ), - 'expectStateContains': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectState, - description: 'Assert app state at path contains subset', - example: const {'path': 'user.name', 'equals': 'Jane'}, - ), - 'expectStateExists': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectStatePath, - description: 'Assert state path resolves without error', - example: const {'path': 'user.id'}, - ), - 'expectStateNotExists': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectStatePath, - description: 'Assert state path is null or absent', - example: const {'path': 'user.id'}, - ), - 'resetState': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.resetStatePath, - description: 'Clear state at path (set to null)', - example: const {'path': 'cart'}, - ), 'setStorage': TestStepRegistryEntry( category: TestStepCategory.storage, tier: TestStepTier.core, @@ -795,13 +746,6 @@ abstract final class TestStepRegistry { description: 'Print the widget tree to the debug console', example: const {}, ), - 'logState': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectStatePath, - description: 'Log resolved state at path', - example: const {'path': 'user.id'}, - ), 'logStorage': TestStepRegistryEntry( category: TestStepCategory.debug, tier: TestStepTier.core, @@ -859,26 +803,5 @@ abstract final class TestStepRegistry { description: 'Assert widget renders without overflow issues', example: const {'id': 'my_widget'}, ), - 'loadFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.fixturePath, - description: 'Load a JSON fixture into the test fixture map', - example: const {'fixture': 'fixtures/user.json'}, - ), - 'setStateFromFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.fixturePath, - description: 'Apply all keys from a JSON fixture to state', - example: const {'fixture': 'fixtures/user.json'}, - ), - 'expectMatchesFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.fixturePath, - description: 'Assert state or path matches a JSON fixture', - example: const {'fixture': 'fixtures/user.json'}, - ), }; } diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart index cffcc153f..f9c505d95 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart @@ -21,7 +21,6 @@ enum TestStepCategory { navigation, apiMock, apiAssertion, - state, storage, runtime, script, diff --git a/tools/ensemble_test_runner/tool/generate_step_registry.dart b/tools/ensemble_test_runner/tool/generate_step_registry.dart index cfd9690a9..955627338 100644 --- a/tools/ensemble_test_runner/tool/generate_step_registry.dart +++ b/tools/ensemble_test_runner/tool/generate_step_registry.dart @@ -237,12 +237,6 @@ void main() { 'waitForNavigation', 'Poll until the given screen is visible', ), - 'waitUntil': step( - 'wait', - 'core', - 'waitUntil', - 'Poll until app state at path equals expected value', - ), 'expectVisible': step( 'uiAssertion', 'core', @@ -471,42 +465,6 @@ void main() { 'apiName', 'Assert the most recent API call name', ), - 'setState': step( - 'state', - 'core', - 'setState', - 'Set app data-context state at path to value', - ), - 'expectState': step( - 'state', - 'core', - 'expectState', - 'Assert app state at path equals expected', - ), - 'expectStateContains': step( - 'state', - 'core', - 'expectState', - 'Assert app state at path contains subset', - ), - 'expectStateExists': step( - 'state', - 'core', - 'expectStatePath', - 'Assert state path resolves without error', - ), - 'expectStateNotExists': step( - 'state', - 'core', - 'expectStatePath', - 'Assert state path is null or absent', - ), - 'resetState': step( - 'state', - 'core', - 'resetStatePath', - 'Clear state at path (set to null)', - ), 'setStorage': step( 'storage', 'core', @@ -633,12 +591,6 @@ void main() { 'empty', 'Print the widget tree to the debug console', ), - 'logState': step( - 'debug', - 'core', - 'expectStatePath', - 'Log resolved state at path', - ), 'logStorage': step( 'debug', 'core', @@ -687,24 +639,6 @@ void main() { 'idRequired', 'Assert widget renders without overflow issues', ), - 'loadFixture': step( - 'fixture', - 'core', - 'fixturePath', - 'Load a JSON fixture into the test fixture map', - ), - 'setStateFromFixture': step( - 'fixture', - 'core', - 'fixturePath', - 'Apply all keys from a JSON fixture to state', - ), - 'expectMatchesFixture': step( - 'fixture', - 'core', - 'fixturePath', - 'Assert state or path matches a JSON fixture', - ), }; const executorAliases = { @@ -823,8 +757,6 @@ Map defaultExampleForArg(String arg) { return {'id': 'loading_spinner', 'timeoutMs': 5000}; case 'waitForNavigation': return {'screen': 'Home', 'timeoutMs': 5000}; - case 'waitUntil': - return {'path': 'user.name', 'equals': 'Jane'}; case 'textRequired': return {'text': 'Welcome'}; case 'expectEquals': @@ -863,10 +795,6 @@ Map defaultExampleForArg(String arg) { return {'name': 'slow_api', 'delayMs': 60000}; case 'apiName': return {'name': 'login', 'times': 1}; - case 'setState': - return {'path': 'user.name', 'value': 'Jane'}; - case 'expectState': - return {'path': 'user.name', 'equals': 'Jane'}; case 'storageKey': return {'key': 'onboarding_done', 'value': true}; case 'optionalStorageKey': @@ -908,10 +836,6 @@ Map defaultExampleForArg(String arg) { }; case 'screenshot': return {'name': 'home_screen'}; - case 'expectStatePath': - return {'path': 'user.id'}; - case 'resetStatePath': - return {'path': 'cart'}; case 'setAuth': return { 'user': {'id': '1', 'email': 'user@test.com'}, @@ -930,8 +854,6 @@ Map defaultExampleForArg(String arg) { return {'contains': 'Screen loaded'}; case 'expectErrorContains': return {'contains': 'overflow'}; - case 'fixturePath': - return {'fixture': 'fixtures/user.json'}; case 'expectApiCallOrder': return { 'names': ['auth', 'profile'] From 47fd48711942264d606114e93849d4716c8bf7bb Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Fri, 10 Jul 2026 16:49:01 +0500 Subject: [PATCH 08/29] refactor(test_runner): enhance wait step and schema definitions for clarity Updated the 'wait' step to utilize real-time delays with runAsync, improving its functionality. Modified the associated schema to include a new 'keychain' property, ensuring better organization and clarity in the test definitions. Adjusted related documentation and tests to reflect these changes. --- .../assets/schema/ensemble_tests_schema.json | 10 ++++-- .../lib/actions/test_execution_config.dart | 2 +- .../lib/actions/test_step_executor.dart | 29 +++++++++------ .../lib/mocks/test_api_provider_overlay.dart | 2 ++ .../lib/runner/ensemble_test_harness.dart | 14 +++++--- .../lib/runner/ensemble_test_runner.dart | 5 +++ .../lib/runner/yaml_test_session.dart | 7 ++++ .../schema/ensemble_test_schema_builder.dart | 1 + .../lib/vocabulary/test_step_registry.dart | 3 +- .../test/ensemble_test_schema_test.dart | 29 +++++++++++---- .../test/test_reporter_test.dart | 36 +++++++++++++++++++ .../tool/generate_step_registry.dart | 3 +- 12 files changed, 112 insertions(+), 29 deletions(-) diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 986e3bdad..81e45713f 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -133,6 +133,10 @@ "type": "object", "additionalProperties": true }, + "keychain": { + "type": "object", + "additionalProperties": true + }, "env": { "type": "object", "additionalProperties": true @@ -693,7 +697,7 @@ }, "additionalProperties": false, "title": "wait", - "description": "Alias for pump — advance frame clock by durationMs", + "description": "Real-time delay using runAsync, followed by a frame pump", "examples": [ { "durationMs": 100 @@ -3024,7 +3028,7 @@ { "type": "object", "title": "wait", - "description": "Alias for pump — advance frame clock by durationMs", + "description": "Real-time delay using runAsync, followed by a frame pump", "examples": [ { "wait": { @@ -3038,7 +3042,7 @@ "properties": { "wait": { "$ref": "#/$defs/args_wait", - "description": "Alias for pump — advance frame clock by durationMs", + "description": "Real-time delay using runAsync, followed by a frame pump", "examples": [ { "durationMs": 100 diff --git a/tools/ensemble_test_runner/lib/actions/test_execution_config.dart b/tools/ensemble_test_runner/lib/actions/test_execution_config.dart index 7ecff1cfc..31c8a0f89 100644 --- a/tools/ensemble_test_runner/lib/actions/test_execution_config.dart +++ b/tools/ensemble_test_runner/lib/actions/test_execution_config.dart @@ -7,7 +7,7 @@ class TestExecutionConfig { const TestExecutionConfig({ this.settleStepDuration = const Duration(milliseconds: 100), - this.settleTimeout = const Duration(seconds: 10), + this.settleTimeout = const Duration(seconds: 2), this.waitPollInterval = const Duration(milliseconds: 100), this.defaultWaitTimeout = const Duration(seconds: 5), }); diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 389efcfcd..5b9197efe 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -73,11 +73,11 @@ class TestStepExecutor { switch (step.type) { case 'wait': - await tester.pump( - Duration( - milliseconds: step.args['durationMs'] as int? ?? 500, - ), - ); + final durationMs = step.args['durationMs'] as int? ?? 500; + await tester.runAsync(() async { + await Future.delayed(Duration(milliseconds: durationMs)); + }); + await tester.pump(); return; case 'waitForText': await _waitFor( @@ -372,11 +372,20 @@ class TestStepExecutor { } Future _settle({Duration? timeout}) async { - await tester.pumpAndSettle( - config.settleStepDuration, - EnginePhase.sendSemanticsUpdate, - timeout ?? config.settleTimeout, - ); + try { + await tester.pumpAndSettle( + config.settleStepDuration, + EnginePhase.sendSemanticsUpdate, + timeout ?? config.settleTimeout, + ); + } catch (e) { + if (e.toString().contains('timed out') || e.toString().contains('timeout')) { + // Swallow timeout error because background streams/listeners (e.g. Firestore) + // might keep the event loop active, but the UI itself has settled. + } else { + rethrow; + } + } await _yieldToLiveApiWork(); } diff --git a/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart b/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart index cdfc3dcd7..fefdf90c4 100644 --- a/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart +++ b/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart @@ -45,6 +45,7 @@ class TestApiProviderOverlay extends HTTPAPIProvider { final Map _mocks; HTTPAPIProvider _delegate; + HTTPAPIProvider get delegate => _delegate; final ApiCallRecorder recorder; Future Function(Future Function())? liveAsyncRunner; @@ -210,6 +211,7 @@ class TestApiOverlay implements APIProvider { final TestApiProviderOverlay _host; final APIProvider _delegate; + APIProvider get delegate => _delegate; @override Future init(String appId, Map config) => diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index de0df32e6..9f4dc59f3 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -250,12 +250,19 @@ class EnsembleTestHarness { final installed = {}; for (final entry in realProviders.entries) { + var delegate = entry.value; + if (delegate is TestApiProviderOverlay) { + delegate = delegate.delegate; + } else if (delegate is TestApiOverlay) { + delegate = delegate.delegate; + } + if (entry.key == 'http') { - mock.bindHttpDelegate(entry.value as HTTPAPIProvider); + mock.bindHttpDelegate(delegate as HTTPAPIProvider); installed['http'] = mock; continue; } - installed[entry.key] = TestApiOverlay(mock, entry.value); + installed[entry.key] = TestApiOverlay(mock, delegate); } if (!installed.containsKey('http')) { @@ -406,8 +413,7 @@ class EnsembleTestHarness { for (final entry in ctx.testCase.mocks.apis.entries) { ctx.apiOverlay.setMock(entry.key, entry.value); } - if (config != null && - config.apiProviders?['http'] is! TestApiProviderOverlay) { + if (config != null) { installTestApiOverlay(config, ctx.apiOverlay); } } diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index a73fb8ac7..5f415a85a 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -1,4 +1,5 @@ import 'package:ensemble/ensemble.dart'; +import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; @@ -101,6 +102,10 @@ class EnsembleTestRunner { context: ctx, ); } + await YamlTestSession.navigationFlow.flushPending(); + YamlTestSession.navigationFlow.beginTest( + ScreenTracker().getCurrentScreenIdentifier(), + ); final result = await _executeSteps( test: test, diff --git a/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart b/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart index 54c5d755d..15bcde7dd 100644 --- a/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart +++ b/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart @@ -24,6 +24,13 @@ class NavigationFlowRecorder { _flushScheduled = false; } + void beginTest(String? currentScreen) { + clear(); + if (currentScreen != null && currentScreen.isNotEmpty) { + _flow.add(currentScreen); + } + } + /// For unit tests only. void seed(Iterable names) { _flow diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index 4bd70ebf4..ba192f0c0 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -36,6 +36,7 @@ class EnsembleTestSchemaBuilder { 'additionalProperties': false, 'properties': { 'storage': {'type': 'object', 'additionalProperties': true}, + 'keychain': {'type': 'object', 'additionalProperties': true}, 'env': {'type': 'object', 'additionalProperties': true}, }, }, diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index 3cdae8878..2d61cf2e5 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -234,9 +234,8 @@ abstract final class TestStepRegistry { category: TestStepCategory.wait, tier: TestStepTier.extended, argKind: TestStepArgKind.pump, - description: 'Alias for pump — advance frame clock by durationMs', + description: 'Real-time delay using runAsync, followed by a frame pump', example: const {'durationMs': 100}, - executorCanonical: 'pump', ), 'pump': TestStepRegistryEntry( category: TestStepCategory.wait, diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart index 6b5098f12..c6d779f86 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart @@ -11,12 +11,10 @@ void main() { final stepDef = schema['\$defs']['step'] as Map; final oneOf = stepDef['oneOf'] as List; - final yamlKeys = oneOf - .map((e) { - final props = (e as Map)['properties'] as Map; - return props.keys.first as String; - }) - .toSet(); + final yamlKeys = oneOf.map((e) { + final props = (e as Map)['properties'] as Map; + return props.keys.first as String; + }).toSet(); for (final name in TestStepRegistry.entries.keys) { expect(yamlKeys, contains(name), reason: 'missing step $name in schema'); @@ -38,7 +36,9 @@ void main() { } }); - test('generated JSON is valid and requires id, steps with XOR start/prerequisite', () { + test( + 'generated JSON is valid and requires id, steps with XOR start/prerequisite', + () { final json = EnsembleTestSchemaBuilder.buildJson(); final decoded = jsonDecode(json) as Map; expect(decoded['\$schema'], EnsembleTestSchemaBuilder.schemaVersion); @@ -50,4 +50,19 @@ void main() { expect(decoded['properties'], contains('prerequisite')); expect(decoded['oneOf'], isA()); }); + + test('initialState schema accepts storage, keychain, and env maps', () { + final schema = EnsembleTestSchemaBuilder.build(); + final initialState = + schema['\$defs']['initialState'] as Map; + final properties = initialState['properties'] as Map; + + expect(properties, contains('storage')); + expect(properties, contains('keychain')); + expect(properties, contains('env')); + expect(properties['keychain'], { + 'type': 'object', + 'additionalProperties': true, + }); + }); } diff --git a/tools/ensemble_test_runner/test/test_reporter_test.dart b/tools/ensemble_test_runner/test/test_reporter_test.dart index b19bc20c1..54626ab6e 100644 --- a/tools/ensemble_test_runner/test/test_reporter_test.dart +++ b/tools/ensemble_test_runner/test/test_reporter_test.dart @@ -1,3 +1,4 @@ +import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:ensemble_test_runner/reporters/test_reporter.dart'; @@ -30,6 +31,41 @@ void main() { ['Hello Home', 'Goodbye', 'Hello Home'], ); }); + + test('beginTest starts a fresh report flow for continuation tests', + () async { + YamlTestSession.navigationFlow.seed([ + 'Login', + 'InitApp', + 'AutoSignIn', + 'AutoSignIn_Gateway', + 'Home', + ]); + + YamlTestSession.navigationFlow.beginTest('Home'); + YamlTestSession.navigationFlow.recordScreenChange( + VisibleScreen(screenName: 'Devices', visibleSince: DateTime.now()), + ); + await YamlTestSession.navigationFlow.flushPending(); + YamlTestSession.navigationFlow.recordScreenChange( + VisibleScreen(screenName: 'Home', visibleSince: DateTime.now()), + ); + await YamlTestSession.navigationFlow.flushPending(); + YamlTestSession.navigationFlow.recordScreenChange( + VisibleScreen( + screenName: 'Settings_Wifi', visibleSince: DateTime.now()), + ); + await YamlTestSession.navigationFlow.flushPending(); + YamlTestSession.navigationFlow.recordScreenChange( + VisibleScreen(screenName: 'Home', visibleSince: DateTime.now()), + ); + await YamlTestSession.navigationFlow.flushPending(); + + expect( + collectScreensVisited('Home'), + ['Home', 'Devices', 'Home', 'Settings_Wifi', 'Home'], + ); + }); }); group('TestReporter', () { diff --git a/tools/ensemble_test_runner/tool/generate_step_registry.dart b/tools/ensemble_test_runner/tool/generate_step_registry.dart index 955627338..68d278c86 100644 --- a/tools/ensemble_test_runner/tool/generate_step_registry.dart +++ b/tools/ensemble_test_runner/tool/generate_step_registry.dart @@ -193,7 +193,7 @@ void main() { 'wait', 'extended', 'pump', - 'Alias for pump — advance frame clock by durationMs', + 'Real-time delay using runAsync, followed by a frame pump', ), 'pump': step( 'wait', @@ -644,7 +644,6 @@ void main() { const executorAliases = { 'waitForText': 'waitFor', 'expectScreen': 'expectNavigateTo', - 'wait': 'pump', 'launchApp': 'restartApp', 'expectScript': 'expectScriptResult', }; From b65837ddca2d924f5004d2982d1ea70c1bd26095 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Fri, 10 Jul 2026 17:03:41 +0500 Subject: [PATCH 09/29] refactor(test_runner): enhance logging and screenshot functionality Updated the ExtendedStepHandlers to log storage and screenshot actions by writing output to files, improving traceability of test results. Introduced a new method for capturing the widget tree and logging API call counts, ensuring better organization of test artifacts. Adjusted related tests to verify the existence and content of generated log files. --- .../lib/actions/extended_step_handlers.dart | 97 ++++++++++++++++--- .../lib/actions/test_step_executor.dart | 17 +++- .../lib/mocks/test_logger.dart | 19 ++++ .../lib/reporters/test_reporter.dart | 2 +- .../test/test_reporter_test.dart | 14 ++- .../test/test_step_executor_test.dart | 73 +++++++++++++- 6 files changed, 197 insertions(+), 25 deletions(-) diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 09b29d692..667bb4f6d 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -1,4 +1,6 @@ import 'dart:convert'; +import 'dart:io'; +import 'dart:ui' as ui; import 'package:ensemble/action/navigation_action.dart'; import 'package:ensemble/screen_controller.dart'; @@ -8,6 +10,7 @@ import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -235,15 +238,31 @@ class ExtendedStepHandlers { final entries = storage.getKeys().map( (key) => MapEntry(key, storage.read(key)), ); + final content = jsonEncode( + Map.fromEntries(entries), + toEncodable: (value) => value.toString(), + ); + final path = await executor.tester.runAsync(() { + return executor.context.logger.writeLogFile( + testId: executor.context.testCase.id, + name: 'storage', + content: content, + ); + }); executor.context.logger.log( - 'storage=${jsonEncode( - Map.fromEntries(entries), - toEncodable: (value) => value.toString(), - )}', + 'storage: $path', ); } else { + final content = '${StorageManager().read(key)}'; + final path = await executor.tester.runAsync(() { + return executor.context.logger.writeLogFile( + testId: executor.context.testCase.id, + name: 'storage_$key', + content: content, + ); + }); executor.context.logger.log( - 'storage[$key]=${StorageManager().read(key)}', + 'storage[$key]: $path', ); } return true; @@ -271,8 +290,14 @@ class ExtendedStepHandlers { await _screenshot(executor, step); return true; case 'dumpTree': - debugDumpApp(); - executor.context.logger.log('dumpTree: see debug console'); + final path = await executor.tester.runAsync(() { + return executor.context.logger.writeLogFile( + testId: executor.context.testCase.id, + name: 'dump_tree', + content: _captureDebugDumpApp(), + ); + }); + executor.context.logger.log('dumpTree: $path'); return true; case 'expectNoConsoleErrors': executor.assertions.expectNoConsoleErrors(); @@ -589,16 +614,60 @@ class ExtendedStepHandlers { } static Future _screenshot(TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - if (name != null && name.isNotEmpty) { - await expectLater( - find.byType(MaterialApp), - matchesGoldenFile('goldens/$name.png'), + await e.tester.pump(); + final rawName = step.args['name']?.toString(); + final name = rawName == null || rawName.isEmpty + ? DateTime.now().microsecondsSinceEpoch.toString() + : rawName; + final fileName = _safeFileName('${e.context.testCase.id}_$name.png'); + final directory = Directory('build/ensemble_test_runner/screenshots'); + final file = File('${directory.path}/$fileName'); + final path = await e.tester.runAsync(() async { + final renderView = e.tester.binding.renderViews.first; + final layer = renderView.debugLayer; + if (layer is! OffsetLayer) { + throw EnsembleTestFailure('screenshot requires a painted render view.'); + } + + final image = await layer.toImage( + renderView.paintBounds, + pixelRatio: renderView.flutterView.devicePixelRatio, ); - } else { + final byteData = await image.toByteData(format: ui.ImageByteFormat.png); + image.dispose(); + if (byteData == null) { + throw EnsembleTestFailure('Failed to encode screenshot as PNG.'); + } + + await directory.create(recursive: true); + await file.writeAsBytes(byteData.buffer.asUint8List()); + return file.path; + }); + + e.context.logger.log('screenshot: $path'); + } + + static String _captureDebugDumpApp() { + final previousDebugPrint = debugPrint; + final lines = []; + debugPrint = (String? message, {int? wrapWidth}) { + if (message != null) { + lines.add(message); + } + }; + try { debugDumpApp(); - e.context.logger.log('screenshot: debug tree dumped (no golden name)'); + } finally { + debugPrint = previousDebugPrint; } + if (lines.isEmpty) { + return 'dumpTree: '; + } + return 'dumpTree:\n${lines.join('\n')}'; + } + + static String _safeFileName(String value) { + return value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); } static bool _deepEquals(dynamic a, dynamic b) { diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 5b9197efe..581dd43f7 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -304,9 +304,17 @@ class TestStepExecutor { for (final call in context.apiOverlay.calls) { counts.update(call.name, (count) => count + 1, ifAbsent: () => 1); } - for (final entry in counts.entries) { - context.logger.log('API ${entry.key} x${entry.value}'); - } + final content = counts.entries + .map((entry) => 'API ${entry.key} x${entry.value}') + .join('\n'); + final path = await tester.runAsync(() { + return context.logger.writeLogFile( + testId: context.testCase.id, + name: 'api_calls', + content: content, + ); + }); + context.logger.log('apiCalls: $path'); break; default: if (await ExtendedStepHandlers.tryExecute(this, step)) { @@ -379,7 +387,8 @@ class TestStepExecutor { timeout ?? config.settleTimeout, ); } catch (e) { - if (e.toString().contains('timed out') || e.toString().contains('timeout')) { + if (e.toString().contains('timed out') || + e.toString().contains('timeout')) { // Swallow timeout error because background streams/listeners (e.g. Firestore) // might keep the event loop active, but the UI itself has settled. } else { diff --git a/tools/ensemble_test_runner/lib/mocks/test_logger.dart b/tools/ensemble_test_runner/lib/mocks/test_logger.dart index ee3ee0b1c..ae71a0300 100644 --- a/tools/ensemble_test_runner/lib/mocks/test_logger.dart +++ b/tools/ensemble_test_runner/lib/mocks/test_logger.dart @@ -1,3 +1,5 @@ +import 'dart:io'; + /// Simple in-memory logger for test runs. class TestLogger { final List logs = []; @@ -6,5 +8,22 @@ class TestLogger { logs.add(message); } + Future writeLogFile({ + required String testId, + required String name, + required String content, + }) async { + final directory = Directory('build/ensemble_test_runner/logs'); + await directory.create(recursive: true); + final fileName = '${_safeFileName(testId)}_${_safeFileName(name)}.log'; + final file = File('${directory.path}/$fileName'); + await file.writeAsString(content); + return file.path; + } + void clear() => logs.clear(); + + static String _safeFileName(String value) { + return value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); + } } diff --git a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart index 89c5e4038..8039b15a8 100644 --- a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart +++ b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart @@ -145,7 +145,7 @@ class TestReporter { } if (r.logs.isNotEmpty) { - buffer.writeln('│ logs:'); + buffer.writeln('│ artifacts:'); for (final log in r.logs) { buffer.writeln('│ $log'); } diff --git a/tools/ensemble_test_runner/test/test_reporter_test.dart b/tools/ensemble_test_runner/test/test_reporter_test.dart index 54626ab6e..07fd44dad 100644 --- a/tools/ensemble_test_runner/test/test_reporter_test.dart +++ b/tools/ensemble_test_runner/test/test_reporter_test.dart @@ -131,14 +131,22 @@ void main() { EnsembleSingleTestResult.passed( testId: 'api_log_test', durationMs: 42, - logs: const ['API getCompleteSchedules x1'], + logs: const [ + 'apiCalls: build/ensemble_test_runner/logs/api_log_test_api_calls.log', + 'dumpTree: build/ensemble_test_runner/logs/api_log_test_dump_tree.log', + ], ), ], ), ); - expect(output, contains('logs:')); - expect(output, contains('API getCompleteSchedules x1')); + expect(output, contains('artifacts:')); + expect( + output, + contains( + 'apiCalls: build/ensemble_test_runner/logs/api_log_test_api_calls.log')); + expect(output, contains('dumpTree:')); + expect(output, isNot(contains('MaterialApp'))); }); }); } diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index 9675d85cc..51fe9401e 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -1,9 +1,12 @@ +import 'dart:io'; + import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble/framework/storage_manager.dart'; +import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { @@ -65,8 +68,72 @@ void main() { await executor.execute(const TestStep(type: 'logStorage', args: {})); expect(context.logger.logs, hasLength(1)); - expect(context.logger.logs.single, startsWith('storage=')); - expect(context.logger.logs.single, contains('"first":"one"')); - expect(context.logger.logs.single, contains('"second":{"nested":true}')); + expect(context.logger.logs.single, startsWith('storage: ')); + final path = context.logger.logs.single.substring('storage: '.length); + final file = File(path); + expect(file.existsSync(), isTrue); + final content = file.readAsStringSync(); + expect(content, contains('"first":"one"')); + expect(content, contains('"second":{"nested":true}')); + }); + + testWidgets('dumpTree writes the widget tree to test logs', (tester) async { + await tester.pumpWidget( + const MaterialApp(home: Text('debug target')), + ); + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 'debug_test', + startScreen: 'Home', + steps: [], + ), + ); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: harness, + ); + + await executor.execute(const TestStep(type: 'dumpTree', args: {})); + + expect(context.logger.logs.single, startsWith('dumpTree: ')); + final path = context.logger.logs.single.substring('dumpTree: '.length); + final file = File(path); + expect(file.existsSync(), isTrue); + expect(file.readAsStringSync(), contains('debug target')); + }); + + testWidgets('screenshot writes a png and logs the path', (tester) async { + await tester.pumpWidget( + const MaterialApp(home: Text('screenshot target')), + ); + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 'debug_test', + startScreen: 'Home', + steps: [], + ), + ); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: harness, + ); + + await executor.execute( + const TestStep(type: 'screenshot', args: {'name': 'home'}), + ); + + expect(context.logger.logs.single, startsWith('screenshot: ')); + final path = context.logger.logs.single.substring('screenshot: '.length); + final file = File(path); + expect(file.existsSync(), isTrue); + expect(file.readAsBytesSync().take(8), [137, 80, 78, 71, 13, 10, 26, 10]); }); } From 9e899ee2e37ffb32ca1feb99e0e133174fe39eb1 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Fri, 10 Jul 2026 17:17:48 +0500 Subject: [PATCH 10/29] feat(test_runner): add font loading functionality to ensure app fonts are available Implemented a new method to load application fonts from the FontManifest.json, enhancing the test harness's ability to handle font dependencies. Added a test to verify that font loading is safe even when the manifest is unavailable, ensuring robustness in the testing environment. --- .../lib/runner/ensemble_test_harness.dart | 54 +++++++++++++++++++ .../test/ensemble_test_harness_test.dart | 7 +++ 2 files changed, 61 insertions(+) diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index 9f4dc59f3..828aa9703 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -1,3 +1,4 @@ +import 'dart:convert'; import 'dart:io'; import 'package:ensemble/ensemble.dart'; @@ -55,6 +56,7 @@ Future applyYamlTestStorageBootstrap(EnsembleTestSetup setup) async { class EnsembleTestHarness { static final String _testStoragePath = Directory.systemTemp.createTempSync('ensemble_test_runner_storage_').path; + static bool _appFontsLoaded = false; static void ensureTestPlugins() { TestWidgetsFlutterBinding.ensureInitialized(); @@ -285,6 +287,7 @@ class EnsembleTestHarness { TestApiProviderOverlay? apiOverlay, }) async { ensureTestPlugins(); + await ensureAppFontsLoaded(); final env = Map.from(config.envOverrides ?? {}); env['firebase_app_check'] = 'false'; @@ -309,6 +312,57 @@ class EnsembleTestHarness { return config; } + static Future ensureAppFontsLoaded() async { + if (_appFontsLoaded) return; + _appFontsLoaded = true; + + List manifest; + try { + final rawManifest = await rootBundle.loadString('FontManifest.json'); + manifest = jsonDecode(rawManifest) as List; + } catch (_) { + return; + } + + for (final familyEntry in manifest.whereType()) { + final family = familyEntry['family']?.toString(); + final fonts = familyEntry['fonts']; + if (family == null || fonts is! List) continue; + + await _loadFontFamily(family, fonts); + + final lowerCaseAlias = family.toLowerCase(); + if (lowerCaseAlias != family) { + await _loadFontFamily(lowerCaseAlias, fonts); + } + } + } + + static Future _loadFontFamily( + String family, + List fontEntries, + ) async { + final loader = FontLoader(family); + var hasFonts = false; + + for (final fontEntry in fontEntries.whereType()) { + final asset = fontEntry['asset']?.toString(); + if (asset == null || asset.isEmpty) continue; + try { + final fontData = await rootBundle.load(asset); + loader.addFont(Future.value(fontData)); + hasFonts = true; + } catch (_) { + // Ignore missing assets so a bad font entry does not fail unrelated + // behavioral tests. + } + } + + if (hasFonts) { + await loader.load(); + } + } + Future loadScreen({ required WidgetTester tester, required EnsembleTestCase testCase, diff --git a/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart index 701fa8b29..bf5d493d9 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart @@ -3,6 +3,13 @@ import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { + testWidgets('app font bootstrap is safe when font manifest is unavailable', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + + await EnsembleTestHarness.ensureAppFontsLoaded(); + }); + testWidgets('initial keychain state is written to secure storage', (tester) async { EnsembleTestHarness.ensureTestPlugins(); From 9398c6bdcf29d0d9fb51d8c26607aa0d8aa4427b Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Sat, 11 Jul 2026 00:37:24 +0500 Subject: [PATCH 11/29] feat(test_runner): enhance screenshot functionality with device frame support Added support for capturing screenshots with device frames in the test runner. Updated the schema to include new properties for device frame, platform, and model in screenshot steps. Introduced a new utility for resolving screenshot devices based on platform and model, and updated related tests to verify the new functionality. This enhancement improves the visual fidelity of screenshots captured during tests. --- .../assets/schema/ensemble_tests_schema.json | 30 +++++-- .../lib/actions/extended_step_handlers.dart | 68 +++++++++++++- .../lib/actions/screenshot_device.dart | 58 ++++++++++++ .../lib/runner/ensemble_test_harness.dart | 31 +++++++ .../lib/vocabulary/test_step_arg_kind.dart | 9 +- .../lib/vocabulary/test_step_registry.dart | 9 +- tools/ensemble_test_runner/pubspec.yaml | 2 + .../test/screenshot_device_test.dart | 49 ++++++++++ .../test/test_step_executor_test.dart | 90 +++++++++++++++++++ .../tool/generate_step_registry.dart | 9 +- 10 files changed, 340 insertions(+), 15 deletions(-) create mode 100644 tools/ensemble_test_runner/lib/actions/screenshot_device.dart create mode 100644 tools/ensemble_test_runner/test/screenshot_device_test.dart diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 81e45713f..0757f93bd 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -2068,14 +2068,26 @@ "properties": { "name": { "type": "string" + }, + "deviceFrame": { + "type": "boolean" + }, + "platform": { + "type": "string" + }, + "model": { + "type": "string" } }, "additionalProperties": false, "title": "screenshot", - "description": "Capture golden or dump widget tree for debugging", + "description": "Capture a screenshot artifact for debugging", "examples": [ { - "name": "home_screen" + "name": "home_screen", + "deviceFrame": true, + "platform": "ios", + "model": "iPhone 15 Pro" } ] }, @@ -5039,11 +5051,14 @@ { "type": "object", "title": "screenshot", - "description": "Capture golden or dump widget tree for debugging", + "description": "Capture a screenshot artifact for debugging", "examples": [ { "screenshot": { - "name": "home_screen" + "name": "home_screen", + "deviceFrame": true, + "platform": "ios", + "model": "iPhone 15 Pro" } } ], @@ -5053,10 +5068,13 @@ "properties": { "screenshot": { "$ref": "#/$defs/args_screenshot", - "description": "Capture golden or dump widget tree for debugging", + "description": "Capture a screenshot artifact for debugging", "examples": [ { - "name": "home_screen" + "name": "home_screen", + "deviceFrame": true, + "platform": "ios", + "model": "iPhone 15 Pro" } ] } diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 667bb4f6d..4c4a07a79 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -6,6 +6,8 @@ import 'package:ensemble/action/navigation_action.dart'; import 'package:ensemble/screen_controller.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_device_preview/ensemble_device_preview.dart'; +import 'package:ensemble_test_runner/actions/screenshot_device.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; @@ -536,9 +538,13 @@ class ExtendedStepHandlers { static Future _setDevice(TestStepExecutor e, TestStep step) async { final width = (step.args['width'] as num?)?.toDouble() ?? 390; final height = (step.args['height'] as num?)?.toDouble() ?? 844; - e.tester.view.physicalSize = Size(width, height); + final size = Size(width, height); + await e.tester.binding.setSurfaceSize(size); + e.tester.view.physicalSize = size; e.tester.view.devicePixelRatio = 1.0; - e.context.runtime.deviceSize = Size(width, height); + e.tester.view.padding = FakeViewPadding.zero; + e.tester.view.viewPadding = FakeViewPadding.zero; + e.context.runtime.deviceSize = size; await e.settle(); } @@ -622,18 +628,27 @@ class ExtendedStepHandlers { final fileName = _safeFileName('${e.context.testCase.id}_$name.png'); final directory = Directory('build/ensemble_test_runner/screenshots'); final file = File('${directory.path}/$fileName'); + final useDeviceFrame = step.args['deviceFrame'] == true; + final device = useDeviceFrame ? resolveScreenshotDevice(step.args) : null; + + await e.tester.pump(); + final path = await e.tester.runAsync(() async { final renderView = e.tester.binding.renderViews.first; final layer = renderView.debugLayer; if (layer is! OffsetLayer) { - throw EnsembleTestFailure('screenshot requires a painted render view.'); + throw EnsembleTestFailure( + 'screenshot requires a painted render view.', + ); } final image = await layer.toImage( renderView.paintBounds, pixelRatio: renderView.flutterView.devicePixelRatio, ); - final byteData = await image.toByteData(format: ui.ImageByteFormat.png); + final byteData = device != null + ? await _addDeviceFrame(image, device) + : await image.toByteData(format: ui.ImageByteFormat.png); image.dispose(); if (byteData == null) { throw EnsembleTestFailure('Failed to encode screenshot as PNG.'); @@ -647,6 +662,51 @@ class ExtendedStepHandlers { e.context.logger.log('screenshot: $path'); } + static Future _addDeviceFrame( + ui.Image screenImage, + DeviceInfo device, + ) async { + final padding = device.frameSize.shortestSide * 0.025; + final outputWidth = (device.frameSize.width + padding * 2).ceil(); + final outputHeight = (device.frameSize.height + padding * 2).ceil(); + + final recorder = ui.PictureRecorder(); + final canvas = Canvas( + recorder, + Rect.fromLTWH(0, 0, outputWidth.toDouble(), outputHeight.toDouble()), + ); + + canvas.drawColor(const Color(0xFFE9EDF3), BlendMode.src); + canvas.save(); + canvas.translate(padding, padding); + device.framePainter.paint(canvas, device.frameSize); + + final screenPath = device.screenPath; + final screenRect = screenPath.getBounds(); + canvas.save(); + canvas.clipPath(screenPath); + canvas.drawImageRect( + screenImage, + Rect.fromLTWH( + 0, + 0, + screenImage.width.toDouble(), + screenImage.height.toDouble(), + ), + screenRect, + Paint()..filterQuality = FilterQuality.high, + ); + canvas.restore(); + canvas.restore(); + + final picture = recorder.endRecording(); + final image = await picture.toImage(outputWidth, outputHeight); + picture.dispose(); + final byteData = await image.toByteData(format: ui.ImageByteFormat.png); + image.dispose(); + return byteData; + } + static String _captureDebugDumpApp() { final previousDebugPrint = debugPrint; final lines = []; diff --git a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart new file mode 100644 index 000000000..e76040a5e --- /dev/null +++ b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart @@ -0,0 +1,58 @@ +import 'package:ensemble_device_preview/ensemble_device_preview.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; + +DeviceInfo? firstScreenshotDevice(List steps) { + for (final step in steps) { + final device = screenshotDeviceForStep(step); + if (device != null) return device; + + final nestedDevice = firstScreenshotDevice(step.nestedSteps); + if (nestedDevice != null) return nestedDevice; + } + return null; +} + +DeviceInfo? screenshotDeviceForStep(TestStep step) { + if (step.type != 'screenshot' || step.args['deviceFrame'] != true) { + return null; + } + return resolveScreenshotDevice(step.args); +} + +DeviceInfo resolveScreenshotDevice(Map args) { + final platform = args['platform']?.toString(); + final model = args['model']?.toString(); + final platformKey = normalizeScreenshotDeviceName(platform); + final candidates = switch (platformKey) { + 'android' => Devices.android.all, + 'ios' || 'iphone' || 'ipad' => Devices.ios.all, + _ => Devices.all, + }; + + if (model != null && model.trim().isNotEmpty) { + final modelKey = normalizeScreenshotDeviceName(model); + for (final device in candidates) { + if (normalizeScreenshotDeviceName(device.name) == modelKey || + normalizeScreenshotDeviceName(device.identifier.name) == modelKey) { + return device; + } + } + for (final device in candidates) { + if (normalizeScreenshotDeviceName(device.name).contains(modelKey) || + normalizeScreenshotDeviceName(device.identifier.name) + .contains(modelKey)) { + return device; + } + } + throw EnsembleTestFailure( + 'Unknown screenshot device model "$model". Available models: ' + '${candidates.map((device) => device.name).join(', ')}', + ); + } + + if (platformKey == 'android') return Devices.android.all.first; + return Devices.ios.iPhone13; +} + +String normalizeScreenshotDeviceName(String? value) => + (value ?? '').toLowerCase().replaceAll(RegExp(r'[^a-z0-9]'), ''); diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index 828aa9703..9ad216dbe 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -9,6 +9,8 @@ import 'package:ensemble/framework/definition_providers/local_provider.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/page_model.dart'; +import 'package:ensemble_device_preview/ensemble_device_preview.dart'; +import 'package:ensemble_test_runner/actions/screenshot_device.dart'; import 'package:ensemble_test_runner/mocks/adobe_test_setup.dart'; import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; @@ -374,6 +376,10 @@ class EnsembleTestHarness { YamlTestSession.navigationFlow.clear(); final ctx = context ?? EnsembleTestContext.fromTestCase(testCase); + final screenshotDevice = firstScreenshotDevice(testCase.steps); + if (screenshotDevice != null) { + await _setViewportForDevice(tester, ctx, screenshotDevice); + } await _ensureDefaultViewport(tester, ctx); var config = existingConfig ?? await buildConfig(); final bootstrapped = await tester.runAsync(() async { @@ -412,11 +418,36 @@ class EnsembleTestHarness { ) async { if (context.runtime.deviceSize != null) return; const size = Size(800, 844); + await tester.binding.setSurfaceSize(size); tester.view.physicalSize = size; tester.view.devicePixelRatio = 1.0; + _setViewPadding(tester, EdgeInsets.zero); context.runtime.deviceSize = size; } + static Future _setViewportForDevice( + WidgetTester tester, + EnsembleTestContext context, + DeviceInfo device, + ) async { + await tester.binding.setSurfaceSize(device.screenSize); + tester.view.devicePixelRatio = 1.0; + tester.view.physicalSize = device.screenSize; + _setViewPadding(tester, device.safeAreas); + context.runtime.deviceSize = device.screenSize; + } + + static void _setViewPadding(WidgetTester tester, EdgeInsets padding) { + final viewPadding = FakeViewPadding( + left: padding.left, + top: padding.top, + right: padding.right, + bottom: padding.bottom, + ); + tester.view.padding = viewPadding; + tester.view.viewPadding = viewPadding; + } + static Future waitForInitialWidgets( WidgetTester tester, { EnsembleTestCase? testCase, diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart index 70e4b84fa..4cce1a3bd 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -286,7 +286,14 @@ extension TestStepArgKindSchema on TestStepArgKind { required: ['id'], ); case TestStepArgKind.screenshot: - return _object(properties: {'name': _string}); + return _object( + properties: { + 'name': _string, + 'deviceFrame': _boolean, + 'platform': _string, + 'model': _string, + }, + ); case TestStepArgKind.setAuth: return _object( properties: { diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index 2d61cf2e5..1f254c7d2 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -735,8 +735,13 @@ abstract final class TestStepRegistry { category: TestStepCategory.debug, tier: TestStepTier.core, argKind: TestStepArgKind.screenshot, - description: 'Capture golden or dump widget tree for debugging', - example: const {'name': 'home_screen'}, + description: 'Capture a screenshot artifact for debugging', + example: const { + 'name': 'home_screen', + 'deviceFrame': true, + 'platform': 'ios', + 'model': 'iPhone 15 Pro' + }, ), 'dumpTree': TestStepRegistryEntry( category: TestStepCategory.debug, diff --git a/tools/ensemble_test_runner/pubspec.yaml b/tools/ensemble_test_runner/pubspec.yaml index 6ba2fe8b6..a93e36fc7 100644 --- a/tools/ensemble_test_runner/pubspec.yaml +++ b/tools/ensemble_test_runner/pubspec.yaml @@ -24,6 +24,8 @@ dependencies: url: https://github.com/EnsembleUI/ensemble.git ref: ensemble-v1.2.49 path: modules/ensemble + device_frame: ^1.3.0 + ensemble_device_preview: ^1.1.3 flutter: sdk: flutter flutter_test: diff --git a/tools/ensemble_test_runner/test/screenshot_device_test.dart b/tools/ensemble_test_runner/test/screenshot_device_test.dart new file mode 100644 index 000000000..a1e38a5e3 --- /dev/null +++ b/tools/ensemble_test_runner/test/screenshot_device_test.dart @@ -0,0 +1,49 @@ +import 'package:ensemble_test_runner/actions/screenshot_device.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('resolves screenshot model by platform and model name', () { + final device = resolveScreenshotDevice({ + 'deviceFrame': true, + 'platform': 'android', + 'model': 'Samsung Galaxy S20', + }); + + expect(device.name, 'Samsung Galaxy S20'); + }); + + test('finds the first framed screenshot in test steps', () { + final device = firstScreenshotDevice([ + const TestStep(type: 'tap', args: {'id': 'open'}), + const TestStep( + type: 'group', + args: {}, + nestedSteps: [ + TestStep( + type: 'screenshot', + args: { + 'name': 'home', + 'deviceFrame': true, + 'platform': 'ios', + 'model': 'iPhone 15 Pro', + }, + ), + ], + ), + ]); + + expect(device?.name, 'iPhone 15 Pro'); + }); + + test('ignores screenshots without a device frame', () { + final device = firstScreenshotDevice([ + const TestStep( + type: 'screenshot', + args: {'name': 'plain'}, + ), + ]); + + expect(device, isNull); + }); +} diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index 51fe9401e..e3eb170ad 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -136,4 +136,94 @@ void main() { expect(file.existsSync(), isTrue); expect(file.readAsBytesSync().take(8), [137, 80, 78, 71, 13, 10, 26, 10]); }); + + testWidgets('screenshot can be wrapped in a device frame', (tester) async { + await tester.pumpWidget( + const MaterialApp( + home: ColoredBox( + color: Colors.green, + child: Center(child: Text('framed screenshot target')), + ), + ), + ); + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 'framed_debug_test', + startScreen: 'Home', + steps: [], + ), + ); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: harness, + ); + + await executor.execute( + const TestStep( + type: 'screenshot', + args: { + 'name': 'home', + 'deviceFrame': true, + 'platform': 'android', + 'model': 'Samsung Galaxy S20', + }, + ), + ); + + final path = context.logger.logs.single.substring('screenshot: '.length); + final dimensions = _pngDimensions(File(path).readAsBytesSync()); + + expect( + dimensions.$1, + greaterThan(tester.binding.renderViews.first.size.width), + ); + expect( + dimensions.$2, + greaterThan(tester.binding.renderViews.first.size.height), + ); + }); + + testWidgets('setDevice updates the render surface size', (tester) async { + await tester.pumpWidget( + const MaterialApp(home: SizedBox.expand()), + ); + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 'set_device_test', + startScreen: 'Home', + steps: [], + ), + ); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: harness, + ); + + await executor.execute( + const TestStep( + type: 'setDevice', + args: {'width': 393, 'height': 852}, + ), + ); + + expect(tester.binding.renderViews.first.size, const Size(393, 852)); + }); +} + +(int, int) _pngDimensions(List bytes) { + int readInt32(int offset) => + bytes[offset] << 24 | + bytes[offset + 1] << 16 | + bytes[offset + 2] << 8 | + bytes[offset + 3]; + + return (readInt32(16), readInt32(20)); } diff --git a/tools/ensemble_test_runner/tool/generate_step_registry.dart b/tools/ensemble_test_runner/tool/generate_step_registry.dart index 68d278c86..63665d353 100644 --- a/tools/ensemble_test_runner/tool/generate_step_registry.dart +++ b/tools/ensemble_test_runner/tool/generate_step_registry.dart @@ -583,7 +583,7 @@ void main() { 'debug', 'core', 'screenshot', - 'Capture golden or dump widget tree for debugging', + 'Capture a screenshot artifact for debugging', ), 'dumpTree': step( 'debug', @@ -834,7 +834,12 @@ Map defaultExampleForArg(String arg) { ], }; case 'screenshot': - return {'name': 'home_screen'}; + return { + 'name': 'home_screen', + 'deviceFrame': true, + 'platform': 'ios', + 'model': 'iPhone 15 Pro', + }; case 'setAuth': return { 'user': {'id': '1', 'email': 'user@test.com'}, From 7939bdea7a414402fa52f73b18fd4d36ecd8c555 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Sat, 11 Jul 2026 00:55:51 +0500 Subject: [PATCH 12/29] refactor(test_runner): remove deviceFrame property from schema and related logic Eliminated the deviceFrame property from the ensemble tests schema and refactored related logic in the screenshot handling to streamline the screenshot functionality. Updated tests to reflect these changes, ensuring that the default device model is now iPhone 15 Pro. This refactor enhances clarity and simplifies the screenshot capturing process. --- .../assets/schema/ensemble_tests_schema.json | 6 ------ .../lib/actions/extended_step_handlers.dart | 7 ++----- .../lib/actions/screenshot_device.dart | 7 ++----- .../lib/vocabulary/test_step_arg_kind.dart | 1 - .../lib/vocabulary/test_step_registry.dart | 1 - .../test/screenshot_device_test.dart | 18 ++++++++++-------- .../test/test_step_executor_test.dart | 15 ++++++++++++--- 7 files changed, 26 insertions(+), 29 deletions(-) diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 0757f93bd..74ab8f3a9 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -2069,9 +2069,6 @@ "name": { "type": "string" }, - "deviceFrame": { - "type": "boolean" - }, "platform": { "type": "string" }, @@ -2085,7 +2082,6 @@ "examples": [ { "name": "home_screen", - "deviceFrame": true, "platform": "ios", "model": "iPhone 15 Pro" } @@ -5056,7 +5052,6 @@ { "screenshot": { "name": "home_screen", - "deviceFrame": true, "platform": "ios", "model": "iPhone 15 Pro" } @@ -5072,7 +5067,6 @@ "examples": [ { "name": "home_screen", - "deviceFrame": true, "platform": "ios", "model": "iPhone 15 Pro" } diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 4c4a07a79..6938eee76 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -628,8 +628,7 @@ class ExtendedStepHandlers { final fileName = _safeFileName('${e.context.testCase.id}_$name.png'); final directory = Directory('build/ensemble_test_runner/screenshots'); final file = File('${directory.path}/$fileName'); - final useDeviceFrame = step.args['deviceFrame'] == true; - final device = useDeviceFrame ? resolveScreenshotDevice(step.args) : null; + final device = resolveScreenshotDevice(step.args); await e.tester.pump(); @@ -646,9 +645,7 @@ class ExtendedStepHandlers { renderView.paintBounds, pixelRatio: renderView.flutterView.devicePixelRatio, ); - final byteData = device != null - ? await _addDeviceFrame(image, device) - : await image.toByteData(format: ui.ImageByteFormat.png); + final byteData = await _addDeviceFrame(image, device); image.dispose(); if (byteData == null) { throw EnsembleTestFailure('Failed to encode screenshot as PNG.'); diff --git a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart index e76040a5e..06877cd7a 100644 --- a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart +++ b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart @@ -13,10 +13,7 @@ DeviceInfo? firstScreenshotDevice(List steps) { } DeviceInfo? screenshotDeviceForStep(TestStep step) { - if (step.type != 'screenshot' || step.args['deviceFrame'] != true) { - return null; - } - return resolveScreenshotDevice(step.args); + return step.type == 'screenshot' ? resolveScreenshotDevice(step.args) : null; } DeviceInfo resolveScreenshotDevice(Map args) { @@ -51,7 +48,7 @@ DeviceInfo resolveScreenshotDevice(Map args) { } if (platformKey == 'android') return Devices.android.all.first; - return Devices.ios.iPhone13; + return Devices.ios.iPhone15Pro; } String normalizeScreenshotDeviceName(String? value) => diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart index 4cce1a3bd..e9cf7c0e3 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -289,7 +289,6 @@ extension TestStepArgKindSchema on TestStepArgKind { return _object( properties: { 'name': _string, - 'deviceFrame': _boolean, 'platform': _string, 'model': _string, }, diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index 1f254c7d2..f5539057e 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -738,7 +738,6 @@ abstract final class TestStepRegistry { description: 'Capture a screenshot artifact for debugging', example: const { 'name': 'home_screen', - 'deviceFrame': true, 'platform': 'ios', 'model': 'iPhone 15 Pro' }, diff --git a/tools/ensemble_test_runner/test/screenshot_device_test.dart b/tools/ensemble_test_runner/test/screenshot_device_test.dart index a1e38a5e3..bb9bac2b0 100644 --- a/tools/ensemble_test_runner/test/screenshot_device_test.dart +++ b/tools/ensemble_test_runner/test/screenshot_device_test.dart @@ -5,7 +5,6 @@ import 'package:flutter_test/flutter_test.dart'; void main() { test('resolves screenshot model by platform and model name', () { final device = resolveScreenshotDevice({ - 'deviceFrame': true, 'platform': 'android', 'model': 'Samsung Galaxy S20', }); @@ -13,7 +12,13 @@ void main() { expect(device.name, 'Samsung Galaxy S20'); }); - test('finds the first framed screenshot in test steps', () { + test('defaults screenshot device to iPhone 15 Pro', () { + final device = resolveScreenshotDevice({}); + + expect(device.name, 'iPhone 15 Pro'); + }); + + test('finds the first screenshot in test steps', () { final device = firstScreenshotDevice([ const TestStep(type: 'tap', args: {'id': 'open'}), const TestStep( @@ -24,9 +29,6 @@ void main() { type: 'screenshot', args: { 'name': 'home', - 'deviceFrame': true, - 'platform': 'ios', - 'model': 'iPhone 15 Pro', }, ), ], @@ -36,11 +38,11 @@ void main() { expect(device?.name, 'iPhone 15 Pro'); }); - test('ignores screenshots without a device frame', () { + test('ignores tests without screenshots', () { final device = firstScreenshotDevice([ const TestStep( - type: 'screenshot', - args: {'name': 'plain'}, + type: 'tap', + args: {'id': 'button'}, ), ]); diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index e3eb170ad..be4f5e77d 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -106,7 +106,7 @@ void main() { expect(file.readAsStringSync(), contains('debug target')); }); - testWidgets('screenshot writes a png and logs the path', (tester) async { + testWidgets('screenshot writes a framed png by default', (tester) async { await tester.pumpWidget( const MaterialApp(home: Text('screenshot target')), ); @@ -135,9 +135,19 @@ void main() { final file = File(path); expect(file.existsSync(), isTrue); expect(file.readAsBytesSync().take(8), [137, 80, 78, 71, 13, 10, 26, 10]); + final dimensions = _pngDimensions(file.readAsBytesSync()); + + expect( + dimensions.$1, + greaterThan(tester.binding.renderViews.first.size.width), + ); + expect( + dimensions.$2, + greaterThan(tester.binding.renderViews.first.size.height), + ); }); - testWidgets('screenshot can be wrapped in a device frame', (tester) async { + testWidgets('screenshot can use a custom device model', (tester) async { await tester.pumpWidget( const MaterialApp( home: ColoredBox( @@ -167,7 +177,6 @@ void main() { type: 'screenshot', args: { 'name': 'home', - 'deviceFrame': true, 'platform': 'android', 'model': 'Samsung Galaxy S20', }, From 6860cb8da0093eae36a3659ee5b66f538b2afb1b Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Sat, 11 Jul 2026 01:27:20 +0500 Subject: [PATCH 13/29] refactor(test_runner): improve logging structure and output formats Refactored the logging functionality in ExtendedStepHandlers and TestStepExecutor to write structured JSON outputs for API calls and storage logs, enhancing readability and traceability. Updated the file extension handling in TestLogger to support dynamic extensions, ensuring logs are saved in appropriate formats. Adjusted related tests to verify the new JSON structure and file extensions, improving the overall logging mechanism in the test runner. --- .../lib/actions/extended_step_handlers.dart | 20 ++++++--- .../lib/actions/test_step_executor.dart | 29 ++++++++++--- .../lib/mocks/test_logger.dart | 9 +++- .../test/test_reporter_test.dart | 6 +-- .../test/test_step_executor_test.dart | 43 +++++++++++++++++-- 5 files changed, 87 insertions(+), 20 deletions(-) diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 6938eee76..516f181d7 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -240,27 +240,27 @@ class ExtendedStepHandlers { final entries = storage.getKeys().map( (key) => MapEntry(key, storage.read(key)), ); - final content = jsonEncode( - Map.fromEntries(entries), - toEncodable: (value) => value.toString(), - ); + final content = + _prettyJson(Map.fromEntries(entries)); final path = await executor.tester.runAsync(() { return executor.context.logger.writeLogFile( testId: executor.context.testCase.id, name: 'storage', content: content, + extension: 'json', ); }); executor.context.logger.log( 'storage: $path', ); } else { - final content = '${StorageManager().read(key)}'; + final content = _prettyJson(StorageManager().read(key)); final path = await executor.tester.runAsync(() { return executor.context.logger.writeLogFile( testId: executor.context.testCase.id, name: 'storage_$key', content: content, + extension: 'json', ); }); executor.context.logger.log( @@ -297,6 +297,7 @@ class ExtendedStepHandlers { testId: executor.context.testCase.id, name: 'dump_tree', content: _captureDebugDumpApp(), + extension: 'txt', ); }); executor.context.logger.log('dumpTree: $path'); @@ -718,9 +719,14 @@ class ExtendedStepHandlers { debugPrint = previousDebugPrint; } if (lines.isEmpty) { - return 'dumpTree: '; + return ''; } - return 'dumpTree:\n${lines.join('\n')}'; + return lines.join('\n'); + } + + static String _prettyJson(Object? value) { + return JsonEncoder.withIndent(' ', (value) => value.toString()) + .convert(value); } static String _safeFileName(String value) { diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 581dd43f7..803593e50 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; @@ -300,18 +301,34 @@ class TestStepExecutor { context.apiOverlay.resetCalls(); break; case 'logApiCalls': - final counts = {}; - for (final call in context.apiOverlay.calls) { - counts.update(call.name, (count) => count + 1, ifAbsent: () => 1); + final callsByName = >{}; + final calls = context.apiOverlay.calls; + for (final call in calls) { + callsByName + .putIfAbsent(call.name, () => []) + .add(call.timestamp); } - final content = counts.entries - .map((entry) => 'API ${entry.key} x${entry.value}') - .join('\n'); + final content = const JsonEncoder.withIndent(' ').convert({ + 'total': calls.length, + 'calls': callsByName.entries + .map( + (entry) => { + 'name': entry.key, + 'count': entry.value.length, + 'timestamps': [ + for (final timestamp in entry.value) + timestamp.toIso8601String(), + ], + }, + ) + .toList(), + }); final path = await tester.runAsync(() { return context.logger.writeLogFile( testId: context.testCase.id, name: 'api_calls', content: content, + extension: 'json', ); }); context.logger.log('apiCalls: $path'); diff --git a/tools/ensemble_test_runner/lib/mocks/test_logger.dart b/tools/ensemble_test_runner/lib/mocks/test_logger.dart index ae71a0300..cae9dc2ec 100644 --- a/tools/ensemble_test_runner/lib/mocks/test_logger.dart +++ b/tools/ensemble_test_runner/lib/mocks/test_logger.dart @@ -12,10 +12,12 @@ class TestLogger { required String testId, required String name, required String content, + String extension = 'log', }) async { final directory = Directory('build/ensemble_test_runner/logs'); await directory.create(recursive: true); - final fileName = '${_safeFileName(testId)}_${_safeFileName(name)}.log'; + final fileName = + '${_safeFileName(testId)}_${_safeFileName(name)}.${_safeExtension(extension)}'; final file = File('${directory.path}/$fileName'); await file.writeAsString(content); return file.path; @@ -26,4 +28,9 @@ class TestLogger { static String _safeFileName(String value) { return value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); } + + static String _safeExtension(String value) { + final extension = value.replaceAll(RegExp(r'[^A-Za-z0-9]+'), ''); + return extension.isEmpty ? 'log' : extension; + } } diff --git a/tools/ensemble_test_runner/test/test_reporter_test.dart b/tools/ensemble_test_runner/test/test_reporter_test.dart index 07fd44dad..89232de41 100644 --- a/tools/ensemble_test_runner/test/test_reporter_test.dart +++ b/tools/ensemble_test_runner/test/test_reporter_test.dart @@ -132,8 +132,8 @@ void main() { testId: 'api_log_test', durationMs: 42, logs: const [ - 'apiCalls: build/ensemble_test_runner/logs/api_log_test_api_calls.log', - 'dumpTree: build/ensemble_test_runner/logs/api_log_test_dump_tree.log', + 'apiCalls: build/ensemble_test_runner/logs/api_log_test_api_calls.json', + 'dumpTree: build/ensemble_test_runner/logs/api_log_test_dump_tree.txt', ], ), ], @@ -144,7 +144,7 @@ void main() { expect( output, contains( - 'apiCalls: build/ensemble_test_runner/logs/api_log_test_api_calls.log')); + 'apiCalls: build/ensemble_test_runner/logs/api_log_test_api_calls.json')); expect(output, contains('dumpTree:')); expect(output, isNot(contains('MaterialApp'))); }); diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index be4f5e77d..ce2e37b2d 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -1,3 +1,4 @@ +import 'dart:convert'; import 'dart:io'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; @@ -70,11 +71,14 @@ void main() { expect(context.logger.logs, hasLength(1)); expect(context.logger.logs.single, startsWith('storage: ')); final path = context.logger.logs.single.substring('storage: '.length); + expect(path, endsWith('.json')); final file = File(path); expect(file.existsSync(), isTrue); final content = file.readAsStringSync(); - expect(content, contains('"first":"one"')); - expect(content, contains('"second":{"nested":true}')); + expect(content, contains('\n "first": "one"')); + final json = jsonDecode(content) as Map; + expect(json['first'], 'one'); + expect(json['second'], {'nested': true}); }); testWidgets('dumpTree writes the widget tree to test logs', (tester) async { @@ -101,9 +105,42 @@ void main() { expect(context.logger.logs.single, startsWith('dumpTree: ')); final path = context.logger.logs.single.substring('dumpTree: '.length); + expect(path, endsWith('.txt')); final file = File(path); expect(file.existsSync(), isTrue); - expect(file.readAsStringSync(), contains('debug target')); + final content = file.readAsStringSync(); + expect(content, isNot(startsWith('dumpTree:'))); + expect(content, contains('debug target')); + }); + + testWidgets('logApiCalls writes structured json', (tester) async { + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 'api_log_test', + startScreen: 'Home', + steps: [], + ), + ); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: harness, + ); + + await executor.execute(const TestStep(type: 'logApiCalls', args: {})); + + expect(context.logger.logs.single, startsWith('apiCalls: ')); + final path = context.logger.logs.single.substring('apiCalls: '.length); + expect(path, endsWith('.json')); + final content = File(path).readAsStringSync(); + expect(content, isNot(contains('API '))); + expect(jsonDecode(content), { + 'total': 0, + 'calls': [], + }); }); testWidgets('screenshot writes a framed png by default', (tester) async { From 56f3a13da9bcd0a1abfa41f1040f719ff303c34f Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Sat, 11 Jul 2026 02:08:19 +0500 Subject: [PATCH 14/29] feat(test_runner): introduce screenshot options in ensemble tests schema Added a new options section to the ensemble tests schema, allowing configuration of screenshot settings such as enabling/disabling screenshots, platform, model, and step inclusion/exclusion. Enhanced the screenshot capturing functionality to utilize these options, ensuring automatic screenshots are taken based on specified criteria. Updated related classes and tests to support the new options structure, improving the flexibility and usability of the screenshot feature in the test runner. --- .../assets/schema/ensemble_tests_schema.json | 36 +++++++++++ .../lib/actions/extended_step_handlers.dart | 62 +++++++++++++------ .../lib/actions/screenshot_device.dart | 11 ++++ .../lib/entry/ensemble_test_entry.dart | 21 +++++-- .../lib/models/ensemble_test_models.dart | 41 ++++++++++++ .../lib/parser/ensemble_test_parser.dart | 23 +++++++ .../lib/reporters/test_reporter.dart | 54 ++++++++++++++++ .../lib/runner/ensemble_test_harness.dart | 2 +- .../lib/runner/ensemble_test_runner.dart | 58 ++++++++++++++++- .../lib/runner/test_runtime_state.dart | 2 + .../schema/ensemble_test_schema_builder.dart | 24 +++++++ .../test/ensemble_test_parser_test.dart | 28 +++++++++ .../test/screenshot_device_test.dart | 21 +++++++ .../test/test_reporter_test.dart | 41 ++++++++++++ 14 files changed, 398 insertions(+), 26 deletions(-) diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 74ab8f3a9..89d1cb4cf 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -58,6 +58,9 @@ "initialState": { "$ref": "#/$defs/initialState" }, + "options": { + "$ref": "#/$defs/options" + }, "mocks": { "$ref": "#/$defs/mocks" }, @@ -155,6 +158,39 @@ } } }, + "options": { + "type": "object", + "additionalProperties": false, + "properties": { + "screenshots": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "platform": { + "type": "string" + }, + "model": { + "type": "string" + }, + "includeSteps": { + "type": "array", + "items": { + "type": "string" + } + }, + "excludeSteps": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, "args_openScreen": { "type": "object", "properties": { diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 516f181d7..d77bcb72f 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -621,41 +621,63 @@ class ExtendedStepHandlers { } static Future _screenshot(TestStepExecutor e, TestStep step) async { - await e.tester.pump(); - final rawName = step.args['name']?.toString(); + final args = + e.context.testCase.options.screenshots.toScreenshotArgs(step.args); + await captureScreenshot(e, args: args); + } + + static Future captureScreenshot( + TestStepExecutor e, { + required Map args, + bool deferWrite = false, + bool pumpBeforeCapture = true, + }) async { + if (pumpBeforeCapture) { + await e.tester.pump(); + } + final rawName = args['name']?.toString(); final name = rawName == null || rawName.isEmpty ? DateTime.now().microsecondsSinceEpoch.toString() : rawName; final fileName = _safeFileName('${e.context.testCase.id}_$name.png'); final directory = Directory('build/ensemble_test_runner/screenshots'); final file = File('${directory.path}/$fileName'); - final device = resolveScreenshotDevice(step.args); - - await e.tester.pump(); + final device = resolveScreenshotDevice(args); - final path = await e.tester.runAsync(() async { - final renderView = e.tester.binding.renderViews.first; - final layer = renderView.debugLayer; - if (layer is! OffsetLayer) { - throw EnsembleTestFailure( - 'screenshot requires a painted render view.', - ); - } + if (pumpBeforeCapture) { + await e.tester.pump(); + } - final image = await layer.toImage( - renderView.paintBounds, - pixelRatio: renderView.flutterView.devicePixelRatio, + final renderView = e.tester.binding.renderViews.first; + final layer = renderView.debugLayer; + if (layer is! OffsetLayer) { + throw EnsembleTestFailure( + 'screenshot requires a painted render view.', ); + } + + final image = layer.toImageSync( + renderView.paintBounds, + pixelRatio: renderView.flutterView.devicePixelRatio, + ); + final path = file.path; + + Future writeImage() async { final byteData = await _addDeviceFrame(image, device); image.dispose(); if (byteData == null) { throw EnsembleTestFailure('Failed to encode screenshot as PNG.'); } - await directory.create(recursive: true); - await file.writeAsBytes(byteData.buffer.asUint8List()); - return file.path; - }); + directory.createSync(recursive: true); + file.writeAsBytesSync(byteData.buffer.asUint8List()); + } + + if (deferWrite) { + e.context.runtime.pendingScreenshotWrites.add(writeImage); + } else { + await e.tester.runAsync(writeImage); + } e.context.logger.log('screenshot: $path'); } diff --git a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart index 06877cd7a..54dfc3fcd 100644 --- a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart +++ b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart @@ -12,6 +12,17 @@ DeviceInfo? firstScreenshotDevice(List steps) { return null; } +DeviceInfo? screenshotDeviceForTestCase(EnsembleTestCase testCase) { + final stepDevice = firstScreenshotDevice(testCase.steps); + if (stepDevice != null) return stepDevice; + if (testCase.options.screenshots.enabled) { + return resolveScreenshotDevice( + testCase.options.screenshots.toScreenshotArgs(), + ); + } + return null; +} + DeviceInfo? screenshotDeviceForStep(TestStep step) { return step.type == 'screenshot' ? resolveScreenshotDevice(step.args) : null; } diff --git a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart index 3f42cb488..546550273 100644 --- a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart +++ b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart @@ -132,7 +132,8 @@ Future runEnsembleYamlTestsWithOptions( } final runResult = EnsembleTestRunResult(results: orderedResults); - final suiteSummary = TestReporter().formatSummary( + final reporter = TestReporter(); + final suiteSummary = reporter.formatSummary( runResult, testFile: '${target.testsAssetPrefix}*.test.yaml', ); @@ -140,10 +141,13 @@ Future runEnsembleYamlTestsWithOptions( _emitMachineReport(runResult); if (failures.isNotEmpty) { + final pendingFrameworkExceptions = _drainPendingExceptions(tester); fail( - 'Failed YAML tests:\n' - '${failures.map((p) => '- $p').join('\n')}\n\n' - '$suiteSummary', + reporter.formatFailureSummary( + runResult, + failedPaths: failures, + pendingFrameworkExceptions: pendingFrameworkExceptions, + ), ); } }, @@ -151,6 +155,15 @@ Future runEnsembleYamlTestsWithOptions( ); } +List _drainPendingExceptions(WidgetTester tester) { + final exceptions = []; + Object? exception; + while ((exception = tester.takeException()) != null) { + exceptions.add(exception); + } + return exceptions; +} + EnsembleTestSelection _selectionFromEnvironment() { return EnsembleTestSelection( ids: _csvSet(const String.fromEnvironment('ensembleTestId')), diff --git a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart index 99863d7fd..1d45952d0 100644 --- a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart +++ b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart @@ -42,6 +42,7 @@ class EnsembleTestCase { /// Test [id] that must run before this one (same app session). final String? prerequisite; final Map initialState; + final EnsembleTestOptions options; final TestMocks mocks; final List steps; @@ -57,6 +58,7 @@ class EnsembleTestCase { this.startScreen, this.prerequisite, this.initialState = const {}, + this.options = const EnsembleTestOptions(), this.mocks = const TestMocks(), required this.steps, }); @@ -74,6 +76,45 @@ class EnsembleTestCase { }; } +class EnsembleTestOptions { + final ScreenshotOptions screenshots; + + const EnsembleTestOptions({ + this.screenshots = const ScreenshotOptions(), + }); +} + +class ScreenshotOptions { + final bool enabled; + final String platform; + final String model; + final List includeSteps; + final List excludeSteps; + + const ScreenshotOptions({ + this.enabled = false, + this.platform = 'ios', + this.model = 'iPhone 15 Pro', + this.includeSteps = const [], + this.excludeSteps = const [], + }); + + bool shouldCaptureStep(String stepType) { + if (!enabled) return false; + if (excludeSteps.contains(stepType)) return false; + if (includeSteps.isNotEmpty && !includeSteps.contains(stepType)) { + return false; + } + return true; + } + + Map toScreenshotArgs([Map? overrides]) => { + 'platform': platform, + 'model': model, + ...?overrides, + }; +} + /// Mock configuration attached to a test case. class TestMocks { final Map apis; diff --git a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart index 7f9ed1100..7cf791b15 100644 --- a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart +++ b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -93,11 +93,34 @@ class EnsembleTestParser { startScreen: hasStartScreen ? startScreen : null, prerequisite: hasPrerequisite ? prerequisite : null, initialState: _toStringDynamicMap(map['initialState']), + options: _parseOptions(map['options']), mocks: _parseMocks(map['mocks']), steps: _parseSteps(stepsNode, testId: id), ); } + static EnsembleTestOptions _parseOptions(dynamic node) { + if (node == null) return const EnsembleTestOptions(); + if (node is! YamlMap) { + throw EnsembleTestFailure('"options" must be a map'); + } + final screenshotsNode = node['screenshots']; + if (screenshotsNode == null) return const EnsembleTestOptions(); + if (screenshotsNode is! YamlMap) { + throw EnsembleTestFailure('"options.screenshots" must be a map'); + } + + return EnsembleTestOptions( + screenshots: ScreenshotOptions( + enabled: screenshotsNode['enabled'] == true, + platform: screenshotsNode['platform']?.toString() ?? 'ios', + model: screenshotsNode['model']?.toString() ?? 'iPhone 15 Pro', + includeSteps: _toStringList(screenshotsNode['includeSteps']), + excludeSteps: _toStringList(screenshotsNode['excludeSteps']), + ), + ); + } + static TestMocks _parseMocks(dynamic node) { if (node == null) return const TestMocks(); if (node is! YamlMap) { diff --git a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart index 8039b15a8..9622d1d48 100644 --- a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart +++ b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart @@ -106,6 +106,55 @@ class TestReporter { return buffer.toString(); } + /// Formats the short failure passed to Flutter's test framework. + /// + /// The full boxed report is printed separately. Keeping this compact avoids + /// Flutter echoing the entire report again with its framework stack trace. + String formatFailureSummary( + EnsembleTestRunResult result, { + Iterable failedPaths = const [], + Iterable pendingFrameworkExceptions = const [], + }) { + final failed = result.results + .where((r) => r.status == TestStatus.failed) + .toList(growable: false); + final paths = failedPaths.toList(growable: false); + final buffer = StringBuffer() + ..writeln( + 'Failed YAML tests (${result.failedCount}/${result.results.length}):', + ); + + for (var i = 0; i < failed.length; i++) { + final r = failed[i]; + final path = i < paths.length ? paths[i] : null; + final label = path == null || r.testId.contains(path) + ? r.testId + : '${r.testId} ($path)'; + final failedStep = r.failedStep != null + ? ' (failed: ${formatStepBrief(r.failedStep!)})' + : r.failedStepIndex != null + ? ' (failed: step ${r.failedStepIndex! + 1})' + : ''; + buffer.writeln('- $label: ${r.message ?? 'failed'}$failedStep'); + } + + final pending = pendingFrameworkExceptions + .map((e) => e?.toString().trim() ?? '') + .where((e) => e.isNotEmpty) + .toList(growable: false); + if (pending.isNotEmpty) { + buffer.writeln(); + buffer.writeln('Pending Flutter framework exceptions:'); + for (final exception in pending) { + buffer.writeln('- ${_firstLine(exception)}'); + } + } + + buffer.writeln(); + buffer.write('See the Ensemble YAML tests report above.'); + return buffer.toString(); + } + void _writeTestCase(StringBuffer buffer, EnsembleSingleTestResult r) { final icon = r.status == TestStatus.passed ? '✓' : '✗'; buffer.writeln('│ $icon ${r.testId} (${r.durationMs}ms)'); @@ -152,3 +201,8 @@ class TestReporter { } } } + +String _firstLine(String value) { + final index = value.indexOf('\n'); + return index == -1 ? value : value.substring(0, index); +} diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index 9ad216dbe..a61fe9764 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -376,7 +376,7 @@ class EnsembleTestHarness { YamlTestSession.navigationFlow.clear(); final ctx = context ?? EnsembleTestContext.fromTestCase(testCase); - final screenshotDevice = firstScreenshotDevice(testCase.steps); + final screenshotDevice = screenshotDeviceForTestCase(testCase); if (screenshotDevice != null) { await _setViewportForDevice(tester, ctx, screenshotDevice); } diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index 5f415a85a..a2a31b591 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -1,5 +1,6 @@ import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; @@ -148,14 +149,19 @@ class EnsembleTestRunner { harness: harness, config: config, ); - for (var i = 0; i < test.steps.length; i++) { final step = test.steps[i]; try { await executor.execute(step); await YamlTestSession.navigationFlow.flushPending(); + await _captureAutomaticScreenshotForStep( + executor: executor, + step: step, + stepIndex: i, + ); } catch (error, stackTrace) { await _settleLiveApiWork(tester, ctx); + await _flushPendingScreenshots(tester, ctx); await YamlTestSession.navigationFlow.flushPending(); return EnsembleSingleTestResult.failed( testId: test.id, @@ -172,6 +178,8 @@ class EnsembleTestRunner { } await YamlTestSession.navigationFlow.flushPending(); + await _settleLiveApiWork(tester, ctx); + await _flushPendingScreenshots(tester, ctx); return EnsembleSingleTestResult.passed( testId: test.id, @@ -182,6 +190,37 @@ class EnsembleTestRunner { ); } + Future _captureAutomaticScreenshotForStep({ + required TestStepExecutor executor, + required TestStep step, + required int stepIndex, + }) async { + final options = executor.context.testCase.options.screenshots; + if (!options.shouldCaptureStep(step.type)) return; + await _captureAutomaticScreenshot( + executor, + name: + 'step_${(stepIndex + 1).toString().padLeft(3, '0')}_${_safeArtifactName(step.type)}', + ); + } + + Future _captureAutomaticScreenshot( + TestStepExecutor executor, { + required String name, + }) { + return ExtendedStepHandlers.captureScreenshot( + executor, + args: executor.context.testCase.options.screenshots.toScreenshotArgs({ + 'name': name, + }), + deferWrite: true, + pumpBeforeCapture: false, + ); + } + + String _safeArtifactName(String value) => + value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); + Future _settleLiveApiWork( WidgetTester tester, EnsembleTestContext ctx, @@ -195,4 +234,21 @@ class EnsembleTestRunner { } } } + + Future _flushPendingScreenshots( + WidgetTester tester, + EnsembleTestContext ctx, + ) async { + final writes = List Function()>.from( + ctx.runtime.pendingScreenshotWrites, + ); + ctx.runtime.pendingScreenshotWrites.clear(); + if (writes.isEmpty) return; + + await tester.runAsync(() async { + for (final write in writes) { + await write(); + } + }); + } } diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index 57194fd94..377e19074 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -6,6 +6,7 @@ class TestRuntimeState { bool networkOffline = false; final List consoleLogs = []; final List flutterErrors = []; + final List Function()> pendingScreenshotWrites = []; Map? authUser; final Map permissions = {}; Size? deviceSize; @@ -16,6 +17,7 @@ class TestRuntimeState { networkOffline = false; consoleLogs.clear(); flutterErrors.clear(); + pendingScreenshotWrites.clear(); authUser = null; permissions.clear(); deviceSize = null; diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index ba192f0c0..d64d98d7d 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -50,6 +50,29 @@ class EnsembleTestSchemaBuilder { }, }, }, + 'options': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'screenshots': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + 'platform': {'type': 'string'}, + 'model': {'type': 'string'}, + 'includeSteps': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'excludeSteps': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + }, + }, + }, + }, 'testCase': { 'type': 'object', 'additionalProperties': false, @@ -97,6 +120,7 @@ class EnsembleTestSchemaBuilder { 'ID of another test that must run before this one in the same app session', }, 'initialState': {'\$ref': '#/\$defs/initialState'}, + 'options': {'\$ref': '#/\$defs/options'}, 'mocks': {'\$ref': '#/\$defs/mocks'}, 'steps': { 'type': 'array', diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 648e6ca67..41180d2e5 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -72,6 +72,34 @@ steps: ); }); + test('parses screenshot options', () { + const yaml = ''' +id: visual_debug +startScreen: Home +options: + screenshots: + enabled: true + platform: android + model: Samsung Galaxy S20 + includeSteps: [tap, waitForNavigation] + excludeSteps: [wait] +steps: + - tap: + id: start_button +'''; + + final test = EnsembleTestParser.parseString(yaml); + final screenshots = test.options.screenshots; + expect(screenshots.enabled, isTrue); + expect(screenshots.platform, 'android'); + expect(screenshots.model, 'Samsung Galaxy S20'); + expect(screenshots.includeSteps, ['tap', 'waitForNavigation']); + expect(screenshots.excludeSteps, ['wait']); + expect(screenshots.shouldCaptureStep('tap'), isTrue); + expect(screenshots.shouldCaptureStep('wait'), isFalse); + expect(screenshots.shouldCaptureStep('settle'), isFalse); + }); + test('parses AI-friendly metadata fields', () { const yaml = ''' id: login_valid diff --git a/tools/ensemble_test_runner/test/screenshot_device_test.dart b/tools/ensemble_test_runner/test/screenshot_device_test.dart index bb9bac2b0..e025593ce 100644 --- a/tools/ensemble_test_runner/test/screenshot_device_test.dart +++ b/tools/ensemble_test_runner/test/screenshot_device_test.dart @@ -48,4 +48,25 @@ void main() { expect(device, isNull); }); + + test('uses screenshot options device when enabled', () { + final device = screenshotDeviceForTestCase( + const EnsembleTestCase( + id: 'screenshots', + startScreen: 'Home', + steps: [ + TestStep(type: 'tap', args: {'id': 'button'}), + ], + options: EnsembleTestOptions( + screenshots: ScreenshotOptions( + enabled: true, + platform: 'android', + model: 'Samsung Galaxy S20', + ), + ), + ), + ); + + expect(device?.name, 'Samsung Galaxy S20'); + }); } diff --git a/tools/ensemble_test_runner/test/test_reporter_test.dart b/tools/ensemble_test_runner/test/test_reporter_test.dart index 89232de41..6fb1e0634 100644 --- a/tools/ensemble_test_runner/test/test_reporter_test.dart +++ b/tools/ensemble_test_runner/test/test_reporter_test.dart @@ -148,5 +148,46 @@ void main() { expect(output, contains('dumpTree:')); expect(output, isNot(contains('MaterialApp'))); }); + + test('formats compact failure summary without repeating boxed report', () { + final output = TestReporter().formatFailureSummary( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.failed( + testId: + 'login_test (ensemble/apps/inhome/tests/signin.test.yaml)', + durationMs: 100, + failedStep: const TestStep( + type: 'waitForNavigation', + args: {'screen': 'Home'}, + ), + error: 'Timed out waiting for navigation', + ), + EnsembleSingleTestResult.failed( + testId: + 'smoke_navigations (ensemble/apps/inhome/tests/smoke.test.yaml)', + durationMs: 0, + error: 'Prerequisite "login_test" failed', + ), + ], + ), + failedPaths: const [ + 'ensemble/apps/inhome/tests/signin.test.yaml', + 'ensemble/apps/inhome/tests/smoke.test.yaml', + ], + pendingFrameworkExceptions: const [ + 'Reentrant call to runAsync() denied.\nextra framework text', + ], + ); + + expect(output, contains('Failed YAML tests (2/2):')); + expect(output, contains('Timed out waiting for navigation')); + expect(output, contains('failed: waitForNavigation(Home)')); + expect(output, contains('Pending Flutter framework exceptions:')); + expect(output, contains('Reentrant call to runAsync() denied.')); + expect(output, contains('See the Ensemble YAML tests report above.')); + expect(output, isNot(contains('┌─ Ensemble YAML tests'))); + expect(output, isNot(contains('extra framework text'))); + }); }); } From 86c0fad79ad973e8aba6f19cbcde5b163b66d3c3 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Mon, 13 Jul 2026 22:06:43 +0500 Subject: [PATCH 15/29] feat(test_runner): add performance logging capabilities to ensemble tests Introduced a new `logPerformance` step to capture and log Flutter app frame timing metrics as JSON artifacts. Updated the ensemble tests schema to include performance options, allowing users to enable or disable performance logging. Enhanced the test runner to automatically write performance logs at the end of test executions when enabled. Updated related classes, tests, and documentation to support this new feature, improving the overall performance monitoring capabilities of the test runner. --- tools/ensemble_test_runner/STEP_VOCABULARY.md | 12 +++- .../assets/schema/ensemble_tests_schema.json | 43 ++++++++++++ .../lib/actions/extended_step_handlers.dart | 7 ++ .../lib/models/ensemble_test_models.dart | 10 +++ .../lib/parser/ensemble_test_parser.dart | 28 +++++--- .../lib/runner/app_performance_log.dart | 53 +++++++++++++++ .../lib/runner/ensemble_test_runner.dart | 27 ++++++++ .../lib/runner/test_runtime_state.dart | 67 +++++++++++++++++++ .../schema/ensemble_test_schema_builder.dart | 7 ++ .../lib/vocabulary/test_step_registry.dart | 7 ++ .../test/ensemble_test_parser_test.dart | 16 +++++ .../test/test_step_executor_test.dart | 45 +++++++++++++ .../tool/generate_step_registry.dart | 7 +- 13 files changed, 318 insertions(+), 11 deletions(-) create mode 100644 tools/ensemble_test_runner/lib/runner/app_performance_log.dart diff --git a/tools/ensemble_test_runner/STEP_VOCABULARY.md b/tools/ensemble_test_runner/STEP_VOCABULARY.md index fa3dbf076..22d92b6c5 100644 --- a/tools/ensemble_test_runner/STEP_VOCABULARY.md +++ b/tools/ensemble_test_runner/STEP_VOCABULARY.md @@ -40,7 +40,7 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense `setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` ### Scripts / fixtures / debug / quality -`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `logStorage`, `screenshot`, `dumpTree`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` +`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `logStorage`, `logPerformance`, `screenshot`, `dumpTree`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` ### Control flow `group`, `repeat`, `optional`, `ifVisible` @@ -72,6 +72,16 @@ steps: Editor validation: `https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json` (committed copy at [`assets/schema/ensemble_tests_schema.json`](assets/schema/ensemble_tests_schema.json), regenerate with `dart run tool/generate_schema.dart`). Arg shapes come from [`TestStepArgKind`](lib/vocabulary/test_step_arg_kind.dart) on each [`TestStepRegistryEntry`](lib/vocabulary/test_step_registry.dart). +`logPerformance` writes Flutter app frame timing metrics to +`build/ensemble_test_runner/logs/_app_performance.json`. To write this +artifact automatically at the end of a test, set: + +```yaml +options: + performance: + enabled: true +``` + Before running a new suite, use `dart run ensemble_test_runner:ensemble_test --doctor` from the Flutter wrapper app root to validate config, test discovery, duplicate IDs, prerequisites, schema comments, and obvious widget IDs. CI can request JSON diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 89d1cb4cf..3986a4356 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -188,6 +188,15 @@ } } } + }, + "performance": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + } + } } } }, @@ -2099,6 +2108,15 @@ {} ] }, + "args_logPerformance": { + "type": "object", + "additionalProperties": false, + "title": "logPerformance", + "description": "Write Flutter app frame timing metrics to a JSON artifact", + "examples": [ + {} + ] + }, "args_screenshot": { "type": "object", "properties": { @@ -5080,6 +5098,31 @@ "logApiCalls" ] }, + { + "type": "object", + "title": "logPerformance", + "description": "Write Flutter app frame timing metrics to a JSON artifact", + "examples": [ + { + "logPerformance": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "logPerformance": { + "$ref": "#/$defs/args_logPerformance", + "description": "Write Flutter app frame timing metrics to a JSON artifact", + "examples": [ + {} + ] + } + }, + "required": [ + "logPerformance" + ] + }, { "type": "object", "title": "screenshot", diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index d77bcb72f..35d113274 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -10,6 +10,7 @@ import 'package:ensemble_device_preview/ensemble_device_preview.dart'; import 'package:ensemble_test_runner/actions/screenshot_device.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/app_performance_log.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; @@ -268,6 +269,12 @@ class ExtendedStepHandlers { ); } return true; + case 'logPerformance': + final path = await executor.tester.runAsync(() { + return writeAppPerformanceLog(executor.context); + }); + executor.context.logger.log('appPerformance: $path'); + return true; case 'expectAccessible': executor.assertions.expectAccessible(executor.requireId(step)); return true; diff --git a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart index 1d45952d0..d0573e7b4 100644 --- a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart +++ b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart @@ -78,9 +78,19 @@ class EnsembleTestCase { class EnsembleTestOptions { final ScreenshotOptions screenshots; + final PerformanceOptions performance; const EnsembleTestOptions({ this.screenshots = const ScreenshotOptions(), + this.performance = const PerformanceOptions(), + }); +} + +class PerformanceOptions { + final bool enabled; + + const PerformanceOptions({ + this.enabled = false, }); } diff --git a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart index 7cf791b15..772aeca58 100644 --- a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart +++ b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -105,19 +105,29 @@ class EnsembleTestParser { throw EnsembleTestFailure('"options" must be a map'); } final screenshotsNode = node['screenshots']; - if (screenshotsNode == null) return const EnsembleTestOptions(); - if (screenshotsNode is! YamlMap) { + final performanceNode = node['performance']; + if (screenshotsNode != null && screenshotsNode is! YamlMap) { throw EnsembleTestFailure('"options.screenshots" must be a map'); } + if (performanceNode != null && performanceNode is! YamlMap) { + throw EnsembleTestFailure('"options.performance" must be a map'); + } return EnsembleTestOptions( - screenshots: ScreenshotOptions( - enabled: screenshotsNode['enabled'] == true, - platform: screenshotsNode['platform']?.toString() ?? 'ios', - model: screenshotsNode['model']?.toString() ?? 'iPhone 15 Pro', - includeSteps: _toStringList(screenshotsNode['includeSteps']), - excludeSteps: _toStringList(screenshotsNode['excludeSteps']), - ), + screenshots: screenshotsNode == null + ? const ScreenshotOptions() + : ScreenshotOptions( + enabled: screenshotsNode['enabled'] == true, + platform: screenshotsNode['platform']?.toString() ?? 'ios', + model: screenshotsNode['model']?.toString() ?? 'iPhone 15 Pro', + includeSteps: _toStringList(screenshotsNode['includeSteps']), + excludeSteps: _toStringList(screenshotsNode['excludeSteps']), + ), + performance: performanceNode == null + ? const PerformanceOptions() + : PerformanceOptions( + enabled: performanceNode['enabled'] == true, + ), ); } diff --git a/tools/ensemble_test_runner/lib/runner/app_performance_log.dart b/tools/ensemble_test_runner/lib/runner/app_performance_log.dart new file mode 100644 index 000000000..5ff8566b1 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/app_performance_log.dart @@ -0,0 +1,53 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; + +Future writeAppPerformanceLog(EnsembleTestContext context) { + final frames = context.runtime.appFrameTimings; + final jankyFrames = frames.where((frame) => frame.isJanky).length; + final slowestFrames = frames.toList() + ..sort((a, b) => b.totalSpanMs.compareTo(a.totalSpanMs)); + final content = const JsonEncoder.withIndent(' ').convert({ + 'testId': context.testCase.id, + 'frameBudgetMs': AppFrameTimingEntry.frameBudgetMs, + 'totalFrames': frames.length, + 'jankyFrames': jankyFrames, + 'jankyFrameRate': _ratio(jankyFrames, frames.length), + 'averageBuildMs': _average(frames.map((frame) => frame.buildMs)), + 'averageRasterMs': _average(frames.map((frame) => frame.rasterMs)), + 'averageTotalSpanMs': _average(frames.map((frame) => frame.totalSpanMs)), + 'maxBuildMs': _max(frames.map((frame) => frame.buildMs)), + 'maxRasterMs': _max(frames.map((frame) => frame.rasterMs)), + 'maxTotalSpanMs': _max(frames.map((frame) => frame.totalSpanMs)), + 'slowestFrames': + slowestFrames.take(10).map((frame) => frame.toJson()).toList(), + 'frames': frames.map((frame) => frame.toJson()).toList(), + }); + + return context.logger.writeLogFile( + testId: context.testCase.id, + name: 'app_performance', + content: content, + extension: 'json', + ); +} + +double _average(Iterable values) { + final list = values.toList(growable: false); + if (list.isEmpty) return 0; + return _round(list.reduce((a, b) => a + b) / list.length); +} + +double _max(Iterable values) { + final list = values.toList(growable: false); + if (list.isEmpty) return 0; + return _round(list.reduce((a, b) => a > b ? a : b)); +} + +double _ratio(int numerator, int denominator) { + if (denominator == 0) return 0; + return _round(numerator / denominator); +} + +double _round(double value) => double.parse(value.toStringAsFixed(3)); diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index a2a31b591..71f5ed1d0 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -1,3 +1,5 @@ +import 'dart:ui' as ui; + import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; @@ -6,11 +8,13 @@ import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/reporters/test_reporter.dart'; +import 'package:ensemble_test_runner/runner/app_performance_log.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/live_async_call.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter/scheduler.dart'; import 'package:flutter_test/flutter_test.dart'; typedef EnsembleTestRunOutput = ({ @@ -77,9 +81,15 @@ class EnsembleTestRunner { bool continuation = false, }) async { final stopwatch = Stopwatch()..start(); + void Function(List)? timingsCallback; try { final ctx = EnsembleTestContext.fromTestCase(test); + timingsCallback = (List timings) { + ctx.runtime.addFrameTimings(timings); + }; + + SchedulerBinding.instance.addTimingsCallback(timingsCallback); ctx.apiOverlay.liveAsyncRunner = tester.runAsync; LiveAsyncCallSupport.runner = tester.runAsync; TestErrorTracker.install(ctx.runtime); @@ -131,6 +141,10 @@ class EnsembleTestRunner { ); } finally { TestErrorTracker.reset(); + final callback = timingsCallback; + if (callback != null) { + SchedulerBinding.instance.removeTimingsCallback(callback); + } } } @@ -162,6 +176,7 @@ class EnsembleTestRunner { } catch (error, stackTrace) { await _settleLiveApiWork(tester, ctx); await _flushPendingScreenshots(tester, ctx); + await _writeAutomaticPerformanceLog(tester, ctx); await YamlTestSession.navigationFlow.flushPending(); return EnsembleSingleTestResult.failed( testId: test.id, @@ -180,6 +195,7 @@ class EnsembleTestRunner { await YamlTestSession.navigationFlow.flushPending(); await _settleLiveApiWork(tester, ctx); await _flushPendingScreenshots(tester, ctx); + await _writeAutomaticPerformanceLog(tester, ctx); return EnsembleSingleTestResult.passed( testId: test.id, @@ -221,6 +237,17 @@ class EnsembleTestRunner { String _safeArtifactName(String value) => value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); + Future _writeAutomaticPerformanceLog( + WidgetTester tester, + EnsembleTestContext ctx, + ) async { + if (!ctx.testCase.options.performance.enabled) return; + final path = await tester.runAsync(() { + return writeAppPerformanceLog(ctx); + }); + ctx.logger.log('appPerformance: $path'); + } + Future _settleLiveApiWork( WidgetTester tester, EnsembleTestContext ctx, diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index 377e19074..c867680cb 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -1,3 +1,5 @@ +import 'dart:ui' as ui; + import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; @@ -6,6 +8,7 @@ class TestRuntimeState { bool networkOffline = false; final List consoleLogs = []; final List flutterErrors = []; + final List appFrameTimings = []; final List Function()> pendingScreenshotWrites = []; Map? authUser; final Map permissions = {}; @@ -17,6 +20,7 @@ class TestRuntimeState { networkOffline = false; consoleLogs.clear(); flutterErrors.clear(); + appFrameTimings.clear(); pendingScreenshotWrites.clear(); authUser = null; permissions.clear(); @@ -24,6 +28,69 @@ class TestRuntimeState { locale = null; themeMode = null; } + + void addFrameTimings(List timings) { + for (final timing in timings) { + appFrameTimings.add( + AppFrameTimingEntry.fromFrameTiming( + frameNumber: appFrameTimings.length + 1, + timing: timing, + ), + ); + } + } +} + +class AppFrameTimingEntry { + static const double frameBudgetMs = 16.67; + + final int frameNumber; + final int buildStartMicros; + final double buildMs; + final double rasterMs; + final double vsyncOverheadMs; + final double totalSpanMs; + + const AppFrameTimingEntry({ + required this.frameNumber, + required this.buildStartMicros, + required this.buildMs, + required this.rasterMs, + required this.vsyncOverheadMs, + required this.totalSpanMs, + }); + + factory AppFrameTimingEntry.fromFrameTiming({ + required int frameNumber, + required ui.FrameTiming timing, + }) { + return AppFrameTimingEntry( + frameNumber: frameNumber, + buildStartMicros: timing.timestampInMicroseconds( + ui.FramePhase.buildStart, + ), + buildMs: _durationMs(timing.buildDuration), + rasterMs: _durationMs(timing.rasterDuration), + vsyncOverheadMs: _durationMs(timing.vsyncOverhead), + totalSpanMs: _durationMs(timing.totalSpan), + ); + } + + bool get isJanky => + totalSpanMs > frameBudgetMs || buildMs + rasterMs > frameBudgetMs; + + Map toJson() => { + 'frameNumber': frameNumber, + 'buildStartMicros': buildStartMicros, + 'buildMs': buildMs, + 'rasterMs': rasterMs, + 'vsyncOverheadMs': vsyncOverheadMs, + 'totalSpanMs': totalSpanMs, + 'janky': isJanky, + }; + + static double _durationMs(Duration duration) => + double.parse((duration.inMicroseconds / 1000).toStringAsFixed(3)); } /// Captures Flutter framework errors for quality assertion steps. diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index d64d98d7d..63266790d 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -71,6 +71,13 @@ class EnsembleTestSchemaBuilder { }, }, }, + 'performance': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + }, + }, }, }, 'testCase': { diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index f5539057e..31b9bd16c 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -731,6 +731,13 @@ abstract final class TestStepRegistry { description: 'Log all recorded API calls to the test log', example: const {}, ), + 'logPerformance': TestStepRegistryEntry( + category: TestStepCategory.debug, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Write Flutter app frame timing metrics to a JSON artifact', + example: const {}, + ), 'screenshot': TestStepRegistryEntry( category: TestStepCategory.debug, tier: TestStepTier.core, diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 41180d2e5..6f67de7ba 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -100,6 +100,22 @@ steps: expect(screenshots.shouldCaptureStep('settle'), isFalse); }); + test('parses performance options', () { + const yaml = ''' +id: perf_debug +startScreen: Home +options: + performance: + enabled: true +steps: + - tap: + id: start_button +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.options.performance.enabled, isTrue); + }); + test('parses AI-friendly metadata fields', () { const yaml = ''' id: login_valid diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index ce2e37b2d..a3e5e9db3 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -6,6 +6,7 @@ import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -143,6 +144,50 @@ void main() { }); }); + testWidgets('logPerformance writes app frame timing json', (tester) async { + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 'performance_log_test', + startScreen: 'Home', + steps: [], + ), + ); + context.runtime.appFrameTimings.add( + const AppFrameTimingEntry( + frameNumber: 1, + buildStartMicros: 1000, + buildMs: 5, + rasterMs: 7, + vsyncOverheadMs: 1, + totalSpanMs: 12, + ), + ); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: harness, + ); + + await executor.execute(const TestStep(type: 'logPerformance', args: {})); + + expect(context.logger.logs.single, startsWith('appPerformance: ')); + final path = + context.logger.logs.single.substring('appPerformance: '.length); + expect(path, endsWith('.json')); + final content = File(path).readAsStringSync(); + final json = jsonDecode(content) as Map; + expect(json['testId'], 'performance_log_test'); + expect(json['frameBudgetMs'], 16.67); + expect(json['totalFrames'], 1); + expect(json['jankyFrames'], 0); + expect(json['averageBuildMs'], 5); + expect(json['averageRasterMs'], 7); + expect((json['frames'] as List).single, containsPair('totalSpanMs', 12)); + }); + testWidgets('screenshot writes a framed png by default', (tester) async { await tester.pumpWidget( const MaterialApp(home: Text('screenshot target')), diff --git a/tools/ensemble_test_runner/tool/generate_step_registry.dart b/tools/ensemble_test_runner/tool/generate_step_registry.dart index 63665d353..c1a629be0 100644 --- a/tools/ensemble_test_runner/tool/generate_step_registry.dart +++ b/tools/ensemble_test_runner/tool/generate_step_registry.dart @@ -579,6 +579,12 @@ void main() { 'empty', 'Log all recorded API calls to the test log', ), + 'logPerformance': step( + 'debug', + 'core', + 'empty', + 'Write Flutter app frame timing metrics to a JSON artifact', + ), 'screenshot': step( 'debug', 'core', @@ -836,7 +842,6 @@ Map defaultExampleForArg(String arg) { case 'screenshot': return { 'name': 'home_screen', - 'deviceFrame': true, 'platform': 'ios', 'model': 'iPhone 15 Pro', }; From 9021039f1b48f06129f8a14323c14d4bf7de4ec5 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Mon, 13 Jul 2026 23:14:19 +0500 Subject: [PATCH 16/29] feat(test_runner): introduce suite-wide configuration for ensemble tests Added a new `config.yaml` file to define suite-wide settings for screenshots, performance logging, and other test artifacts. Updated the ensemble tests schema to support this configuration, allowing users to enable or disable features globally. Refactored related classes and methods to utilize the new configuration structure, enhancing the flexibility and maintainability of the test runner. Updated documentation to reflect these changes and provide examples of the new configuration options. --- tools/ensemble_test_runner/README.md | 22 +++ tools/ensemble_test_runner/STEP_VOCABULARY.md | 20 ++- .../schema/ensemble_test_config_schema.json | 76 ++++++++++ .../assets/schema/ensemble_tests_schema.json | 45 ------ .../doc/TEST_AUTHORING.md | 22 +++ .../lib/actions/extended_step_handlers.dart | 75 ++-------- .../lib/actions/screenshot_device.dart | 9 +- .../lib/actions/test_step_executor.dart | 31 +--- .../lib/cli/ensemble_test_doctor.dart | 31 ++++ .../discovery/ensemble_test_discovery.dart | 17 +++ .../ensemble_test_execution_planner.dart | 18 ++- .../lib/entry/ensemble_test_entry.dart | 8 +- .../lib/mocks/test_logger.dart | 7 +- .../lib/models/ensemble_test_models.dart | 63 ++++++-- .../lib/parser/ensemble_test_parser.dart | 94 +++++++----- .../lib/reporters/test_reporter.dart | 8 ++ .../lib/runner/app_performance_log.dart | 24 +++- .../lib/runner/debug_artifact_logs.dart | 127 ++++++++++++++++ .../lib/runner/ensemble_test_context.dart | 8 +- .../lib/runner/ensemble_test_harness.dart | 9 +- .../lib/runner/ensemble_test_runner.dart | 136 +++++++++++++++--- .../schema/ensemble_test_schema_builder.dart | 98 +++++++++---- .../validation/ensemble_test_validator.dart | 28 ++++ .../test/ensemble_test_parser_test.dart | 62 ++++++-- .../test/ensemble_test_schema_test.dart | 18 +++ .../ensemble_test_schema_up_to_date_test.dart | 26 ++-- .../test/screenshot_device_test.dart | 14 +- .../test/test_reporter_test.dart | 22 +++ .../tool/generate_schema.dart | 9 +- 29 files changed, 839 insertions(+), 288 deletions(-) create mode 100644 tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json create mode 100644 tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart diff --git a/tools/ensemble_test_runner/README.md b/tools/ensemble_test_runner/README.md index 90df23580..1a693a88f 100644 --- a/tools/ensemble_test_runner/README.md +++ b/tools/ensemble_test_runner/README.md @@ -47,6 +47,28 @@ Or per file at the top of a test: # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json ``` +Suite-wide runner config lives in `tests/config.yaml`. The schema is hosted at +`https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json`: + +```yaml +# yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json +screenshots: + enabled: true + platform: ios + model: iPhone 15 Pro + includeSteps: [] + excludeSteps: [] + +performance: + enabled: true +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true +``` + ## App setup 1. Add `*.test.yaml` files under `definitions.local.path/tests/`, for example diff --git a/tools/ensemble_test_runner/STEP_VOCABULARY.md b/tools/ensemble_test_runner/STEP_VOCABULARY.md index 22d92b6c5..ce4e3ab7e 100644 --- a/tools/ensemble_test_runner/STEP_VOCABULARY.md +++ b/tools/ensemble_test_runner/STEP_VOCABULARY.md @@ -74,12 +74,24 @@ Editor validation: `https://cdn.ensembleui.com/schemas/ensemble_tests_schema.jso `logPerformance` writes Flutter app frame timing metrics to `build/ensemble_test_runner/logs/_app_performance.json`. To write this -artifact automatically at the end of a test, set: +artifact automatically once after the full suite, set this in +`tests/config.yaml`: ```yaml -options: - performance: - enabled: true +performance: + enabled: true +``` + +`dumpTree`, `logApiCalls`, and `logStorage` can also be written automatically +once after the full suite from `tests/config.yaml`: + +```yaml +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true ``` Before running a new suite, use `dart run ensemble_test_runner:ensemble_test --doctor` diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json new file mode 100644 index 000000000..2a2089756 --- /dev/null +++ b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json @@ -0,0 +1,76 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json", + "title": "Ensemble declarative test config", + "description": "Suite-wide config for app-local tests/config.yaml", + "type": "object", + "additionalProperties": false, + "properties": { + "screenshots": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "platform": { + "type": "string" + }, + "model": { + "type": "string" + }, + "includeSteps": { + "type": "array", + "items": { + "type": "string" + } + }, + "excludeSteps": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "performance": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "dumpTree": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "logApiCalls": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "logStorage": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "key": { + "type": "string" + } + } + } + } +} diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 3986a4356..d2b234f92 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -58,9 +58,6 @@ "initialState": { "$ref": "#/$defs/initialState" }, - "options": { - "$ref": "#/$defs/options" - }, "mocks": { "$ref": "#/$defs/mocks" }, @@ -158,48 +155,6 @@ } } }, - "options": { - "type": "object", - "additionalProperties": false, - "properties": { - "screenshots": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean" - }, - "platform": { - "type": "string" - }, - "model": { - "type": "string" - }, - "includeSteps": { - "type": "array", - "items": { - "type": "string" - } - }, - "excludeSteps": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "performance": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean" - } - } - } - } - }, "args_openScreen": { "type": "object", "properties": { diff --git a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md index 4805af308..95d55df63 100644 --- a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md +++ b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md @@ -35,6 +35,28 @@ steps: Use `startScreen` for a cold start. Use `prerequisite` when a test should continue from another test in the same app session. +## Suite Config + +Put shared runner settings in `tests/config.yaml`, next to the `*.test.yaml` +files: + +```yaml +# yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json +screenshots: + enabled: true + platform: ios + model: iPhone 15 Pro + +performance: + enabled: true +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true +``` + ## App Context `--inspect-app` emits JSON with screens, widget IDs, APIs, navigation targets, imports, storage/env references, and lifecycle hints. diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 35d113274..5c4d59621 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -11,6 +11,7 @@ import 'package:ensemble_test_runner/actions/screenshot_device.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/app_performance_log.dart'; +import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; @@ -236,38 +237,14 @@ class ExtendedStepHandlers { return true; case 'logStorage': final key = step.args['key']?.toString(); - if (key == null || key.isEmpty) { - final storage = StorageManager(); - final entries = storage.getKeys().map( - (key) => MapEntry(key, storage.read(key)), - ); - final content = - _prettyJson(Map.fromEntries(entries)); - final path = await executor.tester.runAsync(() { - return executor.context.logger.writeLogFile( - testId: executor.context.testCase.id, - name: 'storage', - content: content, - extension: 'json', - ); - }); - executor.context.logger.log( - 'storage: $path', - ); - } else { - final content = _prettyJson(StorageManager().read(key)); - final path = await executor.tester.runAsync(() { - return executor.context.logger.writeLogFile( - testId: executor.context.testCase.id, - name: 'storage_$key', - content: content, - extension: 'json', - ); - }); - executor.context.logger.log( - 'storage[$key]: $path', - ); - } + final path = await executor.tester.runAsync(() { + return writeStorageLog(executor.context, key: key); + }); + executor.context.logger.log( + key == null || key.isEmpty + ? 'storage: $path' + : 'storage[$key]: $path', + ); return true; case 'logPerformance': final path = await executor.tester.runAsync(() { @@ -300,12 +277,7 @@ class ExtendedStepHandlers { return true; case 'dumpTree': final path = await executor.tester.runAsync(() { - return executor.context.logger.writeLogFile( - testId: executor.context.testCase.id, - name: 'dump_tree', - content: _captureDebugDumpApp(), - extension: 'txt', - ); + return writeDumpTreeLog(executor.context); }); executor.context.logger.log('dumpTree: $path'); return true; @@ -628,8 +600,7 @@ class ExtendedStepHandlers { } static Future _screenshot(TestStepExecutor e, TestStep step) async { - final args = - e.context.testCase.options.screenshots.toScreenshotArgs(step.args); + final args = e.context.config.screenshots.toScreenshotArgs(step.args); await captureScreenshot(e, args: args); } @@ -734,30 +705,6 @@ class ExtendedStepHandlers { return byteData; } - static String _captureDebugDumpApp() { - final previousDebugPrint = debugPrint; - final lines = []; - debugPrint = (String? message, {int? wrapWidth}) { - if (message != null) { - lines.add(message); - } - }; - try { - debugDumpApp(); - } finally { - debugPrint = previousDebugPrint; - } - if (lines.isEmpty) { - return ''; - } - return lines.join('\n'); - } - - static String _prettyJson(Object? value) { - return JsonEncoder.withIndent(' ', (value) => value.toString()) - .convert(value); - } - static String _safeFileName(String value) { return value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); } diff --git a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart index 54dfc3fcd..57c6dc036 100644 --- a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart +++ b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart @@ -12,12 +12,15 @@ DeviceInfo? firstScreenshotDevice(List steps) { return null; } -DeviceInfo? screenshotDeviceForTestCase(EnsembleTestCase testCase) { +DeviceInfo? screenshotDeviceForTestCase( + EnsembleTestCase testCase, + EnsembleTestConfig config, +) { final stepDevice = firstScreenshotDevice(testCase.steps); if (stepDevice != null) return stepDevice; - if (testCase.options.screenshots.enabled) { + if (config.screenshots.enabled) { return resolveScreenshotDevice( - testCase.options.screenshots.toScreenshotArgs(), + config.screenshots.toScreenshotArgs(), ); } return null; diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 803593e50..d90935935 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:convert'; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; @@ -7,6 +6,7 @@ import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; import 'package:ensemble_test_runner/actions/test_execution_config.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; @@ -301,35 +301,8 @@ class TestStepExecutor { context.apiOverlay.resetCalls(); break; case 'logApiCalls': - final callsByName = >{}; - final calls = context.apiOverlay.calls; - for (final call in calls) { - callsByName - .putIfAbsent(call.name, () => []) - .add(call.timestamp); - } - final content = const JsonEncoder.withIndent(' ').convert({ - 'total': calls.length, - 'calls': callsByName.entries - .map( - (entry) => { - 'name': entry.key, - 'count': entry.value.length, - 'timestamps': [ - for (final timestamp in entry.value) - timestamp.toIso8601String(), - ], - }, - ) - .toList(), - }); final path = await tester.runAsync(() { - return context.logger.writeLogFile( - testId: context.testCase.id, - name: 'api_calls', - content: content, - extension: 'json', - ); + return writeApiCallsLog(context); }); context.logger.log('apiCalls: $path'); break; diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart index 046957465..53c34deb5 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart @@ -1,10 +1,13 @@ import 'dart:io'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; import 'package:path/path.dart' as p; import 'package:yaml/yaml.dart'; const hostedSchemaUrl = 'https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json'; +const hostedConfigSchemaUrl = + 'https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json'; class EnsembleTestDoctorResult { final List lines; @@ -108,6 +111,24 @@ class EnsembleTestDoctor { } ok('Found ${testFiles.length} YAML test file(s)'); + final testConfigFile = File(p.join(testsDir.path, 'config.yaml')); + if (testConfigFile.existsSync()) { + final relativePath = p.relative(testConfigFile.path, from: appDir); + final content = testConfigFile.readAsStringSync(); + if (!content.contains(hostedConfigSchemaUrl)) { + warn('$relativePath does not reference the hosted config schema URL'); + } + try { + EnsembleTestParser.parseConfigString( + content, + sourcePath: relativePath, + ); + ok('Found tests/config.yaml'); + } catch (failure) { + error('$relativePath: $failure'); + } + } + final ids = {}; final prerequisites = {}; final referencedWidgetIds = {}; @@ -190,6 +211,16 @@ _DoctorTest _parseDoctorTest(String content) { ); } + if (doc.containsKey('options')) { + return ( + id: '', + prerequisite: null, + referencedWidgetIds: {}, + error: + 'Root-level "options" is no longer supported. Move shared settings to tests/config.yaml.', + ); + } + final id = doc['id']?.toString(); if (id == null || id.isEmpty) { return ( diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart index 61aa97af0..4cbf3f8fa 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart @@ -1,5 +1,6 @@ import 'package:ensemble/framework/ensemble_config_service.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter/services.dart'; @@ -36,6 +37,22 @@ class EnsembleTestDiscovery { return files; } + /// Optional suite-level config bundled as `tests/config.yaml`. + static Future findConfigYamlAsset(String testsAssetPrefix) async { + final manifest = await AssetManifest.loadFromAssetBundle(rootBundle); + final path = '${testsAssetPrefix}config.yaml'; + return manifest.listAssets().contains(path) ? path : null; + } + + static Future loadTestConfig( + String testsAssetPrefix, + ) async { + final path = await findConfigYamlAsset(testsAssetPrefix); + if (path == null) return const EnsembleTestConfig(); + final content = await rootBundle.loadString(path); + return EnsembleTestParser.parseConfigString(content, sourcePath: path); + } + /// Reads `definitions.local` from `ensemble/ensemble-config.yaml`. static Future loadAppTarget() async { if (!EnsembleConfigService.isInitialized) { diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart index 54b6d0cbe..1311c975d 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart @@ -1,4 +1,5 @@ import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_discovery.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; @@ -18,8 +19,12 @@ class EnsembleTestDefinition { /// Topologically sorted test run order (each test id appears once). class EnsembleTestExecutionPlan { final List ordered; + final EnsembleTestConfig config; - const EnsembleTestExecutionPlan({required this.ordered}); + const EnsembleTestExecutionPlan({ + required this.ordered, + this.config = const EnsembleTestConfig(), + }); } class EnsembleTestSelection { @@ -51,6 +56,9 @@ class EnsembleTestExecutionPlanner { final paths = await EnsembleTestDiscovery.findTestYamlAssets( resolvedTarget.testsAssetPrefix, ); + final config = await EnsembleTestDiscovery.loadTestConfig( + resolvedTarget.testsAssetPrefix, + ); if (paths.isEmpty) { throw EnsembleTestFailure( 'No declarative tests found. Add *.test.yaml files under ' @@ -60,7 +68,11 @@ class EnsembleTestExecutionPlanner { final byId = {}; for (final path in paths) { - final testCase = await EnsembleTestParser.parseFile(path); + final content = await rootBundle.loadString(path); + final testCase = EnsembleTestParser.parseString( + content, + sourcePath: path, + ); final existing = byId[testCase.id]; if (existing != null) { throw EnsembleTestFailure( @@ -86,7 +98,7 @@ class EnsembleTestExecutionPlanner { } final ordered = _topologicalSort(selectedById); - return EnsembleTestExecutionPlan(ordered: ordered); + return EnsembleTestExecutionPlan(ordered: ordered, config: config); } static Map _applySelection( diff --git a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart index 546550273..1423bf013 100644 --- a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart +++ b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart @@ -102,7 +102,8 @@ Future runEnsembleYamlTestsWithOptions( ); final runner = EnsembleTestRunner(harness: harness); - final resultsById = await runner.runPlan(plan, tester); + final planResult = await runner.runPlan(plan, tester); + final resultsById = planResult.resultsById; await YamlTestSession.navigationFlow.flushPending(); await tester.pump(); @@ -131,7 +132,10 @@ Future runEnsembleYamlTestsWithOptions( } } - final runResult = EnsembleTestRunResult(results: orderedResults); + final runResult = EnsembleTestRunResult( + results: orderedResults, + suiteLogs: planResult.suiteLogs, + ); final reporter = TestReporter(); final suiteSummary = reporter.formatSummary( runResult, diff --git a/tools/ensemble_test_runner/lib/mocks/test_logger.dart b/tools/ensemble_test_runner/lib/mocks/test_logger.dart index cae9dc2ec..d47736b9a 100644 --- a/tools/ensemble_test_runner/lib/mocks/test_logger.dart +++ b/tools/ensemble_test_runner/lib/mocks/test_logger.dart @@ -16,8 +16,11 @@ class TestLogger { }) async { final directory = Directory('build/ensemble_test_runner/logs'); await directory.create(recursive: true); - final fileName = - '${_safeFileName(testId)}_${_safeFileName(name)}.${_safeExtension(extension)}'; + final safeTestId = _safeFileName(testId); + final safeName = _safeFileName(name); + final fileName = safeTestId.isEmpty + ? '$safeName.${_safeExtension(extension)}' + : '${safeTestId}_$safeName.${_safeExtension(extension)}'; final file = File('${directory.path}/$fileName'); await file.writeAsString(content); return file.path; diff --git a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart index d0573e7b4..c95ef48f0 100644 --- a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart +++ b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart @@ -7,6 +7,7 @@ class EnsembleTestRunRequest { final String? appHome; final String? i18nPath; final List tests; + final EnsembleTestConfig config; final EnsembleTestEnvironment environment; const EnsembleTestRunRequest({ @@ -14,6 +15,7 @@ class EnsembleTestRunRequest { this.appHome, this.i18nPath, required this.tests, + this.config = const EnsembleTestConfig(), this.environment = const EnsembleTestEnvironment(), }); } @@ -42,7 +44,6 @@ class EnsembleTestCase { /// Test [id] that must run before this one (same app session). final String? prerequisite; final Map initialState; - final EnsembleTestOptions options; final TestMocks mocks; final List steps; @@ -58,7 +59,6 @@ class EnsembleTestCase { this.startScreen, this.prerequisite, this.initialState = const {}, - this.options = const EnsembleTestOptions(), this.mocks = const TestMocks(), required this.steps, }); @@ -76,32 +76,64 @@ class EnsembleTestCase { }; } -class EnsembleTestOptions { - final ScreenshotOptions screenshots; - final PerformanceOptions performance; +class EnsembleTestConfig { + final ScreenshotConfig screenshots; + final PerformanceConfig performance; + final DumpTreeConfig dumpTree; + final LogApiCallsConfig logApiCalls; + final LogStorageConfig logStorage; + + const EnsembleTestConfig({ + this.screenshots = const ScreenshotConfig(), + this.performance = const PerformanceConfig(), + this.dumpTree = const DumpTreeConfig(), + this.logApiCalls = const LogApiCallsConfig(), + this.logStorage = const LogStorageConfig(), + }); +} + +class PerformanceConfig { + final bool enabled; + + const PerformanceConfig({ + this.enabled = false, + }); +} + +class DumpTreeConfig { + final bool enabled; - const EnsembleTestOptions({ - this.screenshots = const ScreenshotOptions(), - this.performance = const PerformanceOptions(), + const DumpTreeConfig({ + this.enabled = false, }); } -class PerformanceOptions { +class LogApiCallsConfig { final bool enabled; - const PerformanceOptions({ + const LogApiCallsConfig({ this.enabled = false, }); } -class ScreenshotOptions { +class LogStorageConfig { + final bool enabled; + final String? key; + + const LogStorageConfig({ + this.enabled = false, + this.key, + }); +} + +class ScreenshotConfig { final bool enabled; final String platform; final String model; final List includeSteps; final List excludeSteps; - const ScreenshotOptions({ + const ScreenshotConfig({ this.enabled = false, this.platform = 'ios', this.model = 'iPhone 15 Pro', @@ -181,8 +213,12 @@ class TestStep { /// Aggregate result for a YAML test run. class EnsembleTestRunResult { final List results; + final List suiteLogs; - const EnsembleTestRunResult({required this.results}); + const EnsembleTestRunResult({ + required this.results, + this.suiteLogs = const [], + }); int get passedCount => results.where((r) => r.status == TestStatus.passed).length; @@ -198,6 +234,7 @@ class EnsembleTestRunResult { 'passed': passedCount, 'failed': failedCount, 'results': results.map((r) => r.toJson()).toList(), + if (suiteLogs.isNotEmpty) 'suiteLogs': suiteLogs, }; } diff --git a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart index 772aeca58..57ccf45ed 100644 --- a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart +++ b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -1,28 +1,11 @@ import 'dart:io'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -import 'package:flutter/services.dart'; -import 'package:flutter/widgets.dart'; import 'package:yaml/yaml.dart'; class EnsembleTestParser { - /// Loads a test file from [rootBundle] (widget/integration tests) or [File] (CLI). + /// Loads a test file from disk. static Future parseFile(String path) async { - Object? bundleError; - try { - final content = await rootBundle.loadString(path); - return parseString(content, sourcePath: path); - } catch (error) { - bundleError = error; - } - - // dart:io can hang under widget test bindings' fake async zone. - if (_isWidgetTestBinding) { - throw EnsembleTestFailure( - 'Test file not found in asset bundle: $path ($bundleError)', - ); - } - final file = File(path); if (!await file.exists()) { throw EnsembleTestFailure('Test file not found: $path'); @@ -30,12 +13,6 @@ class EnsembleTestParser { return parseString(await file.readAsString(), sourcePath: path); } - static bool get _isWidgetTestBinding { - final type = WidgetsBinding.instance.runtimeType.toString(); - return type.contains('TestWidgetsFlutterBinding') || - type.contains('LiveTestWidgetsFlutterBinding'); - } - static EnsembleTestCase parseString(String content, {String? sourcePath}) { final dynamic doc = loadYaml(content); if (doc is! YamlMap) { @@ -54,6 +31,13 @@ class EnsembleTestParser { } static EnsembleTestCase _parseTestCase(YamlMap map, {String? sourcePath}) { + if (map.containsKey('options')) { + throw EnsembleTestFailure( + 'Root-level "options" is no longer supported in *.test.yaml files. ' + 'Move shared screenshots/performance settings to tests/config.yaml.', + ); + } + final id = map['id']?.toString(); if (id == null || id.isEmpty) { throw EnsembleTestFailure('Each test must have an "id"'); @@ -93,30 +77,52 @@ class EnsembleTestParser { startScreen: hasStartScreen ? startScreen : null, prerequisite: hasPrerequisite ? prerequisite : null, initialState: _toStringDynamicMap(map['initialState']), - options: _parseOptions(map['options']), mocks: _parseMocks(map['mocks']), steps: _parseSteps(stepsNode, testId: id), ); } - static EnsembleTestOptions _parseOptions(dynamic node) { - if (node == null) return const EnsembleTestOptions(); - if (node is! YamlMap) { - throw EnsembleTestFailure('"options" must be a map'); + static EnsembleTestConfig parseConfigString( + String content, { + String? sourcePath, + }) { + if (content.trim().isEmpty) return const EnsembleTestConfig(); + final dynamic doc = loadYaml(content); + if (doc == null) return const EnsembleTestConfig(); + if (doc is! YamlMap) { + throw EnsembleTestFailure( + 'Invalid test config${sourcePath != null ? ' ($sourcePath)' : ''}: root must be a map', + ); } + return _parseConfig(doc); + } + + static EnsembleTestConfig _parseConfig(YamlMap node) { final screenshotsNode = node['screenshots']; final performanceNode = node['performance']; + final dumpTreeNode = node['dumpTree']; + final logApiCallsNode = node['logApiCalls']; + final logStorageNode = node['logStorage']; if (screenshotsNode != null && screenshotsNode is! YamlMap) { - throw EnsembleTestFailure('"options.screenshots" must be a map'); + throw EnsembleTestFailure('"screenshots" must be a map'); } if (performanceNode != null && performanceNode is! YamlMap) { - throw EnsembleTestFailure('"options.performance" must be a map'); + throw EnsembleTestFailure('"performance" must be a map'); + } + if (dumpTreeNode != null && dumpTreeNode is! YamlMap) { + throw EnsembleTestFailure('"dumpTree" must be a map'); + } + if (logApiCallsNode != null && logApiCallsNode is! YamlMap) { + throw EnsembleTestFailure('"logApiCalls" must be a map'); + } + if (logStorageNode != null && logStorageNode is! YamlMap) { + throw EnsembleTestFailure('"logStorage" must be a map'); } - return EnsembleTestOptions( + return EnsembleTestConfig( screenshots: screenshotsNode == null - ? const ScreenshotOptions() - : ScreenshotOptions( + ? const ScreenshotConfig() + : ScreenshotConfig( enabled: screenshotsNode['enabled'] == true, platform: screenshotsNode['platform']?.toString() ?? 'ios', model: screenshotsNode['model']?.toString() ?? 'iPhone 15 Pro', @@ -124,10 +130,26 @@ class EnsembleTestParser { excludeSteps: _toStringList(screenshotsNode['excludeSteps']), ), performance: performanceNode == null - ? const PerformanceOptions() - : PerformanceOptions( + ? const PerformanceConfig() + : PerformanceConfig( enabled: performanceNode['enabled'] == true, ), + dumpTree: dumpTreeNode == null + ? const DumpTreeConfig() + : DumpTreeConfig( + enabled: dumpTreeNode['enabled'] == true, + ), + logApiCalls: logApiCallsNode == null + ? const LogApiCallsConfig() + : LogApiCallsConfig( + enabled: logApiCallsNode['enabled'] == true, + ), + logStorage: logStorageNode == null + ? const LogStorageConfig() + : LogStorageConfig( + enabled: logStorageNode['enabled'] == true, + key: logStorageNode['key']?.toString(), + ), ); } diff --git a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart index 9622d1d48..75979a8ff 100644 --- a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart +++ b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart @@ -99,6 +99,14 @@ class TestReporter { _writeTestCase(buffer, r); } + if (result.suiteLogs.isNotEmpty) { + buffer.writeln('│'); + buffer.writeln('│ suite artifacts:'); + for (final log in result.suiteLogs) { + buffer.writeln('│ $log'); + } + } + buffer.writeln('│'); buffer.writeln( '└─ ${result.summary} · ${totalMs}ms total', diff --git a/tools/ensemble_test_runner/lib/runner/app_performance_log.dart b/tools/ensemble_test_runner/lib/runner/app_performance_log.dart index 5ff8566b1..bee2ab62b 100644 --- a/tools/ensemble_test_runner/lib/runner/app_performance_log.dart +++ b/tools/ensemble_test_runner/lib/runner/app_performance_log.dart @@ -1,15 +1,29 @@ import 'dart:convert'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; Future writeAppPerformanceLog(EnsembleTestContext context) { - final frames = context.runtime.appFrameTimings; + return writePerformanceLog( + logger: context.logger, + filePrefix: context.testCase.id, + name: 'app_performance', + frames: context.runtime.appFrameTimings, + ); +} + +Future writePerformanceLog({ + required TestLogger logger, + required String filePrefix, + required String name, + required List frames, +}) { final jankyFrames = frames.where((frame) => frame.isJanky).length; final slowestFrames = frames.toList() ..sort((a, b) => b.totalSpanMs.compareTo(a.totalSpanMs)); final content = const JsonEncoder.withIndent(' ').convert({ - 'testId': context.testCase.id, + 'testId': filePrefix.isEmpty ? 'suite' : filePrefix, 'frameBudgetMs': AppFrameTimingEntry.frameBudgetMs, 'totalFrames': frames.length, 'jankyFrames': jankyFrames, @@ -25,9 +39,9 @@ Future writeAppPerformanceLog(EnsembleTestContext context) { 'frames': frames.map((frame) => frame.toJson()).toList(), }); - return context.logger.writeLogFile( - testId: context.testCase.id, - name: 'app_performance', + return logger.writeLogFile( + testId: filePrefix, + name: name, content: content, extension: 'json', ); diff --git a/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart b/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart new file mode 100644 index 000000000..bb268cb57 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart @@ -0,0 +1,127 @@ +import 'dart:convert'; + +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:flutter/material.dart'; + +Future writeDumpTreeLog(EnsembleTestContext context) { + return writeDumpTreeLogFile( + logger: context.logger, + filePrefix: context.testCase.id, + ); +} + +Future writeDumpTreeLogFile({ + required TestLogger logger, + required String filePrefix, +}) { + return logger.writeLogFile( + testId: filePrefix, + name: 'dump_tree', + content: captureDebugDumpApp(), + extension: 'txt', + ); +} + +Future writeApiCallsLog(EnsembleTestContext context) { + return writeApiCallsLogFile( + logger: context.logger, + filePrefix: context.testCase.id, + calls: context.apiOverlay.calls, + ); +} + +Future writeApiCallsLogFile({ + required TestLogger logger, + required String filePrefix, + required List calls, +}) { + final callsByName = >{}; + for (final call in calls) { + callsByName.putIfAbsent(call.name, () => []).add(call.timestamp); + } + final content = const JsonEncoder.withIndent(' ').convert({ + 'total': calls.length, + 'calls': callsByName.entries + .map( + (entry) => { + 'name': entry.key, + 'count': entry.value.length, + 'timestamps': [ + for (final timestamp in entry.value) timestamp.toIso8601String(), + ], + }, + ) + .toList(), + }); + + return logger.writeLogFile( + testId: filePrefix, + name: 'api_calls', + content: content, + extension: 'json', + ); +} + +Future writeStorageLog( + EnsembleTestContext context, { + String? key, +}) { + return writeStorageLogFile( + logger: context.logger, + filePrefix: context.testCase.id, + key: key, + ); +} + +Future writeStorageLogFile({ + required TestLogger logger, + required String filePrefix, + String? key, +}) { + if (key == null || key.isEmpty) { + final storage = StorageManager(); + final entries = storage.getKeys().map( + (key) => MapEntry(key, storage.read(key)), + ); + return logger.writeLogFile( + testId: filePrefix, + name: 'storage', + content: _prettyJson(Map.fromEntries(entries)), + extension: 'json', + ); + } + + return logger.writeLogFile( + testId: filePrefix, + name: 'storage_$key', + content: _prettyJson(StorageManager().read(key)), + extension: 'json', + ); +} + +String captureDebugDumpApp() { + final previousDebugPrint = debugPrint; + final lines = []; + debugPrint = (String? message, {int? wrapWidth}) { + if (message != null) { + lines.add(message); + } + }; + try { + debugDumpApp(); + } finally { + debugPrint = previousDebugPrint; + } + if (lines.isEmpty) { + return ''; + } + return lines.join('\n'); +} + +String _prettyJson(Object? value) { + return JsonEncoder.withIndent(' ', (value) => value.toString()) + .convert(value); +} diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart index 63ad7a41b..a3c2386fa 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart @@ -8,6 +8,7 @@ import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; class EnsembleTestContext { final EnsembleTestCase testCase; + final EnsembleTestConfig config; final TestApiProviderOverlay apiOverlay; final TestLogger logger; final EnsembleTestSetup setup; @@ -19,12 +20,16 @@ class EnsembleTestContext { EnsembleTestContext({ required this.testCase, + this.config = const EnsembleTestConfig(), required this.apiOverlay, required this.logger, required this.setup, }); - factory EnsembleTestContext.fromTestCase(EnsembleTestCase testCase) { + factory EnsembleTestContext.fromTestCase( + EnsembleTestCase testCase, { + EnsembleTestConfig config = const EnsembleTestConfig(), + }) { final logger = TestLogger(); final mockApi = TestApiProviderOverlay( mocks: Map.from(testCase.mocks.apis), @@ -47,6 +52,7 @@ class EnsembleTestContext { final ctx = EnsembleTestContext( testCase: testCase, + config: config, apiOverlay: mockApi, logger: logger, setup: setup, diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index a61fe9764..cde687d50 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -370,13 +370,18 @@ class EnsembleTestHarness { required EnsembleTestCase testCase, EnsembleConfig? existingConfig, EnsembleTestContext? context, + EnsembleTestConfig suiteConfig = const EnsembleTestConfig(), }) async { resetTestRuntime(); ScreenTracker().clearAll(); YamlTestSession.navigationFlow.clear(); - final ctx = context ?? EnsembleTestContext.fromTestCase(testCase); - final screenshotDevice = screenshotDeviceForTestCase(testCase); + final ctx = context ?? + EnsembleTestContext.fromTestCase( + testCase, + config: suiteConfig, + ); + final screenshotDevice = screenshotDeviceForTestCase(testCase, ctx.config); if (screenshotDevice != null) { await _setViewportForDevice(tester, ctx, screenshotDevice); } diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index 71f5ed1d0..b9ecb0c9d 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -7,8 +7,11 @@ import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; import 'package:ensemble_test_runner/reporters/test_reporter.dart'; import 'package:ensemble_test_runner/runner/app_performance_log.dart'; +import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/live_async_call.dart'; @@ -20,8 +23,19 @@ import 'package:flutter_test/flutter_test.dart'; typedef EnsembleTestRunOutput = ({ EnsembleSingleTestResult result, EnsembleConfig config, + EnsembleTestContext context, }); +class EnsembleTestPlanRunResult { + final Map resultsById; + final List suiteLogs; + + const EnsembleTestPlanRunResult({ + required this.resultsById, + this.suiteLogs = const [], + }); +} + /// Executes parsed Ensemble YAML test plans against a widget tester. class EnsembleTestRunner { /// Harness used to boot and reset the real Ensemble runtime. @@ -31,11 +45,15 @@ class EnsembleTestRunner { EnsembleTestRunner({required this.harness}); /// Runs every test in [plan] and returns results keyed by test id. - Future> runPlan( + Future runPlan( EnsembleTestExecutionPlan plan, WidgetTester tester, ) async { final resultsById = {}; + final suiteLogger = TestLogger(); + final suiteFrames = []; + final suiteApiCalls = []; + EnsembleTestContext? lastContext; var config = await harness.buildConfig(); for (final def in plan.ordered) { @@ -63,28 +81,48 @@ class EnsembleTestRunner { final out = await runOne( test, tester, + suiteConfig: plan.config, existingConfig: config, continuation: test.hasPrerequisite, ); resultsById[test.id] = out.result; config = out.config; + lastContext = out.context; + _appendSuiteFrames(suiteFrames, out.context.runtime.appFrameTimings); + suiteApiCalls.addAll(out.context.apiOverlay.calls); } - return resultsById; + final suiteLogs = await _writeSuiteLogs( + tester: tester, + config: plan.config, + logger: suiteLogger, + frames: suiteFrames, + apiCalls: suiteApiCalls, + lastContext: lastContext, + ); + + return EnsembleTestPlanRunResult( + resultsById: resultsById, + suiteLogs: suiteLogs, + ); } /// Runs a single [test], optionally continuing an existing app session. Future runOne( EnsembleTestCase test, WidgetTester tester, { + EnsembleTestConfig suiteConfig = const EnsembleTestConfig(), EnsembleConfig? existingConfig, bool continuation = false, }) async { final stopwatch = Stopwatch()..start(); void Function(List)? timingsCallback; + final ctx = EnsembleTestContext.fromTestCase( + test, + config: suiteConfig, + ); try { - final ctx = EnsembleTestContext.fromTestCase(test); timingsCallback = (List timings) { ctx.runtime.addFrameTimings(timings); }; @@ -111,6 +149,7 @@ class EnsembleTestRunner { testCase: test, existingConfig: existingConfig, context: ctx, + suiteConfig: suiteConfig, ); } await YamlTestSession.navigationFlow.flushPending(); @@ -125,7 +164,7 @@ class EnsembleTestRunner { config: config, stopwatch: stopwatch, ); - return (result: result, config: config); + return (result: result, config: config, context: ctx); } catch (error, stackTrace) { final config = existingConfig ?? Ensemble().getConfig(); return ( @@ -138,6 +177,7 @@ class EnsembleTestRunner { report: buildTestReportDetails(test), ), config: config ?? await harness.buildConfig(), + context: ctx, ); } finally { TestErrorTracker.reset(); @@ -176,7 +216,6 @@ class EnsembleTestRunner { } catch (error, stackTrace) { await _settleLiveApiWork(tester, ctx); await _flushPendingScreenshots(tester, ctx); - await _writeAutomaticPerformanceLog(tester, ctx); await YamlTestSession.navigationFlow.flushPending(); return EnsembleSingleTestResult.failed( testId: test.id, @@ -195,7 +234,6 @@ class EnsembleTestRunner { await YamlTestSession.navigationFlow.flushPending(); await _settleLiveApiWork(tester, ctx); await _flushPendingScreenshots(tester, ctx); - await _writeAutomaticPerformanceLog(tester, ctx); return EnsembleSingleTestResult.passed( testId: test.id, @@ -211,7 +249,7 @@ class EnsembleTestRunner { required TestStep step, required int stepIndex, }) async { - final options = executor.context.testCase.options.screenshots; + final options = executor.context.config.screenshots; if (!options.shouldCaptureStep(step.type)) return; await _captureAutomaticScreenshot( executor, @@ -226,7 +264,7 @@ class EnsembleTestRunner { }) { return ExtendedStepHandlers.captureScreenshot( executor, - args: executor.context.testCase.options.screenshots.toScreenshotArgs({ + args: executor.context.config.screenshots.toScreenshotArgs({ 'name': name, }), deferWrite: true, @@ -237,15 +275,79 @@ class EnsembleTestRunner { String _safeArtifactName(String value) => value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); - Future _writeAutomaticPerformanceLog( - WidgetTester tester, - EnsembleTestContext ctx, - ) async { - if (!ctx.testCase.options.performance.enabled) return; - final path = await tester.runAsync(() { - return writeAppPerformanceLog(ctx); - }); - ctx.logger.log('appPerformance: $path'); + void _appendSuiteFrames( + List suiteFrames, + List testFrames, + ) { + for (final frame in testFrames) { + suiteFrames.add( + AppFrameTimingEntry( + frameNumber: suiteFrames.length + 1, + buildStartMicros: frame.buildStartMicros, + buildMs: frame.buildMs, + rasterMs: frame.rasterMs, + vsyncOverheadMs: frame.vsyncOverheadMs, + totalSpanMs: frame.totalSpanMs, + ), + ); + } + } + + Future> _writeSuiteLogs({ + required WidgetTester tester, + required EnsembleTestConfig config, + required TestLogger logger, + required List frames, + required List apiCalls, + required EnsembleTestContext? lastContext, + }) async { + final logs = []; + + if (config.performance.enabled) { + final path = await tester.runAsync(() { + return writePerformanceLog( + logger: logger, + filePrefix: '', + name: 'app_performance', + frames: frames, + ); + }); + logs.add('appPerformance: $path'); + } + + if (config.dumpTree.enabled && lastContext != null) { + final path = await tester.runAsync(() { + return writeDumpTreeLogFile(logger: logger, filePrefix: ''); + }); + logs.add('dumpTree: $path'); + } + + if (config.logApiCalls.enabled) { + final path = await tester.runAsync(() { + return writeApiCallsLogFile( + logger: logger, + filePrefix: '', + calls: apiCalls, + ); + }); + logs.add('apiCalls: $path'); + } + + if (config.logStorage.enabled && lastContext != null) { + final key = config.logStorage.key; + final path = await tester.runAsync(() { + return writeStorageLogFile( + logger: logger, + filePrefix: '', + key: key, + ); + }); + logs.add( + key == null || key.isEmpty ? 'storage: $path' : 'storage[$key]: $path', + ); + } + + return logs; } Future _settleLiveApiWork( diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index 63266790d..ab3b47de4 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -6,6 +6,8 @@ import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; class EnsembleTestSchemaBuilder { static const schemaId = 'https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json'; + static const configSchemaId = + 'https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json'; static const schemaVersion = 'https://json-schema.org/draft/2020-12/schema'; static Map build() { @@ -50,36 +52,6 @@ class EnsembleTestSchemaBuilder { }, }, }, - 'options': { - 'type': 'object', - 'additionalProperties': false, - 'properties': { - 'screenshots': { - 'type': 'object', - 'additionalProperties': false, - 'properties': { - 'enabled': {'type': 'boolean'}, - 'platform': {'type': 'string'}, - 'model': {'type': 'string'}, - 'includeSteps': { - 'type': 'array', - 'items': {'type': 'string'}, - }, - 'excludeSteps': { - 'type': 'array', - 'items': {'type': 'string'}, - }, - }, - }, - 'performance': { - 'type': 'object', - 'additionalProperties': false, - 'properties': { - 'enabled': {'type': 'boolean'}, - }, - }, - }, - }, 'testCase': { 'type': 'object', 'additionalProperties': false, @@ -127,7 +99,6 @@ class EnsembleTestSchemaBuilder { 'ID of another test that must run before this one in the same app session', }, 'initialState': {'\$ref': '#/\$defs/initialState'}, - 'options': {'\$ref': '#/\$defs/options'}, 'mocks': {'\$ref': '#/\$defs/mocks'}, 'steps': { 'type': 'array', @@ -215,4 +186,69 @@ class EnsembleTestSchemaBuilder { pretty ? const JsonEncoder.withIndent(' ') : const JsonEncoder(); return encoder.convert(build()); } + + static Map buildConfig() { + return { + '\$schema': schemaVersion, + '\$id': configSchemaId, + 'title': 'Ensemble declarative test config', + 'description': 'Suite-wide config for app-local tests/config.yaml', + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'screenshots': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + 'platform': {'type': 'string'}, + 'model': {'type': 'string'}, + 'includeSteps': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'excludeSteps': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + }, + }, + 'performance': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + }, + }, + 'dumpTree': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + }, + }, + 'logApiCalls': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + }, + }, + 'logStorage': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + 'key': {'type': 'string'}, + }, + }, + }, + }; + } + + static String buildConfigJson({bool pretty = true}) { + final encoder = + pretty ? const JsonEncoder.withIndent(' ') : const JsonEncoder(); + return encoder.convert(buildConfig()); + } } diff --git a/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart b/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart index 3f49b7692..ab968048f 100644 --- a/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart +++ b/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart @@ -2,6 +2,7 @@ import 'dart:convert'; import 'dart:io'; import 'package:ensemble_test_runner/inspect/ensemble_app_inspector.dart'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; import 'package:path/path.dart' as p; import 'package:yaml/yaml.dart'; @@ -122,6 +123,23 @@ class EnsembleTestValidator { return EnsembleTestValidationResult(issues); } + final configFile = File(p.join(testsDir.path, 'config.yaml')); + if (configFile.existsSync()) { + try { + EnsembleTestParser.parseConfigString( + configFile.readAsStringSync(), + sourcePath: p.relative(configFile.path, from: appDir), + ); + } catch (error) { + add( + ValidationSeverity.error, + 'config', + error.toString(), + path: p.relative(configFile.path, from: appDir), + ); + } + } + final screensByName = { for (final screen in inspection.screens) screen.name: screen }; @@ -141,6 +159,16 @@ class EnsembleTestValidator { continue; } + if (doc.containsKey('options')) { + add( + ValidationSeverity.error, + 'options', + 'Root-level "options" is no longer supported. Move shared settings to tests/config.yaml.', + path: relativePath, + ); + continue; + } + final id = doc['id']?.toString(); if (id == null || id.isEmpty) { add(ValidationSeverity.error, 'missingId', 'Each test must have an id', diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 6f67de7ba..40c6ba867 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -72,7 +72,7 @@ steps: ); }); - test('parses screenshot options', () { + test('rejects root-level options in test files', () { const yaml = ''' id: visual_debug startScreen: Home @@ -88,8 +88,30 @@ steps: id: start_button '''; - final test = EnsembleTestParser.parseString(yaml); - final screenshots = test.options.screenshots; + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Move shared screenshots/performance settings'), + ), + ), + ); + }); + + test('parses suite config screenshots', () { + const yaml = ''' +screenshots: + enabled: true + platform: android + model: Samsung Galaxy S20 + includeSteps: [tap, waitForNavigation] + excludeSteps: [wait] +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + final screenshots = config.screenshots; expect(screenshots.enabled, isTrue); expect(screenshots.platform, 'android'); expect(screenshots.model, 'Samsung Galaxy S20'); @@ -100,20 +122,32 @@ steps: expect(screenshots.shouldCaptureStep('settle'), isFalse); }); - test('parses performance options', () { + test('parses suite config performance', () { const yaml = ''' -id: perf_debug -startScreen: Home -options: - performance: - enabled: true -steps: - - tap: - id: start_button +performance: + enabled: true '''; - final test = EnsembleTestParser.parseString(yaml); - expect(test.options.performance.enabled, isTrue); + final config = EnsembleTestParser.parseConfigString(yaml); + expect(config.performance.enabled, isTrue); + }); + + test('parses suite config debug artifacts', () { + const yaml = ''' +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true + key: auth +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + expect(config.dumpTree.enabled, isTrue); + expect(config.logApiCalls.enabled, isTrue); + expect(config.logStorage.enabled, isTrue); + expect(config.logStorage.key, 'auth'); }); test('parses AI-friendly metadata fields', () { diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart index c6d779f86..4ac7906ea 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart @@ -46,11 +46,29 @@ void main() { expect((decoded['required'] as List), isNot(contains('startScreen'))); expect(decoded['properties'], contains('id')); expect(decoded['properties'], isNot(contains('tests'))); + expect(decoded['properties'], isNot(contains('options'))); expect(decoded['properties'], contains('startScreen')); expect(decoded['properties'], contains('prerequisite')); expect(decoded['oneOf'], isA()); }); + test('config schema includes suite screenshots and performance settings', () { + final json = EnsembleTestSchemaBuilder.buildConfigJson(); + final decoded = jsonDecode(json) as Map; + final properties = decoded['properties'] as Map; + + expect(decoded['\$schema'], EnsembleTestSchemaBuilder.schemaVersion); + expect(properties, contains('screenshots')); + expect(properties, contains('performance')); + expect(properties, contains('dumpTree')); + expect(properties, contains('logApiCalls')); + expect(properties, contains('logStorage')); + expect( + (properties['screenshots'] as Map)['properties'], + containsPair('model', {'type': 'string'}), + ); + }); + test('initialState schema accepts storage, keychain, and env maps', () { final schema = EnsembleTestSchemaBuilder.build(); final initialState = diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart index 04ed2ad9a..fa41e8b31 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart @@ -5,14 +5,22 @@ import 'package:flutter_test/flutter_test.dart'; void main() { test('committed schema matches generator output', () { - final path = 'assets/schema/ensemble_tests_schema.json'; - final committed = File(path).readAsStringSync().trim(); - final generated = EnsembleTestSchemaBuilder.buildJson().trim(); - expect( - committed, - generated, - reason: - 'Run: cd tools/ensemble_test_runner && dart run tool/generate_schema.dart', - ); + final schemas = { + 'assets/schema/ensemble_tests_schema.json': + EnsembleTestSchemaBuilder.buildJson(), + 'assets/schema/ensemble_test_config_schema.json': + EnsembleTestSchemaBuilder.buildConfigJson(), + }; + + for (final entry in schemas.entries) { + final committed = File(entry.key).readAsStringSync().trim(); + final generated = entry.value.trim(); + expect( + committed, + generated, + reason: + 'Run: cd tools/ensemble_test_runner && dart run tool/generate_schema.dart', + ); + } }); } diff --git a/tools/ensemble_test_runner/test/screenshot_device_test.dart b/tools/ensemble_test_runner/test/screenshot_device_test.dart index e025593ce..d0d79714b 100644 --- a/tools/ensemble_test_runner/test/screenshot_device_test.dart +++ b/tools/ensemble_test_runner/test/screenshot_device_test.dart @@ -49,7 +49,7 @@ void main() { expect(device, isNull); }); - test('uses screenshot options device when enabled', () { + test('uses suite screenshot config device when enabled', () { final device = screenshotDeviceForTestCase( const EnsembleTestCase( id: 'screenshots', @@ -57,12 +57,12 @@ void main() { steps: [ TestStep(type: 'tap', args: {'id': 'button'}), ], - options: EnsembleTestOptions( - screenshots: ScreenshotOptions( - enabled: true, - platform: 'android', - model: 'Samsung Galaxy S20', - ), + ), + const EnsembleTestConfig( + screenshots: ScreenshotConfig( + enabled: true, + platform: 'android', + model: 'Samsung Galaxy S20', ), ), ); diff --git a/tools/ensemble_test_runner/test/test_reporter_test.dart b/tools/ensemble_test_runner/test/test_reporter_test.dart index 6fb1e0634..b1f78d01b 100644 --- a/tools/ensemble_test_runner/test/test_reporter_test.dart +++ b/tools/ensemble_test_runner/test/test_reporter_test.dart @@ -149,6 +149,28 @@ void main() { expect(output, isNot(contains('MaterialApp'))); }); + test('prints suite logs separately from test logs', () { + final output = TestReporter().formatSummary( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.passed( + testId: 'login_test', + durationMs: 42, + ), + ], + suiteLogs: const [ + 'storage: build/ensemble_test_runner/logs/storage.json', + ], + ), + ); + + expect(output, contains('suite artifacts:')); + expect( + output, + contains('storage: build/ensemble_test_runner/logs/storage.json'), + ); + }); + test('formats compact failure summary without repeating boxed report', () { final output = TestReporter().formatFailureSummary( EnsembleTestRunResult( diff --git a/tools/ensemble_test_runner/tool/generate_schema.dart b/tools/ensemble_test_runner/tool/generate_schema.dart index 669e77528..63de542bc 100644 --- a/tools/ensemble_test_runner/tool/generate_schema.dart +++ b/tools/ensemble_test_runner/tool/generate_schema.dart @@ -2,7 +2,7 @@ import 'dart:io'; import 'package:ensemble_test_runner/schema/ensemble_test_schema_builder.dart'; -/// Regenerates [assets/schema/ensemble_tests_schema.json] from the step registry. +/// Regenerates JSON Schemas from the step registry. /// /// Run from this package: /// dart run tool/generate_schema.dart @@ -17,4 +17,11 @@ void main() { outFile.parent.createSync(recursive: true); outFile.writeAsStringSync('${EnsembleTestSchemaBuilder.buildJson()}\n'); stdout.writeln('Wrote ${outFile.path}'); + + final configOutFile = File('assets/schema/ensemble_test_config_schema.json'); + configOutFile.parent.createSync(recursive: true); + configOutFile.writeAsStringSync( + '${EnsembleTestSchemaBuilder.buildConfigJson()}\n', + ); + stdout.writeln('Wrote ${configOutFile.path}'); } From b564ceb16a1bae695503abc6ce5dcda7b3d2c54c Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Tue, 14 Jul 2026 01:32:14 +0500 Subject: [PATCH 17/29] feat(test_runner): enhance performance logging with detailed metrics and correlation Expanded the performance logging capabilities in the ensemble test runner by introducing new metrics and correlations. Added attributes for tracking performance markers, including worst steps and screens, jank clusters, and API call correlations. Updated the `writePerformanceLog` function to include these new metrics in the output JSON. Enhanced the `EnsembleTestRunner` to record performance markers during test execution, providing a comprehensive overview of performance data. Updated tests to validate the new logging features and ensure accurate performance reporting. --- .../lib/runner/app_performance_log.dart | 204 +++++++++++++++++- .../lib/runner/ensemble_test_runner.dart | 109 ++++++++++ .../lib/runner/test_runtime_state.dart | 63 ++++++ .../test/test_step_executor_test.dart | 89 +++++++- 4 files changed, 461 insertions(+), 4 deletions(-) diff --git a/tools/ensemble_test_runner/lib/runner/app_performance_log.dart b/tools/ensemble_test_runner/lib/runner/app_performance_log.dart index bee2ab62b..794a7f462 100644 --- a/tools/ensemble_test_runner/lib/runner/app_performance_log.dart +++ b/tools/ensemble_test_runner/lib/runner/app_performance_log.dart @@ -1,5 +1,6 @@ import 'dart:convert'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/mocks/test_logger.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; @@ -10,6 +11,8 @@ Future writeAppPerformanceLog(EnsembleTestContext context) { filePrefix: context.testCase.id, name: 'app_performance', frames: context.runtime.appFrameTimings, + markers: context.runtime.performanceMarkers, + apiCalls: context.apiOverlay.calls, ); } @@ -18,13 +21,35 @@ Future writePerformanceLog({ required String filePrefix, required String name, required List frames, + List markers = const [], + List apiCalls = const [], }) { final jankyFrames = frames.where((frame) => frame.isJanky).length; - final slowestFrames = frames.toList() - ..sort((a, b) => b.totalSpanMs.compareTo(a.totalSpanMs)); + final attributedFrames = frames + .map((frame) => _AttributedFrame(frame, _markerForFrame(frame, markers))) + .toList(growable: false); + final slowestFrames = attributedFrames.toList() + ..sort((a, b) => b.frame.totalSpanMs.compareTo(a.frame.totalSpanMs)); + final worstSteps = _worstSteps(attributedFrames); + final worstScreens = _worstScreens(attributedFrames); + final jankClusters = _jankClusters(attributedFrames); + final apiCorrelation = _apiCorrelation(attributedFrames, apiCalls); final content = const JsonEncoder.withIndent(' ').convert({ 'testId': filePrefix.isEmpty ? 'suite' : filePrefix, 'frameBudgetMs': AppFrameTimingEntry.frameBudgetMs, + 'summary': { + 'totalFrames': frames.length, + 'jankyFrames': jankyFrames, + 'jankyFrameRate': _ratio(jankyFrames, frames.length), + 'averageBuildMs': _average(frames.map((frame) => frame.buildMs)), + 'averageRasterMs': _average(frames.map((frame) => frame.rasterMs)), + 'averageTotalSpanMs': _average(frames.map((frame) => frame.totalSpanMs)), + 'maxBuildMs': _max(frames.map((frame) => frame.buildMs)), + 'maxRasterMs': _max(frames.map((frame) => frame.rasterMs)), + 'maxTotalSpanMs': _max(frames.map((frame) => frame.totalSpanMs)), + if (worstSteps.isNotEmpty) 'worstStep': worstSteps.first['step'], + if (worstScreens.isNotEmpty) 'worstScreen': worstScreens.first['screen'], + }, 'totalFrames': frames.length, 'jankyFrames': jankyFrames, 'jankyFrameRate': _ratio(jankyFrames, frames.length), @@ -34,9 +59,13 @@ Future writePerformanceLog({ 'maxBuildMs': _max(frames.map((frame) => frame.buildMs)), 'maxRasterMs': _max(frames.map((frame) => frame.rasterMs)), 'maxTotalSpanMs': _max(frames.map((frame) => frame.totalSpanMs)), + 'worstSteps': worstSteps, + 'worstScreens': worstScreens, + 'jankClusters': jankClusters, + 'apiCorrelation': apiCorrelation, 'slowestFrames': slowestFrames.take(10).map((frame) => frame.toJson()).toList(), - 'frames': frames.map((frame) => frame.toJson()).toList(), + 'frames': attributedFrames.map((frame) => frame.toJson()).toList(), }); return logger.writeLogFile( @@ -47,6 +76,175 @@ Future writePerformanceLog({ ); } +PerformanceMarker? _markerForFrame( + AppFrameTimingEntry frame, + List markers, +) { + for (final marker in markers.reversed) { + if (marker.containsFrame(frame.frameNumber)) return marker; + } + return null; +} + +List> _worstSteps(List<_AttributedFrame> frames) { + final byStep = >{}; + for (final frame in frames) { + final marker = frame.marker; + if (marker == null) continue; + byStep.putIfAbsent(marker.label, () => []).add(frame); + } + final rows = byStep.entries.map((entry) { + final stepFrames = entry.value; + final janky = stepFrames.where((frame) => frame.frame.isJanky).toList(); + return { + 'step': entry.key, + 'testId': stepFrames.first.marker?.testId, + if (stepFrames.first.marker?.stepIndex != null) + 'stepIndex': stepFrames.first.marker?.stepIndex, + 'screen': stepFrames.first.marker?.screen, + 'phase': stepFrames.first.marker?.phase, + 'totalFrames': stepFrames.length, + 'jankyFrames': janky.length, + 'jankyFrameRate': _ratio(janky.length, stepFrames.length), + 'maxTotalSpanMs': + _max(stepFrames.map((frame) => frame.frame.totalSpanMs)), + 'totalJankOverBudgetMs': _jankOverBudget(janky), + }; + }).toList() + ..sort(_rankJankRows); + return rows.take(10).toList(); +} + +List> _worstScreens(List<_AttributedFrame> frames) { + final byScreen = >{}; + for (final frame in frames) { + final screen = frame.marker?.screen; + if (screen == null || screen.isEmpty) continue; + byScreen.putIfAbsent(screen, () => []).add(frame); + } + final rows = byScreen.entries.map((entry) { + final screenFrames = entry.value; + final janky = screenFrames.where((frame) => frame.frame.isJanky).toList(); + return { + 'screen': entry.key, + 'totalFrames': screenFrames.length, + 'jankyFrames': janky.length, + 'jankyFrameRate': _ratio(janky.length, screenFrames.length), + 'maxTotalSpanMs': + _max(screenFrames.map((frame) => frame.frame.totalSpanMs)), + 'totalJankOverBudgetMs': _jankOverBudget(janky), + }; + }).toList() + ..sort(_rankJankRows); + return rows.take(10).toList(); +} + +int _rankJankRows(Map a, Map b) { + final byJank = (b['jankyFrames'] as int).compareTo(a['jankyFrames'] as int); + if (byJank != 0) return byJank; + return (b['maxTotalSpanMs'] as double) + .compareTo(a['maxTotalSpanMs'] as double); +} + +List> _jankClusters(List<_AttributedFrame> frames) { + final clusters = >[]; + var current = <_AttributedFrame>[]; + for (final frame in frames) { + if (!frame.frame.isJanky) { + if (current.isNotEmpty) { + clusters.add(current); + current = <_AttributedFrame>[]; + } + continue; + } + if (current.isEmpty || + frame.frame.frameNumber - current.last.frame.frameNumber <= 2) { + current.add(frame); + } else { + clusters.add(current); + current = [frame]; + } + } + if (current.isNotEmpty) clusters.add(current); + + final rows = clusters.map((cluster) { + final slowest = cluster.toList() + ..sort((a, b) => b.frame.totalSpanMs.compareTo(a.frame.totalSpanMs)); + final marker = slowest.first.marker; + return { + 'startFrame': cluster.first.frame.frameNumber, + 'endFrame': cluster.last.frame.frameNumber, + 'jankyFrames': cluster.length, + 'maxTotalSpanMs': slowest.first.frame.totalSpanMs, + 'totalJankOverBudgetMs': _jankOverBudget(cluster), + if (marker != null) ..._markerJson(marker), + }; + }).toList() + ..sort(_rankJankRows); + return rows.take(10).toList(); +} + +List> _apiCorrelation( + List<_AttributedFrame> frames, + List apiCalls, +) { + final rows = >[]; + final jankyMarkers = frames + .where((frame) => frame.frame.isJanky && frame.marker != null) + .map((frame) => frame.marker!) + .toSet(); + for (final marker in jankyMarkers) { + final calls = apiCalls + .where((call) => marker.overlaps(call.timestamp, + tolerance: const Duration(milliseconds: 250))) + .toList(); + if (calls.isEmpty) continue; + rows.add({ + ..._markerJson(marker), + 'apiCalls': [ + for (final call in calls) + { + 'name': call.name, + 'timestamp': call.timestamp.toIso8601String(), + }, + ], + }); + } + return rows.take(25).toList(); +} + +double _jankOverBudget(Iterable<_AttributedFrame> frames) { + return _round( + frames.fold( + 0, + (sum, frame) => + sum + + (frame.frame.totalSpanMs - AppFrameTimingEntry.frameBudgetMs) + .clamp(0, double.infinity), + ), + ); +} + +Map _markerJson(PerformanceMarker marker) => { + 'testId': marker.testId, + if (marker.stepIndex != null) 'stepIndex': marker.stepIndex, + 'step': marker.label, + if (marker.screen != null) 'screen': marker.screen, + 'phase': marker.phase, + }; + +class _AttributedFrame { + final AppFrameTimingEntry frame; + final PerformanceMarker? marker; + + const _AttributedFrame(this.frame, this.marker); + + Map toJson() => { + ...frame.toJson(), + if (marker != null) ..._markerJson(marker!), + }; +} + double _average(Iterable values) { final list = values.toList(growable: false); if (list.isEmpty) return 0; diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index b9ecb0c9d..af85eea36 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -52,6 +52,7 @@ class EnsembleTestRunner { final resultsById = {}; final suiteLogger = TestLogger(); final suiteFrames = []; + final suiteMarkers = []; final suiteApiCalls = []; EnsembleTestContext? lastContext; var config = await harness.buildConfig(); @@ -88,7 +89,13 @@ class EnsembleTestRunner { resultsById[test.id] = out.result; config = out.config; lastContext = out.context; + final frameOffset = suiteFrames.length; _appendSuiteFrames(suiteFrames, out.context.runtime.appFrameTimings); + suiteMarkers.addAll( + out.context.runtime.performanceMarkers.map( + (marker) => marker.shiftedFrames(frameOffset), + ), + ); suiteApiCalls.addAll(out.context.apiOverlay.calls); } @@ -97,6 +104,7 @@ class EnsembleTestRunner { config: plan.config, logger: suiteLogger, frames: suiteFrames, + markers: suiteMarkers, apiCalls: suiteApiCalls, lastContext: lastContext, ); @@ -131,6 +139,8 @@ class EnsembleTestRunner { ctx.apiOverlay.liveAsyncRunner = tester.runAsync; LiveAsyncCallSupport.runner = tester.runAsync; TestErrorTracker.install(ctx.runtime); + final startupStartFrame = ctx.runtime.appFrameTimings.length + 1; + final startupStartTime = DateTime.now(); late final EnsembleConfig config; if (continuation) { @@ -156,6 +166,15 @@ class EnsembleTestRunner { YamlTestSession.navigationFlow.beginTest( ScreenTracker().getCurrentScreenIdentifier(), ); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: null, + label: '${test.id} startup', + phase: 'startup', + startFrame: startupStartFrame, + startTime: startupStartTime, + ); final result = await _executeSteps( test: test, @@ -205,6 +224,8 @@ class EnsembleTestRunner { ); for (var i = 0; i < test.steps.length; i++) { final step = test.steps[i]; + final startFrame = ctx.runtime.appFrameTimings.length + 1; + final startTime = DateTime.now(); try { await executor.execute(step); await YamlTestSession.navigationFlow.flushPending(); @@ -213,10 +234,39 @@ class EnsembleTestRunner { step: step, stepIndex: i, ); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: i + 1, + label: '${test.id} step ${i + 1} ${formatStepBrief(step)}', + phase: _phaseForStep(step), + startFrame: startFrame, + startTime: startTime, + ); } catch (error, stackTrace) { + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: i + 1, + label: '${test.id} step ${i + 1} ${formatStepBrief(step)}', + phase: _phaseForStep(step), + startFrame: startFrame, + startTime: startTime, + ); + final idleStartFrame = ctx.runtime.appFrameTimings.length + 1; + final idleStartTime = DateTime.now(); await _settleLiveApiWork(tester, ctx); await _flushPendingScreenshots(tester, ctx); await YamlTestSession.navigationFlow.flushPending(); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: null, + label: '${test.id} failure cleanup', + phase: 'idle', + startFrame: idleStartFrame, + startTime: idleStartTime, + ); return EnsembleSingleTestResult.failed( testId: test.id, metadata: test.metadataJson, @@ -232,8 +282,19 @@ class EnsembleTestRunner { } await YamlTestSession.navigationFlow.flushPending(); + final idleStartFrame = ctx.runtime.appFrameTimings.length + 1; + final idleStartTime = DateTime.now(); await _settleLiveApiWork(tester, ctx); await _flushPendingScreenshots(tester, ctx); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: null, + label: '${test.id} idle', + phase: 'idle', + startFrame: idleStartFrame, + startTime: idleStartTime, + ); return EnsembleSingleTestResult.passed( testId: test.id, @@ -275,6 +336,51 @@ class EnsembleTestRunner { String _safeArtifactName(String value) => value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); + void _recordPerformanceMarker({ + required EnsembleTestContext ctx, + required String testId, + required int? stepIndex, + required String label, + required String phase, + required int startFrame, + required DateTime startTime, + }) { + final endFrame = ctx.runtime.appFrameTimings.length; + if (endFrame < startFrame) return; + ctx.runtime.recordPerformanceMarker( + PerformanceMarker( + testId: testId, + stepIndex: stepIndex, + label: label, + screen: ScreenTracker().getCurrentScreenIdentifier(), + phase: phase, + startFrame: startFrame, + endFrame: endFrame, + startTime: startTime, + endTime: DateTime.now(), + ), + ); + } + + String _phaseForStep(TestStep step) { + switch (step.type) { + case 'waitForNavigation': + case 'openScreen': + case 'goBack': + case 'restartApp': + case 'reloadScreen': + case 'launchApp': + return 'navigation'; + case 'settle': + case 'wait': + case 'waitFor': + case 'waitForApi': + return 'settle'; + default: + return 'step'; + } + } + void _appendSuiteFrames( List suiteFrames, List testFrames, @@ -298,6 +404,7 @@ class EnsembleTestRunner { required EnsembleTestConfig config, required TestLogger logger, required List frames, + required List markers, required List apiCalls, required EnsembleTestContext? lastContext, }) async { @@ -310,6 +417,8 @@ class EnsembleTestRunner { filePrefix: '', name: 'app_performance', frames: frames, + markers: markers, + apiCalls: apiCalls, ); }); logs.add('appPerformance: $path'); diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index c867680cb..30d6709fa 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -9,6 +9,7 @@ class TestRuntimeState { final List consoleLogs = []; final List flutterErrors = []; final List appFrameTimings = []; + final List performanceMarkers = []; final List Function()> pendingScreenshotWrites = []; Map? authUser; final Map permissions = {}; @@ -21,6 +22,7 @@ class TestRuntimeState { consoleLogs.clear(); flutterErrors.clear(); appFrameTimings.clear(); + performanceMarkers.clear(); pendingScreenshotWrites.clear(); authUser = null; permissions.clear(); @@ -39,6 +41,67 @@ class TestRuntimeState { ); } } + + void recordPerformanceMarker(PerformanceMarker marker) { + performanceMarkers.add(marker); + } +} + +class PerformanceMarker { + final String testId; + final int? stepIndex; + final String label; + final String? screen; + final String phase; + final int startFrame; + final int endFrame; + final DateTime startTime; + final DateTime endTime; + + const PerformanceMarker({ + required this.testId, + required this.stepIndex, + required this.label, + required this.screen, + required this.phase, + required this.startFrame, + required this.endFrame, + required this.startTime, + required this.endTime, + }); + + PerformanceMarker shiftedFrames(int offset) => PerformanceMarker( + testId: testId, + stepIndex: stepIndex, + label: label, + screen: screen, + phase: phase, + startFrame: startFrame + offset, + endFrame: endFrame + offset, + startTime: startTime, + endTime: endTime, + ); + + bool containsFrame(int frameNumber) => + frameNumber >= startFrame && frameNumber <= endFrame; + + bool overlaps(DateTime timestamp, {Duration tolerance = Duration.zero}) { + final lower = startTime.subtract(tolerance); + final upper = endTime.add(tolerance); + return !timestamp.isBefore(lower) && !timestamp.isAfter(upper); + } + + Map toJson() => { + 'testId': testId, + if (stepIndex != null) 'stepIndex': stepIndex, + 'label': label, + if (screen != null) 'screen': screen, + 'phase': phase, + 'startFrame': startFrame, + 'endFrame': endFrame, + 'startTime': startTime.toIso8601String(), + 'endTime': endTime.toIso8601String(), + }; } class AppFrameTimingEntry { diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index a3e5e9db3..4be663faa 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -4,12 +4,16 @@ import 'dart:io'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; +import 'package:ensemble_test_runner/runner/app_performance_log.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:yaml/yaml.dart'; void main() { testWidgets('waitFor requires id or text', (tester) async { @@ -181,6 +185,7 @@ void main() { final json = jsonDecode(content) as Map; expect(json['testId'], 'performance_log_test'); expect(json['frameBudgetMs'], 16.67); + expect(json['summary'], containsPair('totalFrames', 1)); expect(json['totalFrames'], 1); expect(json['jankyFrames'], 0); expect(json['averageBuildMs'], 5); @@ -188,6 +193,88 @@ void main() { expect((json['frames'] as List).single, containsPair('totalSpanMs', 12)); }); + test('performance log attributes frames and ranks jank context', () async { + final logger = TestLogger(); + final start = DateTime.now(); + final apiTimestamp = start.add(const Duration(milliseconds: 20)); + final path = await writePerformanceLog( + logger: logger, + filePrefix: 'suite', + name: 'app_performance', + frames: const [ + AppFrameTimingEntry( + frameNumber: 1, + buildStartMicros: 1000, + buildMs: 30, + rasterMs: 0, + vsyncOverheadMs: 1, + totalSpanMs: 45, + ), + AppFrameTimingEntry( + frameNumber: 2, + buildStartMicros: 2000, + buildMs: 5, + rasterMs: 0, + vsyncOverheadMs: 1, + totalSpanMs: 10, + ), + AppFrameTimingEntry( + frameNumber: 3, + buildStartMicros: 3000, + buildMs: 40, + rasterMs: 0, + vsyncOverheadMs: 1, + totalSpanMs: 60, + ), + ], + markers: [ + PerformanceMarker( + testId: 'login_test', + stepIndex: 1, + label: 'login_test step 1 tap(login_button)', + screen: 'Login', + phase: 'step', + startFrame: 1, + endFrame: 3, + startTime: start, + endTime: start.add(const Duration(milliseconds: 100)), + ), + ], + apiCalls: [ + APICallRecord( + name: 'loginApi', + apiDefinition: loadYaml('{}') as YamlMap, + timestamp: apiTimestamp, + ), + ], + ); + + final json = jsonDecode(File(path).readAsStringSync()) as Map; + expect(json['summary'], containsPair('worstScreen', 'Login')); + expect( + json['summary'], + containsPair('worstStep', 'login_test step 1 tap(login_button)'), + ); + expect((json['frames'] as List).first, containsPair('screen', 'Login')); + expect((json['frames'] as List).first, containsPair('phase', 'step')); + expect( + ((json['worstSteps'] as List).first as Map)['step'], + 'login_test step 1 tap(login_button)', + ); + expect( + ((json['worstScreens'] as List).first as Map)['screen'], + 'Login', + ); + expect((json['jankClusters'] as List), isNotEmpty); + expect( + (((json['apiCorrelation'] as List).first as Map)['apiCalls'] as List) + .single, + containsPair('name', 'loginApi'), + ); + expect( + (json['slowestFrames'] as List).first, containsPair('screen', 'Login')); + }); + testWidgets('screenshot writes a framed png by default', (tester) async { await tester.pumpWidget( const MaterialApp(home: Text('screenshot target')), From b7dc4eb40d5a7f910eb63c4456bca99deccfad52 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Tue, 14 Jul 2026 03:24:50 +0500 Subject: [PATCH 18/29] feat(view_util): add support for testId in widget model construction Enhanced the ViewUtil class to include a new property, testId, in the model construction process. This allows for the preservation of explicit testId values without requiring an id. Additionally, added a new test case to verify this functionality, ensuring that the testId is correctly assigned and that the id property is not included when not specified. --- .../lib/framework/widget/view_util.dart | 3 +++ modules/ensemble/test/custom_widget_test.dart | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 modules/ensemble/test/custom_widget_test.dart diff --git a/modules/ensemble/lib/framework/widget/view_util.dart b/modules/ensemble/lib/framework/widget/view_util.dart index dc15e4809..f03fc447a 100644 --- a/modules/ensemble/lib/framework/widget/view_util.dart +++ b/modules/ensemble/lib/framework/widget/view_util.dart @@ -171,6 +171,9 @@ class ViewUtil { if (callerPayload?['id'] != null) { props["id"] = callerPayload?['id']; } + if (callerPayload?['testId'] != null) { + props["testId"] = callerPayload?['testId']; + } Map inputPayload = {}; if (callerPayload?['inputs'] is Map) { diff --git a/modules/ensemble/test/custom_widget_test.dart b/modules/ensemble/test/custom_widget_test.dart new file mode 100644 index 000000000..c3fb31fe7 --- /dev/null +++ b/modules/ensemble/test/custom_widget_test.dart @@ -0,0 +1,27 @@ +import 'package:ensemble/framework/widget/view_util.dart'; +import 'package:ensemble/widget/custom_widget/custom_widget_model.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + test('custom widget call preserves explicit testId without requiring id', () { + final customWidgets = { + 'MiniCard': loadYaml(''' +body: + Text: + text: Devices +''') as YamlMap, + }; + + final model = ViewUtil.buildModel( + loadYaml(''' +MiniCard: + testId: devices_mini_card +'''), + customWidgets, + ) as CustomWidgetModel; + + expect(model.props['testId'], 'devices_mini_card'); + expect(model.props.containsKey('id'), isFalse); + }); +} From 6adf5bf408850ddb84cf2aa2a6e52443731521e5 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Tue, 14 Jul 2026 03:39:56 +0500 Subject: [PATCH 19/29] feat(test_runner): add support for CLI inputs in ensemble tests Introduced repeatable `--input key=value` flags for passing test inputs via the command line. Updated the test runner to reference these inputs in the test YAML using `${inputs.key}`. Enhanced the `EnsembleTestParser` and related classes to handle input values, ensuring they can be utilized in initial states, mocks, and steps. Added tests to validate the correct resolution of CLI inputs and error handling for missing inputs, improving the flexibility and usability of the test framework. --- tools/ensemble_test_runner/README.md | 18 +++ .../doc/TEST_AUTHORING.md | 22 +++ .../lib/cli/ensemble_test_cli.dart | 49 ++++++ .../lib/cli/ensemble_test_cli_output.dart | 48 +++--- .../ensemble_test_execution_planner.dart | 2 + .../lib/entry/ensemble_test_entry.dart | 16 ++ .../lib/parser/ensemble_test_parser.dart | 142 +++++++++++++----- .../test/ensemble_test_cli_output_test.dart | 3 + .../test/ensemble_test_parser_test.dart | 51 +++++++ 9 files changed, 297 insertions(+), 54 deletions(-) diff --git a/tools/ensemble_test_runner/README.md b/tools/ensemble_test_runner/README.md index 1a693a88f..9252c5b12 100644 --- a/tools/ensemble_test_runner/README.md +++ b/tools/ensemble_test_runner/README.md @@ -97,6 +97,24 @@ By default, output is quiet: no `pub get` package list, no Flutter test progress Optional: `--app-dir=` when not running from the app root. +Pass test-runner inputs with repeatable `--input key=value` flags. Tests can +reference them as `${inputs.key}` in `initialState`, mocks, and steps: + +```bash +dart run ensemble_test_runner:ensemble_test \ + --input adminPassword='s4C>M7U6t~' \ + --input expectedDeviceCount=2 +``` + +```yaml +initialState: + keychain: + adminPassword: ${inputs.adminPassword} +steps: + - expectText: + text: ${inputs.expectedDeviceCount} +``` + The default suite timeout is 10 minutes. Override it when a flow should fail faster or when a long chain needs more time: diff --git a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md index 95d55df63..f089c391c 100644 --- a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md +++ b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md @@ -112,6 +112,28 @@ dart run ensemble_test_runner:ensemble_test --id=login_valid Prerequisites are included automatically when a selected test depends on them. +## CLI Inputs + +Use repeatable `--input key=value` flags for values that should come from the +command line: + +```sh +dart run ensemble_test_runner:ensemble_test \ + --input adminPassword='s4C>M7U6t~' \ + --input expectedDeviceCount=2 +``` + +Reference them in test YAML with `${inputs.key}`: + +```yaml +initialState: + keychain: + adminPassword: ${inputs.adminPassword} +steps: + - expectText: + text: ${inputs.expectedDeviceCount} +``` + ## CI Output Stable exit codes: diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart index ec09e50fb..61463c31c 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart @@ -1,3 +1,4 @@ +import 'dart:convert'; import 'dart:io'; import 'package:ensemble_test_runner/cli/ensemble_test_doctor.dart'; @@ -23,6 +24,7 @@ import 'package:ensemble_test_runner/validation/ensemble_test_validator.dart'; /// --feature= Run matching feature(s); repeatable /// --tag= Run matching tag(s); repeatable /// --path= Run matching test asset path(s); repeatable +/// --input key=value Provide a test input for ${inputs.key}; repeatable /// --timeout= Test suite timeout, e.g. 30s, 5m, 1h (default: 10m) /// --verbose Full `flutter pub get` / `flutter test` output Future runEnsembleYamlTestsCli(List arguments) async { @@ -129,6 +131,7 @@ Future runEnsembleYamlTestsCli(List arguments) async { '--dart-define=ensembleTestReportFile=$reportFile', if (timeoutSeconds != null) '--dart-define=ensembleTestTimeoutSeconds=$timeoutSeconds', + ..._inputDartDefines(arguments), ..._selectionDartDefines(arguments), '--reporter', verbose ? 'expanded' : 'silent', @@ -202,6 +205,52 @@ List _selectionDartDefines(List arguments) { ]; } +List _inputDartDefines(List arguments) { + final inputs = _inputValues(arguments); + if (inputs.isEmpty) return const []; + final encoded = base64Url.encode(utf8.encode(jsonEncode(inputs))); + return ['--dart-define=ensembleTestInputs=$encoded']; +} + +Map _inputValues(List arguments) { + final inputs = {}; + for (var i = 0; i < arguments.length; i++) { + final arg = arguments[i]; + String? raw; + if (arg == '--input') { + if (i + 1 >= arguments.length) { + throw StateError('--input requires key=value'); + } + raw = arguments[++i]; + } else if (arg.startsWith('--input=')) { + raw = arg.substring('--input='.length); + } + if (raw == null) continue; + + final separator = raw.indexOf('='); + if (separator <= 0) { + throw StateError('Invalid --input "$raw". Use --input key=value.'); + } + final key = raw.substring(0, separator).trim(); + if (key.isEmpty) { + throw StateError('Invalid --input "$raw". Input key cannot be empty.'); + } + inputs[key] = _parseInputValue(raw.substring(separator + 1)); + } + return inputs; +} + +dynamic _parseInputValue(String value) { + final trimmed = value.trim(); + if (trimmed == 'true') return true; + if (trimmed == 'false') return false; + final intValue = int.tryParse(trimmed); + if (intValue != null) return intValue; + final doubleValue = double.tryParse(trimmed); + if (doubleValue != null) return doubleValue; + return value; +} + List _optionValues(List arguments, String name) { return arguments .where((arg) => arg.startsWith('$name=')) diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart index b4037e4ca..10497babb 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart @@ -67,26 +67,34 @@ String _extractPrefixedReport(String output, String prefix) { /// Arguments forwarded to `flutter test` (CLI-only flags removed). List flutterTestArguments(List arguments) { - return arguments - .where( - (a) => - !a.startsWith('--app-dir') && - !a.startsWith('--report') && - a != '--doctor' && - a != '--inspect-app' && - a != '--validate-only' && - a != '--scaffold-test' && - !a.startsWith('--scaffold-test=') && - !a.startsWith('--screen=') && - !a.startsWith('--id=') && - !a.startsWith('--feature=') && - !a.startsWith('--tag=') && - !a.startsWith('--path=') && - !a.startsWith('--timeout=') && - a != '--verbose' && - a != '--quiet', - ) - .toList(); + final forwarded = []; + for (var i = 0; i < arguments.length; i++) { + final a = arguments[i]; + if (a == '--input') { + i++; + continue; + } + if (a.startsWith('--input=')) continue; + if (a.startsWith('--app-dir') || + a.startsWith('--report') || + a == '--doctor' || + a == '--inspect-app' || + a == '--validate-only' || + a == '--scaffold-test' || + a.startsWith('--scaffold-test=') || + a.startsWith('--screen=') || + a.startsWith('--id=') || + a.startsWith('--feature=') || + a.startsWith('--tag=') || + a.startsWith('--path=') || + a.startsWith('--timeout=') || + a == '--verbose' || + a == '--quiet') { + continue; + } + forwarded.add(a); + } + return forwarded; } /// Flutter sometimes prints asset cleanup warnings after a successful run. diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart index 1311c975d..08c1ee51f 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart @@ -50,6 +50,7 @@ class EnsembleTestExecutionPlanner { static Future build({ EnsembleTestAppTarget? target, EnsembleTestSelection selection = const EnsembleTestSelection(), + Map inputs = const {}, }) async { final resolvedTarget = target ?? await EnsembleTestDiscovery.loadAppTarget(); @@ -72,6 +73,7 @@ class EnsembleTestExecutionPlanner { final testCase = EnsembleTestParser.parseString( content, sourcePath: path, + inputs: inputs, ); final existing = byId[testCase.id]; if (existing != null) { diff --git a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart index 1423bf013..486c016d8 100644 --- a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart +++ b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart @@ -93,6 +93,7 @@ Future runEnsembleYamlTestsWithOptions( final plan = await EnsembleTestExecutionPlanner.build( target: target, selection: _selectionFromEnvironment(), + inputs: _inputsFromEnvironment(), ); final harness = EnsembleTestHarness( appPath: target.appPath, @@ -186,6 +187,21 @@ Set _csvSet(String value) { .toSet(); } +Map _inputsFromEnvironment() { + const encoded = String.fromEnvironment('ensembleTestInputs'); + if (encoded.isEmpty) return const {}; + try { + final decoded = utf8.decode(base64Url.decode(encoded)); + final value = json.decode(decoded); + if (value is Map) { + return value.map((key, value) => MapEntry(key.toString(), value)); + } + } catch (_) { + // Fall through to the explicit failure below. + } + throw EnsembleTestFailure('Invalid ensembleTestInputs dart-define payload.'); +} + void _emitMachineReport(EnsembleTestRunResult result) { const reportMode = String.fromEnvironment('ensembleTestReport'); const reportFile = String.fromEnvironment('ensembleTestReportFile'); diff --git a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart index 57ccf45ed..359f7f44a 100644 --- a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart +++ b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -5,15 +5,26 @@ import 'package:yaml/yaml.dart'; class EnsembleTestParser { /// Loads a test file from disk. - static Future parseFile(String path) async { + static Future parseFile( + String path, { + Map inputs = const {}, + }) async { final file = File(path); if (!await file.exists()) { throw EnsembleTestFailure('Test file not found: $path'); } - return parseString(await file.readAsString(), sourcePath: path); + return parseString( + await file.readAsString(), + sourcePath: path, + inputs: inputs, + ); } - static EnsembleTestCase parseString(String content, {String? sourcePath}) { + static EnsembleTestCase parseString( + String content, { + String? sourcePath, + Map inputs = const {}, + }) { final dynamic doc = loadYaml(content); if (doc is! YamlMap) { throw EnsembleTestFailure( @@ -27,10 +38,14 @@ class EnsembleTestParser { ); } - return _parseTestCase(doc, sourcePath: sourcePath); + return _parseTestCase(doc, sourcePath: sourcePath, inputs: inputs); } - static EnsembleTestCase _parseTestCase(YamlMap map, {String? sourcePath}) { + static EnsembleTestCase _parseTestCase( + YamlMap map, { + String? sourcePath, + required Map inputs, + }) { if (map.containsKey('options')) { throw EnsembleTestFailure( 'Root-level "options" is no longer supported in *.test.yaml files. ' @@ -76,9 +91,9 @@ class EnsembleTestParser { priority: map['priority']?.toString(), startScreen: hasStartScreen ? startScreen : null, prerequisite: hasPrerequisite ? prerequisite : null, - initialState: _toStringDynamicMap(map['initialState']), - mocks: _parseMocks(map['mocks']), - steps: _parseSteps(stepsNode, testId: id), + initialState: _toStringDynamicMap(map['initialState'], inputs: inputs), + mocks: _parseMocks(map['mocks'], inputs: inputs), + steps: _parseSteps(stepsNode, testId: id, inputs: inputs), ); } @@ -153,7 +168,10 @@ class EnsembleTestParser { ); } - static TestMocks _parseMocks(dynamic node) { + static TestMocks _parseMocks( + dynamic node, { + required Map inputs, + }) { if (node == null) return const TestMocks(); if (node is! YamlMap) { throw EnsembleTestFailure('"mocks" must be a map'); @@ -170,13 +188,16 @@ class EnsembleTestParser { if (value is! YamlMap) { throw EnsembleTestFailure('Mock for API "$key" must be a map'); } - apis[key.toString()] = _parseMockApiResponse(value); + apis[key.toString()] = _parseMockApiResponse(value, inputs: inputs); }); return TestMocks(apis: apis); } - static MockAPIResponse _parseMockApiResponse(YamlMap map) { + static MockAPIResponse _parseMockApiResponse( + YamlMap map, { + required Map inputs, + }) { final response = map['response']; if (response is! YamlMap) { throw EnsembleTestFailure('API mock must include a "response" map'); @@ -184,32 +205,45 @@ class EnsembleTestParser { return MockAPIResponse( statusCode: response['statusCode'] as int? ?? 200, - body: _unwrapYaml(response['body']), + body: _unwrapYaml(response['body'], inputs: inputs), headers: response['headers'] is YamlMap - ? _toStringDynamicMap(response['headers']) + ? _toStringDynamicMap(response['headers'], inputs: inputs) : null, delayMs: map['delayMs'] as int?, ); } - static List _parseSteps(YamlList steps, {required String testId}) => - _parseStepsList(steps, testId: testId); + static List _parseSteps( + YamlList steps, { + required String testId, + required Map inputs, + }) => + _parseStepsList(steps, testId: testId, inputs: inputs); - static List _parseStepsList(dynamic steps, - {required String testId}) { + static List _parseStepsList( + dynamic steps, { + required String testId, + required Map inputs, + }) { if (steps is! List || steps.isEmpty) { throw EnsembleTestFailure( 'Test "$testId" requires a non-empty "steps" list'); } final result = []; for (var i = 0; i < steps.length; i++) { - result.add(_parseStep(steps[i], testId: testId, index: i)); + result.add( + _parseStep(steps[i], testId: testId, index: i, inputs: inputs), + ); } return result; } - static TestStep _parseStep(dynamic step, - {required String testId, int? index}) { + static TestStep _parseStep( + dynamic step, { + required String testId, + int? index, + required Map inputs, + }) { final String type; final dynamic argsNode; @@ -226,53 +260,62 @@ class EnsembleTestParser { ); } - final args = _argsFromNode(argsNode); + final args = _argsFromNode(argsNode, inputs: inputs); List nested = const []; if (type == 'group' || type == 'repeat') { final stepsNode = args['steps']; if (stepsNode is List && stepsNode.isNotEmpty) { - nested = _parseStepsList(stepsNode, testId: testId); + nested = _parseStepsList(stepsNode, testId: testId, inputs: inputs); } else { throw EnsembleTestFailure('"$type" requires a non-empty "steps" list'); } } else if (type == 'optional' || type == 'ifVisible') { final single = args['step']; if (single is YamlMap && single.length == 1) { - nested = [_parseStep(single, testId: testId)]; + nested = [_parseStep(single, testId: testId, inputs: inputs)]; } else if (single is Map && single.length == 1) { - nested = [_parseStep(single, testId: testId)]; + nested = [_parseStep(single, testId: testId, inputs: inputs)]; } else if (args['steps'] is List && (args['steps'] as List).isNotEmpty) { - nested = _parseStepsList(args['steps'], testId: testId); + nested = _parseStepsList(args['steps'], testId: testId, inputs: inputs); } } return TestStep(type: type, args: args, nestedSteps: nested); } - static Map _argsFromNode(dynamic node) { + static Map _argsFromNode( + dynamic node, { + required Map inputs, + }) { if (node is YamlMap) { - return _toStringDynamicMap(node); + return _toStringDynamicMap(node, inputs: inputs); } if (node is Map) { return node.map( - (key, value) => MapEntry(key.toString(), _unwrapYaml(value)), + (key, value) => MapEntry( + key.toString(), + _unwrapYaml(value, inputs: inputs), + ), ); } if (node != null) { - return {'value': _unwrapYaml(node)}; + return {'value': _unwrapYaml(node, inputs: inputs)}; } return {}; } - static Map _toStringDynamicMap(dynamic node) { + static Map _toStringDynamicMap( + dynamic node, { + required Map inputs, + }) { if (node == null) return {}; if (node is! YamlMap) { throw EnsembleTestFailure('Expected a map'); } final out = {}; node.forEach((key, value) { - out[key.toString()] = _unwrapYaml(value); + out[key.toString()] = _unwrapYaml(value, inputs: inputs); }); return out; } @@ -285,13 +328,44 @@ class EnsembleTestParser { return node.map((value) => value.toString()).toList(); } - static dynamic _unwrapYaml(dynamic value) { + static dynamic _unwrapYaml( + dynamic value, { + required Map inputs, + }) { if (value is YamlMap) { - return _toStringDynamicMap(value); + return _toStringDynamicMap(value, inputs: inputs); } if (value is YamlList) { - return value.map(_unwrapYaml).toList(); + return value.map((item) => _unwrapYaml(item, inputs: inputs)).toList(); + } + if (value is String) { + return _resolveInputPlaceholders(value, inputs); } return value; } + + static dynamic _resolveInputPlaceholders( + String value, + Map inputs, + ) { + final exact = + RegExp(r'^\$\{inputs\.([A-Za-z0-9_.-]+)\}$').firstMatch(value); + if (exact != null) { + return _inputValue(exact.group(1)!, inputs); + } + + return value.replaceAllMapped( + RegExp(r'\$\{inputs\.([A-Za-z0-9_.-]+)\}'), + (match) => _inputValue(match.group(1)!, inputs).toString(), + ); + } + + static dynamic _inputValue(String key, Map inputs) { + if (!inputs.containsKey(key)) { + throw EnsembleTestFailure( + 'Missing CLI input "$key". Pass it with --input $key=value.', + ); + } + return inputs[key]; + } } diff --git a/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart b/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart index 388592e5c..01810a87a 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart @@ -41,6 +41,9 @@ name is ={first: John} '--tag=smoke', '--path=auth/', '--timeout=30s', + '--input', + 'adminPassword=s4C>M7U6t~', + '--input=expectedDeviceCount=2', '--name', 'x', ]), diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 40c6ba867..5544d5888 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -72,6 +72,57 @@ steps: ); }); + test('resolves CLI inputs in initial state and steps', () { + const yaml = ''' +id: input_test +startScreen: Login +initialState: + keychain: + adminPassword: \${inputs.adminPassword} +steps: + - expectText: + text: \${inputs.expectedDeviceCount} + - expectText: + text: "Devices: \${inputs.expectedDeviceCount}" +'''; + + final test = EnsembleTestParser.parseString( + yaml, + inputs: { + 'adminPassword': 's4C>M7U6t~', + 'expectedDeviceCount': 2, + }, + ); + + expect( + (test.initialState['keychain'] as Map)['adminPassword'], + 's4C>M7U6t~', + ); + expect(test.steps[0].args['text'], 2); + expect(test.steps[1].args['text'], 'Devices: 2'); + }); + + test('fails clearly when CLI input is missing', () { + const yaml = ''' +id: input_test +startScreen: Login +steps: + - expectText: + text: \${inputs.expectedDeviceCount} +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Missing CLI input "expectedDeviceCount"'), + ), + ), + ); + }); + test('rejects root-level options in test files', () { const yaml = ''' id: visual_debug From f7bade8d58c875d740f7452d46ccb63f62da6fca Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Tue, 14 Jul 2026 17:59:09 +0500 Subject: [PATCH 20/29] feat(test_runner): enhance mock handling and scenario support in ensemble tests Refactored the ensemble test runner to improve mock handling by introducing a new `mocks` section in the test schema, allowing for the use of `.mock.json` files for API responses. Updated the `EnsembleTestCase` model to include `mockFiles` and `scenarios`, enabling scenario-based testing with variable substitutions. Enhanced the `EnsembleTestParser` to parse scenarios and their variables, ensuring proper integration with the test execution flow. Updated documentation and tests to reflect these changes, improving the flexibility and usability of the test framework. --- tools/ensemble_test_runner/README.md | 12 +- tools/ensemble_test_runner/STEP_VOCABULARY.md | 12 +- .../assets/schema/ensemble_tests_schema.json | 402 ++---------------- .../doc/TEST_AUTHORING.md | 68 ++- .../lib/actions/extended_step_handlers.dart | 86 ---- .../lib/actions/test_step_executor.dart | 26 +- .../lib/cli/ensemble_test_scaffold.dart | 2 +- .../lib/cli/yaml_test_app_patcher.dart | 51 ++- .../ensemble_test_execution_planner.dart | 294 ++++++++++++- .../lib/models/ensemble_test_models.dart | 24 +- .../lib/parser/ensemble_test_parser.dart | 314 +++++++++++--- .../lib/runner/ensemble_test_context.dart | 19 +- .../lib/runner/ensemble_test_runner.dart | 46 +- .../schema/ensemble_test_schema_builder.dart | 41 +- .../validation/ensemble_test_validator.dart | 55 +-- .../lib/vocabulary/test_step_arg_kind.dart | 43 -- .../lib/vocabulary/test_step_registry.dart | 52 --- .../lib/vocabulary/test_step_vocabulary.dart | 1 - .../ensemble_test_execution_planner_test.dart | 130 ++++++ .../test/ensemble_test_parser_test.dart | 76 +++- .../test/ensemble_test_scaffold_test.dart | 3 +- .../test/ensemble_test_schema_test.dart | 13 + .../test/ensemble_test_validator_test.dart | 26 +- .../test/yaml_test_app_patcher_test.dart | 38 ++ .../tool/generate_step_registry.dart | 56 --- 25 files changed, 966 insertions(+), 924 deletions(-) diff --git a/tools/ensemble_test_runner/README.md b/tools/ensemble_test_runner/README.md index 9252c5b12..d34fd436a 100644 --- a/tools/ensemble_test_runner/README.md +++ b/tools/ensemble_test_runner/README.md @@ -29,7 +29,7 @@ Widget YAML must set `testId` (or `id`, which maps to the same `ValueKey`). ### Step vocabulary -The full official catalog (lifecycle, gestures, API mocks, fixtures, debug, etc.) is in **[STEP_VOCABULARY.md](STEP_VOCABULARY.md)**. +The full official catalog (lifecycle, gestures, API assertions, debug, etc.) is in **[STEP_VOCABULARY.md](STEP_VOCABULARY.md)**. Machine-readable registry (single source): `lib/vocabulary/test_step_registry.dart`. @@ -157,7 +157,7 @@ Create a starter test under `definitions.local.path/tests/`: dart run ensemble_test_runner:ensemble_test --scaffold-test=login_valid --feature=login --tag=smoke --screen=Login ``` -See [`docs/TEST_AUTHORING.md`](docs/TEST_AUTHORING.md) for the test authoring workflow, fixture conventions, validation rules, and repair-loop output. +See [`docs/TEST_AUTHORING.md`](docs/TEST_AUTHORING.md) for the test authoring workflow, mock file conventions, validation rules, and repair-loop output. ### CI output @@ -197,13 +197,9 @@ On success the console prints one consolidated boxed report for the suite: each # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json id: login_flow startScreen: Login +mocks: + - mocks/login_success.mock.json steps: - - mockApi: - name: login - response: - statusCode: 200 - body: - token: test-token - enterText: id: email_field value: user@test.com diff --git a/tools/ensemble_test_runner/STEP_VOCABULARY.md b/tools/ensemble_test_runner/STEP_VOCABULARY.md index ce4e3ab7e..ce4c1b2d4 100644 --- a/tools/ensemble_test_runner/STEP_VOCABULARY.md +++ b/tools/ensemble_test_runner/STEP_VOCABULARY.md @@ -33,13 +33,13 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense ### Navigation `expectScreen` (alias), `expectNavigateTo`, `expectVisited`, `expectNotVisited`, `expectBackStack`, `expectCanGoBack`, `goBack` -### API mock / assert -`mockApi`, `mockApiError`, `mockApiFromFixture`, `mockApiException`, `mockTimeout`, `resetApiCalls`, `clearApiMocks`, `expectApiCalled`, `expectApiNotCalled`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` +### API assert / logs +`resetApiCalls`, `expectApiCalled`, `expectApiNotCalled`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` ### Storage / runtime `setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` -### Scripts / fixtures / debug / quality +### Scripts / debug / quality `runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `logStorage`, `logPerformance`, `screenshot`, `dumpTree`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` ### Control flow @@ -51,12 +51,6 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense id: login_flow startScreen: Login steps: - - mockApi: - name: login - response: - statusCode: 200 - body: - token: test - enterText: id: email_field value: user@test.com diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index d2b234f92..7bdf08c9c 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -59,7 +59,18 @@ "$ref": "#/$defs/initialState" }, "mocks": { - "$ref": "#/$defs/mocks" + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "scenarios": { + "type": "array", + "items": { + "$ref": "#/$defs/scenario" + }, + "minItems": 1 }, "steps": { "type": "array", @@ -96,35 +107,6 @@ } ], "$defs": { - "mockResponse": { - "type": "object", - "additionalProperties": false, - "properties": { - "statusCode": { - "type": "integer" - }, - "body": true, - "headers": { - "type": "object", - "additionalProperties": true - } - } - }, - "mockApiEntry": { - "type": "object", - "additionalProperties": false, - "properties": { - "response": { - "$ref": "#/$defs/mockResponse" - }, - "delayMs": { - "type": "integer" - } - }, - "required": [ - "response" - ] - }, "initialState": { "type": "object", "additionalProperties": false, @@ -143,17 +125,25 @@ } } }, - "mocks": { + "scenario": { "type": "object", "additionalProperties": false, "properties": { - "apis": { + "id": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "vars": { "type": "object", - "additionalProperties": { - "$ref": "#/$defs/mockApiEntry" - } + "additionalProperties": true } - } + }, + "required": [ + "id" + ] }, "args_openScreen": { "type": "object", @@ -1406,141 +1396,6 @@ {} ] }, - "args_mockApi": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "response": { - "$ref": "#/$defs/mockResponse" - }, - "delayMs": { - "type": "integer" - } - }, - "required": [ - "name", - "response" - ], - "additionalProperties": false, - "title": "mockApi", - "description": "Register a mock HTTP API response by API name", - "examples": [ - { - "name": "login", - "response": { - "statusCode": 200, - "body": { - "token": "test-token" - } - } - } - ] - }, - "args_mockApiError": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "statusCode": { - "type": "integer" - }, - "body": true, - "delayMs": { - "type": "integer" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "mockApiError", - "description": "Mock an API to return an error status/body", - "examples": [ - { - "name": "login", - "statusCode": 401, - "body": { - "error": "Unauthorized" - } - } - ] - }, - "args_mockApiFromFixture": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statusCode": { - "type": "integer" - } - }, - "required": [ - "name", - "fixture" - ], - "additionalProperties": false, - "title": "mockApiFromFixture", - "description": "Load mock response body from a JSON fixture asset", - "examples": [ - { - "name": "users", - "fixture": "fixtures/users.json" - } - ] - }, - "args_mockApiException": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "message": { - "type": "string" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "mockApiException", - "description": "Force an API call to throw an exception", - "examples": [ - { - "name": "login", - "message": "Network error" - } - ] - }, - "args_mockTimeout": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "delayMs": { - "type": "integer" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "mockTimeout", - "description": "Mock an API with a long delay (simulate timeout)", - "examples": [ - { - "name": "slow_api", - "delayMs": 60000 - } - ] - }, "args_resetApiCalls": { "type": "object", "additionalProperties": false, @@ -1550,15 +1405,6 @@ {} ] }, - "args_clearApiMocks": { - "type": "object", - "additionalProperties": false, - "title": "clearApiMocks", - "description": "Remove all registered API mocks", - "examples": [ - {} - ] - }, "args_expectApiCalled": { "type": "object", "properties": { @@ -4089,177 +3935,6 @@ "goBack" ] }, - { - "type": "object", - "title": "mockApi", - "description": "Register a mock HTTP API response by API name", - "examples": [ - { - "mockApi": { - "name": "login", - "response": { - "statusCode": 200, - "body": { - "token": "test-token" - } - } - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockApi": { - "$ref": "#/$defs/args_mockApi", - "description": "Register a mock HTTP API response by API name", - "examples": [ - { - "name": "login", - "response": { - "statusCode": 200, - "body": { - "token": "test-token" - } - } - } - ] - } - }, - "required": [ - "mockApi" - ] - }, - { - "type": "object", - "title": "mockApiError", - "description": "Mock an API to return an error status/body", - "examples": [ - { - "mockApiError": { - "name": "login", - "statusCode": 401, - "body": { - "error": "Unauthorized" - } - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockApiError": { - "$ref": "#/$defs/args_mockApiError", - "description": "Mock an API to return an error status/body", - "examples": [ - { - "name": "login", - "statusCode": 401, - "body": { - "error": "Unauthorized" - } - } - ] - } - }, - "required": [ - "mockApiError" - ] - }, - { - "type": "object", - "title": "mockApiFromFixture", - "description": "Load mock response body from a JSON fixture asset", - "examples": [ - { - "mockApiFromFixture": { - "name": "users", - "fixture": "fixtures/users.json" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockApiFromFixture": { - "$ref": "#/$defs/args_mockApiFromFixture", - "description": "Load mock response body from a JSON fixture asset", - "examples": [ - { - "name": "users", - "fixture": "fixtures/users.json" - } - ] - } - }, - "required": [ - "mockApiFromFixture" - ] - }, - { - "type": "object", - "title": "mockApiException", - "description": "Force an API call to throw an exception", - "examples": [ - { - "mockApiException": { - "name": "login", - "message": "Network error" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockApiException": { - "$ref": "#/$defs/args_mockApiException", - "description": "Force an API call to throw an exception", - "examples": [ - { - "name": "login", - "message": "Network error" - } - ] - } - }, - "required": [ - "mockApiException" - ] - }, - { - "type": "object", - "title": "mockTimeout", - "description": "Mock an API with a long delay (simulate timeout)", - "examples": [ - { - "mockTimeout": { - "name": "slow_api", - "delayMs": 60000 - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "mockTimeout": { - "$ref": "#/$defs/args_mockTimeout", - "description": "Mock an API with a long delay (simulate timeout)", - "examples": [ - { - "name": "slow_api", - "delayMs": 60000 - } - ] - } - }, - "required": [ - "mockTimeout" - ] - }, { "type": "object", "title": "resetApiCalls", @@ -4285,31 +3960,6 @@ "resetApiCalls" ] }, - { - "type": "object", - "title": "clearApiMocks", - "description": "Remove all registered API mocks", - "examples": [ - { - "clearApiMocks": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "clearApiMocks": { - "$ref": "#/$defs/args_clearApiMocks", - "description": "Remove all registered API mocks", - "examples": [ - {} - ] - } - }, - "required": [ - "clearApiMocks" - ] - }, { "type": "object", "title": "expectApiCalled", diff --git a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md index f089c391c..5e584943a 100644 --- a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md +++ b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md @@ -61,41 +61,65 @@ logStorage: `--inspect-app` emits JSON with screens, widget IDs, APIs, navigation targets, imports, storage/env references, and lifecycle hints. -## Fixtures And Mocks +## Mocks -Put JSON fixtures under: - -```text -ensemble/apps//tests/fixtures/.json -``` - -Reference fixtures by filename: +Use `mocks` for API responses. The value is an ordered list of `.mock.json` +files. Later files override earlier files. ```yaml -steps: - - mockApiFromFixture: - name: profile - fixture: profile_success.json +mocks: + - mocks/common/base.mock.json ``` -Use root `mocks.apis` for APIs that may run during `onLoad` or startup. Use step-level `mockApi` or `mockApiFromFixture` for APIs triggered after user actions. +For scenario-based suites, keep reusable API data in mock files. The runner does +not know what your scenario variables mean; it only substitutes `${scenario.*}` +values and merges mock files in order. ```yaml +id: home_scenarios +prerequisite: signin_to_gateway + mocks: - apis: - profile: - response: - body: {name: Jane} + - mocks/common/base.mock.json + - mocks/devices/${scenario.device}.mock.json + - mocks/behaviors/${scenario.behavior}.mock.json + +scenarios: + - id: v14_online + vars: + device: v14 + behavior: online + expectedDeviceCount: 2 + steps: - - mockApi: - name: login - response: - body: {token: test-token} + - expectText: + text: ${scenario.expectedDeviceCount} ``` +Each scenario expands to its own test id, for example +`home_scenarios[v14_online]`. When a scenario suite has a `prerequisite`, the +first scenario depends on that prerequisite and later scenarios run after the +previous scenario in declaration order. + +Mock files are JSON files. The root object maps API names to response +overrides: + +```json +{ + "getDevices": { + "statusCode": 200, + "body": { + "status": [] + } + } +} +``` + +Later files override earlier files. + ## Validation -`--validate-only` checks generated tests without running Flutter. It reports blocking errors for invalid YAML shape, missing tests, duplicate IDs, unknown prerequisites, unknown screens, and missing fixtures. It reports warnings for likely unknown widget IDs/APIs and mock placement issues. +`--validate-only` checks generated tests without running Flutter. It reports blocking errors for invalid YAML shape, missing tests, duplicate IDs, unknown prerequisites, and unknown screens. It reports warnings for likely unknown widget IDs/APIs. Warnings do not fail the command. Errors exit with code `2`. diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 5c4d59621..4476e78d3 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -1,4 +1,3 @@ -import 'dart:convert'; import 'dart:io'; import 'dart:ui' as ui; @@ -175,18 +174,6 @@ class ExtendedStepHandlers { case 'goBack': await _goBack(executor); return true; - case 'mockApiFromFixture': - await _mockApiFromFixture(executor, step); - return true; - case 'clearApiMocks': - executor.context.apiOverlay.clearMocks(); - return true; - case 'mockApiException': - await _mockApiException(executor, step); - return true; - case 'mockTimeout': - await _mockTimeout(executor, step); - return true; case 'expectApiCallOrder': executor.assertions.expectApiCallOrder( (step.args['names'] as List?)?.map((e) => e.toString()).toList() ?? @@ -449,48 +436,6 @@ class ExtendedStepHandlers { throw EnsembleTestFailure('goBack: no navigator or active scope'); } - static Future _mockApiFromFixture( - TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - final fixture = step.args['fixture']?.toString(); - if (name == null || fixture == null) { - throw EnsembleTestFailure( - 'mockApiFromFixture requires "name" and "fixture"'); - } - final body = await _readFixture(e, fixture); - e.context.apiOverlay.setMock( - name, - MockAPIResponse( - statusCode: step.args['statusCode'] as int? ?? 200, - body: body, - ), - ); - } - - static Future _mockApiException( - TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - if (name == null) - throw EnsembleTestFailure('mockApiException requires "name"'); - e.context.apiOverlay.setApiException( - name, - Exception(step.args['message']?.toString() ?? 'API exception (test)'), - ); - } - - static Future _mockTimeout(TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - if (name == null) throw EnsembleTestFailure('mockTimeout requires "name"'); - e.context.apiOverlay.setMock( - name, - MockAPIResponse( - statusCode: 200, - body: {}, - delayMs: step.args['delayMs'] as int? ?? 60000, - ), - ); - } - static void _setAuth(TestStepExecutor e, TestStep step) { final user = step.args['user']; if (user is Map) { @@ -568,37 +513,6 @@ class ExtendedStepHandlers { } } - static Future _readFixture(TestStepExecutor e, String path) async { - Object? lastError; - for (final candidate in _fixtureAssetCandidates(e, path)) { - try { - final raw = await rootBundle.loadString(candidate); - return json.decode(raw); - } catch (error) { - lastError = error; - } - } - throw EnsembleTestFailure( - 'Fixture not found: $path. Use tests/fixtures/.json. $lastError', - ); - } - - static List _fixtureAssetCandidates(TestStepExecutor e, String path) { - final sourcePath = e.context.testCase.sourcePath; - final candidates = []; - if (sourcePath != null && sourcePath.contains('/')) { - final testDir = sourcePath.substring(0, sourcePath.lastIndexOf('/')); - candidates.add( - path.startsWith('fixtures/') - ? '$testDir/$path' - : '$testDir/fixtures/$path', - ); - } - candidates.add(path); - if (!path.startsWith('ensemble/')) candidates.add('ensemble/$path'); - return candidates.toSet().toList(); - } - static Future _screenshot(TestStepExecutor e, TestStep step) async { final args = e.context.config.screenshots.toScreenshotArgs(step.args); await captureScreenshot(e, args: args); diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index d90935935..f42f8e941 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -273,30 +273,6 @@ class TestStepExecutor { } context.setEnv(key, step.args['value']); break; - case 'mockApi': - final name = step.args['name']?.toString(); - if (name == null) { - throw EnsembleTestFailure('mockApi requires "name"'); - } - context.apiOverlay.setMock( - name, - context.mockFromStepArgs(step.args), - ); - break; - case 'mockApiError': - final name = step.args['name']?.toString(); - if (name == null) { - throw EnsembleTestFailure('mockApiError requires "name"'); - } - context.apiOverlay.setMock( - name, - MockAPIResponse( - statusCode: step.args['statusCode'] as int? ?? 500, - body: step.args['body'], - delayMs: step.args['delayMs'] as int?, - ), - ); - break; case 'resetApiCalls': context.apiOverlay.resetCalls(); break; @@ -351,6 +327,7 @@ class TestStepExecutor { testCase: EnsembleTestCase( id: tc.id, startScreen: screen, + mockFiles: tc.mockFiles, initialState: tc.initialState, mocks: tc.mocks, steps: const [], @@ -527,6 +504,7 @@ class TestStepExecutor { testCase: EnsembleTestCase( id: tc.id, startScreen: screen, + mockFiles: tc.mockFiles, initialState: tc.initialState, mocks: tc.mocks, steps: const [], diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart index f29a8b8ad..ef9942b7b 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart @@ -32,7 +32,7 @@ class EnsembleTestScaffold { final testsDir = Directory(p.join(appDir, inspection.appPath, 'tests')); testsDir.createSync(recursive: true); - Directory(p.join(testsDir.path, 'fixtures')).createSync(recursive: true); + Directory(p.join(testsDir.path, 'mocks')).createSync(recursive: true); final file = File(p.join(testsDir.path, '$normalizedId.test.yaml')); if (file.existsSync()) { diff --git a/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart b/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart index 7e1c43c8a..31c7d52ec 100644 --- a/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart +++ b/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart @@ -156,11 +156,31 @@ Future main() async { String _activatePubspec(String content) { final testsDir = testsDirRelative; if (testsDir != null && _hasTestYamlOnDisk(testsDir)) { - return _activateTestAssets(content, testsDir); + return _activateTestAssets(content, testsDir, _testsAssetLines(testsDir)); } return content; } + List _testsAssetLines(String testsDirRelative) { + final testsDir = Directory(p.join(appDir, testsDirRelative)); + final dirs = {_withTrailingSlash(testsDirRelative)}; + if (testsDir.existsSync()) { + for (final entity in testsDir.listSync(recursive: true)) { + if (entity is! File) continue; + final dir = p.dirname(p.relative(entity.path, from: appDir)); + dirs.add(_withTrailingSlash(p.split(dir).join('/'))); + } + } + final sortedDirs = dirs.toList() + ..sort((a, b) { + final depthCompare = '/'.allMatches(a).length.compareTo( + '/'.allMatches(b).length, + ); + return depthCompare == 0 ? a.compareTo(b) : depthCompare; + }); + return sortedDirs.map(testsAssetLineFor).toList(); + } + String? get _testsDirRelative { final configFile = File(_ensembleConfigPath); if (!configFile.existsSync()) return null; @@ -188,20 +208,37 @@ Future main() async { .any((entity) => entity.path.endsWith('.test.yaml')); } - static String _activateTestAssets(String content, String testsDirRelative) { + static String _activateTestAssets( + String content, + String testsDirRelative, + List testAssetLines, + ) { final normalizedTestsDir = _withTrailingSlash(testsDirRelative); final testsAssetLine = testsAssetLineFor(normalizedTestsDir); - if (content.contains(testsAssetLine) || - content.contains('- $normalizedTestsDir')) { + final missingLines = testAssetLines + .where( + (line) => + !content.contains(line) && !content.contains(line.trimLeft()), + ) + .toList(); + if (missingLines.isEmpty) { return content; } + final insertion = missingLines.join('\n'); + + if (content.contains(testsAssetLine)) { + return content.replaceFirst( + testsAssetLine, + '$testsAssetLine\n$insertion', + ); + } final localAppLine = ' - ${_withTrailingSlash(p.posix.dirname(_withoutTrailingSlash(normalizedTestsDir)))}\n'; if (content.contains(localAppLine)) { return content.replaceFirst( localAppLine, - '$localAppLine$testsAssetLine\n', + '$localAppLine$insertion\n', ); } @@ -209,7 +246,7 @@ Future main() async { if (content.contains(ensembleDirLine)) { return content.replaceFirst( ensembleDirLine, - '$ensembleDirLine$testsAssetLine\n', + '$ensembleDirLine$insertion\n', ); } @@ -217,7 +254,7 @@ Future main() async { if (content.contains(assetsMarker)) { return content.replaceFirst( assetsMarker, - '$assetsMarker$testsAssetLine\n', + '$assetsMarker$insertion\n', ); } diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart index 08c1ee51f..0bbf14d5a 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; @@ -5,6 +7,8 @@ import 'package:ensemble_test_runner/discovery/ensemble_test_discovery.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; +typedef _AssetStringLoader = Future Function(String assetPath); + /// A parsed `*.test.yaml` file with its asset path. class EnsembleTestDefinition { final String assetPath; @@ -70,21 +74,21 @@ class EnsembleTestExecutionPlanner { final byId = {}; for (final path in paths) { final content = await rootBundle.loadString(path); - final testCase = EnsembleTestParser.parseString( + final definitions = await _parseDefinitionsFromAsset( + path, content, - sourcePath: path, inputs: inputs, ); - final existing = byId[testCase.id]; - if (existing != null) { - throw EnsembleTestFailure( - 'Duplicate test id "${testCase.id}" in ${existing.assetPath} and $path', - ); + for (final definition in definitions) { + final existing = byId[definition.testCase.id]; + if (existing != null) { + throw EnsembleTestFailure( + 'Duplicate test id "${definition.testCase.id}" in ' + '${existing.assetPath} and $path', + ); + } + byId[definition.testCase.id] = definition; } - byId[testCase.id] = EnsembleTestDefinition( - assetPath: path, - testCase: testCase, - ); } final selectedById = _applySelection(byId, selection); @@ -103,6 +107,268 @@ class EnsembleTestExecutionPlanner { return EnsembleTestExecutionPlan(ordered: ordered, config: config); } + /// Exposed for unit tests only. + @visibleForTesting + static Future> parseDefinitionsForTest( + String path, + String content, { + Map inputs = const {}, + Future Function(String assetPath)? assetLoader, + }) { + return _parseDefinitionsFromAsset( + path, + content, + inputs: inputs, + assetLoader: assetLoader ?? _rootBundleAssetLoader, + ); + } + + static Future> _parseDefinitionsFromAsset( + String path, + String content, { + required Map inputs, + _AssetStringLoader assetLoader = _rootBundleAssetLoader, + }) async { + final base = EnsembleTestParser.parseString( + content, + sourcePath: path, + inputs: inputs, + ); + if (base.scenarios.isEmpty) { + final mocks = await _mergedMocksFor( + assetPath: path, + mockFiles: base.mockFiles, + assetLoader: assetLoader, + ); + return [ + EnsembleTestDefinition( + assetPath: path, + testCase: _withRuntimeFields( + base, + id: base.id, + startScreen: base.startScreen, + prerequisite: base.prerequisite, + mocks: mocks, + ), + ), + ]; + } + + final definitions = []; + String? previousScenarioId; + for (final scenario in base.scenarios) { + final parsed = EnsembleTestParser.parseString( + content, + sourcePath: path, + inputs: inputs, + scenario: scenario.vars, + scenarioId: scenario.id, + ); + final parsedScenario = parsed.scenarios.firstWhere( + (item) => item.id == scenario.id, + orElse: () => scenario, + ); + final id = '${base.id}[${scenario.id}]'; + final prerequisite = parsed.prerequisite == null + ? null + : previousScenarioId ?? parsed.prerequisite; + final mocks = await _mergedMocksFor( + assetPath: path, + mockFiles: parsed.mockFiles, + assetLoader: assetLoader, + ); + + definitions.add( + EnsembleTestDefinition( + assetPath: path, + testCase: _withRuntimeFields( + parsed, + id: id, + description: parsedScenario.description ?? parsed.description, + startScreen: + parsed.prerequisite == null ? parsed.startScreen : null, + prerequisite: prerequisite, + mocks: mocks, + ), + ), + ); + previousScenarioId = id; + } + return definitions; + } + + static EnsembleTestCase _withRuntimeFields( + EnsembleTestCase test, { + required String id, + String? description, + String? startScreen, + String? prerequisite, + required TestMocks mocks, + }) { + return EnsembleTestCase( + id: id, + sourcePath: test.sourcePath, + type: test.type, + feature: test.feature, + tags: test.tags, + description: description ?? test.description, + owner: test.owner, + priority: test.priority, + startScreen: startScreen, + prerequisite: prerequisite, + mockFiles: test.mockFiles, + initialState: test.initialState, + mocks: mocks, + steps: test.steps, + ); + } + + static Future _mergedMocksFor({ + required String assetPath, + required List mockFiles, + required _AssetStringLoader assetLoader, + }) async { + final apis = {}; + for (final file in mockFiles) { + final fileMocks = await _loadMockFile( + assetPath, + file, + assetLoader: assetLoader, + ); + apis.addAll(fileMocks.apis); + } + return TestMocks(apis: apis); + } + + static Future _loadMockFile( + String testAssetPath, + String mockFilePath, { + required _AssetStringLoader assetLoader, + }) async { + final assetPath = _resolveAssetPath(testAssetPath, mockFilePath); + final content = await _loadRequiredAsset( + assetPath, + 'Mock file "$mockFilePath" referenced by $testAssetPath was not found.', + assetLoader: assetLoader, + ); + if (!assetPath.endsWith('.mock.json')) { + throw EnsembleTestFailure( + 'Mock file "$mockFilePath" must be a .mock.json file.', + ); + } + + final dynamic doc = _parseMockFileDocument(content, assetPath); + if (doc == null) return const TestMocks(); + if (doc is! Map) { + throw EnsembleTestFailure('Mock file "$assetPath" root must be a map'); + } + + final apis = {}; + for (final entry in doc.entries) { + if (entry.value is! Map) { + throw EnsembleTestFailure( + 'Mock for API "${entry.key}" in "$assetPath" must be a map', + ); + } + apis[entry.key.toString()] = _parseLayerMockApiResponse( + Map.from(entry.value as Map), + layerAssetPath: assetPath, + apiName: entry.key.toString(), + ); + } + return TestMocks(apis: apis); + } + + static dynamic _parseMockFileDocument(String content, String assetPath) { + try { + return jsonDecode(content); + } on FormatException catch (error) { + throw EnsembleTestFailure( + 'Mock file "$assetPath" is not valid JSON: ${error.message}', + ); + } + } + + static MockAPIResponse _parseLayerMockApiResponse( + Map map, { + required String layerAssetPath, + required String apiName, + }) { + if (map.isEmpty) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$layerAssetPath" must include a response', + ); + } + final unknownKeys = map.keys + .map((key) => key.toString()) + .where( + (key) => + !const {'statusCode', 'body', 'headers', 'delayMs'}.contains(key), + ) + .toList(); + if (unknownKeys.isNotEmpty) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$layerAssetPath" has unsupported keys: ' + '${unknownKeys.join(", ")}. Use direct JSON shape: ' + '{"statusCode": 200, "body": ...}.', + ); + } + + return MockAPIResponse( + statusCode: map['statusCode'] as int? ?? 200, + body: map.containsKey('body') ? _unwrapJsonValue(map['body']) : null, + headers: map['headers'] is Map + ? Map.from( + _unwrapJsonValue(map['headers']) as Map, + ) + : null, + delayMs: map['delayMs'] as int?, + ); + } + + static dynamic _unwrapJsonValue(dynamic value) { + if (value is Map) { + return { + for (final entry in value.entries) + entry.key.toString(): _unwrapJsonValue(entry.value), + }; + } + if (value is List) { + return value.map(_unwrapJsonValue).toList(); + } + return value; + } + + static Future _loadRequiredAsset( + String assetPath, + String missingMessage, { + required _AssetStringLoader assetLoader, + }) async { + try { + return await assetLoader(assetPath); + } on FlutterError { + throw EnsembleTestFailure(missingMessage); + } + } + + static Future _rootBundleAssetLoader(String assetPath) { + return rootBundle.loadString(assetPath); + } + + static String _resolveAssetPath(String fromAssetPath, String relativePath) { + if (relativePath.startsWith('/')) return relativePath.substring(1); + final segments = fromAssetPath.split('/')..removeLast(); + for (final part in relativePath.split('/')) { + if (part.isEmpty || part == '.') continue; + if (part == '..') { + if (segments.isNotEmpty) segments.removeLast(); + } else { + segments.add(part); + } + } + return segments.join('/'); + } + static Map _applySelection( Map byId, EnsembleTestSelection selection, @@ -144,8 +410,10 @@ class EnsembleTestExecutionPlanner { EnsembleTestSelection selection, ) { final test = def.testCase; - final idMatches = - selection.ids.isNotEmpty && selection.ids.contains(test.id); + final idMatches = selection.ids.isNotEmpty && + selection.ids.any( + (id) => test.id == id || test.id.startsWith('$id['), + ); final featureMatches = selection.features.isNotEmpty && selection.features.contains(test.feature); final tagMatches = selection.tags.isNotEmpty && diff --git a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart index c95ef48f0..d08f47ba4 100644 --- a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart +++ b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart @@ -43,7 +43,11 @@ class EnsembleTestCase { /// Test [id] that must run before this one (same app session). final String? prerequisite; + final List mockFiles; + final List scenarios; final Map initialState; + + /// Runtime API mocks resolved from [mockFiles]. final TestMocks mocks; final List steps; @@ -58,6 +62,8 @@ class EnsembleTestCase { this.priority, this.startScreen, this.prerequisite, + this.mockFiles = const [], + this.scenarios = const [], this.initialState = const {}, this.mocks = const TestMocks(), required this.steps, @@ -76,6 +82,19 @@ class EnsembleTestCase { }; } +/// One scenario from a scenario-based test suite. +class TestScenario { + final String id; + final String? description; + final Map vars; + + const TestScenario({ + required this.id, + this.description, + this.vars = const {}, + }); +} + class EnsembleTestConfig { final ScreenshotConfig screenshots; final PerformanceConfig performance; @@ -341,7 +360,6 @@ class EnsembleSingleTestResult { String _failureKind(String text) { final lower = text.toLowerCase(); - if (lower.contains('fixture')) return 'parseError'; if (lower.contains('timeout') || lower.contains('timed out')) { return 'timeout'; } @@ -376,7 +394,7 @@ class EnsembleSingleTestResult { case 'apiMismatch': return [ 'Check API name spelling against --inspect-app output.', - 'Use root mocks.apis for onLoad APIs and mockApi for later user-triggered APIs.', + 'Use root mocks with .mock.json files for mocked API responses.', ]; case 'navigationMismatch': return [ @@ -390,7 +408,7 @@ class EnsembleSingleTestResult { ]; case 'parseError': return [ - 'Run --validate-only to find schema, fixture, and prerequisite issues.', + 'Run --validate-only to find schema and prerequisite issues.', ]; default: return [ diff --git a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart index 359f7f44a..41ff4bdc2 100644 --- a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart +++ b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -8,6 +8,8 @@ class EnsembleTestParser { static Future parseFile( String path, { Map inputs = const {}, + Map scenario = const {}, + String? scenarioId, }) async { final file = File(path); if (!await file.exists()) { @@ -17,6 +19,8 @@ class EnsembleTestParser { await file.readAsString(), sourcePath: path, inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, ); } @@ -24,6 +28,8 @@ class EnsembleTestParser { String content, { String? sourcePath, Map inputs = const {}, + Map scenario = const {}, + String? scenarioId, }) { final dynamic doc = loadYaml(content); if (doc is! YamlMap) { @@ -38,13 +44,21 @@ class EnsembleTestParser { ); } - return _parseTestCase(doc, sourcePath: sourcePath, inputs: inputs); + return _parseTestCase( + doc, + sourcePath: sourcePath, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } static EnsembleTestCase _parseTestCase( YamlMap map, { String? sourcePath, required Map inputs, + required Map scenario, + String? scenarioId, }) { if (map.containsKey('options')) { throw EnsembleTestFailure( @@ -52,6 +66,11 @@ class EnsembleTestParser { 'Move shared screenshots/performance settings to tests/config.yaml.', ); } + if (map.containsKey('mocks') && map['mocks'] is! YamlList) { + throw EnsembleTestFailure( + 'Root-level "mocks" must be a list of .mock.json files.', + ); + } final id = map['id']?.toString(); if (id == null || id.isEmpty) { @@ -91,9 +110,26 @@ class EnsembleTestParser { priority: map['priority']?.toString(), startScreen: hasStartScreen ? startScreen : null, prerequisite: hasPrerequisite ? prerequisite : null, - initialState: _toStringDynamicMap(map['initialState'], inputs: inputs), - mocks: _parseMocks(map['mocks'], inputs: inputs), - steps: _parseSteps(stepsNode, testId: id, inputs: inputs), + mockFiles: _toStringList( + map['mocks'], + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + scenarios: _parseScenarios(map['scenarios'], inputs: inputs), + initialState: _toStringDynamicMap( + map['initialState'], + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + steps: _parseSteps( + stepsNode, + testId: id, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), ); } @@ -168,62 +204,65 @@ class EnsembleTestParser { ); } - static TestMocks _parseMocks( + static List _parseScenarios( dynamic node, { required Map inputs, }) { - if (node == null) return const TestMocks(); - if (node is! YamlMap) { - throw EnsembleTestFailure('"mocks" must be a map'); - } - - final apisNode = node['apis']; - if (apisNode == null) return const TestMocks(); - if (apisNode is! YamlMap) { - throw EnsembleTestFailure('"mocks.apis" must be a map'); + if (node == null) return const []; + if (node is! YamlList) { + throw EnsembleTestFailure('"scenarios" must be a list'); } - - final apis = {}; - apisNode.forEach((key, value) { - if (value is! YamlMap) { - throw EnsembleTestFailure('Mock for API "$key" must be a map'); + final scenarios = []; + final ids = {}; + for (final item in node) { + if (item is! YamlMap) { + throw EnsembleTestFailure('Each scenario must be a map'); } - apis[key.toString()] = _parseMockApiResponse(value, inputs: inputs); - }); - - return TestMocks(apis: apis); - } - - static MockAPIResponse _parseMockApiResponse( - YamlMap map, { - required Map inputs, - }) { - final response = map['response']; - if (response is! YamlMap) { - throw EnsembleTestFailure('API mock must include a "response" map'); + final id = item['id']?.toString(); + if (id == null || id.isEmpty) { + throw EnsembleTestFailure('Each scenario must have an "id"'); + } + if (!ids.add(id)) { + throw EnsembleTestFailure('Duplicate scenario id "$id"'); + } + final vars = _toStringDynamicMap( + item['vars'], + inputs: inputs, + scenario: const {}, + scenarioId: id, + ); + scenarios.add( + TestScenario( + id: id, + description: item['description']?.toString(), + vars: vars, + ), + ); } - - return MockAPIResponse( - statusCode: response['statusCode'] as int? ?? 200, - body: _unwrapYaml(response['body'], inputs: inputs), - headers: response['headers'] is YamlMap - ? _toStringDynamicMap(response['headers'], inputs: inputs) - : null, - delayMs: map['delayMs'] as int?, - ); + return scenarios; } static List _parseSteps( YamlList steps, { required String testId, required Map inputs, + required Map scenario, + String? scenarioId, }) => - _parseStepsList(steps, testId: testId, inputs: inputs); + _parseStepsList( + steps, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); static List _parseStepsList( dynamic steps, { required String testId, required Map inputs, + required Map scenario, + String? scenarioId, }) { if (steps is! List || steps.isEmpty) { throw EnsembleTestFailure( @@ -232,7 +271,14 @@ class EnsembleTestParser { final result = []; for (var i = 0; i < steps.length; i++) { result.add( - _parseStep(steps[i], testId: testId, index: i, inputs: inputs), + _parseStep( + steps[i], + testId: testId, + index: i, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), ); } return result; @@ -243,6 +289,8 @@ class EnsembleTestParser { required String testId, int? index, required Map inputs, + Map scenario = const {}, + String? scenarioId, }) { final String type; final dynamic argsNode; @@ -260,24 +308,57 @@ class EnsembleTestParser { ); } - final args = _argsFromNode(argsNode, inputs: inputs); + final args = _argsFromNode( + argsNode, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); List nested = const []; if (type == 'group' || type == 'repeat') { final stepsNode = args['steps']; if (stepsNode is List && stepsNode.isNotEmpty) { - nested = _parseStepsList(stepsNode, testId: testId, inputs: inputs); + nested = _parseStepsList( + stepsNode, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } else { throw EnsembleTestFailure('"$type" requires a non-empty "steps" list'); } } else if (type == 'optional' || type == 'ifVisible') { final single = args['step']; if (single is YamlMap && single.length == 1) { - nested = [_parseStep(single, testId: testId, inputs: inputs)]; + nested = [ + _parseStep( + single, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ]; } else if (single is Map && single.length == 1) { - nested = [_parseStep(single, testId: testId, inputs: inputs)]; + nested = [ + _parseStep( + single, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ]; } else if (args['steps'] is List && (args['steps'] as List).isNotEmpty) { - nested = _parseStepsList(args['steps'], testId: testId, inputs: inputs); + nested = _parseStepsList( + args['steps'], + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } } @@ -287,20 +368,39 @@ class EnsembleTestParser { static Map _argsFromNode( dynamic node, { required Map inputs, + required Map scenario, + String? scenarioId, }) { if (node is YamlMap) { - return _toStringDynamicMap(node, inputs: inputs); + return _toStringDynamicMap( + node, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } if (node is Map) { return node.map( (key, value) => MapEntry( key.toString(), - _unwrapYaml(value, inputs: inputs), + _unwrapYaml( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), ), ); } if (node != null) { - return {'value': _unwrapYaml(node, inputs: inputs)}; + return { + 'value': _unwrapYaml( + node, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + }; } return {}; } @@ -308,6 +408,8 @@ class EnsembleTestParser { static Map _toStringDynamicMap( dynamic node, { required Map inputs, + Map scenario = const {}, + String? scenarioId, }) { if (node == null) return {}; if (node is! YamlMap) { @@ -315,57 +417,129 @@ class EnsembleTestParser { } final out = {}; node.forEach((key, value) { - out[key.toString()] = _unwrapYaml(value, inputs: inputs); + out[key.toString()] = _unwrapYaml( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); }); return out; } - static List _toStringList(dynamic node) { + static List _toStringList( + dynamic node, { + Map inputs = const {}, + Map scenario = const {}, + String? scenarioId, + }) { if (node == null) return const []; if (node is! Iterable) { throw EnsembleTestFailure('Expected a list'); } - return node.map((value) => value.toString()).toList(); + return node + .map( + (value) => _unwrapYaml( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ).toString(), + ) + .toList(); } static dynamic _unwrapYaml( dynamic value, { required Map inputs, + Map scenario = const {}, + String? scenarioId, }) { if (value is YamlMap) { - return _toStringDynamicMap(value, inputs: inputs); + return _toStringDynamicMap( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } if (value is YamlList) { - return value.map((item) => _unwrapYaml(item, inputs: inputs)).toList(); + return value + .map( + (item) => _unwrapYaml( + item, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ) + .toList(); } if (value is String) { - return _resolveInputPlaceholders(value, inputs); + return _resolvePlaceholders( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } return value; } - static dynamic _resolveInputPlaceholders( - String value, - Map inputs, - ) { - final exact = - RegExp(r'^\$\{inputs\.([A-Za-z0-9_.-]+)\}$').firstMatch(value); + static dynamic _resolvePlaceholders( + String value, { + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + final exact = RegExp(r'^\$\{(inputs|scenario)\.([A-Za-z0-9_.-]+)\}$') + .firstMatch(value); if (exact != null) { - return _inputValue(exact.group(1)!, inputs); + return _placeholderValue( + exact.group(1)!, + exact.group(2)!, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } return value.replaceAllMapped( - RegExp(r'\$\{inputs\.([A-Za-z0-9_.-]+)\}'), - (match) => _inputValue(match.group(1)!, inputs).toString(), + RegExp(r'\$\{(inputs|scenario)\.([A-Za-z0-9_.-]+)\}'), + (match) => _placeholderValue( + match.group(1)!, + match.group(2)!, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ).toString(), ); } - static dynamic _inputValue(String key, Map inputs) { - if (!inputs.containsKey(key)) { + static dynamic _placeholderValue( + String namespace, + String key, { + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + if (namespace == 'inputs') { + if (!inputs.containsKey(key)) { + throw EnsembleTestFailure( + 'Missing CLI input "$key". Pass it with --input $key=value.', + ); + } + return inputs[key]; + } + if (scenarioId == null && scenario.isEmpty) { + return '\${scenario.$key}'; + } + if (!scenario.containsKey(key)) { throw EnsembleTestFailure( - 'Missing CLI input "$key". Pass it with --input $key=value.', + 'Missing scenario value "$key"' + '${scenarioId != null ? ' in scenario "$scenarioId"' : ''}.', ); } - return inputs[key]; + return scenario[key]; } } diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart index a3c2386fa..5f69bacce 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart @@ -31,7 +31,7 @@ class EnsembleTestContext { EnsembleTestConfig config = const EnsembleTestConfig(), }) { final logger = TestLogger(); - final mockApi = TestApiProviderOverlay( + final apiOverlay = TestApiProviderOverlay( mocks: Map.from(testCase.mocks.apis), ); @@ -53,7 +53,7 @@ class EnsembleTestContext { final ctx = EnsembleTestContext( testCase: testCase, config: config, - apiOverlay: mockApi, + apiOverlay: apiOverlay, logger: logger, setup: setup, ); @@ -82,19 +82,4 @@ class EnsembleTestContext { Future clearStorage() async { await StorageManager().clearPublicStorage(); } - - MockAPIResponse mockFromStepArgs(Map args) { - final response = args['response']; - if (response is! Map) { - throw EnsembleTestFailure('mockApi requires a "response" map'); - } - return MockAPIResponse( - statusCode: response['statusCode'] as int? ?? 200, - body: response['body'], - headers: response['headers'] is Map - ? Map.from(response['headers'] as Map) - : null, - delayMs: args['delayMs'] as int?, - ); - } } diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index af85eea36..1a6188029 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -411,46 +411,38 @@ class EnsembleTestRunner { final logs = []; if (config.performance.enabled) { - final path = await tester.runAsync(() { - return writePerformanceLog( - logger: logger, - filePrefix: '', - name: 'app_performance', - frames: frames, - markers: markers, - apiCalls: apiCalls, - ); - }); + final path = await writePerformanceLog( + logger: logger, + filePrefix: '', + name: 'app_performance', + frames: frames, + markers: markers, + apiCalls: apiCalls, + ); logs.add('appPerformance: $path'); } if (config.dumpTree.enabled && lastContext != null) { - final path = await tester.runAsync(() { - return writeDumpTreeLogFile(logger: logger, filePrefix: ''); - }); + final path = await writeDumpTreeLogFile(logger: logger, filePrefix: ''); logs.add('dumpTree: $path'); } if (config.logApiCalls.enabled) { - final path = await tester.runAsync(() { - return writeApiCallsLogFile( - logger: logger, - filePrefix: '', - calls: apiCalls, - ); - }); + final path = await writeApiCallsLogFile( + logger: logger, + filePrefix: '', + calls: apiCalls, + ); logs.add('apiCalls: $path'); } if (config.logStorage.enabled && lastContext != null) { final key = config.logStorage.key; - final path = await tester.runAsync(() { - return writeStorageLogFile( - logger: logger, - filePrefix: '', - key: key, - ); - }); + final path = await writeStorageLogFile( + logger: logger, + filePrefix: '', + key: key, + ); logs.add( key == null || key.isEmpty ? 'storage: $path' : 'storage[$key]: $path', ); diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index ab3b47de4..328d176b2 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -12,27 +12,6 @@ class EnsembleTestSchemaBuilder { static Map build() { final defs = { - 'mockResponse': { - 'type': 'object', - 'additionalProperties': false, - 'properties': { - 'statusCode': {'type': 'integer'}, - 'body': true, - 'headers': { - 'type': 'object', - 'additionalProperties': true, - }, - }, - }, - 'mockApiEntry': { - 'type': 'object', - 'additionalProperties': false, - 'properties': { - 'response': {'\$ref': '#/\$defs/mockResponse'}, - 'delayMs': {'type': 'integer'}, - }, - 'required': ['response'], - }, 'initialState': { 'type': 'object', 'additionalProperties': false, @@ -42,15 +21,15 @@ class EnsembleTestSchemaBuilder { 'env': {'type': 'object', 'additionalProperties': true}, }, }, - 'mocks': { + 'scenario': { 'type': 'object', 'additionalProperties': false, 'properties': { - 'apis': { - 'type': 'object', - 'additionalProperties': {'\$ref': '#/\$defs/mockApiEntry'}, - }, + 'id': {'type': 'string', 'minLength': 1}, + 'description': {'type': 'string'}, + 'vars': {'type': 'object', 'additionalProperties': true}, }, + 'required': ['id'], }, 'testCase': { 'type': 'object', @@ -99,7 +78,15 @@ class EnsembleTestSchemaBuilder { 'ID of another test that must run before this one in the same app session', }, 'initialState': {'\$ref': '#/\$defs/initialState'}, - 'mocks': {'\$ref': '#/\$defs/mocks'}, + 'mocks': { + 'type': 'array', + 'items': {'type': 'string', 'minLength': 1}, + }, + 'scenarios': { + 'type': 'array', + 'items': {'\$ref': '#/\$defs/scenario'}, + 'minItems': 1, + }, 'steps': { 'type': 'array', 'minItems': 1, diff --git a/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart b/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart index ab968048f..3e5998c4e 100644 --- a/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart +++ b/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart @@ -219,6 +219,7 @@ class EnsembleTestValidator { final stepInfo = _collectStepInfo(steps); for (final widgetId in stepInfo.widgetIds) { + if (_isPlaceholder(widgetId)) continue; if (!knownWidgetIds.contains(widgetId)) { add( ValidationSeverity.warning, @@ -230,6 +231,7 @@ class EnsembleTestValidator { } } for (final api in stepInfo.apiNames) { + if (_isPlaceholder(api)) continue; if (!knownApis.contains(api)) { add( ValidationSeverity.warning, @@ -240,39 +242,6 @@ class EnsembleTestValidator { ); } } - for (final fixture in stepInfo.fixtures) { - final fixtureFile = File( - fixture.startsWith('fixtures/') - ? p.join(testsDir.path, fixture) - : p.join(testsDir.path, 'fixtures', fixture), - ); - if (!fixtureFile.existsSync()) { - add( - ValidationSeverity.error, - 'missingFixture', - 'Fixture not found: tests/fixtures/$fixture', - path: relativePath, - testId: id, - ); - } - } - - final rootMocks = _rootMockApis(doc); - final onLoadApis = - startScreen != null && screensByName[startScreen] != null - ? screensByName[startScreen]!.apis - : const []; - for (final api in onLoadApis) { - if (stepInfo.mockApis.contains(api) && !rootMocks.contains(api)) { - add( - ValidationSeverity.warning, - 'mockPlacement', - 'API "$api" may run during onLoad. Prefer root mocks.apis for startup APIs.', - path: relativePath, - testId: id, - ); - } - } } for (final entry in prerequisites.entries) { @@ -294,15 +263,11 @@ class EnsembleTestValidator { typedef _StepInfo = ({ Set widgetIds, Set apiNames, - Set mockApis, - Set fixtures, }); _StepInfo _collectStepInfo(dynamic steps) { final widgetIds = {}; final apiNames = {}; - final mockApis = {}; - final fixtures = {}; void visit(dynamic node) { if (node is! Iterable) return; @@ -317,11 +282,6 @@ _StepInfo _collectStepInfo(dynamic steps) { if (name != null && type.toLowerCase().contains('api')) { apiNames.add(name.toString()); } - if (type == 'mockApi' || type == 'mockApiFromFixture') { - if (name != null) mockApis.add(name.toString()); - } - final fixture = args['fixture']; - if (fixture != null) fixtures.add(fixture.toString()); visit(args['steps']); final single = args['step']; if (single != null) visit([single]); @@ -333,15 +293,8 @@ _StepInfo _collectStepInfo(dynamic steps) { return ( widgetIds: widgetIds, apiNames: apiNames, - mockApis: mockApis, - fixtures: fixtures, ); } -Set _rootMockApis(YamlMap doc) { - final mocks = doc['mocks']; - if (mocks is! YamlMap) return const {}; - final apis = mocks['apis']; - if (apis is! YamlMap) return const {}; - return apis.keys.map((key) => key.toString()).toSet(); -} +bool _isPlaceholder(String value) => + RegExp(r'\$\{(inputs|scenario)\.[A-Za-z0-9_.-]+\}').hasMatch(value); diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart index e9cf7c0e3..777f1e595 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -26,11 +26,6 @@ enum TestStepArgKind { expectListCount, screenRequired, expectVisited, - mockApi, - mockApiError, - mockApiFromFixture, - mockApiException, - mockTimeout, apiName, storageKey, optionalStorageKey, @@ -201,44 +196,6 @@ extension TestStepArgKindSchema on TestStepArgKind { properties: {'screen': _string}, required: ['screen'], ); - case TestStepArgKind.mockApi: - return _object( - properties: { - 'name': _string, - 'response': _ref('mockResponse'), - 'delayMs': _integer, - }, - required: ['name', 'response'], - ); - case TestStepArgKind.mockApiError: - return _object( - properties: { - 'name': _string, - 'statusCode': _integer, - 'body': _any, - 'delayMs': _integer, - }, - required: ['name'], - ); - case TestStepArgKind.mockApiFromFixture: - return _object( - properties: { - 'name': _string, - 'fixture': _string, - 'statusCode': _integer, - }, - required: ['name', 'fixture'], - ); - case TestStepArgKind.mockApiException: - return _object( - properties: {'name': _string, 'message': _string}, - required: ['name'], - ); - case TestStepArgKind.mockTimeout: - return _object( - properties: {'name': _string, 'delayMs': _integer}, - required: ['name'], - ); case TestStepArgKind.apiName: return _object( properties: {'name': _string, 'times': _integer}, diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index 31b9bd16c..b26aaf221 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -480,51 +480,6 @@ abstract final class TestStepRegistry { description: 'Navigate back (Ensemble navigateBack or Navigator.pop)', example: const {}, ), - 'mockApi': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApi, - description: 'Register a mock HTTP API response by API name', - example: const { - 'name': 'login', - 'response': const { - 'statusCode': 200, - 'body': const {'token': 'test-token'} - } - }, - ), - 'mockApiError': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApiError, - description: 'Mock an API to return an error status/body', - example: const { - 'name': 'login', - 'statusCode': 401, - 'body': const {'error': 'Unauthorized'} - }, - ), - 'mockApiFromFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApiFromFixture, - description: 'Load mock response body from a JSON fixture asset', - example: const {'name': 'users', 'fixture': 'fixtures/users.json'}, - ), - 'mockApiException': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApiException, - description: 'Force an API call to throw an exception', - example: const {'name': 'login', 'message': 'Network error'}, - ), - 'mockTimeout': TestStepRegistryEntry( - category: TestStepCategory.network, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockTimeout, - description: 'Mock an API with a long delay (simulate timeout)', - example: const {'name': 'slow_api', 'delayMs': 60000}, - ), 'resetApiCalls': TestStepRegistryEntry( category: TestStepCategory.apiMock, tier: TestStepTier.core, @@ -532,13 +487,6 @@ abstract final class TestStepRegistry { description: 'Clear recorded API call history', example: const {}, ), - 'clearApiMocks': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Remove all registered API mocks', - example: const {}, - ), 'expectApiCalled': TestStepRegistryEntry( category: TestStepCategory.apiAssertion, tier: TestStepTier.core, diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart index f9c505d95..a23be9a53 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart @@ -25,7 +25,6 @@ enum TestStepCategory { runtime, script, network, - fixture, control, debug, quality, diff --git a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart index 9cd4f2a42..c55c29472 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart @@ -98,6 +98,136 @@ steps: expect(order, ['login', 'profile']); }); + + test('expands scenarios with bracketed ids and prerequisite chain', + () async { + const yaml = ''' +id: home_scenarios +prerequisite: signin_to_gateway +scenarios: + - id: v12_online + vars: + expectedDeviceCount: 2 + - id: v14_empty + vars: + expectedDeviceCount: 0 +steps: + - expectText: + text: \${scenario.expectedDeviceCount} +'''; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'ensemble/apps/inhome/tests/home.test.yaml', + yaml, + ); + + expect( + definitions.map((definition) => definition.testCase.id), + [ + 'home_scenarios[v12_online]', + 'home_scenarios[v14_empty]', + ], + ); + expect( + definitions[0].testCase.prerequisite, + 'signin_to_gateway', + ); + expect( + definitions[1].testCase.prerequisite, + 'home_scenarios[v12_online]', + ); + expect(definitions[0].testCase.steps.single.args['text'], 2); + expect(definitions[1].testCase.steps.single.args['text'], 0); + }); + + test('loads JSON mock files and applies override order', () async { + const yaml = ''' +id: home_scenarios +startScreen: Home +mocks: + - mocks/common.mock.json + - mocks/\${scenario.behavior}.mock.json +scenarios: + - id: online + vars: + behavior: online +steps: + - expectVisible: + id: home +'''; + final assets = { + 'suite/tests/mocks/common.mock.json': ''' +{ + "getDevices": { + "statusCode": 200, + "body": {"count": 1} + }, + "rootApi": { + "body": {"from": "layer"} + } +} +''', + 'suite/tests/mocks/online.mock.json': ''' +{ + "getDevices": { + "statusCode": 200, + "body": {"count": 2} + }, + "scenarioApi": { + "body": {"from": "scenario-layer"} + } +} +''', + }; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + assetLoader: (path) async => assets[path]!, + ); + + final mocks = definitions.single.testCase.mocks.apis; + expect((mocks['getDevices']!.body as Map)['count'], 2); + expect((mocks['rootApi']!.body as Map)['from'], 'layer'); + expect((mocks['scenarioApi']!.body as Map)['from'], 'scenario-layer'); + }); + + test('selection by base scenario suite id includes expanded scenarios', + () async { + const yaml = ''' +id: home_scenarios +startScreen: Home +scenarios: + - id: first + vars: {} + - id: second + vars: {} +steps: + - expectVisible: + id: home +'''; + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + ); + final byId = { + for (final definition in definitions) + definition.testCase.id: definition, + }; + + final order = EnsembleTestExecutionPlanner.selectAndOrderIdsForTest( + byId, + const EnsembleTestSelection(ids: {'home_scenarios'}), + ); + + expect(order, [ + 'home_scenarios[first]', + 'home_scenarios[second]', + ]); + }); }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 5544d5888..a5973eb34 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -20,7 +20,7 @@ steps: expect(test.steps.first.args['id'], 'emailInput'); }); - test('parses API mocks', () { + test('rejects inline root-level mocks', () { const yaml = ''' id: login_success startScreen: Login @@ -38,12 +38,15 @@ steps: times: 1 '''; - final test = EnsembleTestParser.parseString(yaml); - expect(test.mocks.apis['loginApi']?.statusCode, 200); - expect(test.mocks.apis['loginApi']?.delayMs, 100); expect( - (test.mocks.apis['loginApi']?.body as Map)['token'], - 'test-token', + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Root-level "mocks" must be a list of .mock.json files'), + ), + ), ); }); @@ -123,6 +126,67 @@ steps: ); }); + test('parses scenarios and resolves scenario placeholders', () { + const yaml = ''' +id: home_scenarios +startScreen: Home +mocks: + - mocks/\${scenario.device}.mock.json +scenarios: + - id: v14_online + vars: + device: v14 + expectedDeviceCount: 2 +steps: + - expectText: + text: \${scenario.expectedDeviceCount} +'''; + + final base = EnsembleTestParser.parseString(yaml); + expect(base.scenarios.single.id, 'v14_online'); + expect(base.scenarios.single.vars['device'], 'v14'); + expect(base.mockFiles.single, 'mocks/\${scenario.device}.mock.json'); + + final expanded = EnsembleTestParser.parseString( + yaml, + scenario: base.scenarios.single.vars, + scenarioId: base.scenarios.single.id, + ); + expect(expanded.mockFiles.single, 'mocks/v14.mock.json'); + expect(expanded.steps.single.args['text'], 2); + }); + + test('fails clearly when scenario value is missing', () { + const yaml = ''' +id: home_scenarios +startScreen: Home +scenarios: + - id: missing_value + vars: {} +steps: + - expectText: + text: \${scenario.expectedDeviceCount} +'''; + + final base = EnsembleTestParser.parseString(yaml); + expect( + () => EnsembleTestParser.parseString( + yaml, + scenario: base.scenarios.single.vars, + scenarioId: base.scenarios.single.id, + ), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains( + 'Missing scenario value "expectedDeviceCount" in scenario "missing_value"', + ), + ), + ), + ); + }); + test('rejects root-level options in test files', () { const yaml = ''' id: visual_debug diff --git a/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart b/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart index 8c25ee42b..ffad03202 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart @@ -26,8 +26,7 @@ void main() { expect(content, contains('tags: [smoke]')); expect(content, contains('startScreen: Login')); expect( - Directory('${dir.path}/ensemble/apps/inhome/tests/fixtures') - .existsSync(), + Directory('${dir.path}/ensemble/apps/inhome/tests/mocks').existsSync(), isTrue); }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart index 4ac7906ea..22dc15db0 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart @@ -47,11 +47,24 @@ void main() { expect(decoded['properties'], contains('id')); expect(decoded['properties'], isNot(contains('tests'))); expect(decoded['properties'], isNot(contains('options'))); + expect(decoded['properties'], contains('mocks')); expect(decoded['properties'], contains('startScreen')); expect(decoded['properties'], contains('prerequisite')); + expect(decoded['properties'], isNot(contains('mockLayers'))); + expect(decoded['properties'], contains('scenarios')); expect(decoded['oneOf'], isA()); }); + test('schema keeps mocks at the root only', () { + final schema = EnsembleTestSchemaBuilder.build(); + final defs = schema['\$defs'] as Map; + final scenario = defs['scenario'] as Map; + final scenarioProperties = scenario['properties'] as Map; + + expect(scenarioProperties, contains('vars')); + expect(scenarioProperties, isNot(contains('mocks'))); + }); + test('config schema includes suite screenshots and performance settings', () { final json = EnsembleTestSchemaBuilder.buildConfigJson(); final decoded = jsonDecode(json) as Map; diff --git a/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart b/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart index 95bde0f52..01092a772 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart @@ -4,22 +4,14 @@ import 'package:ensemble_test_runner/validation/ensemble_test_validator.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { - test('validates generated tests with structured issues', () { + test('validates generated tests with structured warning issues', () { final dir = Directory.systemTemp.createTempSync('ensemble_validate_'); addTearDown(() => dir.deleteSync(recursive: true)); _writeApp(dir); _writeTest(dir, 'login.test.yaml', ''' id: login_happy startScreen: Login -mocks: - apis: - login: - response: - body: {ok: true} steps: - - mockApiFromFixture: - name: profile - fixture: missing.json - tap: id: missing_button - expectApiCalled: @@ -29,35 +21,23 @@ steps: final result = EnsembleTestValidator(dir.path).validate(); final codes = result.issues.map((issue) => issue.code).toList(); - expect(result.hasErrors, isTrue); - expect(codes, contains('missingFixture')); + expect(result.hasErrors, isFalse); expect(codes, contains('unknownWidgetId')); expect(codes, contains('unknownApi')); - expect(codes, contains('mockPlacement')); }); test('passes with warnings only as non-blocking', () { final dir = Directory.systemTemp.createTempSync('ensemble_validate_ok_'); addTearDown(() => dir.deleteSync(recursive: true)); _writeApp(dir); - Directory('${dir.path}/ensemble/apps/inhome/tests/fixtures') - .createSync(recursive: true); - File('${dir.path}/ensemble/apps/inhome/tests/fixtures/profile.json') - .writeAsStringSync('{"ok": true}'); _writeTest(dir, 'login.test.yaml', ''' id: login_happy startScreen: Login -mocks: - apis: - profile: - response: - body: {ok: true} steps: - tap: id: login_button - - mockApiFromFixture: + - expectApiCalled: name: login - fixture: profile.json '''); final result = EnsembleTestValidator(dir.path).validate(); diff --git a/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart b/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart index e51bfb9a7..bb01aab9e 100644 --- a/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart +++ b/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart @@ -182,6 +182,44 @@ steps: [] patcher.restore(); }); + test('enable adds nested test asset directories for mock files', () { + final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); + addTearDown(() => dir.deleteSync(recursive: true)); + + const pubspec = ''' +name: sample_app +flutter: + assets: + - ensemble/ + - ensemble/apps/helloApp/tests/ +'''; + File('${dir.path}/pubspec.yaml').writeAsStringSync(pubspec); + _writeConfig(dir); + Directory('${dir.path}/ensemble/apps/helloApp/tests/mocks/common') + .createSync(recursive: true); + File('${dir.path}/ensemble/apps/helloApp/tests/sample.test.yaml') + .writeAsStringSync(''' +id: sample +startScreen: Home +steps: + - expectVisible: + id: home +'''); + File('${dir.path}/ensemble/apps/helloApp/tests/mocks/common/base.mock.json') + .writeAsStringSync('{}'); + + final patcher = YamlTestAppPatcher(dir.path); + patcher.enable(); + + final enabled = File('${dir.path}/pubspec.yaml').readAsStringSync(); + expect( + enabled, + contains(' - ensemble/apps/helloApp/tests/mocks/common/'), + ); + + patcher.restore(); + }); + test('exposes configured tests directory and test presence', () { final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); addTearDown(() => dir.deleteSync(recursive: true)); diff --git a/tools/ensemble_test_runner/tool/generate_step_registry.dart b/tools/ensemble_test_runner/tool/generate_step_registry.dart index c1a629be0..5137fe865 100644 --- a/tools/ensemble_test_runner/tool/generate_step_registry.dart +++ b/tools/ensemble_test_runner/tool/generate_step_registry.dart @@ -399,48 +399,12 @@ void main() { 'empty', 'Navigate back (Ensemble navigateBack or Navigator.pop)', ), - 'mockApi': step( - 'apiMock', - 'core', - 'mockApi', - 'Register a mock HTTP API response by API name', - ), - 'mockApiError': step( - 'apiMock', - 'core', - 'mockApiError', - 'Mock an API to return an error status/body', - ), - 'mockApiFromFixture': step( - 'fixture', - 'core', - 'mockApiFromFixture', - 'Load mock response body from a JSON fixture asset', - ), - 'mockApiException': step( - 'apiMock', - 'core', - 'mockApiException', - 'Force an API call to throw an exception', - ), - 'mockTimeout': step( - 'network', - 'core', - 'mockTimeout', - 'Mock an API with a long delay (simulate timeout)', - ), 'resetApiCalls': step( 'apiMock', 'core', 'empty', 'Clear recorded API call history', ), - 'clearApiMocks': step( - 'apiMock', - 'core', - 'empty', - 'Remove all registered API mocks', - ), 'expectApiCalled': step( 'apiAssertion', 'core', @@ -778,26 +742,6 @@ Map defaultExampleForArg(String arg) { return {'screen': 'Home'}; case 'expectVisited': return {'screen': 'Login'}; - case 'mockApi': - return { - 'name': 'login', - 'response': { - 'statusCode': 200, - 'body': {'token': 'test-token'}, - }, - }; - case 'mockApiError': - return { - 'name': 'login', - 'statusCode': 401, - 'body': {'error': 'Unauthorized'} - }; - case 'mockApiFromFixture': - return {'name': 'users', 'fixture': 'fixtures/users.json'}; - case 'mockApiException': - return {'name': 'login', 'message': 'Network error'}; - case 'mockTimeout': - return {'name': 'slow_api', 'delayMs': 60000}; case 'apiName': return {'name': 'login', 'times': 1}; case 'storageKey': From 416d4ce7bf25b45edd1bc6c67931f3fd563fb244 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Tue, 14 Jul 2026 18:05:46 +0500 Subject: [PATCH 21/29] feat(test_runner): add delayMs support for mocked APIs in ensemble tests Enhanced the ensemble test framework by introducing a `delayMs` property for mocked API responses, allowing for simulated latency in tests. Updated the documentation to reflect this new feature and added tests to ensure that mocked APIs honor the specified delay before returning responses. This improvement aligns mock behavior with real API loading times, enhancing the realism of test scenarios. --- .../doc/TEST_AUTHORING.md | 4 +- .../ensemble_test_execution_planner_test.dart | 2 + .../test/test_api_provider_overlay_test.dart | 37 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md index 5e584943a..b6302af51 100644 --- a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md +++ b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md @@ -108,6 +108,7 @@ overrides: { "getDevices": { "statusCode": 200, + "delayMs": 300, "body": { "status": [] } @@ -115,7 +116,8 @@ overrides: } ``` -Later files override earlier files. +Later files override earlier files. Use `delayMs` when a mock should stay +pending briefly before returning, matching the loading behavior of a real API. ## Validation diff --git a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart index c55c29472..80c1803a1 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart @@ -172,6 +172,7 @@ steps: { "getDevices": { "statusCode": 200, + "delayMs": 125, "body": {"count": 2} }, "scenarioApi": { @@ -190,6 +191,7 @@ steps: final mocks = definitions.single.testCase.mocks.apis; expect((mocks['getDevices']!.body as Map)['count'], 2); + expect(mocks['getDevices']!.delayMs, 125); expect((mocks['rootApi']!.body as Map)['from'], 'layer'); expect((mocks['scenarioApi']!.body as Map)['from'], 'scenario-layer'); }); diff --git a/tools/ensemble_test_runner/test/test_api_provider_overlay_test.dart b/tools/ensemble_test_runner/test/test_api_provider_overlay_test.dart index aebf8f0e5..0e3e19d2d 100644 --- a/tools/ensemble_test_runner/test/test_api_provider_overlay_test.dart +++ b/tools/ensemble_test_runner/test/test_api_provider_overlay_test.dart @@ -53,6 +53,43 @@ void main() { expect(response.body, {'token': 'abc'}); }); + test('mocked APIs honor delayMs before returning', () async { + final delegate = _RecordingAPIProvider(); + final provider = TestApiProviderOverlay( + mocks: { + 'slowProfile': MockAPIResponse( + statusCode: 200, + body: {'name': 'Jane'}, + delayMs: 50, + ), + }, + delegate: delegate, + ); + final context = _FakeBuildContext(); + var completed = false; + + final responseFuture = provider + .invokeApiWithDelegate( + delegate, + context, + YamlMap.wrap({'type': 'http', 'url': 'https://example.test/profile'}), + DataContext(buildContext: context), + 'slowProfile', + ) + .then((response) { + completed = true; + return response; + }); + + await Future.delayed(const Duration(milliseconds: 10)); + expect(completed, isFalse); + + final response = await responseFuture; + expect(completed, isTrue); + expect(delegate.invoked, isFalse); + expect(response.body, {'name': 'Jane'}); + }); + test('serializes live runner calls to avoid reentrant runAsync', () async { var inFlight = 0; var maxInFlight = 0; From 578e562b261ce3e3019fd17a002ee52271de0d2d Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Wed, 15 Jul 2026 02:16:08 +0500 Subject: [PATCH 22/29] feat(test_runner): add recording and screenshot contact sheet features Enhanced the ensemble test runner by introducing a new `record` configuration option, allowing users to enable session recordings during tests. Implemented functionality to capture and save recordings as GIFs along with a JSON manifest. Additionally, added a feature to generate a screenshot contact sheet, compiling automatic screenshots into a single image for easier review. Updated the schema, models, and documentation to reflect these new capabilities, improving the overall testing experience and artifact management. --- tools/ensemble_test_runner/STEP_VOCABULARY.md | 13 +- .../schema/ensemble_test_config_schema.json | 15 ++ .../assets/schema/ensemble_tests_schema.json | 170 -------------- .../doc/TEST_AUTHORING.md | 13 ++ .../lib/actions/extended_step_handlers.dart | 99 ++------ .../lib/actions/screenshot_device.dart | 24 +- .../lib/actions/test_step_executor.dart | 68 ++++-- .../ensemble_test_execution_planner.dart | 5 + .../lib/models/ensemble_test_models.dart | 20 ++ .../lib/parser/ensemble_test_parser.dart | 11 + .../lib/runner/ensemble_test_harness.dart | 11 +- .../lib/runner/ensemble_test_runner.dart | 97 +++++--- .../lib/runner/screenshot_contact_sheet.dart | 87 +++++++ .../lib/runner/session_recording.dart | 128 ++++++++++ .../lib/runner/test_runtime_state.dart | 44 +++- .../schema/ensemble_test_schema_builder.dart | 9 + .../lib/vocabulary/test_step_arg_kind.dart | 14 -- .../lib/vocabulary/test_step_registry.dart | 33 --- tools/ensemble_test_runner/pubspec.yaml | 1 + .../ensemble_test_execution_planner_test.dart | 16 ++ .../test/ensemble_test_parser_test.dart | 15 ++ .../test/ensemble_test_schema_test.dart | 9 + .../test/screenshot_device_test.dart | 46 ++-- .../test/session_recording_test.dart | 59 +++++ .../test/test_step_executor_test.dart | 219 ------------------ .../tool/generate_step_registry.dart | 32 --- 26 files changed, 605 insertions(+), 653 deletions(-) create mode 100644 tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart create mode 100644 tools/ensemble_test_runner/lib/runner/session_recording.dart create mode 100644 tools/ensemble_test_runner/test/session_recording_test.dart diff --git a/tools/ensemble_test_runner/STEP_VOCABULARY.md b/tools/ensemble_test_runner/STEP_VOCABULARY.md index ce4c1b2d4..90e0715c1 100644 --- a/tools/ensemble_test_runner/STEP_VOCABULARY.md +++ b/tools/ensemble_test_runner/STEP_VOCABULARY.md @@ -40,7 +40,7 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense `setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` ### Scripts / debug / quality -`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `logStorage`, `logPerformance`, `screenshot`, `dumpTree`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` +`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` ### Control flow `group`, `repeat`, `optional`, `ifVisible` @@ -66,18 +66,17 @@ steps: Editor validation: `https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json` (committed copy at [`assets/schema/ensemble_tests_schema.json`](assets/schema/ensemble_tests_schema.json), regenerate with `dart run tool/generate_schema.dart`). Arg shapes come from [`TestStepArgKind`](lib/vocabulary/test_step_arg_kind.dart) on each [`TestStepRegistryEntry`](lib/vocabulary/test_step_registry.dart). -`logPerformance` writes Flutter app frame timing metrics to -`build/ensemble_test_runner/logs/_app_performance.json`. To write this -artifact automatically once after the full suite, set this in -`tests/config.yaml`: +Performance logging writes Flutter app frame timing metrics to +`build/ensemble_test_runner/logs/app_performance.json`. Enable it once for the +full suite in `tests/config.yaml`: ```yaml performance: enabled: true ``` -`dumpTree`, `logApiCalls`, and `logStorage` can also be written automatically -once after the full suite from `tests/config.yaml`: +`dumpTree`, `logApiCalls`, and `logStorage` are also suite-level config +artifacts: ```yaml dumpTree: diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json index 2a2089756..0ac1e6afa 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json @@ -33,6 +33,21 @@ } } }, + "record": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "platform": { + "type": "string" + }, + "model": { + "type": "string" + } + } + }, "performance": { "type": "object", "additionalProperties": false, diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 7bdf08c9c..555f7467c 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -1909,64 +1909,6 @@ {} ] }, - "args_logPerformance": { - "type": "object", - "additionalProperties": false, - "title": "logPerformance", - "description": "Write Flutter app frame timing metrics to a JSON artifact", - "examples": [ - {} - ] - }, - "args_screenshot": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "model": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "screenshot", - "description": "Capture a screenshot artifact for debugging", - "examples": [ - { - "name": "home_screen", - "platform": "ios", - "model": "iPhone 15 Pro" - } - ] - }, - "args_dumpTree": { - "type": "object", - "additionalProperties": false, - "title": "dumpTree", - "description": "Print the widget tree to the debug console", - "examples": [ - {} - ] - }, - "args_logStorage": { - "type": "object", - "properties": { - "key": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "logStorage", - "description": "Log one public storage value, or all public storage when key is omitted", - "examples": [ - { - "key": "onboarding_done" - } - ] - }, "args_expectNoConsoleErrors": { "type": "object", "additionalProperties": false, @@ -4703,118 +4645,6 @@ "logApiCalls" ] }, - { - "type": "object", - "title": "logPerformance", - "description": "Write Flutter app frame timing metrics to a JSON artifact", - "examples": [ - { - "logPerformance": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "logPerformance": { - "$ref": "#/$defs/args_logPerformance", - "description": "Write Flutter app frame timing metrics to a JSON artifact", - "examples": [ - {} - ] - } - }, - "required": [ - "logPerformance" - ] - }, - { - "type": "object", - "title": "screenshot", - "description": "Capture a screenshot artifact for debugging", - "examples": [ - { - "screenshot": { - "name": "home_screen", - "platform": "ios", - "model": "iPhone 15 Pro" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "screenshot": { - "$ref": "#/$defs/args_screenshot", - "description": "Capture a screenshot artifact for debugging", - "examples": [ - { - "name": "home_screen", - "platform": "ios", - "model": "iPhone 15 Pro" - } - ] - } - }, - "required": [ - "screenshot" - ] - }, - { - "type": "object", - "title": "dumpTree", - "description": "Print the widget tree to the debug console", - "examples": [ - { - "dumpTree": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "dumpTree": { - "$ref": "#/$defs/args_dumpTree", - "description": "Print the widget tree to the debug console", - "examples": [ - {} - ] - } - }, - "required": [ - "dumpTree" - ] - }, - { - "type": "object", - "title": "logStorage", - "description": "Log one public storage value, or all public storage when key is omitted", - "examples": [ - { - "logStorage": { - "key": "onboarding_done" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "logStorage": { - "$ref": "#/$defs/args_logStorage", - "description": "Log one public storage value, or all public storage when key is omitted", - "examples": [ - { - "key": "onboarding_done" - } - ] - } - }, - "required": [ - "logStorage" - ] - }, { "type": "object", "title": "expectNoConsoleErrors", diff --git a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md index b6302af51..46b07cf2e 100644 --- a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md +++ b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md @@ -47,6 +47,11 @@ screenshots: platform: ios model: iPhone 15 Pro +record: + enabled: true + platform: ios + model: iPhone 15 Pro + performance: enabled: true dumpTree: @@ -57,6 +62,14 @@ logStorage: enabled: true ``` +When `screenshots.enabled` is true, the runner captures automatic step +screenshots into one contact sheet per test case under +`build/ensemble_test_runner/screenshots/`. + +When `record.enabled` is true, the runner writes +`build/ensemble_test_runner/recordings/recording.gif` and a matching +`recording.json` frame manifest. + ## App Context `--inspect-app` emits JSON with screens, widget IDs, APIs, navigation targets, imports, storage/env references, and lifecycle hints. diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 4476e78d3..6dfb11635 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -1,4 +1,3 @@ -import 'dart:io'; import 'dart:ui' as ui; import 'package:ensemble/action/navigation_action.dart'; @@ -6,11 +5,8 @@ import 'package:ensemble/screen_controller.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble_device_preview/ensemble_device_preview.dart'; -import 'package:ensemble_test_runner/actions/screenshot_device.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -import 'package:ensemble_test_runner/runner/app_performance_log.dart'; -import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; @@ -222,23 +218,6 @@ class ExtendedStepHandlers { executor.assertions .expectConsoleLog(step.args['contains']?.toString() ?? ''); return true; - case 'logStorage': - final key = step.args['key']?.toString(); - final path = await executor.tester.runAsync(() { - return writeStorageLog(executor.context, key: key); - }); - executor.context.logger.log( - key == null || key.isEmpty - ? 'storage: $path' - : 'storage[$key]: $path', - ); - return true; - case 'logPerformance': - final path = await executor.tester.runAsync(() { - return writeAppPerformanceLog(executor.context); - }); - executor.context.logger.log('appPerformance: $path'); - return true; case 'expectAccessible': executor.assertions.expectAccessible(executor.requireId(step)); return true; @@ -259,15 +238,6 @@ class ExtendedStepHandlers { case 'expectNoErrors': executor.assertions.expectNoRenderErrors(); return true; - case 'screenshot': - await _screenshot(executor, step); - return true; - case 'dumpTree': - final path = await executor.tester.runAsync(() { - return writeDumpTreeLog(executor.context); - }); - executor.context.logger.log('dumpTree: $path'); - return true; case 'expectNoConsoleErrors': executor.assertions.expectNoConsoleErrors(); return true; @@ -513,34 +483,20 @@ class ExtendedStepHandlers { } } - static Future _screenshot(TestStepExecutor e, TestStep step) async { - final args = e.context.config.screenshots.toScreenshotArgs(step.args); - await captureScreenshot(e, args: args); - } - - static Future captureScreenshot( - TestStepExecutor e, { - required Map args, - bool deferWrite = false, - bool pumpBeforeCapture = true, + static Future captureScreenshotBytes( + WidgetTester tester, { + required DeviceInfo device, }) async { - if (pumpBeforeCapture) { - await e.tester.pump(); - } - final rawName = args['name']?.toString(); - final name = rawName == null || rawName.isEmpty - ? DateTime.now().microsecondsSinceEpoch.toString() - : rawName; - final fileName = _safeFileName('${e.context.testCase.id}_$name.png'); - final directory = Directory('build/ensemble_test_runner/screenshots'); - final file = File('${directory.path}/$fileName'); - final device = resolveScreenshotDevice(args); - - if (pumpBeforeCapture) { - await e.tester.pump(); + final image = captureScreenshotImage(tester); + try { + return await encodeScreenshotImage(image, device); + } finally { + image.dispose(); } + } - final renderView = e.tester.binding.renderViews.first; + static ui.Image captureScreenshotImage(WidgetTester tester) { + final renderView = tester.binding.renderViews.first; final layer = renderView.debugLayer; if (layer is! OffsetLayer) { throw EnsembleTestFailure( @@ -548,30 +504,21 @@ class ExtendedStepHandlers { ); } - final image = layer.toImageSync( + return layer.toImageSync( renderView.paintBounds, pixelRatio: renderView.flutterView.devicePixelRatio, ); - final path = file.path; - - Future writeImage() async { - final byteData = await _addDeviceFrame(image, device); - image.dispose(); - if (byteData == null) { - throw EnsembleTestFailure('Failed to encode screenshot as PNG.'); - } - - directory.createSync(recursive: true); - file.writeAsBytesSync(byteData.buffer.asUint8List()); - } + } - if (deferWrite) { - e.context.runtime.pendingScreenshotWrites.add(writeImage); - } else { - await e.tester.runAsync(writeImage); + static Future encodeScreenshotImage( + ui.Image image, + DeviceInfo device, + ) async { + final byteData = await _addDeviceFrame(image, device); + if (byteData == null) { + throw EnsembleTestFailure('Failed to encode screenshot as PNG.'); } - - e.context.logger.log('screenshot: $path'); + return byteData.buffer.asUint8List(); } static Future _addDeviceFrame( @@ -619,10 +566,6 @@ class ExtendedStepHandlers { return byteData; } - static String _safeFileName(String value) { - return value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); - } - static bool _deepEquals(dynamic a, dynamic b) { if (a == b) return true; if (a is Map && b is Map) { diff --git a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart index 57c6dc036..de276fe37 100644 --- a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart +++ b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart @@ -1,35 +1,23 @@ import 'package:ensemble_device_preview/ensemble_device_preview.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -DeviceInfo? firstScreenshotDevice(List steps) { - for (final step in steps) { - final device = screenshotDeviceForStep(step); - if (device != null) return device; - - final nestedDevice = firstScreenshotDevice(step.nestedSteps); - if (nestedDevice != null) return nestedDevice; - } - return null; -} - DeviceInfo? screenshotDeviceForTestCase( - EnsembleTestCase testCase, + EnsembleTestCase _, EnsembleTestConfig config, ) { - final stepDevice = firstScreenshotDevice(testCase.steps); - if (stepDevice != null) return stepDevice; if (config.screenshots.enabled) { return resolveScreenshotDevice( config.screenshots.toScreenshotArgs(), ); } + if (config.record.enabled) { + return resolveScreenshotDevice( + config.record.toScreenshotArgs(), + ); + } return null; } -DeviceInfo? screenshotDeviceForStep(TestStep step) { - return step.type == 'screenshot' ? resolveScreenshotDevice(step.args) : null; -} - DeviceInfo resolveScreenshotDevice(Map args) { final platform = args['platform']?.toString(); final model = args['model']?.toString(); diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index f42f8e941..8c10344a5 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -9,6 +9,7 @@ import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/runner/session_recording.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; import 'package:flutter/material.dart'; @@ -78,7 +79,7 @@ class TestStepExecutor { await tester.runAsync(() async { await Future.delayed(Duration(milliseconds: durationMs)); }); - await tester.pump(); + await _pumpAndMaybeRecord(label: 'wait'); return; case 'waitForText': await _waitFor( @@ -171,11 +172,12 @@ class TestStepExecutor { ); break; case 'pump': - await tester.pump( - Duration( + await _pumpAndMaybeRecord( + duration: Duration( milliseconds: step.args['durationMs'] as int? ?? config.waitPollInterval.inMilliseconds, ), + label: 'pump', ); break; case 'settle': @@ -348,11 +350,25 @@ class TestStepExecutor { Future _settle({Duration? timeout}) async { try { - await tester.pumpAndSettle( - config.settleStepDuration, - EnginePhase.sendSemanticsUpdate, - timeout ?? config.settleTimeout, - ); + if (context.config.record.enabled) { + final deadline = DateTime.now().add(timeout ?? config.settleTimeout); + do { + if (DateTime.now().isAfter(deadline)) { + throw TimeoutException('settle timed out'); + } + await _pumpAndMaybeRecord( + duration: config.settleStepDuration, + phase: EnginePhase.sendSemanticsUpdate, + label: 'settle', + ); + } while (tester.binding.hasScheduledFrame); + } else { + await tester.pumpAndSettle( + config.settleStepDuration, + EnginePhase.sendSemanticsUpdate, + timeout ?? config.settleTimeout, + ); + } } catch (e) { if (e.toString().contains('timed out') || e.toString().contains('timeout')) { @@ -380,7 +396,7 @@ class TestStepExecutor { // Keep polling; HTTP may still be in flight inside runAsync. } } - await tester.pump(); + await _pumpAndMaybeRecord(label: 'liveApi'); if (!hadPending) { return; } @@ -397,7 +413,7 @@ class TestStepExecutor { } _expectSingleWidget(finder, id, 'tap'); await tester.ensureVisible(finder); - await tester.pump(); + await _pumpAndMaybeRecord(label: 'tap:before'); await tester.tap(finder); await _settle(); } @@ -462,7 +478,8 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); + await _pumpAndMaybeRecord( + duration: config.waitPollInterval, label: 'waitFor'); if (id != null && assertions.finderForId(id).evaluate().isNotEmpty) { return; } @@ -527,7 +544,8 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { await _yieldToLiveApiWork(); - await tester.pump(config.waitPollInterval); + await _pumpAndMaybeRecord( + duration: config.waitPollInterval, label: 'waitForApi'); if (context.apiOverlay.callCount(name) >= times) { await _yieldToLiveApiWork(); return; @@ -556,10 +574,13 @@ class TestStepExecutor { return; } await _yieldToLiveApiWork(); - await tester.pump(config.waitPollInterval); + await _pumpAndMaybeRecord( + duration: config.waitPollInterval, + label: 'waitForNavigation', + ); } await _yieldToLiveApiWork(); - await tester.pump(); + await _pumpAndMaybeRecord(label: 'waitForNavigation'); await YamlTestSession.navigationFlow.flushPending(); if (hasNavigated()) { return; @@ -579,7 +600,8 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); + await _pumpAndMaybeRecord( + duration: config.waitPollInterval, label: 'waitForGone'); if (assertions.finderForId(id).evaluate().isEmpty) { return; } @@ -588,4 +610,20 @@ class TestStepExecutor { 'Timed out after ${timeoutMs}ms waiting for id "$id" to disappear', ); } + + Future _pumpAndMaybeRecord({ + Duration? duration, + EnginePhase phase = EnginePhase.sendSemanticsUpdate, + required String label, + }) async { + await tester.pump(duration, phase); + if (!context.config.record.enabled) { + return; + } + await captureRecordingFrame( + tester, + context, + label: label, + ); + } } diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart index 0bbf14d5a..786c3a3c7 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart @@ -6,6 +6,7 @@ import 'package:flutter/services.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_discovery.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; +import 'package:yaml/yaml.dart'; typedef _AssetStringLoader = Future Function(String assetPath); @@ -129,6 +130,10 @@ class EnsembleTestExecutionPlanner { required Map inputs, _AssetStringLoader assetLoader = _rootBundleAssetLoader, }) async { + if (loadYaml(content) == null) { + return const []; + } + final base = EnsembleTestParser.parseString( content, sourcePath: path, diff --git a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart index d08f47ba4..84d87c5e3 100644 --- a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart +++ b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart @@ -97,6 +97,7 @@ class TestScenario { class EnsembleTestConfig { final ScreenshotConfig screenshots; + final RecordConfig record; final PerformanceConfig performance; final DumpTreeConfig dumpTree; final LogApiCallsConfig logApiCalls; @@ -104,6 +105,7 @@ class EnsembleTestConfig { const EnsembleTestConfig({ this.screenshots = const ScreenshotConfig(), + this.record = const RecordConfig(), this.performance = const PerformanceConfig(), this.dumpTree = const DumpTreeConfig(), this.logApiCalls = const LogApiCallsConfig(), @@ -145,6 +147,24 @@ class LogStorageConfig { }); } +class RecordConfig { + final bool enabled; + final String platform; + final String model; + + const RecordConfig({ + this.enabled = false, + this.platform = 'ios', + this.model = 'iPhone 15 Pro', + }); + + Map toScreenshotArgs([Map? overrides]) => { + 'platform': platform, + 'model': model, + ...?overrides, + }; +} + class ScreenshotConfig { final bool enabled; final String platform; diff --git a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart index 41ff4bdc2..8989942d2 100644 --- a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart +++ b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -150,6 +150,7 @@ class EnsembleTestParser { static EnsembleTestConfig _parseConfig(YamlMap node) { final screenshotsNode = node['screenshots']; + final recordNode = node['record']; final performanceNode = node['performance']; final dumpTreeNode = node['dumpTree']; final logApiCallsNode = node['logApiCalls']; @@ -157,6 +158,9 @@ class EnsembleTestParser { if (screenshotsNode != null && screenshotsNode is! YamlMap) { throw EnsembleTestFailure('"screenshots" must be a map'); } + if (recordNode != null && recordNode is! YamlMap) { + throw EnsembleTestFailure('"record" must be a map'); + } if (performanceNode != null && performanceNode is! YamlMap) { throw EnsembleTestFailure('"performance" must be a map'); } @@ -180,6 +184,13 @@ class EnsembleTestParser { includeSteps: _toStringList(screenshotsNode['includeSteps']), excludeSteps: _toStringList(screenshotsNode['excludeSteps']), ), + record: recordNode == null + ? const RecordConfig() + : RecordConfig( + enabled: recordNode['enabled'] == true, + platform: recordNode['platform']?.toString() ?? 'ios', + model: recordNode['model']?.toString() ?? 'iPhone 15 Pro', + ), performance: performanceNode == null ? const PerformanceConfig() : PerformanceConfig( diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index cde687d50..7f4594273 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -16,6 +16,7 @@ import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; @@ -489,9 +490,15 @@ class EnsembleTestHarness { } static Future _yieldToRealAsyncWork(WidgetTester tester) async { - await tester.runAsync(() async { + Future yieldOnce() async { await Future.delayed(Duration.zero); - }); + } + + if (LiveAsyncCallSupport.runner != null) { + await LiveAsyncCallSupport.run(yieldOnce); + } else { + await tester.runAsync(yieldOnce); + } } static Future applyInPlaceSetup(EnsembleTestContext ctx) async { diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index 1a6188029..f48adcddf 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -15,6 +15,8 @@ import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:ensemble_test_runner/runner/screenshot_contact_sheet.dart'; +import 'package:ensemble_test_runner/runner/session_recording.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter/scheduler.dart'; @@ -54,6 +56,7 @@ class EnsembleTestRunner { final suiteFrames = []; final suiteMarkers = []; final suiteApiCalls = []; + final suiteRecordingFrames = []; EnsembleTestContext? lastContext; var config = await harness.buildConfig(); @@ -97,6 +100,7 @@ class EnsembleTestRunner { ), ); suiteApiCalls.addAll(out.context.apiOverlay.calls); + suiteRecordingFrames.addAll(out.context.runtime.recordingFrames); } final suiteLogs = await _writeSuiteLogs( @@ -106,6 +110,7 @@ class EnsembleTestRunner { frames: suiteFrames, markers: suiteMarkers, apiCalls: suiteApiCalls, + recordingFrames: suiteRecordingFrames, lastContext: lastContext, ); @@ -222,18 +227,23 @@ class EnsembleTestRunner { harness: harness, config: config, ); + await _captureRecordingFrame(executor, label: '${test.id} startup'); for (var i = 0; i < test.steps.length; i++) { final step = test.steps[i]; final startFrame = ctx.runtime.appFrameTimings.length + 1; final startTime = DateTime.now(); try { - await executor.execute(step); - await YamlTestSession.navigationFlow.flushPending(); await _captureAutomaticScreenshotForStep( executor: executor, step: step, stepIndex: i, ); + await executor.execute(step); + await YamlTestSession.navigationFlow.flushPending(); + await _captureRecordingFrame( + executor, + label: '${test.id} step ${i + 1} ${formatStepBrief(step)}', + ); _recordPerformanceMarker( ctx: ctx, testId: test.id, @@ -256,8 +266,12 @@ class EnsembleTestRunner { final idleStartFrame = ctx.runtime.appFrameTimings.length + 1; final idleStartTime = DateTime.now(); await _settleLiveApiWork(tester, ctx); - await _flushPendingScreenshots(tester, ctx); + await _flushPendingScreenshots(ctx); await YamlTestSession.navigationFlow.flushPending(); + await _captureRecordingFrame( + executor, + label: '${test.id} failure cleanup', + ); _recordPerformanceMarker( ctx: ctx, testId: test.id, @@ -285,7 +299,8 @@ class EnsembleTestRunner { final idleStartFrame = ctx.runtime.appFrameTimings.length + 1; final idleStartTime = DateTime.now(); await _settleLiveApiWork(tester, ctx); - await _flushPendingScreenshots(tester, ctx); + await _flushPendingScreenshots(ctx); + await _captureRecordingFrame(executor, label: '${test.id} idle'); _recordPerformanceMarker( ctx: ctx, testId: test.id, @@ -312,30 +327,29 @@ class EnsembleTestRunner { }) async { final options = executor.context.config.screenshots; if (!options.shouldCaptureStep(step.type)) return; - await _captureAutomaticScreenshot( - executor, - name: - 'step_${(stepIndex + 1).toString().padLeft(3, '0')}_${_safeArtifactName(step.type)}', + if (executor.context.apiOverlay.hasPendingLiveCalls) return; + + executor.context.runtime.addScreenshotSheetFrame( + ScreenshotSheetFrame( + label: '${stepIndex + 1}. ${formatStepBrief(step)}', + image: ExtendedStepHandlers.captureScreenshotImage(executor.tester), + ), ); } - Future _captureAutomaticScreenshot( + Future _captureRecordingFrame( TestStepExecutor executor, { - required String name, - }) { - return ExtendedStepHandlers.captureScreenshot( - executor, - args: executor.context.config.screenshots.toScreenshotArgs({ - 'name': name, - }), - deferWrite: true, - pumpBeforeCapture: false, + required String label, + }) async { + if (!executor.context.config.record.enabled) return; + await captureRecordingFrame( + executor.tester, + executor.context, + label: label, + force: true, ); } - String _safeArtifactName(String value) => - value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); - void _recordPerformanceMarker({ required EnsembleTestContext ctx, required String testId, @@ -406,6 +420,7 @@ class EnsembleTestRunner { required List frames, required List markers, required List apiCalls, + required List recordingFrames, required EnsembleTestContext? lastContext, }) async { final logs = []; @@ -448,6 +463,16 @@ class EnsembleTestRunner { ); } + if (config.record.enabled) { + final path = await writeSessionRecording( + config: config.record, + frames: recordingFrames, + ); + if (path != null) { + logs.add('recording: $path'); + } + } + return logs; } @@ -465,20 +490,22 @@ class EnsembleTestRunner { } } - Future _flushPendingScreenshots( - WidgetTester tester, - EnsembleTestContext ctx, - ) async { - final writes = List Function()>.from( - ctx.runtime.pendingScreenshotWrites, + Future _flushPendingScreenshots(EnsembleTestContext ctx) async { + final sheetFrames = List.from( + ctx.runtime.screenshotSheetFrames, ); - ctx.runtime.pendingScreenshotWrites.clear(); - if (writes.isEmpty) return; - - await tester.runAsync(() async { - for (final write in writes) { - await write(); - } - }); + ctx.runtime.screenshotSheetFrames.clear(); + if (sheetFrames.isEmpty) return; + + final path = await LiveAsyncCallSupport.run( + () => writeScreenshotContactSheet( + testId: ctx.testCase.id, + config: ctx.config.screenshots, + frames: sheetFrames, + ), + ); + if (path != null) { + ctx.logger.log('screenshots: $path'); + } } } diff --git a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart new file mode 100644 index 000000000..3490fb418 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart @@ -0,0 +1,87 @@ +import 'dart:io'; +import 'dart:math' as math; + +import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; +import 'package:ensemble_test_runner/actions/screenshot_device.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:image/image.dart' as img; + +Future writeScreenshotContactSheet({ + required String testId, + required ScreenshotConfig config, + required List frames, +}) async { + if (frames.isEmpty) return null; + + final tiles = []; + final device = resolveScreenshotDevice(config.toScreenshotArgs()); + try { + for (final frame in frames) { + final pngBytes = await ExtendedStepHandlers.encodeScreenshotImage( + frame.image, + device, + ); + final source = img.decodePng(pngBytes); + if (source == null) continue; + tiles.add(_buildTile(source, frame.label)); + } + } finally { + for (final frame in frames) { + frame.image.dispose(); + } + } + + if (tiles.isEmpty) return null; + + const columns = 5; + const gap = 16; + final rows = (tiles.length / columns).ceil(); + final tileWidth = tiles.map((tile) => tile.width).reduce(math.max); + final tileHeight = tiles.map((tile) => tile.height).reduce(math.max); + final sheet = img.Image( + width: columns * tileWidth + (columns + 1) * gap, + height: rows * tileHeight + (rows + 1) * gap, + )..clear(img.ColorRgb8(240, 240, 240)); + + for (var i = 0; i < tiles.length; i++) { + final tile = tiles[i]; + final x = gap + (i % columns) * (tileWidth + gap); + final y = gap + (i ~/ columns) * (tileHeight + gap); + img.compositeImage(sheet, tile, dstX: x, dstY: y); + } + + final directory = Directory('build/ensemble_test_runner/screenshots'); + directory.createSync(recursive: true); + final file = File('${directory.path}/${_safeFileName(testId)}_sheet.jpg'); + file.writeAsBytesSync(img.encodeJpg(sheet, quality: 88)); + return file.path; +} + +img.Image _buildTile(img.Image source, String label) { + const width = 240; + const labelHeight = 44; + final thumbnail = img.copyResize( + source, + width: width, + interpolation: img.Interpolation.average, + ); + final tile = img.Image(width: width, height: thumbnail.height + labelHeight) + ..clear(img.ColorRgb8(255, 255, 255)); + img.compositeImage(tile, thumbnail, dstX: 0, dstY: labelHeight); + img.drawString( + tile, + _shortLabel(label), + font: img.arial14, + x: 4, + y: 4, + color: img.ColorRgb8(0, 0, 0), + ); + return tile; +} + +String _shortLabel(String label) => + label.length <= 34 ? label : '${label.substring(0, 31)}...'; + +String _safeFileName(String value) => + value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); diff --git a/tools/ensemble_test_runner/lib/runner/session_recording.dart b/tools/ensemble_test_runner/lib/runner/session_recording.dart new file mode 100644 index 000000000..84e9c220c --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/session_recording.dart @@ -0,0 +1,128 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; +import 'package:ensemble_test_runner/actions/screenshot_device.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:image/image.dart' as img; + +const _recordingFrameDurationMs = 800; +const _recordingFrameIntervalMs = 75; +const _recordingMaxFrames = 600; + +Future captureRecordingFrame( + WidgetTester tester, + EnsembleTestContext context, { + required String label, + bool force = false, +}) async { + final config = context.config.record; + if (!config.enabled) return; + if (context.runtime.recordingFrames.length >= _recordingMaxFrames) return; + + final now = DateTime.now(); + final lastTime = context.runtime.lastRecordingFrameTime; + if (!force && + lastTime != null && + now.difference(lastTime).inMilliseconds < _recordingFrameIntervalMs) { + return; + } + + final screenImage = ExtendedStepHandlers.captureScreenshotImage(tester); + + final durationMs = lastTime == null + ? _recordingFrameDurationMs + : now.difference(lastTime).inMilliseconds.clamp( + _recordingFrameIntervalMs, + _recordingFrameDurationMs * 4, + ); + context.runtime + ..lastRecordingFrameTime = now + ..addRecordingFrame( + RecordingFrame( + testId: context.testCase.id, + label: label, + image: screenImage, + timestamp: now, + durationMs: durationMs, + ), + ); +} + +Future writeSessionRecording({ + required RecordConfig config, + required List frames, +}) async { + if (!config.enabled || frames.isEmpty) return null; + + final directory = Directory('build/ensemble_test_runner/recordings'); + directory.createSync(recursive: true); + + final encoder = img.GifEncoder(repeat: 0); + for (final frame in frames) { + var image = await _decodeRecordingFrame(frame, config); + if (image != null) { + if (image.width > 480) { + image = img.copyResize( + image, + width: 480, + interpolation: img.Interpolation.average, + ); + } + final duration = (frame.durationMs / 10).round().clamp(1, 6000); + encoder.addFrame(image, duration: duration); + } + } + + final bytes = encoder.finish(); + if (bytes == null) return null; + + final gifFile = File('${directory.path}/recording.gif'); + gifFile.writeAsBytesSync(bytes); + + final manifestFile = File('${directory.path}/recording.json'); + manifestFile.writeAsStringSync( + const JsonEncoder.withIndent(' ').convert({ + 'file': gifFile.path, + 'frames': [ + for (var i = 0; i < frames.length; i++) + { + 'index': i + 1, + 'testId': frames[i].testId, + 'label': frames[i].label, + 'timestamp': frames[i].timestamp.toIso8601String(), + 'durationMs': frames[i].durationMs, + }, + ], + }), + ); + + return gifFile.path; +} + +Future _decodeRecordingFrame( + RecordingFrame frame, + RecordConfig config, +) async { + final pngBytes = frame.pngBytes; + if (pngBytes != null) { + return img.decodePng(pngBytes); + } + + final screenImage = frame.image; + if (screenImage == null) return null; + + try { + final device = resolveScreenshotDevice(config.toScreenshotArgs()); + final framedBytes = await ExtendedStepHandlers.encodeScreenshotImage( + screenImage, + device, + ); + return img.decodePng(framedBytes); + } finally { + screenImage.dispose(); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index 30d6709fa..36a311e26 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -10,7 +10,9 @@ class TestRuntimeState { final List flutterErrors = []; final List appFrameTimings = []; final List performanceMarkers = []; - final List Function()> pendingScreenshotWrites = []; + final List screenshotSheetFrames = []; + final List recordingFrames = []; + DateTime? lastRecordingFrameTime; Map? authUser; final Map permissions = {}; Size? deviceSize; @@ -23,7 +25,9 @@ class TestRuntimeState { flutterErrors.clear(); appFrameTimings.clear(); performanceMarkers.clear(); - pendingScreenshotWrites.clear(); + screenshotSheetFrames.clear(); + recordingFrames.clear(); + lastRecordingFrameTime = null; authUser = null; permissions.clear(); deviceSize = null; @@ -45,6 +49,42 @@ class TestRuntimeState { void recordPerformanceMarker(PerformanceMarker marker) { performanceMarkers.add(marker); } + + void addRecordingFrame(RecordingFrame frame) { + recordingFrames.add(frame); + } + + void addScreenshotSheetFrame(ScreenshotSheetFrame frame) { + screenshotSheetFrames.add(frame); + } +} + +class ScreenshotSheetFrame { + final String label; + final ui.Image image; + + const ScreenshotSheetFrame({ + required this.label, + required this.image, + }); +} + +class RecordingFrame { + final String testId; + final String label; + final Uint8List? pngBytes; + final ui.Image? image; + final DateTime timestamp; + final int durationMs; + + const RecordingFrame({ + required this.testId, + required this.label, + this.pngBytes, + this.image, + required this.timestamp, + required this.durationMs, + }); } class PerformanceMarker { diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index 328d176b2..c029b25b0 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -200,6 +200,15 @@ class EnsembleTestSchemaBuilder { }, }, }, + 'record': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + 'platform': {'type': 'string'}, + 'model': {'type': 'string'}, + }, + }, 'performance': { 'type': 'object', 'additionalProperties': false, diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart index 777f1e595..fcec134c5 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -28,12 +28,10 @@ enum TestStepArgKind { expectVisited, apiName, storageKey, - optionalStorageKey, group, repeat, optional, ifVisible, - screenshot, setAuth, setPermission, setDevice, @@ -206,10 +204,6 @@ extension TestStepArgKindSchema on TestStepArgKind { properties: {'key': _string, 'equals': _any, 'value': _any}, required: ['key'], ); - case TestStepArgKind.optionalStorageKey: - return _object( - properties: {'key': _string}, - ); case TestStepArgKind.group: return _object( properties: { @@ -242,14 +236,6 @@ extension TestStepArgKindSchema on TestStepArgKind { }, required: ['id'], ); - case TestStepArgKind.screenshot: - return _object( - properties: { - 'name': _string, - 'platform': _string, - 'model': _string, - }, - ); case TestStepArgKind.setAuth: return _object( properties: { diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index b26aaf221..1156648b5 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -679,39 +679,6 @@ abstract final class TestStepRegistry { description: 'Log all recorded API calls to the test log', example: const {}, ), - 'logPerformance': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Write Flutter app frame timing metrics to a JSON artifact', - example: const {}, - ), - 'screenshot': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.screenshot, - description: 'Capture a screenshot artifact for debugging', - example: const { - 'name': 'home_screen', - 'platform': 'ios', - 'model': 'iPhone 15 Pro' - }, - ), - 'dumpTree': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Print the widget tree to the debug console', - example: const {}, - ), - 'logStorage': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.optionalStorageKey, - description: - 'Log one public storage value, or all public storage when key is omitted', - example: const {'key': 'onboarding_done'}, - ), 'expectNoConsoleErrors': TestStepRegistryEntry( category: TestStepCategory.quality, tier: TestStepTier.core, diff --git a/tools/ensemble_test_runner/pubspec.yaml b/tools/ensemble_test_runner/pubspec.yaml index a93e36fc7..084c995ef 100644 --- a/tools/ensemble_test_runner/pubspec.yaml +++ b/tools/ensemble_test_runner/pubspec.yaml @@ -32,6 +32,7 @@ dependencies: sdk: flutter yaml: ^3.1.2 path: ^1.9.0 + image: ^4.8.0 firebase_core_platform_interface: ^7.0.1 cloud_functions_platform_interface: ^6.0.1 cloud_firestore_platform_interface: ^8.0.1 diff --git a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart index 80c1803a1..4e898a319 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart @@ -141,6 +141,22 @@ steps: expect(definitions[1].testCase.steps.single.args['text'], 0); }); + test('skips empty discovered test files', () async { + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'ensemble/apps/inhome/tests/commented.test.yaml', + ''' +# id: disabled_test +# startScreen: Home +# steps: +# - expectVisible: +# id: home_body +''', + ); + + expect(definitions, isEmpty); + }); + test('loads JSON mock files and applies override order', () async { const yaml = ''' id: home_scenarios diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index a5973eb34..2c9f1063c 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -247,6 +247,21 @@ performance: expect(config.performance.enabled, isTrue); }); + test('parses suite config recording', () { + const yaml = ''' +record: + enabled: true + platform: android + model: Samsung Galaxy S20 +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + final record = config.record; + expect(record.enabled, isTrue); + expect(record.platform, 'android'); + expect(record.model, 'Samsung Galaxy S20'); + }); + test('parses suite config debug artifacts', () { const yaml = ''' dumpTree: diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart index 22dc15db0..25e361ef6 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart @@ -72,6 +72,7 @@ void main() { expect(decoded['\$schema'], EnsembleTestSchemaBuilder.schemaVersion); expect(properties, contains('screenshots')); + expect(properties, contains('record')); expect(properties, contains('performance')); expect(properties, contains('dumpTree')); expect(properties, contains('logApiCalls')); @@ -80,6 +81,14 @@ void main() { (properties['screenshots'] as Map)['properties'], containsPair('model', {'type': 'string'}), ); + final recordProperties = + (properties['record'] as Map)['properties']; + expect(recordProperties, contains('enabled')); + expect(recordProperties, contains('platform')); + expect(recordProperties, contains('model')); + expect(recordProperties, isNot(contains('frameDurationMs'))); + expect(recordProperties, isNot(contains('frameIntervalMs'))); + expect(recordProperties, isNot(contains('maxFrames'))); }); test('initialState schema accepts storage, keychain, and env maps', () { diff --git a/tools/ensemble_test_runner/test/screenshot_device_test.dart b/tools/ensemble_test_runner/test/screenshot_device_test.dart index d0d79714b..364dc454f 100644 --- a/tools/ensemble_test_runner/test/screenshot_device_test.dart +++ b/tools/ensemble_test_runner/test/screenshot_device_test.dart @@ -18,48 +18,38 @@ void main() { expect(device.name, 'iPhone 15 Pro'); }); - test('finds the first screenshot in test steps', () { - final device = firstScreenshotDevice([ - const TestStep(type: 'tap', args: {'id': 'open'}), - const TestStep( - type: 'group', - args: {}, - nestedSteps: [ - TestStep( - type: 'screenshot', - args: { - 'name': 'home', - }, - ), + test('uses suite screenshot config device when enabled', () { + final device = screenshotDeviceForTestCase( + const EnsembleTestCase( + id: 'screenshots', + startScreen: 'Home', + steps: [ + TestStep(type: 'tap', args: {'id': 'button'}), ], ), - ]); - - expect(device?.name, 'iPhone 15 Pro'); - }); - - test('ignores tests without screenshots', () { - final device = firstScreenshotDevice([ - const TestStep( - type: 'tap', - args: {'id': 'button'}, + const EnsembleTestConfig( + screenshots: ScreenshotConfig( + enabled: true, + platform: 'android', + model: 'Samsung Galaxy S20', + ), ), - ]); + ); - expect(device, isNull); + expect(device?.name, 'Samsung Galaxy S20'); }); - test('uses suite screenshot config device when enabled', () { + test('uses suite record config device when enabled', () { final device = screenshotDeviceForTestCase( const EnsembleTestCase( - id: 'screenshots', + id: 'recording', startScreen: 'Home', steps: [ TestStep(type: 'tap', args: {'id': 'button'}), ], ), const EnsembleTestConfig( - screenshots: ScreenshotConfig( + record: RecordConfig( enabled: true, platform: 'android', model: 'Samsung Galaxy S20', diff --git a/tools/ensemble_test_runner/test/session_recording_test.dart b/tools/ensemble_test_runner/test/session_recording_test.dart new file mode 100644 index 000000000..c0c6f74b7 --- /dev/null +++ b/tools/ensemble_test_runner/test/session_recording_test.dart @@ -0,0 +1,59 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/session_recording.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:image/image.dart' as img; + +void main() { + test('writes suite recording gif and manifest', () async { + final directory = Directory('build/ensemble_test_runner/recordings'); + if (directory.existsSync()) { + directory.deleteSync(recursive: true); + } + + final png = img.encodePng( + img.Image(width: 2, height: 2)..clear(img.ColorRgb8(0, 255, 0)), + ); + + final path = await writeSessionRecording( + config: const RecordConfig(enabled: true), + frames: [ + RecordingFrame( + testId: 'login_test', + label: 'login_test startup', + pngBytes: Uint8List.fromList(png), + timestamp: DateTime.parse('2026-07-14T10:00:00Z'), + durationMs: 250, + ), + RecordingFrame( + testId: 'login_test', + label: 'login_test step 1 tap(login_button)', + pngBytes: Uint8List.fromList(png), + timestamp: DateTime.parse('2026-07-14T10:00:01Z'), + durationMs: 1000, + ), + ], + ); + + expect(path, 'build/ensemble_test_runner/recordings/recording.gif'); + final gif = File(path!); + expect(gif.existsSync(), isTrue); + expect(gif.readAsBytesSync().take(6), utf8.encode('GIF89a')); + + final manifest = jsonDecode( + File('build/ensemble_test_runner/recordings/recording.json') + .readAsStringSync(), + ) as Map; + expect(manifest, isNot(contains('frameDurationMs'))); + expect(manifest, isNot(contains('frameIntervalMs'))); + expect(manifest, isNot(contains('maxFrames'))); + expect(manifest['frames'], hasLength(2)); + expect((manifest['frames'] as List).last['label'], + 'login_test step 1 tap(login_button)'); + expect((manifest['frames'] as List).last['durationMs'], 1000); + }); +} diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index 4be663faa..da10139d0 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -10,7 +10,6 @@ import 'package:ensemble_test_runner/runner/app_performance_log.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; -import 'package:ensemble/framework/storage_manager.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:yaml/yaml.dart'; @@ -45,79 +44,6 @@ void main() { ); }); - testWidgets('logStorage logs all public storage when key is omitted', - (tester) async { - EnsembleTestHarness.ensureTestPlugins(); - await tester.runAsync(() async { - await StorageManager().init(); - await StorageManager().clearPublicStorage(); - await StorageManager().write('first', 'one'); - await StorageManager().write('second', {'nested': true}); - }); - - final context = EnsembleTestContext.fromTestCase( - const EnsembleTestCase( - id: 't', - startScreen: 'Home', - steps: [], - ), - ); - final harness = - EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); - final executor = TestStepExecutor( - tester: tester, - context: context, - assertions: AssertionEngine(tester: tester, context: context), - harness: harness, - ); - - await executor.execute(const TestStep(type: 'logStorage', args: {})); - - expect(context.logger.logs, hasLength(1)); - expect(context.logger.logs.single, startsWith('storage: ')); - final path = context.logger.logs.single.substring('storage: '.length); - expect(path, endsWith('.json')); - final file = File(path); - expect(file.existsSync(), isTrue); - final content = file.readAsStringSync(); - expect(content, contains('\n "first": "one"')); - final json = jsonDecode(content) as Map; - expect(json['first'], 'one'); - expect(json['second'], {'nested': true}); - }); - - testWidgets('dumpTree writes the widget tree to test logs', (tester) async { - await tester.pumpWidget( - const MaterialApp(home: Text('debug target')), - ); - final context = EnsembleTestContext.fromTestCase( - const EnsembleTestCase( - id: 'debug_test', - startScreen: 'Home', - steps: [], - ), - ); - final harness = - EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); - final executor = TestStepExecutor( - tester: tester, - context: context, - assertions: AssertionEngine(tester: tester, context: context), - harness: harness, - ); - - await executor.execute(const TestStep(type: 'dumpTree', args: {})); - - expect(context.logger.logs.single, startsWith('dumpTree: ')); - final path = context.logger.logs.single.substring('dumpTree: '.length); - expect(path, endsWith('.txt')); - final file = File(path); - expect(file.existsSync(), isTrue); - final content = file.readAsStringSync(); - expect(content, isNot(startsWith('dumpTree:'))); - expect(content, contains('debug target')); - }); - testWidgets('logApiCalls writes structured json', (tester) async { final context = EnsembleTestContext.fromTestCase( const EnsembleTestCase( @@ -148,51 +74,6 @@ void main() { }); }); - testWidgets('logPerformance writes app frame timing json', (tester) async { - final context = EnsembleTestContext.fromTestCase( - const EnsembleTestCase( - id: 'performance_log_test', - startScreen: 'Home', - steps: [], - ), - ); - context.runtime.appFrameTimings.add( - const AppFrameTimingEntry( - frameNumber: 1, - buildStartMicros: 1000, - buildMs: 5, - rasterMs: 7, - vsyncOverheadMs: 1, - totalSpanMs: 12, - ), - ); - final harness = - EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); - final executor = TestStepExecutor( - tester: tester, - context: context, - assertions: AssertionEngine(tester: tester, context: context), - harness: harness, - ); - - await executor.execute(const TestStep(type: 'logPerformance', args: {})); - - expect(context.logger.logs.single, startsWith('appPerformance: ')); - final path = - context.logger.logs.single.substring('appPerformance: '.length); - expect(path, endsWith('.json')); - final content = File(path).readAsStringSync(); - final json = jsonDecode(content) as Map; - expect(json['testId'], 'performance_log_test'); - expect(json['frameBudgetMs'], 16.67); - expect(json['summary'], containsPair('totalFrames', 1)); - expect(json['totalFrames'], 1); - expect(json['jankyFrames'], 0); - expect(json['averageBuildMs'], 5); - expect(json['averageRasterMs'], 7); - expect((json['frames'] as List).single, containsPair('totalSpanMs', 12)); - }); - test('performance log attributes frames and ranks jank context', () async { final logger = TestLogger(); final start = DateTime.now(); @@ -275,96 +156,6 @@ void main() { (json['slowestFrames'] as List).first, containsPair('screen', 'Login')); }); - testWidgets('screenshot writes a framed png by default', (tester) async { - await tester.pumpWidget( - const MaterialApp(home: Text('screenshot target')), - ); - final context = EnsembleTestContext.fromTestCase( - const EnsembleTestCase( - id: 'debug_test', - startScreen: 'Home', - steps: [], - ), - ); - final harness = - EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); - final executor = TestStepExecutor( - tester: tester, - context: context, - assertions: AssertionEngine(tester: tester, context: context), - harness: harness, - ); - - await executor.execute( - const TestStep(type: 'screenshot', args: {'name': 'home'}), - ); - - expect(context.logger.logs.single, startsWith('screenshot: ')); - final path = context.logger.logs.single.substring('screenshot: '.length); - final file = File(path); - expect(file.existsSync(), isTrue); - expect(file.readAsBytesSync().take(8), [137, 80, 78, 71, 13, 10, 26, 10]); - final dimensions = _pngDimensions(file.readAsBytesSync()); - - expect( - dimensions.$1, - greaterThan(tester.binding.renderViews.first.size.width), - ); - expect( - dimensions.$2, - greaterThan(tester.binding.renderViews.first.size.height), - ); - }); - - testWidgets('screenshot can use a custom device model', (tester) async { - await tester.pumpWidget( - const MaterialApp( - home: ColoredBox( - color: Colors.green, - child: Center(child: Text('framed screenshot target')), - ), - ), - ); - final context = EnsembleTestContext.fromTestCase( - const EnsembleTestCase( - id: 'framed_debug_test', - startScreen: 'Home', - steps: [], - ), - ); - final harness = - EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); - final executor = TestStepExecutor( - tester: tester, - context: context, - assertions: AssertionEngine(tester: tester, context: context), - harness: harness, - ); - - await executor.execute( - const TestStep( - type: 'screenshot', - args: { - 'name': 'home', - 'platform': 'android', - 'model': 'Samsung Galaxy S20', - }, - ), - ); - - final path = context.logger.logs.single.substring('screenshot: '.length); - final dimensions = _pngDimensions(File(path).readAsBytesSync()); - - expect( - dimensions.$1, - greaterThan(tester.binding.renderViews.first.size.width), - ); - expect( - dimensions.$2, - greaterThan(tester.binding.renderViews.first.size.height), - ); - }); - testWidgets('setDevice updates the render surface size', (tester) async { await tester.pumpWidget( const MaterialApp(home: SizedBox.expand()), @@ -395,13 +186,3 @@ void main() { expect(tester.binding.renderViews.first.size, const Size(393, 852)); }); } - -(int, int) _pngDimensions(List bytes) { - int readInt32(int offset) => - bytes[offset] << 24 | - bytes[offset + 1] << 16 | - bytes[offset + 2] << 8 | - bytes[offset + 3]; - - return (readInt32(16), readInt32(20)); -} diff --git a/tools/ensemble_test_runner/tool/generate_step_registry.dart b/tools/ensemble_test_runner/tool/generate_step_registry.dart index 5137fe865..e1704115b 100644 --- a/tools/ensemble_test_runner/tool/generate_step_registry.dart +++ b/tools/ensemble_test_runner/tool/generate_step_registry.dart @@ -543,30 +543,6 @@ void main() { 'empty', 'Log all recorded API calls to the test log', ), - 'logPerformance': step( - 'debug', - 'core', - 'empty', - 'Write Flutter app frame timing metrics to a JSON artifact', - ), - 'screenshot': step( - 'debug', - 'core', - 'screenshot', - 'Capture a screenshot artifact for debugging', - ), - 'dumpTree': step( - 'debug', - 'core', - 'empty', - 'Print the widget tree to the debug console', - ), - 'logStorage': step( - 'debug', - 'core', - 'optionalStorageKey', - 'Log one public storage value, or all public storage when key is omitted', - ), 'expectNoConsoleErrors': step( 'quality', 'core', @@ -746,8 +722,6 @@ Map defaultExampleForArg(String arg) { return {'name': 'login', 'times': 1}; case 'storageKey': return {'key': 'onboarding_done', 'value': true}; - case 'optionalStorageKey': - return {'key': 'onboarding_done'}; case 'group': return { 'name': 'login_flow', @@ -783,12 +757,6 @@ Map defaultExampleForArg(String arg) { }, ], }; - case 'screenshot': - return { - 'name': 'home_screen', - 'platform': 'ios', - 'model': 'iPhone 15 Pro', - }; case 'setAuth': return { 'user': {'id': '1', 'email': 'user@test.com'}, From 58d2762423fb374140429a53fbd272a822a9b091 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Wed, 15 Jul 2026 02:34:47 +0500 Subject: [PATCH 23/29] feat(test_runner): enhance screenshot functionality with highlighting and layout adjustments Improved the screenshot capturing process in the ensemble test runner by adding functionality to highlight specific UI elements during screenshot capture. Implemented methods to ensure the target elements are visible before capturing, and adjusted the layout of the screenshot contact sheet to center tiles dynamically. These enhancements provide clearer visual feedback in test results and improve the overall usability of the screenshot features. --- .../lib/runner/ensemble_test_runner.dart | 157 +++++++++++++++++- .../lib/runner/screenshot_contact_sheet.dart | 10 +- 2 files changed, 164 insertions(+), 3 deletions(-) diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index f48adcddf..24f0fcb14 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -19,6 +19,7 @@ import 'package:ensemble_test_runner/runner/screenshot_contact_sheet.dart'; import 'package:ensemble_test_runner/runner/session_recording.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter/rendering.dart'; import 'package:flutter/scheduler.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -329,14 +330,168 @@ class EnsembleTestRunner { if (!options.shouldCaptureStep(step.type)) return; if (executor.context.apiOverlay.hasPendingLiveCalls) return; + await _ensureHighlightTargetVisible(executor, step); + + var image = ExtendedStepHandlers.captureScreenshotImage(executor.tester); + final highlightRect = _highlightRectForStep(executor, step); + if (highlightRect != null) { + try { + final highlighted = await _highlightScreenshotImage( + tester: executor.tester, + image: image, + rect: highlightRect, + ); + image.dispose(); + image = highlighted; + } catch (_) { + // Screenshot annotations must never change test behavior. + } + } + executor.context.runtime.addScreenshotSheetFrame( ScreenshotSheetFrame( label: '${stepIndex + 1}. ${formatStepBrief(step)}', - image: ExtendedStepHandlers.captureScreenshotImage(executor.tester), + image: image, ), ); } + ui.Rect? _highlightRectForStep(TestStepExecutor executor, TestStep step) { + if (!_shouldHighlightStep(step)) return null; + + final id = step.args['id']?.toString(); + if (id == null || id.isEmpty) return null; + + final elements = executor.assertions.finderForId(id).evaluate(); + if (elements.isEmpty) return null; + + final renderObject = elements.first.renderObject; + if (renderObject == null) return null; + return _rectForRenderObject(renderObject); + } + + Future _ensureHighlightTargetVisible( + TestStepExecutor executor, + TestStep step, + ) async { + if (!_shouldHighlightStep(step)) return; + + final id = step.args['id']?.toString(); + if (id == null || id.isEmpty) return; + + final finder = executor.assertions.finderForId(id); + if (finder.evaluate().isEmpty) return; + + await executor.tester.ensureVisible(finder); + await executor.tester.pump(); + } + + ui.Rect? _rectForRenderObject(RenderObject renderObject) { + if (renderObject is RenderBox && + renderObject.hasSize && + renderObject.size.longestSide > 0) { + final topLeft = renderObject.localToGlobal(ui.Offset.zero); + final rect = topLeft & renderObject.size; + if (rect.isFinite && !rect.isEmpty) { + return rect; + } + } + + final rects = []; + + void collect(RenderObject object) { + if (object is RenderBox && + object.hasSize && + object.size.longestSide > 0) { + final topLeft = object.localToGlobal(ui.Offset.zero); + final rect = topLeft & object.size; + if (rect.isFinite && !rect.isEmpty) { + rects.add(rect); + } + } + object.visitChildren(collect); + } + + collect(renderObject); + if (rects.isEmpty) return null; + + return rects.reduce((a, b) => a.expandToInclude(b)); + } + + bool _shouldHighlightStep(TestStep step) { + switch (step.type) { + case 'tap': + case 'doubleTap': + case 'longPress': + case 'toggle': + case 'check': + case 'uncheck': + case 'enterText': + case 'clearText': + case 'replaceText': + case 'submitText': + case 'focus': + case 'select': + case 'selectIndex': + case 'setSlider': + return true; + default: + return false; + } + } + + Future _highlightScreenshotImage({ + required WidgetTester tester, + required ui.Image image, + required ui.Rect rect, + }) async { + final renderView = tester.binding.renderViews.first; + final paintBounds = renderView.paintBounds; + final scaleX = image.width / paintBounds.width; + final scaleY = image.height / paintBounds.height; + final scaledRect = ui.Rect.fromLTRB( + (rect.left - paintBounds.left) * scaleX, + (rect.top - paintBounds.top) * scaleY, + (rect.right - paintBounds.left) * scaleX, + (rect.bottom - paintBounds.top) * scaleY, + ).inflate(6 * scaleX); + + final recorder = ui.PictureRecorder(); + final canvas = ui.Canvas( + recorder, + ui.Rect.fromLTWH(0, 0, image.width.toDouble(), image.height.toDouble()), + ); + canvas.drawImage(image, ui.Offset.zero, ui.Paint()); + + final radius = ui.Radius.circular(10 * scaleX); + final rrect = ui.RRect.fromRectAndRadius(scaledRect, radius); + canvas.drawRRect( + rrect, + ui.Paint() + ..color = const ui.Color(0x33FFD400) + ..style = ui.PaintingStyle.fill, + ); + canvas.drawRRect( + rrect, + ui.Paint() + ..color = const ui.Color(0xFF006BFF) + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 5 * scaleX, + ); + canvas.drawCircle( + scaledRect.center, + 9 * scaleX, + ui.Paint() + ..color = const ui.Color(0xFF006BFF) + ..style = ui.PaintingStyle.fill, + ); + + final picture = recorder.endRecording(); + final highlighted = await picture.toImage(image.width, image.height); + picture.dispose(); + return highlighted; + } + Future _captureRecordingFrame( TestStepExecutor executor, { required String label, diff --git a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart index 3490fb418..3fd69f609 100644 --- a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart +++ b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart @@ -46,8 +46,14 @@ Future writeScreenshotContactSheet({ for (var i = 0; i < tiles.length; i++) { final tile = tiles[i]; - final x = gap + (i % columns) * (tileWidth + gap); - final y = gap + (i ~/ columns) * (tileHeight + gap); + final row = i ~/ columns; + final column = i % columns; + final remainingTiles = tiles.length - row * columns; + final rowTiles = math.min(columns, remainingTiles); + final rowWidth = rowTiles * tileWidth + (rowTiles - 1) * gap; + final rowStartX = ((sheet.width - rowWidth) / 2).round(); + final x = rowStartX + column * (tileWidth + gap); + final y = gap + row * (tileHeight + gap); img.compositeImage(sheet, tile, dstX: x, dstY: y); } From 65318a657a6a682599164997b4c4e50f95dd501d Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Wed, 15 Jul 2026 02:46:43 +0500 Subject: [PATCH 24/29] refactor(test_runner): update screenshot handling and layout adjustments Removed the check for pending live calls during screenshot capture to streamline the process. Changed the screenshot contact sheet format from JPG to PNG for better quality and adjusted the layout by increasing tile dimensions and label sizes. Enhanced the label truncation logic for improved readability. These changes aim to enhance the overall screenshot functionality and visual presentation in the ensemble test runner. --- .../lib/runner/ensemble_test_runner.dart | 1 - .../lib/runner/screenshot_contact_sheet.dart | 18 +++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index 24f0fcb14..78a875a6b 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -328,7 +328,6 @@ class EnsembleTestRunner { }) async { final options = executor.context.config.screenshots; if (!options.shouldCaptureStep(step.type)) return; - if (executor.context.apiOverlay.hasPendingLiveCalls) return; await _ensureHighlightTargetVisible(executor, step); diff --git a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart index 3fd69f609..8a43ee288 100644 --- a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart +++ b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart @@ -59,18 +59,18 @@ Future writeScreenshotContactSheet({ final directory = Directory('build/ensemble_test_runner/screenshots'); directory.createSync(recursive: true); - final file = File('${directory.path}/${_safeFileName(testId)}_sheet.jpg'); - file.writeAsBytesSync(img.encodeJpg(sheet, quality: 88)); + final file = File('${directory.path}/${_safeFileName(testId)}_sheet.png'); + file.writeAsBytesSync(img.encodePng(sheet)); return file.path; } img.Image _buildTile(img.Image source, String label) { - const width = 240; - const labelHeight = 44; + const width = 420; + const labelHeight = 56; final thumbnail = img.copyResize( source, width: width, - interpolation: img.Interpolation.average, + interpolation: img.Interpolation.cubic, ); final tile = img.Image(width: width, height: thumbnail.height + labelHeight) ..clear(img.ColorRgb8(255, 255, 255)); @@ -78,16 +78,16 @@ img.Image _buildTile(img.Image source, String label) { img.drawString( tile, _shortLabel(label), - font: img.arial14, - x: 4, - y: 4, + font: img.arial24, + x: 8, + y: 8, color: img.ColorRgb8(0, 0, 0), ); return tile; } String _shortLabel(String label) => - label.length <= 34 ? label : '${label.substring(0, 31)}...'; + label.length <= 30 ? label : '${label.substring(0, 27)}...'; String _safeFileName(String value) => value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); From 1d44c132982757898711752125a8682be27dbfdf Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Wed, 15 Jul 2026 03:01:52 +0500 Subject: [PATCH 25/29] refactor(test_runner): remove recording feature and update schema Eliminated the recording functionality from the ensemble test runner, including the associated configuration and related code. Updated the schema and documentation to reflect the removal of the `record` option. Adjusted tests to ensure compatibility with the new configuration structure, enhancing the clarity and maintainability of the test framework. --- .../schema/ensemble_test_config_schema.json | 15 -- .../doc/TEST_AUTHORING.md | 9 -- .../lib/actions/screenshot_device.dart | 5 - .../lib/actions/test_step_executor.dart | 56 ++------ .../lib/models/ensemble_test_models.dart | 20 --- .../lib/parser/ensemble_test_parser.dart | 27 ++-- .../lib/runner/ensemble_test_runner.dart | 38 ------ .../lib/runner/screenshot_contact_sheet.dart | 129 ++++++++++++++++-- .../lib/runner/session_recording.dart | 128 ----------------- .../lib/runner/test_runtime_state.dart | 26 ---- .../schema/ensemble_test_schema_builder.dart | 9 -- .../test/ensemble_test_parser_test.dart | 19 +-- .../test/ensemble_test_schema_test.dart | 10 +- .../test/screenshot_device_test.dart | 21 --- .../test/session_recording_test.dart | 59 -------- 15 files changed, 162 insertions(+), 409 deletions(-) delete mode 100644 tools/ensemble_test_runner/lib/runner/session_recording.dart delete mode 100644 tools/ensemble_test_runner/test/session_recording_test.dart diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json index 0ac1e6afa..2a2089756 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json @@ -33,21 +33,6 @@ } } }, - "record": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean" - }, - "platform": { - "type": "string" - }, - "model": { - "type": "string" - } - } - }, "performance": { "type": "object", "additionalProperties": false, diff --git a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md index 46b07cf2e..d2a66c560 100644 --- a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md +++ b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md @@ -47,11 +47,6 @@ screenshots: platform: ios model: iPhone 15 Pro -record: - enabled: true - platform: ios - model: iPhone 15 Pro - performance: enabled: true dumpTree: @@ -66,10 +61,6 @@ When `screenshots.enabled` is true, the runner captures automatic step screenshots into one contact sheet per test case under `build/ensemble_test_runner/screenshots/`. -When `record.enabled` is true, the runner writes -`build/ensemble_test_runner/recordings/recording.gif` and a matching -`recording.json` frame manifest. - ## App Context `--inspect-app` emits JSON with screens, widget IDs, APIs, navigation targets, imports, storage/env references, and lifecycle hints. diff --git a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart index de276fe37..d3172f6f7 100644 --- a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart +++ b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart @@ -10,11 +10,6 @@ DeviceInfo? screenshotDeviceForTestCase( config.screenshots.toScreenshotArgs(), ); } - if (config.record.enabled) { - return resolveScreenshotDevice( - config.record.toScreenshotArgs(), - ); - } return null; } diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 8c10344a5..0661ddbc7 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -9,7 +9,6 @@ import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; -import 'package:ensemble_test_runner/runner/session_recording.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; import 'package:flutter/material.dart'; @@ -79,7 +78,7 @@ class TestStepExecutor { await tester.runAsync(() async { await Future.delayed(Duration(milliseconds: durationMs)); }); - await _pumpAndMaybeRecord(label: 'wait'); + await _pump(label: 'wait'); return; case 'waitForText': await _waitFor( @@ -172,7 +171,7 @@ class TestStepExecutor { ); break; case 'pump': - await _pumpAndMaybeRecord( + await _pump( duration: Duration( milliseconds: step.args['durationMs'] as int? ?? config.waitPollInterval.inMilliseconds, @@ -350,25 +349,11 @@ class TestStepExecutor { Future _settle({Duration? timeout}) async { try { - if (context.config.record.enabled) { - final deadline = DateTime.now().add(timeout ?? config.settleTimeout); - do { - if (DateTime.now().isAfter(deadline)) { - throw TimeoutException('settle timed out'); - } - await _pumpAndMaybeRecord( - duration: config.settleStepDuration, - phase: EnginePhase.sendSemanticsUpdate, - label: 'settle', - ); - } while (tester.binding.hasScheduledFrame); - } else { - await tester.pumpAndSettle( - config.settleStepDuration, - EnginePhase.sendSemanticsUpdate, - timeout ?? config.settleTimeout, - ); - } + await tester.pumpAndSettle( + config.settleStepDuration, + EnginePhase.sendSemanticsUpdate, + timeout ?? config.settleTimeout, + ); } catch (e) { if (e.toString().contains('timed out') || e.toString().contains('timeout')) { @@ -396,7 +381,7 @@ class TestStepExecutor { // Keep polling; HTTP may still be in flight inside runAsync. } } - await _pumpAndMaybeRecord(label: 'liveApi'); + await _pump(label: 'liveApi'); if (!hadPending) { return; } @@ -413,7 +398,7 @@ class TestStepExecutor { } _expectSingleWidget(finder, id, 'tap'); await tester.ensureVisible(finder); - await _pumpAndMaybeRecord(label: 'tap:before'); + await _pump(label: 'tap:before'); await tester.tap(finder); await _settle(); } @@ -478,8 +463,7 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { - await _pumpAndMaybeRecord( - duration: config.waitPollInterval, label: 'waitFor'); + await _pump(duration: config.waitPollInterval, label: 'waitFor'); if (id != null && assertions.finderForId(id).evaluate().isNotEmpty) { return; } @@ -544,8 +528,7 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { await _yieldToLiveApiWork(); - await _pumpAndMaybeRecord( - duration: config.waitPollInterval, label: 'waitForApi'); + await _pump(duration: config.waitPollInterval, label: 'waitForApi'); if (context.apiOverlay.callCount(name) >= times) { await _yieldToLiveApiWork(); return; @@ -574,13 +557,13 @@ class TestStepExecutor { return; } await _yieldToLiveApiWork(); - await _pumpAndMaybeRecord( + await _pump( duration: config.waitPollInterval, label: 'waitForNavigation', ); } await _yieldToLiveApiWork(); - await _pumpAndMaybeRecord(label: 'waitForNavigation'); + await _pump(label: 'waitForNavigation'); await YamlTestSession.navigationFlow.flushPending(); if (hasNavigated()) { return; @@ -600,8 +583,7 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { - await _pumpAndMaybeRecord( - duration: config.waitPollInterval, label: 'waitForGone'); + await tester.pump(config.waitPollInterval); if (assertions.finderForId(id).evaluate().isEmpty) { return; } @@ -611,19 +593,11 @@ class TestStepExecutor { ); } - Future _pumpAndMaybeRecord({ + Future _pump({ Duration? duration, EnginePhase phase = EnginePhase.sendSemanticsUpdate, required String label, }) async { await tester.pump(duration, phase); - if (!context.config.record.enabled) { - return; - } - await captureRecordingFrame( - tester, - context, - label: label, - ); } } diff --git a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart index 84d87c5e3..d08f47ba4 100644 --- a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart +++ b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart @@ -97,7 +97,6 @@ class TestScenario { class EnsembleTestConfig { final ScreenshotConfig screenshots; - final RecordConfig record; final PerformanceConfig performance; final DumpTreeConfig dumpTree; final LogApiCallsConfig logApiCalls; @@ -105,7 +104,6 @@ class EnsembleTestConfig { const EnsembleTestConfig({ this.screenshots = const ScreenshotConfig(), - this.record = const RecordConfig(), this.performance = const PerformanceConfig(), this.dumpTree = const DumpTreeConfig(), this.logApiCalls = const LogApiCallsConfig(), @@ -147,24 +145,6 @@ class LogStorageConfig { }); } -class RecordConfig { - final bool enabled; - final String platform; - final String model; - - const RecordConfig({ - this.enabled = false, - this.platform = 'ios', - this.model = 'iPhone 15 Pro', - }); - - Map toScreenshotArgs([Map? overrides]) => { - 'platform': platform, - 'model': model, - ...?overrides, - }; -} - class ScreenshotConfig { final bool enabled; final String platform; diff --git a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart index 8989942d2..5df554a1a 100644 --- a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart +++ b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -149,8 +149,23 @@ class EnsembleTestParser { } static EnsembleTestConfig _parseConfig(YamlMap node) { + const allowedKeys = { + 'screenshots', + 'performance', + 'dumpTree', + 'logApiCalls', + 'logStorage', + }; + for (final key in node.keys) { + if (!allowedKeys.contains(key)) { + throw EnsembleTestFailure( + 'Unsupported test config key "$key". Supported keys: ' + '${allowedKeys.join(', ')}', + ); + } + } + final screenshotsNode = node['screenshots']; - final recordNode = node['record']; final performanceNode = node['performance']; final dumpTreeNode = node['dumpTree']; final logApiCallsNode = node['logApiCalls']; @@ -158,9 +173,6 @@ class EnsembleTestParser { if (screenshotsNode != null && screenshotsNode is! YamlMap) { throw EnsembleTestFailure('"screenshots" must be a map'); } - if (recordNode != null && recordNode is! YamlMap) { - throw EnsembleTestFailure('"record" must be a map'); - } if (performanceNode != null && performanceNode is! YamlMap) { throw EnsembleTestFailure('"performance" must be a map'); } @@ -184,13 +196,6 @@ class EnsembleTestParser { includeSteps: _toStringList(screenshotsNode['includeSteps']), excludeSteps: _toStringList(screenshotsNode['excludeSteps']), ), - record: recordNode == null - ? const RecordConfig() - : RecordConfig( - enabled: recordNode['enabled'] == true, - platform: recordNode['platform']?.toString() ?? 'ios', - model: recordNode['model']?.toString() ?? 'iPhone 15 Pro', - ), performance: performanceNode == null ? const PerformanceConfig() : PerformanceConfig( diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index 78a875a6b..20d46db10 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -16,7 +16,6 @@ import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/live_async_call.dart'; import 'package:ensemble_test_runner/runner/screenshot_contact_sheet.dart'; -import 'package:ensemble_test_runner/runner/session_recording.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter/rendering.dart'; @@ -57,7 +56,6 @@ class EnsembleTestRunner { final suiteFrames = []; final suiteMarkers = []; final suiteApiCalls = []; - final suiteRecordingFrames = []; EnsembleTestContext? lastContext; var config = await harness.buildConfig(); @@ -101,7 +99,6 @@ class EnsembleTestRunner { ), ); suiteApiCalls.addAll(out.context.apiOverlay.calls); - suiteRecordingFrames.addAll(out.context.runtime.recordingFrames); } final suiteLogs = await _writeSuiteLogs( @@ -111,7 +108,6 @@ class EnsembleTestRunner { frames: suiteFrames, markers: suiteMarkers, apiCalls: suiteApiCalls, - recordingFrames: suiteRecordingFrames, lastContext: lastContext, ); @@ -228,7 +224,6 @@ class EnsembleTestRunner { harness: harness, config: config, ); - await _captureRecordingFrame(executor, label: '${test.id} startup'); for (var i = 0; i < test.steps.length; i++) { final step = test.steps[i]; final startFrame = ctx.runtime.appFrameTimings.length + 1; @@ -241,10 +236,6 @@ class EnsembleTestRunner { ); await executor.execute(step); await YamlTestSession.navigationFlow.flushPending(); - await _captureRecordingFrame( - executor, - label: '${test.id} step ${i + 1} ${formatStepBrief(step)}', - ); _recordPerformanceMarker( ctx: ctx, testId: test.id, @@ -269,10 +260,6 @@ class EnsembleTestRunner { await _settleLiveApiWork(tester, ctx); await _flushPendingScreenshots(ctx); await YamlTestSession.navigationFlow.flushPending(); - await _captureRecordingFrame( - executor, - label: '${test.id} failure cleanup', - ); _recordPerformanceMarker( ctx: ctx, testId: test.id, @@ -301,7 +288,6 @@ class EnsembleTestRunner { final idleStartTime = DateTime.now(); await _settleLiveApiWork(tester, ctx); await _flushPendingScreenshots(ctx); - await _captureRecordingFrame(executor, label: '${test.id} idle'); _recordPerformanceMarker( ctx: ctx, testId: test.id, @@ -491,19 +477,6 @@ class EnsembleTestRunner { return highlighted; } - Future _captureRecordingFrame( - TestStepExecutor executor, { - required String label, - }) async { - if (!executor.context.config.record.enabled) return; - await captureRecordingFrame( - executor.tester, - executor.context, - label: label, - force: true, - ); - } - void _recordPerformanceMarker({ required EnsembleTestContext ctx, required String testId, @@ -574,7 +547,6 @@ class EnsembleTestRunner { required List frames, required List markers, required List apiCalls, - required List recordingFrames, required EnsembleTestContext? lastContext, }) async { final logs = []; @@ -617,16 +589,6 @@ class EnsembleTestRunner { ); } - if (config.record.enabled) { - final path = await writeSessionRecording( - config: config.record, - frames: recordingFrames, - ); - if (path != null) { - logs.add('recording: $path'); - } - } - return logs; } diff --git a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart index 8a43ee288..338a29cfa 100644 --- a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart +++ b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart @@ -66,7 +66,7 @@ Future writeScreenshotContactSheet({ img.Image _buildTile(img.Image source, String label) { const width = 420; - const labelHeight = 56; + const labelHeight = 72; final thumbnail = img.copyResize( source, width: width, @@ -74,20 +74,129 @@ img.Image _buildTile(img.Image source, String label) { ); final tile = img.Image(width: width, height: thumbnail.height + labelHeight) ..clear(img.ColorRgb8(255, 255, 255)); - img.compositeImage(tile, thumbnail, dstX: 0, dstY: labelHeight); - img.drawString( + img.fillRect( + tile, + x1: 0, + y1: 0, + x2: width - 1, + y2: labelHeight - 1, + color: img.ColorRgb8(255, 255, 255), + ); + img.drawLine( tile, - _shortLabel(label), - font: img.arial24, - x: 8, - y: 8, - color: img.ColorRgb8(0, 0, 0), + x1: 0, + y1: labelHeight - 1, + x2: width - 1, + y2: labelHeight - 1, + color: img.ColorRgb8(224, 229, 236), ); + img.compositeImage(tile, thumbnail, dstX: 0, dstY: labelHeight); + _drawLabel(tile, label, width: width, height: labelHeight); return tile; } -String _shortLabel(String label) => - label.length <= 30 ? label : '${label.substring(0, 27)}...'; +void _drawLabel( + img.Image image, + String label, { + required int width, + required int height, +}) { + const horizontalPadding = 12; + final availableWidth = width - horizontalPadding * 2; + final lines = _fitLabelLines(label, img.arial24, availableWidth); + final lineHeight = img.arial24.lineHeight; + final contentHeight = lines.length * lineHeight; + final startY = ((height - contentHeight) / 2).round(); + + for (var i = 0; i < lines.length; i++) { + final x = ((width - _textWidth(lines[i], img.arial24)) / 2).round(); + img.drawString( + image, + lines[i], + font: img.arial24, + x: math.max(horizontalPadding, x), + y: startY + i * lineHeight, + color: img.ColorRgb8(20, 26, 36), + ); + } +} + +List _fitLabelLines(String label, img.BitmapFont font, int maxWidth) { + if (_textWidth(label, font) <= maxWidth) { + return [label]; + } + + final breakIndex = _bestBreakIndex(label, font, maxWidth); + if (breakIndex != null) { + final first = label.substring(0, breakIndex).trimRight(); + final second = label.substring(breakIndex).trimLeft(); + if (_textWidth(second, font) <= maxWidth) { + return [first, second]; + } + return [first, _ellipsis(second, font, maxWidth)]; + } + + return [_ellipsis(label, font, maxWidth)]; +} + +int? _bestBreakIndex(String label, img.BitmapFont font, int maxWidth) { + final candidates = []; + for (var i = 1; i < label.length; i++) { + final previous = label[i - 1]; + final current = label[i]; + if (previous == ' ' || + previous == '_' || + previous == '(' || + current == ')' || + current == '_' || + current == '(') { + candidates.add(i); + } + } + + int? best; + var bestScore = double.infinity; + for (final index in candidates) { + final first = label.substring(0, index).trimRight(); + final second = label.substring(index).trimLeft(); + if (first.isEmpty || second.isEmpty) continue; + if (_textWidth(first, font) > maxWidth) continue; + + final overflow = math.max(0, _textWidth(second, font) - maxWidth); + final balance = (first.length - second.length).abs(); + final score = overflow * 100 + balance; + if (score < bestScore) { + bestScore = score.toDouble(); + best = index; + } + } + return best; +} + +String _ellipsis(String text, img.BitmapFont font, int maxWidth) { + const suffix = '...'; + if (_textWidth(text, font) <= maxWidth) return text; + if (_textWidth(suffix, font) > maxWidth) return ''; + + var end = text.length; + while (end > 0) { + final candidate = '${text.substring(0, end).trimRight()}$suffix'; + if (_textWidth(candidate, font) <= maxWidth) { + return candidate; + } + end--; + } + return suffix; +} + +int _textWidth(String text, img.BitmapFont font) { + var width = 0; + for (final codeUnit in text.codeUnits) { + final character = font.characters[codeUnit]; + width += character?.xAdvance ?? font.base ~/ 2; + } + return width; +} String _safeFileName(String value) => value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); diff --git a/tools/ensemble_test_runner/lib/runner/session_recording.dart b/tools/ensemble_test_runner/lib/runner/session_recording.dart deleted file mode 100644 index 84e9c220c..000000000 --- a/tools/ensemble_test_runner/lib/runner/session_recording.dart +++ /dev/null @@ -1,128 +0,0 @@ -import 'dart:convert'; -import 'dart:io'; - -import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; -import 'package:ensemble_test_runner/actions/screenshot_device.dart'; -import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; -import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:image/image.dart' as img; - -const _recordingFrameDurationMs = 800; -const _recordingFrameIntervalMs = 75; -const _recordingMaxFrames = 600; - -Future captureRecordingFrame( - WidgetTester tester, - EnsembleTestContext context, { - required String label, - bool force = false, -}) async { - final config = context.config.record; - if (!config.enabled) return; - if (context.runtime.recordingFrames.length >= _recordingMaxFrames) return; - - final now = DateTime.now(); - final lastTime = context.runtime.lastRecordingFrameTime; - if (!force && - lastTime != null && - now.difference(lastTime).inMilliseconds < _recordingFrameIntervalMs) { - return; - } - - final screenImage = ExtendedStepHandlers.captureScreenshotImage(tester); - - final durationMs = lastTime == null - ? _recordingFrameDurationMs - : now.difference(lastTime).inMilliseconds.clamp( - _recordingFrameIntervalMs, - _recordingFrameDurationMs * 4, - ); - context.runtime - ..lastRecordingFrameTime = now - ..addRecordingFrame( - RecordingFrame( - testId: context.testCase.id, - label: label, - image: screenImage, - timestamp: now, - durationMs: durationMs, - ), - ); -} - -Future writeSessionRecording({ - required RecordConfig config, - required List frames, -}) async { - if (!config.enabled || frames.isEmpty) return null; - - final directory = Directory('build/ensemble_test_runner/recordings'); - directory.createSync(recursive: true); - - final encoder = img.GifEncoder(repeat: 0); - for (final frame in frames) { - var image = await _decodeRecordingFrame(frame, config); - if (image != null) { - if (image.width > 480) { - image = img.copyResize( - image, - width: 480, - interpolation: img.Interpolation.average, - ); - } - final duration = (frame.durationMs / 10).round().clamp(1, 6000); - encoder.addFrame(image, duration: duration); - } - } - - final bytes = encoder.finish(); - if (bytes == null) return null; - - final gifFile = File('${directory.path}/recording.gif'); - gifFile.writeAsBytesSync(bytes); - - final manifestFile = File('${directory.path}/recording.json'); - manifestFile.writeAsStringSync( - const JsonEncoder.withIndent(' ').convert({ - 'file': gifFile.path, - 'frames': [ - for (var i = 0; i < frames.length; i++) - { - 'index': i + 1, - 'testId': frames[i].testId, - 'label': frames[i].label, - 'timestamp': frames[i].timestamp.toIso8601String(), - 'durationMs': frames[i].durationMs, - }, - ], - }), - ); - - return gifFile.path; -} - -Future _decodeRecordingFrame( - RecordingFrame frame, - RecordConfig config, -) async { - final pngBytes = frame.pngBytes; - if (pngBytes != null) { - return img.decodePng(pngBytes); - } - - final screenImage = frame.image; - if (screenImage == null) return null; - - try { - final device = resolveScreenshotDevice(config.toScreenshotArgs()); - final framedBytes = await ExtendedStepHandlers.encodeScreenshotImage( - screenImage, - device, - ); - return img.decodePng(framedBytes); - } finally { - screenImage.dispose(); - } -} diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index 36a311e26..9b90dcb31 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -11,8 +11,6 @@ class TestRuntimeState { final List appFrameTimings = []; final List performanceMarkers = []; final List screenshotSheetFrames = []; - final List recordingFrames = []; - DateTime? lastRecordingFrameTime; Map? authUser; final Map permissions = {}; Size? deviceSize; @@ -26,8 +24,6 @@ class TestRuntimeState { appFrameTimings.clear(); performanceMarkers.clear(); screenshotSheetFrames.clear(); - recordingFrames.clear(); - lastRecordingFrameTime = null; authUser = null; permissions.clear(); deviceSize = null; @@ -50,10 +46,6 @@ class TestRuntimeState { performanceMarkers.add(marker); } - void addRecordingFrame(RecordingFrame frame) { - recordingFrames.add(frame); - } - void addScreenshotSheetFrame(ScreenshotSheetFrame frame) { screenshotSheetFrames.add(frame); } @@ -69,24 +61,6 @@ class ScreenshotSheetFrame { }); } -class RecordingFrame { - final String testId; - final String label; - final Uint8List? pngBytes; - final ui.Image? image; - final DateTime timestamp; - final int durationMs; - - const RecordingFrame({ - required this.testId, - required this.label, - this.pngBytes, - this.image, - required this.timestamp, - required this.durationMs, - }); -} - class PerformanceMarker { final String testId; final int? stepIndex; diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index c029b25b0..328d176b2 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -200,15 +200,6 @@ class EnsembleTestSchemaBuilder { }, }, }, - 'record': { - 'type': 'object', - 'additionalProperties': false, - 'properties': { - 'enabled': {'type': 'boolean'}, - 'platform': {'type': 'string'}, - 'model': {'type': 'string'}, - }, - }, 'performance': { 'type': 'object', 'additionalProperties': false, diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 2c9f1063c..7415a7939 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -247,19 +247,22 @@ performance: expect(config.performance.enabled, isTrue); }); - test('parses suite config recording', () { + test('rejects removed record config', () { const yaml = ''' record: enabled: true - platform: android - model: Samsung Galaxy S20 '''; - final config = EnsembleTestParser.parseConfigString(yaml); - final record = config.record; - expect(record.enabled, isTrue); - expect(record.platform, 'android'); - expect(record.model, 'Samsung Galaxy S20'); + expect( + () => EnsembleTestParser.parseConfigString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Unsupported test config key "record"'), + ), + ), + ); }); test('parses suite config debug artifacts', () { diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart index 25e361ef6..e7cfa008c 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart @@ -72,7 +72,7 @@ void main() { expect(decoded['\$schema'], EnsembleTestSchemaBuilder.schemaVersion); expect(properties, contains('screenshots')); - expect(properties, contains('record')); + expect(properties, isNot(contains('record'))); expect(properties, contains('performance')); expect(properties, contains('dumpTree')); expect(properties, contains('logApiCalls')); @@ -81,14 +81,6 @@ void main() { (properties['screenshots'] as Map)['properties'], containsPair('model', {'type': 'string'}), ); - final recordProperties = - (properties['record'] as Map)['properties']; - expect(recordProperties, contains('enabled')); - expect(recordProperties, contains('platform')); - expect(recordProperties, contains('model')); - expect(recordProperties, isNot(contains('frameDurationMs'))); - expect(recordProperties, isNot(contains('frameIntervalMs'))); - expect(recordProperties, isNot(contains('maxFrames'))); }); test('initialState schema accepts storage, keychain, and env maps', () { diff --git a/tools/ensemble_test_runner/test/screenshot_device_test.dart b/tools/ensemble_test_runner/test/screenshot_device_test.dart index 364dc454f..7c8939136 100644 --- a/tools/ensemble_test_runner/test/screenshot_device_test.dart +++ b/tools/ensemble_test_runner/test/screenshot_device_test.dart @@ -38,25 +38,4 @@ void main() { expect(device?.name, 'Samsung Galaxy S20'); }); - - test('uses suite record config device when enabled', () { - final device = screenshotDeviceForTestCase( - const EnsembleTestCase( - id: 'recording', - startScreen: 'Home', - steps: [ - TestStep(type: 'tap', args: {'id': 'button'}), - ], - ), - const EnsembleTestConfig( - record: RecordConfig( - enabled: true, - platform: 'android', - model: 'Samsung Galaxy S20', - ), - ), - ); - - expect(device?.name, 'Samsung Galaxy S20'); - }); } diff --git a/tools/ensemble_test_runner/test/session_recording_test.dart b/tools/ensemble_test_runner/test/session_recording_test.dart deleted file mode 100644 index c0c6f74b7..000000000 --- a/tools/ensemble_test_runner/test/session_recording_test.dart +++ /dev/null @@ -1,59 +0,0 @@ -import 'dart:convert'; -import 'dart:io'; -import 'dart:typed_data'; - -import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -import 'package:ensemble_test_runner/runner/session_recording.dart'; -import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:image/image.dart' as img; - -void main() { - test('writes suite recording gif and manifest', () async { - final directory = Directory('build/ensemble_test_runner/recordings'); - if (directory.existsSync()) { - directory.deleteSync(recursive: true); - } - - final png = img.encodePng( - img.Image(width: 2, height: 2)..clear(img.ColorRgb8(0, 255, 0)), - ); - - final path = await writeSessionRecording( - config: const RecordConfig(enabled: true), - frames: [ - RecordingFrame( - testId: 'login_test', - label: 'login_test startup', - pngBytes: Uint8List.fromList(png), - timestamp: DateTime.parse('2026-07-14T10:00:00Z'), - durationMs: 250, - ), - RecordingFrame( - testId: 'login_test', - label: 'login_test step 1 tap(login_button)', - pngBytes: Uint8List.fromList(png), - timestamp: DateTime.parse('2026-07-14T10:00:01Z'), - durationMs: 1000, - ), - ], - ); - - expect(path, 'build/ensemble_test_runner/recordings/recording.gif'); - final gif = File(path!); - expect(gif.existsSync(), isTrue); - expect(gif.readAsBytesSync().take(6), utf8.encode('GIF89a')); - - final manifest = jsonDecode( - File('build/ensemble_test_runner/recordings/recording.json') - .readAsStringSync(), - ) as Map; - expect(manifest, isNot(contains('frameDurationMs'))); - expect(manifest, isNot(contains('frameIntervalMs'))); - expect(manifest, isNot(contains('maxFrames'))); - expect(manifest['frames'], hasLength(2)); - expect((manifest['frames'] as List).last['label'], - 'login_test step 1 tap(login_button)'); - expect((manifest['frames'] as List).last['durationMs'], 1000); - }); -} From 09d8be5b0573eb75930a7fec338591fcf6f75710 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Wed, 15 Jul 2026 03:18:38 +0500 Subject: [PATCH 26/29] feat(test_runner): enhance font handling and package info resolution Improved the EnsembleTestHarness by adding support for font family aliases and asset candidates, allowing for more flexible font loading. Implemented methods to resolve package information from app metadata files, with fallbacks for missing data. Added comprehensive tests to validate the new functionality, ensuring robust handling of font assets and package information in the test framework. --- .../lib/runner/ensemble_test_harness.dart | 144 +++++++++++++++--- .../test/ensemble_test_harness_test.dart | 74 +++++++++ 2 files changed, 201 insertions(+), 17 deletions(-) diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index 7f4594273..52d125bf8 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -21,6 +21,7 @@ import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:yaml/yaml.dart'; /// Per-test bootstrap data applied before the widget tree mounts. class EnsembleTestSetup { @@ -85,12 +86,7 @@ class EnsembleTestHarness { TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger .setMockMethodCallHandler(packageInfoChannel, (call) async { if (call.method == 'getAll') { - return { - 'appName': 'EnsembleTest', - 'packageName': 'com.ensemble.test', - 'version': '1.0.0', - 'buildNumber': '1', - }; + return _resolvePackageInfo(Directory.current); } return null; }); @@ -332,13 +328,123 @@ class EnsembleTestHarness { final fonts = familyEntry['fonts']; if (family == null || fonts is! List) continue; - await _loadFontFamily(family, fonts); + for (final alias in _fontFamilyAliases(family)) { + await _loadFontFamily(alias, fonts); + } + } + } + + static List fontFamilyAliasesForTest(String family) => + _fontFamilyAliases(family); + + static List fontAssetCandidatesForTest(String asset) => + _fontAssetCandidates(asset); + + static Map packageInfoForTest(String appDir) => + _resolvePackageInfo(Directory(appDir)); + + static Map _resolvePackageInfo(Directory appDir) { + final pubspec = _readPubspec(appDir); + final properties = _readProperties( + File('${appDir.path}/ensemble/ensemble.properties'), + ); + + final packageName = + properties['appId'] ?? pubspec['name'] ?? 'com.ensemble.test'; + final appName = properties['appName'] ?? + _titleFromPackageName(pubspec['name']) ?? + 'EnsembleTest'; + final versionParts = _splitVersion(pubspec['version']); + + return { + 'appName': appName, + 'packageName': packageName, + 'version': versionParts.version, + 'buildNumber': versionParts.buildNumber, + }; + } + + static Map _readPubspec(Directory appDir) { + final file = File('${appDir.path}/pubspec.yaml'); + if (!file.existsSync()) return const {}; + + try { + final yaml = loadYaml(file.readAsStringSync()); + if (yaml is! YamlMap) return const {}; + return { + for (final entry in yaml.entries) + if (entry.value != null) entry.key.toString(): entry.value.toString(), + }; + } catch (_) { + return const {}; + } + } + + static Map _readProperties(File file) { + if (!file.existsSync()) return const {}; + + final values = {}; + for (final rawLine in file.readAsLinesSync()) { + final line = rawLine.trim(); + if (line.isEmpty || line.startsWith('#')) continue; + final separator = line.indexOf('='); + if (separator <= 0) continue; + values[line.substring(0, separator).trim()] = + line.substring(separator + 1).trim(); + } + return values; + } + + static ({String version, String buildNumber}) _splitVersion(String? raw) { + if (raw == null || raw.trim().isEmpty) { + return (version: '1.0.0', buildNumber: '1'); + } + final parts = raw.trim().split('+'); + final version = parts.first.trim(); + final buildNumber = parts.length > 1 ? parts.sublist(1).join('+') : '1'; + return ( + version: version.isEmpty ? '1.0.0' : version, + buildNumber: buildNumber.trim().isEmpty ? '1' : buildNumber.trim(), + ); + } + + static String? _titleFromPackageName(String? name) { + if (name == null || name.trim().isEmpty) return null; + return name + .trim() + .split(RegExp(r'[_\s]+')) + .where((part) => part.isNotEmpty) + .map((part) => '${part[0].toUpperCase()}${part.substring(1)}') + .join(' '); + } + + static List _fontFamilyAliases(String family) { + final aliases = {family, family.toLowerCase()}; + + const packagePrefix = 'packages/'; + if (family.startsWith(packagePrefix)) { + final slashIndex = family.lastIndexOf('/'); + if (slashIndex != -1 && slashIndex < family.length - 1) { + final unqualifiedFamily = family.substring(slashIndex + 1); + aliases.add(unqualifiedFamily); + aliases.add(unqualifiedFamily.toLowerCase()); + } + } + + return aliases.toList(growable: false); + } - final lowerCaseAlias = family.toLowerCase(); - if (lowerCaseAlias != family) { - await _loadFontFamily(lowerCaseAlias, fonts); + static List _fontAssetCandidates(String asset) { + final candidates = {asset}; + if (asset.contains('%')) { + try { + candidates.add(Uri.decodeFull(asset)); + } catch (_) { + // Keep the manifest-provided asset key if decoding is not valid URI + // escaping. } } + return candidates.toList(growable: false); } static Future _loadFontFamily( @@ -351,13 +457,17 @@ class EnsembleTestHarness { for (final fontEntry in fontEntries.whereType()) { final asset = fontEntry['asset']?.toString(); if (asset == null || asset.isEmpty) continue; - try { - final fontData = await rootBundle.load(asset); - loader.addFont(Future.value(fontData)); - hasFonts = true; - } catch (_) { - // Ignore missing assets so a bad font entry does not fail unrelated - // behavioral tests. + + for (final assetKey in _fontAssetCandidates(asset)) { + try { + final fontData = await rootBundle.load(assetKey); + loader.addFont(Future.value(fontData)); + hasFonts = true; + break; + } catch (_) { + // Ignore missing assets so a bad font entry does not fail unrelated + // behavioral tests. + } } } diff --git a/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart index bf5d493d9..9a3b49780 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart @@ -1,8 +1,82 @@ +import 'dart:io'; + import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { + test('app font bootstrap aliases package icon font families', () { + final aliases = EnsembleTestHarness.fontFamilyAliasesForTest( + 'packages/font_awesome_flutter/FontAwesomeSolid', + ); + + expect( + aliases, + containsAll([ + 'packages/font_awesome_flutter/FontAwesomeSolid', + 'FontAwesomeSolid', + 'fontawesomesolid', + ]), + ); + }); + + test('app font bootstrap can try decoded asset keys', () { + final candidates = EnsembleTestHarness.fontAssetCandidatesForTest( + 'packages/font_awesome_flutter/lib/fonts/Font%20Awesome%207%20Free-Solid-900.otf', + ); + + expect( + candidates, + contains( + 'packages/font_awesome_flutter/lib/fonts/Font Awesome 7 Free-Solid-900.otf', + ), + ); + }); + + test('package info bootstrap reads app metadata from app files', () { + final dir = Directory.systemTemp.createTempSync('ensemble_package_info_'); + try { + File('${dir.path}/pubspec.yaml').writeAsStringSync(''' +name: inhome +version: 0.6.0+60 +'''); + Directory('${dir.path}/ensemble').createSync(); + File('${dir.path}/ensemble/ensemble.properties').writeAsStringSync(''' +appId=com.kpn.inhome.dev +appName=KPN InHome Dev +'''); + + expect( + EnsembleTestHarness.packageInfoForTest(dir.path), + { + 'appName': 'KPN InHome Dev', + 'packageName': 'com.kpn.inhome.dev', + 'version': '0.6.0', + 'buildNumber': '60', + }, + ); + } finally { + dir.deleteSync(recursive: true); + } + }); + + test('package info bootstrap falls back without app metadata files', () { + final dir = Directory.systemTemp.createTempSync('ensemble_package_info_'); + try { + expect( + EnsembleTestHarness.packageInfoForTest(dir.path), + { + 'appName': 'EnsembleTest', + 'packageName': 'com.ensemble.test', + 'version': '1.0.0', + 'buildNumber': '1', + }, + ); + } finally { + dir.deleteSync(recursive: true); + } + }); + testWidgets('app font bootstrap is safe when font manifest is unavailable', (tester) async { EnsembleTestHarness.ensureTestPlugins(); From e292dd91c410dda14e09a1bcad3cf6de05c58dc7 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Wed, 15 Jul 2026 04:15:56 +0500 Subject: [PATCH 27/29] feat(test_runner): enhance live output filtering and report extraction Introduced a new LiveFlutterTestOutputFilter to manage the output of running Flutter tests, allowing for selective emission of logs during test execution. Updated the extractSuiteReport function to conditionally include or exclude screen tracker lines based on parameters. Enhanced the extractKnownFailure function to capture actionable error messages from the Flutter test framework. Added tests to validate the new filtering behavior and report extraction logic, improving the clarity and usability of test outputs in the ensemble test runner. --- .../lib/cli/ensemble_test_cli.dart | 119 +++++++++++++++++- .../lib/cli/ensemble_test_cli_output.dart | 52 +++++++- .../lib/runner/screenshot_contact_sheet.dart | 18 ++- .../test/ensemble_test_cli_output_test.dart | 67 ++++++++++ 4 files changed, 245 insertions(+), 11 deletions(-) diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart index 61463c31c..9a74df34a 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:convert'; import 'dart:io'; @@ -29,9 +30,12 @@ import 'package:ensemble_test_runner/validation/ensemble_test_validator.dart'; /// --verbose Full `flutter pub get` / `flutter test` output Future runEnsembleYamlTestsCli(List arguments) async { final verbose = isVerboseCli(arguments); + final quiet = arguments.contains('--quiet'); final reportMode = _resolveReportMode(arguments); final jsonReport = reportMode == 'json'; final junitReport = reportMode == 'junit'; + final machineReport = jsonReport || junitReport; + final streamLiveOutput = !quiet && !machineReport; final reportFile = _resolveReportFile(arguments); final appDir = _resolveAppDir(arguments); final timeoutSeconds = _resolveTimeoutSeconds(arguments); @@ -104,9 +108,19 @@ Future runEnsembleYamlTestsCli(List arguments) async { var exitCode = 0; try { + _writeStatus( + 'Preparing Ensemble YAML tests...', + quiet: quiet, + machineReport: machineReport, + ); patcher.enable(); if (patcher.pubspecChanged) { + _writeStatus( + 'Resolving Flutter dependencies...', + quiet: quiet, + machineReport: machineReport, + ); final pubGet = await _runProcess( 'flutter', ['pub', 'get', '--suppress-analytics'], @@ -138,28 +152,46 @@ Future runEnsembleYamlTestsCli(List arguments) async { ...flutterTestArguments(arguments), ]; - final testRun = await _runProcess( + _writeStatus( + 'Running Ensemble YAML tests...', + quiet: quiet, + machineReport: machineReport, + ); + final testRun = await _runFlutterTestProcess( 'flutter', testArgs, workingDirectory: appDir, + streamOutput: streamLiveOutput, + verbose: verbose, ); if (testRun.exitCode != 0 && !verbose) { final output = '${testRun.stdout ?? ''}\n${testRun.stderr ?? ''}'; final json = jsonReport ? extractJsonReport(output) : ''; final junit = junitReport ? extractJunitReport(output) : ''; + final report = machineReport + ? '' + : extractSuiteReport( + output, + includeScreenTracker: !streamLiveOutput, + ); final knownFailure = extractKnownFailure(output); if (junit.isNotEmpty) { stdout.writeln(junit); } else if (json.isNotEmpty) { stdout.writeln(json); + } else if (report.isNotEmpty) { + stdout.write(report); + if (!report.endsWith('\n')) stdout.writeln(); } else if (knownFailure.isNotEmpty) { stderr.writeln(knownFailure); } else { _writeProcessStreams(testRun); } - } else if (verbose || testRun.exitCode != 0) { + } else if (!verbose && testRun.exitCode != 0) { _writeProcessStreams(testRun); + } else if (verbose) { + // Output was already streamed live. } else { final out = testRun.stdout?.toString() ?? ''; final json = jsonReport ? extractJsonReport(out) : ''; @@ -168,7 +200,10 @@ Future runEnsembleYamlTestsCli(List arguments) async { ? junit : json.isNotEmpty ? json - : extractSuiteReport(out); + : extractSuiteReport( + out, + includeScreenTracker: !streamLiveOutput, + ); if (report.isNotEmpty) { stdout.write(report); if (!report.endsWith('\n')) stdout.writeln(); @@ -259,6 +294,75 @@ List _optionValues(List arguments, String name) { .toList(); } +Future _runFlutterTestProcess( + String executable, + List arguments, { + required String workingDirectory, + required bool streamOutput, + required bool verbose, +}) async { + final process = await Process.start( + executable, + arguments, + workingDirectory: workingDirectory, + runInShell: false, + ); + final stdoutBuffer = StringBuffer(); + final stderrBuffer = StringBuffer(); + final liveFilter = LiveFlutterTestOutputFilter(); + final elapsed = Stopwatch()..start(); + var lastLiveOutput = DateTime.now(); + Timer? heartbeat; + + if (streamOutput && !verbose) { + heartbeat = Timer.periodic(const Duration(seconds: 10), (_) { + final idleFor = DateTime.now().difference(lastLiveOutput); + if (idleFor.inSeconds >= 10) { + stderr.writeln( + 'Still running Flutter tests (${elapsed.elapsed.inSeconds}s)...', + ); + } + }); + } + + try { + final stdoutDone = process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((line) { + stdoutBuffer.writeln(line); + if (verbose) { + stdout.writeln(line); + } else if (streamOutput && liveFilter.shouldEmit(line)) { + lastLiveOutput = DateTime.now(); + stderr.writeln(line); + } + }).asFuture(); + + final stderrDone = process.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((line) { + stderrBuffer.writeln(line); + if (verbose) { + stderr.writeln(line); + } + }).asFuture(); + + final exitCode = await process.exitCode; + await Future.wait([stdoutDone, stderrDone]); + + return ProcessResult( + process.pid, + exitCode, + stdoutBuffer.toString(), + stderrBuffer.toString(), + ); + } finally { + heartbeat?.cancel(); + } +} + Future _runProcess( String executable, List arguments, { @@ -331,6 +435,15 @@ int? _resolveTimeoutSeconds(List arguments) { return seconds; } +void _writeStatus( + String message, { + required bool quiet, + required bool machineReport, +}) { + if (quiet || machineReport) return; + stderr.writeln(message); +} + void _writeProcessStreams(ProcessResult result) { final out = result.stdout?.toString() ?? ''; final err = result.stderr?.toString() ?? ''; diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart index 10497babb..7c5ab8b63 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart @@ -6,15 +6,45 @@ const screenTrackerPrefix = 'SCREEN TRACKER:'; const noDeclarativeTestsPrefix = 'No declarative tests found.'; const jsonReportPrefix = 'ENSEMBLE_TEST_JSON_REPORT:'; const junitReportPrefix = 'ENSEMBLE_TEST_JUNIT_REPORT:'; +const flutterTestExceptionStart = + '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK'; +const flutterTakeExceptionHint = + '(The following exception is now available via WidgetTester.takeException:)'; + +/// Filters stdout from a running `flutter test` process for live CLI display. +/// +/// The full stdout is still captured and parsed at the end. This filter only +/// decides which lines are safe to show while the process is still running. +class LiveFlutterTestOutputFilter { + var _suppressRest = false; + + bool shouldEmit(String line) { + if (_suppressRest) return false; + if (line.startsWith(jsonReportPrefix) || + line.startsWith(junitReportPrefix) || + line.startsWith(flutterTakeExceptionHint)) { + return false; + } + if (line.startsWith(suiteReportStart) || + line.startsWith(flutterTestExceptionStart)) { + _suppressRest = true; + return false; + } + return line.trim().isNotEmpty; + } +} /// Strips Flutter test framework noise; keeps navigation logs and the suite report. -String extractSuiteReport(String output) { +String extractSuiteReport( + String output, { + bool includeScreenTracker = true, +}) { final lines = output.split('\n'); final kept = []; var inReport = false; for (final line in lines) { - if (line.startsWith(screenTrackerPrefix)) { + if (includeScreenTracker && line.startsWith(screenTrackerPrefix)) { kept.add(line); continue; } @@ -38,6 +68,14 @@ bool isVerboseCli(List arguments) => arguments.contains('--verbose'); /// Extracts known actionable failures from Flutter's framework error output. String extractKnownFailure(String output) { + final ensembleFailure = _extractFrameworkFailureMessage( + output, + 'The following EnsembleTestFailure was thrown running a test:', + ); + if (ensembleFailure.isNotEmpty) { + return ensembleFailure; + } + final noTests = RegExp( r'No declarative tests found\. Add \*\.test\.yaml files under [^\r\n]+', ).firstMatch(output); @@ -47,6 +85,16 @@ String extractKnownFailure(String output) { return ''; } +String _extractFrameworkFailureMessage(String output, String marker) { + final start = output.indexOf(marker); + if (start < 0) return ''; + final messageStart = start + marker.length; + final messageEnd = + output.indexOf('\n\nWhen the exception was thrown', messageStart); + if (messageEnd < 0) return ''; + return output.substring(messageStart, messageEnd).trim(); +} + String extractJsonReport(String output) { return _extractPrefixedReport(output, jsonReportPrefix); } diff --git a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart index 338a29cfa..775d64833 100644 --- a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart +++ b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart @@ -34,6 +34,16 @@ Future writeScreenshotContactSheet({ if (tiles.isEmpty) return null; + final sheet = _composeSheet(tiles); + + final directory = Directory('build/ensemble_test_runner/screenshots'); + directory.createSync(recursive: true); + final file = File('${directory.path}/${_safeFileName(testId)}_sheet.png'); + file.writeAsBytesSync(img.encodePng(sheet, level: 1)); + return file.path; +} + +img.Image _composeSheet(List tiles) { const columns = 5; const gap = 16; final rows = (tiles.length / columns).ceil(); @@ -57,11 +67,7 @@ Future writeScreenshotContactSheet({ img.compositeImage(sheet, tile, dstX: x, dstY: y); } - final directory = Directory('build/ensemble_test_runner/screenshots'); - directory.createSync(recursive: true); - final file = File('${directory.path}/${_safeFileName(testId)}_sheet.png'); - file.writeAsBytesSync(img.encodePng(sheet)); - return file.path; + return sheet; } img.Image _buildTile(img.Image source, String label) { @@ -70,7 +76,7 @@ img.Image _buildTile(img.Image source, String label) { final thumbnail = img.copyResize( source, width: width, - interpolation: img.Interpolation.cubic, + interpolation: img.Interpolation.linear, ); final tile = img.Image(width: width, height: thumbnail.height + labelHeight) ..clear(img.ColorRgb8(255, 255, 255)); diff --git a/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart b/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart index 01810a87a..e0270105a 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart @@ -24,6 +24,23 @@ name is ={first: John} ); }); + test('extractSuiteReport can omit streamed screen tracker lines', () { + const noisy = ''' +SCREEN TRACKER: Login +┌─ Ensemble YAML tests ───────────────────────────── +│ ✓ login_test +└─ 1 passed, 0 failed (1 total) · 653ms total +'''; + + expect( + extractSuiteReport(noisy, includeScreenTracker: false), + '''┌─ Ensemble YAML tests ───────────────────────────── +│ ✓ login_test +└─ 1 passed, 0 failed (1 total) · 653ms total +''', + ); + }); + test('flutterTestArguments strips CLI-only flags', () { expect( flutterTestArguments([ @@ -86,4 +103,54 @@ The test description was: 'No declarative tests found. Add *.test.yaml files under ensemble/apps/inhome/tests/', ); }); + + test('extractKnownFailure keeps actionable EnsembleTestFailure message', () { + const noisy = ''' +══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════ +The following EnsembleTestFailure was thrown running a test: +Missing CLI input "password". Pass it with --input password=value. + +When the exception was thrown, this was the stack: +#0 EnsembleTestParser._placeholderValue +════════════════════════════════════════════════════════════════════════════════ +'''; + + expect( + extractKnownFailure(noisy), + 'Missing CLI input "password". Pass it with --input password=value.', + ); + }); + + test('live output filter emits app logs before final report only', () { + final filter = LiveFlutterTestOutputFilter(); + + expect(filter.shouldEmit('SCREEN TRACKER: Login'), isTrue); + expect( + filter.shouldEmit('DEBUG [InitApp] token stored successfully'), isTrue); + expect( + filter.shouldEmit('ENSEMBLE_TEST_JSON_REPORT:{"status":"passed"}'), + isFalse, + ); + expect( + filter.shouldEmit( + '(The following exception is now available via WidgetTester.takeException:)', + ), + isFalse, + ); + expect(filter.shouldEmit('┌─ Ensemble YAML tests ─────'), isFalse); + expect(filter.shouldEmit('SCREEN TRACKER: Home'), isFalse); + }); + + test('live output filter suppresses framework exception noise', () { + final filter = LiveFlutterTestOutputFilter(); + + expect(filter.shouldEmit('SCREEN TRACKER: Login'), isTrue); + expect( + filter.shouldEmit( + '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════', + ), + isFalse, + ); + expect(filter.shouldEmit('The following TestFailure was thrown'), isFalse); + }); } From bc3d3c171dbbde7cb979128dd97f55aa1f39c100 Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Wed, 15 Jul 2026 19:07:50 +0500 Subject: [PATCH 28/29] feat(test_runner): implement toggle functionality for UI elements Added a new `_toggle` method in the TestStepExecutor to handle toggling of UI elements such as Switches and Checkboxes. Updated the visibility handling in the EnsembleTestRunner to ensure the correct element is targeted for interaction. Introduced a test case to validate the toggle functionality within a keyed input wrapper, enhancing the test framework's capability to interact with various UI components. --- .../lib/actions/test_step_executor.dart | 26 +++++++++- .../lib/runner/ensemble_test_runner.dart | 27 +++++++++- .../test/test_step_executor_test.dart | 49 +++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 0661ddbc7..2328619e3 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -11,6 +11,7 @@ import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; +import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -160,7 +161,7 @@ class TestStepExecutor { await _select(_requireId(step), step.args['value']?.toString()); break; case 'toggle': - await _tap(_requireId(step)); + await _toggle(_requireId(step)); break; case 'waitFor': await _waitFor( @@ -403,6 +404,29 @@ class TestStepExecutor { await _settle(); } + Future _toggle(String id) async { + final finder = assertions.finderForId(id); + if (finder.evaluate().isEmpty) { + await _waitFor( + id: id, + timeoutMs: config.defaultWaitTimeout.inMilliseconds, + ); + } + _expectSingleWidget(finder, id, 'toggle'); + await tester.ensureVisible(finder); + await _pump(label: 'toggle:before'); + + final control = find.descendant( + of: finder, + matching: find.byWidgetPredicate( + (widget) => + widget is Switch || widget is CupertinoSwitch || widget is Checkbox, + ), + ); + await tester.tap(control.evaluate().isNotEmpty ? control.first : finder); + await _settle(); + } + Future _enterText(String id, String value, {bool submit = false}) async { final finder = assertions.finderForId(id); diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index 20d46db10..be6cec03e 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -18,6 +18,8 @@ import 'package:ensemble_test_runner/runner/live_async_call.dart'; import 'package:ensemble_test_runner/runner/screenshot_contact_sheet.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter/cupertino.dart'; +import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter/scheduler.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -229,6 +231,9 @@ class EnsembleTestRunner { final startFrame = ctx.runtime.appFrameTimings.length + 1; final startTime = DateTime.now(); try { + if (i == 0 && ctx.config.screenshots.enabled) { + await executor.settle(); + } await _captureAutomaticScreenshotForStep( executor: executor, step: step, @@ -367,7 +372,27 @@ class EnsembleTestRunner { final finder = executor.assertions.finderForId(id); if (finder.evaluate().isEmpty) return; - await executor.tester.ensureVisible(finder); + var visibilityTarget = finder; + if (step.type == 'toggle' || + step.type == 'check' || + step.type == 'uncheck') { + final control = find.descendant( + of: finder, + matching: find.byWidgetPredicate( + (widget) => + widget is Switch || + widget is CupertinoSwitch || + widget is Checkbox, + ), + ); + if (control.evaluate().isNotEmpty) { + visibilityTarget = control.first; + } + } + + if (visibilityTarget.hitTestable().evaluate().isNotEmpty) return; + + await executor.tester.ensureVisible(visibilityTarget); await executor.tester.pump(); } diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index da10139d0..5b7ac9d5d 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -10,11 +10,60 @@ import 'package:ensemble_test_runner/runner/app_performance_log.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:yaml/yaml.dart'; void main() { + testWidgets('toggle taps the switch inside a keyed input wrapper', + (tester) async { + var value = false; + await tester.pumpWidget( + MaterialApp( + home: StatefulBuilder( + builder: (context, setState) => Scaffold( + body: KeyedSubtree( + key: const ValueKey('notifications'), + child: SizedBox( + width: 400, + child: Row( + children: [ + const Expanded(child: Text('Notifications')), + CupertinoSwitch( + value: value, + onChanged: (next) => setState(() => value = next), + ), + ], + ), + ), + ), + ), + ), + ), + ); + + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 't', + startScreen: 'Home', + steps: [], + ), + ); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'), + ); + + await executor.execute( + const TestStep(type: 'toggle', args: {'id': 'notifications'}), + ); + + expect(value, isTrue); + }); + testWidgets('waitFor requires id or text', (tester) async { final context = EnsembleTestContext.fromTestCase( const EnsembleTestCase( From f256e09007965818999ac70931f2b5ea666ab69c Mon Sep 17 00:00:00 2001 From: Sharjeel Yunus Date: Fri, 17 Jul 2026 06:20:35 +0500 Subject: [PATCH 29/29] feat(test_runner): enhance HTTP request and command execution capabilities Introduced new actions for executing HTTP requests and running commands within the ensemble test runner. The `HttpRequestAction` class allows for sending HTTP requests with customizable parameters, while the `RunCommandAction` class enables executing external commands with specified arguments and environment variables. Updated the test step executor to support these new actions, enhancing the framework's ability to interact with external services and perform setup tasks. Additionally, improved the assertion engine to handle visibility checks more effectively, ensuring robust test execution. --- modules/ensemble/lib/util/ensemble_utils.dart | 16 +- tools/ensemble_test_runner/README.md | 72 +- tools/ensemble_test_runner/STEP_VOCABULARY.md | 61 +- .../schema/ensemble_test_config_schema.json | 62 + .../assets/schema/ensemble_tests_schema.json | 326 ++++ .../doc/TEST_AUTHORING.md | 11 + .../lib/actions/extended_step_handlers.dart | 2 +- .../lib/actions/http_request_action.dart | 101 ++ .../lib/actions/run_command_action.dart | 124 ++ .../lib/actions/test_step_executor.dart | 59 +- .../lib/assertions/assertion_engine.dart | 132 +- .../lib/cli/ensemble_test_cli.dart | 1470 ++++++++++++++++- .../lib/cli/ensemble_test_cli_output.dart | 1 + .../lib/cli/ensemble_test_doctor.dart | 18 + .../lib/cli/yaml_test_app_patcher.dart | 90 + .../discovery/ensemble_test_discovery.dart | 108 +- .../ensemble_test_execution_planner.dart | 97 +- .../lib/entry/ensemble_test_entry.dart | 297 +++- .../lib/mocks/test_logger.dart | 18 +- .../lib/models/ensemble_test_models.dart | 93 ++ .../lib/parser/ensemble_test_parser.dart | 225 ++- .../lib/reporters/test_reporter.dart | 7 + .../lib/runner/app_session_snapshot.dart | 56 + .../lib/runner/ensemble_test_context.dart | 12 + .../lib/runner/ensemble_test_harness.dart | 67 +- .../lib/runner/ensemble_test_runner.dart | 711 +++++++- .../lib/runner/screenshot_contact_sheet.dart | 659 +++++++- .../lib/runner/test_artifacts.dart | 26 + .../lib/runner/test_runtime_state.dart | 22 +- .../lib/runner/test_service_manager.dart | 160 ++ .../schema/ensemble_test_schema_builder.dart | 79 +- .../validation/ensemble_test_validator.dart | 16 + .../lib/vocabulary/test_step_arg_kind.dart | 47 + .../lib/vocabulary/test_step_registry.dart | 70 +- tools/ensemble_test_runner/pubspec.yaml | 1 + .../scratch/check_image.dart | 6 + .../test/app_session_snapshot_test.dart | 42 + .../ensemble_test_execution_planner_test.dart | 51 + .../test/ensemble_test_parser_test.dart | 188 +++ .../test/ensemble_test_schema_test.dart | 18 +- .../test/screenshot_contact_sheet_test.dart | 114 ++ .../test/test_reporter_test.dart | 17 + .../test/test_service_manager_test.dart | 50 + .../test/test_step_executor_test.dart | 144 ++ .../test/yaml_test_app_patcher_test.dart | 77 + .../tool/generate_step_registry.dart | 26 +- 46 files changed, 5655 insertions(+), 394 deletions(-) create mode 100644 tools/ensemble_test_runner/lib/actions/http_request_action.dart create mode 100644 tools/ensemble_test_runner/lib/actions/run_command_action.dart create mode 100644 tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart create mode 100644 tools/ensemble_test_runner/lib/runner/test_artifacts.dart create mode 100644 tools/ensemble_test_runner/lib/runner/test_service_manager.dart create mode 100644 tools/ensemble_test_runner/scratch/check_image.dart create mode 100644 tools/ensemble_test_runner/test/app_session_snapshot_test.dart create mode 100644 tools/ensemble_test_runner/test/screenshot_contact_sheet_test.dart create mode 100644 tools/ensemble_test_runner/test/test_service_manager_test.dart diff --git a/modules/ensemble/lib/util/ensemble_utils.dart b/modules/ensemble/lib/util/ensemble_utils.dart index 2ddb62880..62d87911a 100644 --- a/modules/ensemble/lib/util/ensemble_utils.dart +++ b/modules/ensemble/lib/util/ensemble_utils.dart @@ -24,8 +24,13 @@ class EnsembleUtils { // Fallback to RouteObserver method final route = Ensemble().getCurrentRoute(); - if (route is PopupRoute && route.isCurrent && route.navigator != null) { - return route.navigator!.maybePop(payload); + final navigator = route?.navigator; + if (route is PopupRoute && + route.isCurrent && + navigator != null && + navigator.canPop()) { + navigator.pop(payload); + return Future.value(true); } return Future.value(false); } @@ -54,10 +59,13 @@ class EnsembleUtils { // Fallback to RouteObserver method final route = Ensemble().getCurrentRoute(); + final navigator = route?.navigator; if (route is ModalBottomSheetRoute && route.isCurrent && - route.navigator != null) { - return route.navigator!.maybePop(payload); + navigator != null && + navigator.canPop()) { + navigator.pop(payload); + return Future.value(true); } return Future.value(false); } diff --git a/tools/ensemble_test_runner/README.md b/tools/ensemble_test_runner/README.md index d34fd436a..c5997c5b9 100644 --- a/tools/ensemble_test_runner/README.md +++ b/tools/ensemble_test_runner/README.md @@ -23,7 +23,25 @@ steps: id: greeting_text ``` -Each `*.test.yaml` file is **one** test — `id`, `steps`, and **either** `startScreen` **or** `prerequisite` are at the root (no `tests:` array). A test with `prerequisite: ` runs after that test on the **same** app session, applying only `initialState`/`mocks` in-place before executing its steps. +Each `*.test.yaml` file is **one** test (no `tests:` array). It either +cold-starts with `startScreen`, continues a mounted flow with `prerequisite`, or +restores reusable app state with `session` and starts on its own `startScreen`. + +Use root-level `setup` for commands and HTTP requests that must complete before +the screen is mounted. This is useful for resetting or configuring a stub server: + +```yaml +id: authenticated_home +session: signin +startScreen: Home +setup: + - httpRequest: + method: POST + url: ${services.modemStub.url}/api/v1/stub/scenario + body: {testcase: home, responsename: offline} +steps: + - expectVisible: {id: offline_message} +``` Widget YAML must set `testId` (or `id`, which maps to the same `ValueKey`). @@ -52,6 +70,16 @@ Suite-wide runner config lives in `tests/config.yaml`. The schema is hosted at ```yaml # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json +services: + - name: modemStub + command: .venv/bin/python + arguments: [modemstub/app.py] + workingDirectory: ensemble/apps/inhome/autotests + readyUrl: /ping + +The runner assigns a free local port. Tests can reference that resolved endpoint +as `${services.modemStub.url}`. + screenshots: enabled: true platform: ios @@ -61,6 +89,10 @@ screenshots: performance: enabled: true +timers: + enabled: true + maxStartAfterSeconds: 1 + maxRepeatIntervalSeconds: 1 dumpTree: enabled: true logApiCalls: @@ -115,8 +147,8 @@ steps: text: ${inputs.expectedDeviceCount} ``` -The default suite timeout is 10 minutes. Override it when a flow should fail -faster or when a long chain needs more time: +There is no implicit whole-suite timeout; individual steps and services keep +their own bounded timeouts. Add one when CI should enforce a suite deadline: ```bash dart run ensemble_test_runner:ensemble_test --timeout=30s @@ -187,7 +219,7 @@ dart run ensemble_test_runner:ensemble_test --path=auth/ Prerequisite tests are included automatically for selected continuation tests. -On success the console prints one consolidated boxed report for the suite: each test id (with YAML path), timing, **start screen** or **prerequisite**, **navigation flow**, and a numbered **step outline**. +On success the console prints one consolidated boxed report for the suite: each test id (with YAML path), timing, **start screen** or **prerequisite**, **navigation flow**, and a numbered **step outline**. Raw app console output is written to `build/ensemble_test_runner/logs/app_console*.log` and listed as an `appLogs` suite artifact. ## Examples @@ -197,6 +229,7 @@ On success the console prints one consolidated boxed report for the suite: each # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json id: login_flow startScreen: Login +retry: 3 mocks: - mocks/login_success.mock.json steps: @@ -251,6 +284,36 @@ steps: id: dashboard_title ``` +### Reusable authenticated session + +The session producer runs once. After it passes, the runner captures public +storage, keychain values, and locale in memory. Each consumer restores that +snapshot, runs its `setup`, and mounts a fresh requested screen. + +```yaml +id: signin +startScreen: Login +steps: + - tap: {id: login_button} + - waitForNavigation: {screen: Home} +``` + +```yaml +id: devices +session: signin +startScreen: Home +setup: + - httpRequest: + method: POST + url: ${services.modemStub.url}/api/v1/stub/reset +steps: + - tap: {id: devices_button} +``` + +Use `prerequisite` when the second test must continue the exact mounted UI +state. Use `session` when tests need the same signed-in data but should otherwise +start independently. Session snapshots are not written to disk. + ## Package layout ``` @@ -276,7 +339,6 @@ The runner uses small, optional hooks in the core module — not a package depen - Test harness applies `EnsembleTestSetup` (storage seeds, env overrides) before `EnsembleApp` mounts - Test harness installs `MockAPIProvider` on `EnsembleConfig.apiProviders['http']` -- Test mode via `--dart-define=testmode=true` (added automatically by the CLI) - Navigation flow for `expectVisited` is recorded in the test runner via `ScreenTracker.onScreenChange` `EnsembleTestHarness` runs storage init inside `tester.runAsync()` so `GetStorage` can finish under the widget test binding. diff --git a/tools/ensemble_test_runner/STEP_VOCABULARY.md b/tools/ensemble_test_runner/STEP_VOCABULARY.md index 90e0715c1..ebde82903 100644 --- a/tools/ensemble_test_runner/STEP_VOCABULARY.md +++ b/tools/ensemble_test_runner/STEP_VOCABULARY.md @@ -34,7 +34,7 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense `expectScreen` (alias), `expectNavigateTo`, `expectVisited`, `expectNotVisited`, `expectBackStack`, `expectCanGoBack`, `goBack` ### API assert / logs -`resetApiCalls`, `expectApiCalled`, `expectApiNotCalled`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` +`httpRequest`, `resetApiCalls`, `expectApiCalled`, `expectApiNotCalled`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` ### Storage / runtime `setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` @@ -75,6 +75,16 @@ performance: enabled: true ``` +Long app timers can be capped during the test run without changing the checked +in screen YAML: + +```yaml +timers: + enabled: true + maxStartAfterSeconds: 1 + maxRepeatIntervalSeconds: 1 +``` + `dumpTree`, `logApiCalls`, and `logStorage` are also suite-level config artifacts: @@ -87,6 +97,45 @@ logStorage: enabled: true ``` +Long-running test support processes belong in `tests/config.yaml`. They start +once before the suite, must answer the optional readiness URL, and are stopped +after the suite: + +```yaml +services: + - name: modemStub + command: .venv/bin/python + arguments: [modemstub/app.py] + workingDirectory: ensemble/apps/inhome/autotests + readyUrl: /ping +``` + +The runner assigns a free local port. Use `${services.modemStub.url}` in test +steps instead of repeating the endpoint. + +Use `httpRequest` for finite setup or state changes during a test: + +```yaml +- httpRequest: + method: POST + url: ${services.modemStub.url}/api/v1/stub/hard-reset + body: + loadDefaults: true + expectStatus: 200 +``` + +Use `runCommand` only in root-level `setup` for a finite setup command. It does +not invoke a shell; pass arguments separately. Long-running servers belong in +`services` instead: + +```yaml +setup: + - runCommand: + command: .venv/bin/python + arguments: [tools/clear_test_state.py] + timeoutMs: 30000 +``` + Before running a new suite, use `dart run ensemble_test_runner:ensemble_test --doctor` from the Flutter wrapper app root to validate config, test discovery, duplicate IDs, prerequisites, schema comments, and obvious widget IDs. CI can request JSON @@ -96,8 +145,16 @@ Each `*.test.yaml` file is a single test case and must provide **exactly one** o - `startScreen` — cold-starts the app on the given screen and runs steps - `prerequisite` — ID of another test that must run first; the runner reuses the same app session, applies `initialState`/`mocks` in-place, and then runs this test's steps only +- `session` with `startScreen` — runs the referenced test once, restores its captured storage/keychain/locale for this test, runs `setup`, and mounts a fresh requested screen +- `retry` — number of additional attempts after a failed run, e.g. `retry: 3` + +Root-level `setup` supports `httpRequest`, `runCommand`, `group`, and `optional`. +It runs before the test screen is mounted; it is intended for external service +and stub configuration, not widget actions. -When multiple tests declare `prerequisite` chains, the runner discovers all YAML files, builds a dependency graph by `id`/`prerequisite`, and executes tests once each in topological order. +The runner discovers all YAML files, builds a dependency graph from +`prerequisite` and `session` references, and executes each test once in +topological order. ## Adding a step diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json index 2a2089756..f4edfc0e9 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json @@ -6,6 +6,51 @@ "type": "object", "additionalProperties": false, "properties": { + "services": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "command" + ], + "properties": { + "name": { + "type": "string" + }, + "command": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "arguments": { + "type": "array", + "items": { + "type": "string" + } + }, + "workingDirectory": { + "type": "string" + }, + "environment": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "readyUrl": { + "type": "string" + }, + "readyTimeoutMs": { + "type": "integer", + "minimum": 1 + } + } + } + }, "screenshots": { "type": "object", "additionalProperties": false, @@ -42,6 +87,23 @@ } } }, + "timers": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "maxStartAfterSeconds": { + "type": "integer", + "minimum": 0 + }, + "maxRepeatIntervalSeconds": { + "type": "integer", + "minimum": 0 + } + } + }, "dumpTree": { "type": "object", "additionalProperties": false, diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 555f7467c..1ad3719f2 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -45,19 +45,46 @@ "low" ] }, + "parallel": { + "type": "boolean", + "description": "Set false for tests that mutate shared external state and must not run in a parallel worker shard." + }, + "retry": { + "type": "integer", + "minimum": 0, + "description": "Number of additional attempts after the first failure." + }, "startScreen": { "type": "string", "minLength": 1, "description": "Ensemble screen name or id to load first" }, + "startScreenInputs": { + "type": "object", + "additionalProperties": true, + "description": "Inputs passed to startScreen" + }, "prerequisite": { "type": "string", "minLength": 1, "description": "ID of another test that must run before this one in the same app session" }, + "session": { + "type": "string", + "minLength": 1, + "description": "ID of a successful test whose captured app state is restored before startScreen" + }, "initialState": { "$ref": "#/$defs/initialState" }, + "setup": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/setupStep" + }, + "description": "Headless httpRequest or runCommand actions executed before startScreen mounts" + }, "mocks": { "type": "array", "items": { @@ -106,6 +133,12 @@ } } ], + "not": { + "required": [ + "prerequisite", + "session" + ] + }, "$defs": { "initialState": { "type": "object", @@ -818,6 +851,96 @@ } ] }, + "args_httpRequest": { + "type": "object", + "properties": { + "url": { + "type": "string" + }, + "method": { + "type": "string", + "enum": [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE", + "HEAD", + "OPTIONS" + ] + }, + "headers": { + "type": "object", + "additionalProperties": true + }, + "body": true, + "timeoutMs": { + "type": "integer" + }, + "expectStatus": { + "type": "integer" + }, + "expectBodyContains": { + "type": "string" + } + }, + "required": [ + "url" + ], + "additionalProperties": false, + "title": "httpRequest", + "description": "Send an HTTP request to a test support service", + "examples": [ + { + "method": "POST", + "url": "http://127.0.0.1:5001/api/test/reset", + "body": { + "enabled": true + }, + "expectStatus": 200 + } + ] + }, + "args_runCommand": { + "type": "object", + "properties": { + "command": { + "type": "string" + }, + "arguments": { + "type": "array", + "items": true + }, + "workingDirectory": { + "type": "string" + }, + "environment": { + "type": "object", + "additionalProperties": true + }, + "timeoutMs": { + "type": "integer" + }, + "expectExitCode": { + "type": "integer" + } + }, + "required": [ + "command" + ], + "additionalProperties": false, + "title": "runCommand", + "description": "Run a finite external test setup command", + "examples": [ + { + "command": "dart", + "arguments": [ + "--version" + ], + "expectExitCode": 0 + } + ] + }, "args_waitForNavigation": { "type": "object", "properties": { @@ -3041,6 +3164,45 @@ "waitForApi" ] }, + { + "type": "object", + "title": "httpRequest", + "description": "Send an HTTP request to a test support service", + "examples": [ + { + "httpRequest": { + "method": "POST", + "url": "http://127.0.0.1:5001/api/test/reset", + "body": { + "enabled": true + }, + "expectStatus": 200 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "httpRequest": { + "$ref": "#/$defs/args_httpRequest", + "description": "Send an HTTP request to a test support service", + "examples": [ + { + "method": "POST", + "url": "http://127.0.0.1:5001/api/test/reset", + "body": { + "enabled": true + }, + "expectStatus": 200 + } + ] + } + }, + "required": [ + "httpRequest" + ] + }, { "type": "object", "title": "waitForNavigation", @@ -4839,6 +5001,170 @@ ] } ] + }, + "setupStep": { + "oneOf": [ + { + "type": "object", + "title": "httpRequest", + "description": "Send an HTTP request to a test support service", + "examples": [ + { + "httpRequest": { + "method": "POST", + "url": "http://127.0.0.1:5001/api/test/reset", + "body": { + "enabled": true + }, + "expectStatus": 200 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "httpRequest": { + "$ref": "#/$defs/args_httpRequest", + "description": "Send an HTTP request to a test support service", + "examples": [ + { + "method": "POST", + "url": "http://127.0.0.1:5001/api/test/reset", + "body": { + "enabled": true + }, + "expectStatus": 200 + } + ] + } + }, + "required": [ + "httpRequest" + ] + }, + { + "type": "object", + "title": "runCommand", + "description": "Run a finite external test setup command", + "examples": [ + { + "runCommand": { + "command": "dart", + "arguments": [ + "--version" + ], + "expectExitCode": 0 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "runCommand": { + "$ref": "#/$defs/args_runCommand", + "description": "Run a finite external test setup command", + "examples": [ + { + "command": "dart", + "arguments": [ + "--version" + ], + "expectExitCode": 0 + } + ] + } + }, + "required": [ + "runCommand" + ] + }, + { + "type": "object", + "title": "group", + "description": "Run nested steps as a named group", + "examples": [ + { + "group": { + "name": "login_flow", + "steps": [ + { + "tap": { + "id": "login_button" + } + } + ] + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "group": { + "$ref": "#/$defs/args_group", + "description": "Run nested steps as a named group", + "examples": [ + { + "name": "login_flow", + "steps": [ + { + "tap": { + "id": "login_button" + } + } + ] + } + ] + } + }, + "required": [ + "group" + ] + }, + { + "type": "object", + "title": "optional", + "description": "Run nested steps; swallow failures", + "examples": [ + { + "optional": { + "steps": [ + { + "tap": { + "id": "dismiss_banner" + } + } + ] + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "optional": { + "$ref": "#/$defs/args_optional", + "description": "Run nested steps; swallow failures", + "examples": [ + { + "steps": [ + { + "tap": { + "id": "dismiss_banner" + } + } + ] + } + ] + } + }, + "required": [ + "optional" + ] + } + ] } } } diff --git a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md index d2a66c560..ec380d401 100644 --- a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md +++ b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md @@ -28,12 +28,15 @@ tags: [smoke, auth] description: Valid user can log in and reach Home priority: high startScreen: Login +retry: 3 steps: - expectVisible: id: email_field ``` Use `startScreen` for a cold start. Use `prerequisite` when a test should continue from another test in the same app session. +Use `retry` for known flaky external flows; `retry: 3` means one initial attempt +plus up to three additional attempts. ## Suite Config @@ -49,6 +52,10 @@ screenshots: performance: enabled: true +timers: + enabled: true + maxStartAfterSeconds: 1 + maxRepeatIntervalSeconds: 1 dumpTree: enabled: true logApiCalls: @@ -61,6 +68,10 @@ When `screenshots.enabled` is true, the runner captures automatic step screenshots into one contact sheet per test case under `build/ensemble_test_runner/screenshots/`. +When `timers.enabled` is true, the CLI temporarily caps numeric +`startAfter` and `repeatInterval` values in local app screen YAML while the test +process runs, then restores the original files. + ## App Context `--inspect-app` emits JSON with screens, widget IDs, APIs, navigation targets, imports, storage/env references, and lifecycle hints. diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 6dfb11635..ae0fde9a7 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -535,7 +535,7 @@ class ExtendedStepHandlers { Rect.fromLTWH(0, 0, outputWidth.toDouble(), outputHeight.toDouble()), ); - canvas.drawColor(const Color(0xFFE9EDF3), BlendMode.src); + canvas.drawColor(const Color(0x00000000), BlendMode.src); canvas.save(); canvas.translate(padding, padding); device.framePainter.paint(canvas, device.frameSize); diff --git a/tools/ensemble_test_runner/lib/actions/http_request_action.dart b/tools/ensemble_test_runner/lib/actions/http_request_action.dart new file mode 100644 index 000000000..fe89f4554 --- /dev/null +++ b/tools/ensemble_test_runner/lib/actions/http_request_action.dart @@ -0,0 +1,101 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:http/http.dart' as http; + +abstract final class HttpRequestAction { + static Future execute(Map args) async { + final rawUrl = args['url']?.toString(); + if (rawUrl == null || rawUrl.isEmpty) { + throw EnsembleTestFailure('httpRequest requires "url"'); + } + final uri = Uri.tryParse(rawUrl); + if (uri == null || !uri.hasScheme || !uri.hasAuthority) { + throw EnsembleTestFailure('httpRequest has an invalid URL: $rawUrl'); + } + + final method = args['method']?.toString().toUpperCase() ?? 'GET'; + final headers = _headers(args['headers']); + final request = http.Request(method, uri)..headers.addAll(headers); + if (args.containsKey('body')) { + final body = args['body']; + if (body is String) { + request.body = body; + } else { + if (!_hasContentType(request.headers)) { + request.headers['content-type'] = 'application/json'; + } + request.body = jsonEncode(body); + } + } + + final timeoutMs = _positiveInt(args['timeoutMs'], fallback: 10000); + final client = http.Client(); + try { + final response = await LiveAsyncCallSupport.run(() async { + final streamed = await client + .send(request) + .timeout(Duration(milliseconds: timeoutMs)); + return http.Response.fromStream(streamed); + }); + if (response == null) { + throw EnsembleTestFailure('httpRequest did not return a response'); + } + + final expectedStatus = args['expectStatus']; + if (expectedStatus != null) { + final expected = _positiveInt(expectedStatus); + if (response.statusCode != expected) { + throw EnsembleTestFailure( + 'httpRequest $method $rawUrl expected status $expected, got ' + '${response.statusCode}: ${response.body}', + ); + } + } else if (response.statusCode < 200 || response.statusCode >= 300) { + throw EnsembleTestFailure( + 'httpRequest $method $rawUrl failed with status ' + '${response.statusCode}: ${response.body}', + ); + } + + final expectedBody = args['expectBodyContains']?.toString(); + if (expectedBody != null && !response.body.contains(expectedBody)) { + throw EnsembleTestFailure( + 'httpRequest $method $rawUrl response did not contain ' + '"$expectedBody": ${response.body}', + ); + } + } on EnsembleTestFailure { + rethrow; + } catch (error) { + throw EnsembleTestFailure('httpRequest $method $rawUrl failed: $error'); + } finally { + client.close(); + } + } + + static Map _headers(dynamic node) { + if (node == null) return const {}; + if (node is! Map) { + throw EnsembleTestFailure('httpRequest "headers" must be a map'); + } + return { + for (final entry in node.entries) + entry.key.toString(): entry.value.toString(), + }; + } + + static int _positiveInt(dynamic value, {int? fallback}) { + if (value == null && fallback != null) return fallback; + final parsed = value is int ? value : int.tryParse(value.toString()); + if (parsed == null || parsed <= 0) { + throw EnsembleTestFailure('Expected a positive integer, got "$value"'); + } + return parsed; + } + + static bool _hasContentType(Map headers) => headers.keys.any( + (key) => key.toLowerCase() == 'content-type', + ); +} diff --git a/tools/ensemble_test_runner/lib/actions/run_command_action.dart b/tools/ensemble_test_runner/lib/actions/run_command_action.dart new file mode 100644 index 000000000..919457b85 --- /dev/null +++ b/tools/ensemble_test_runner/lib/actions/run_command_action.dart @@ -0,0 +1,124 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; + +abstract final class RunCommandAction { + static Future execute(Map args) async { + final command = args['command']?.toString(); + if (command == null || command.isEmpty) { + throw EnsembleTestFailure('runCommand requires "command"'); + } + final argumentsNode = args['arguments']; + if (argumentsNode != null && argumentsNode is! List) { + throw EnsembleTestFailure('runCommand "arguments" must be a list'); + } + final environmentNode = args['environment']; + if (environmentNode != null && environmentNode is! Map) { + throw EnsembleTestFailure('runCommand "environment" must be a map'); + } + + final arguments = argumentsNode == null + ? const [] + : List.from( + argumentsNode.map((value) => value.toString()), + ); + final environment = environmentNode == null + ? null + : { + for (final entry in environmentNode.entries) + entry.key.toString(): entry.value.toString(), + }; + final timeoutMs = _positiveInt(args['timeoutMs'], fallback: 30000); + final expectedExitCode = _nonNegativeInt( + args['expectExitCode'], + fallback: 0, + ); + + try { + final result = await LiveAsyncCallSupport.run<_CommandResult>(() async { + final process = await Process.start( + command, + arguments, + workingDirectory: args['workingDirectory']?.toString(), + environment: environment, + includeParentEnvironment: true, + ); + final stdout = process.stdout.transform(systemEncoding.decoder).join(); + final stderr = process.stderr.transform(systemEncoding.decoder).join(); + try { + final exitCode = await process.exitCode.timeout( + Duration(milliseconds: timeoutMs), + ); + return _CommandResult( + exitCode: exitCode, + stdout: await stdout, + stderr: await stderr, + ); + } on TimeoutException { + process.kill(); + try { + await process.exitCode.timeout(const Duration(seconds: 1)); + } on TimeoutException { + if (!Platform.isWindows) process.kill(ProcessSignal.sigkill); + await process.exitCode; + } + await Future.wait([stdout, stderr]); + rethrow; + } + }); + if (result == null) { + throw EnsembleTestFailure('runCommand did not return a result'); + } + if (result.exitCode != expectedExitCode) { + throw EnsembleTestFailure( + 'runCommand "$command" expected exit code $expectedExitCode, got ' + '${result.exitCode}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}', + ); + } + } on EnsembleTestFailure { + rethrow; + } catch (error) { + throw EnsembleTestFailure('runCommand "$command" failed: $error'); + } + } + + static int _positiveInt(dynamic value, {required int fallback}) { + final parsed = value == null + ? fallback + : value is int + ? value + : int.tryParse(value.toString()); + if (parsed == null || parsed <= 0) { + throw EnsembleTestFailure('Expected a positive integer, got "$value"'); + } + return parsed; + } + + static int _nonNegativeInt(dynamic value, {required int fallback}) { + final parsed = value == null + ? fallback + : value is int + ? value + : int.tryParse(value.toString()); + if (parsed == null || parsed < 0) { + throw EnsembleTestFailure( + 'Expected a non-negative integer, got "$value"', + ); + } + return parsed; + } +} + +class _CommandResult { + const _CommandResult({ + required this.exitCode, + required this.stdout, + required this.stderr, + }); + + final int exitCode; + final String stdout; + final String stderr; +} diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 2328619e3..3bd987166 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -3,6 +3,7 @@ import 'dart:async'; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; +import 'package:ensemble_test_runner/actions/http_request_action.dart'; import 'package:ensemble_test_runner/actions/test_execution_config.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; @@ -23,6 +24,7 @@ class TestStepExecutor { final EnsembleTestHarness harness; final TestExecutionConfig config; EnsembleConfig? _config; + FutureOr Function(TestStep step)? onWaitForTextMatched; TestStepExecutor({ required this.tester, @@ -74,6 +76,9 @@ class TestStepExecutor { } switch (step.type) { + case 'httpRequest': + await HttpRequestAction.execute(step.args); + return; case 'wait': final durationMs = step.args['durationMs'] as int? ?? 500; await tester.runAsync(() async { @@ -83,6 +88,7 @@ class TestStepExecutor { return; case 'waitForText': await _waitFor( + step: step, text: step.args['text']?.toString(), timeoutMs: step.args['timeoutMs'] as int? ?? config.defaultWaitTimeout.inMilliseconds, @@ -165,6 +171,7 @@ class TestStepExecutor { break; case 'waitFor': await _waitFor( + step: step, id: step.args['id']?.toString(), text: step.args['text']?.toString(), timeoutMs: step.args['timeoutMs'] as int? ?? @@ -390,31 +397,49 @@ class TestStepExecutor { } Future _tap(String id) async { - final finder = assertions.finderForId(id); - if (finder.evaluate().isEmpty) { + final baseFinder = assertions.finderForId(id); + if (baseFinder.evaluate().isEmpty) { await _waitFor( id: id, timeoutMs: config.defaultWaitTimeout.inMilliseconds, ); } + var finder = _interactiveFinder(baseFinder); _expectSingleWidget(finder, id, 'tap'); await tester.ensureVisible(finder); - await _pump(label: 'tap:before'); + await _pump(label: 'tap.ensureVisible'); + finder = _hitTestableFinderForTap(finder, id); await tester.tap(finder); await _settle(); } + Finder _hitTestableFinderForTap(Finder finder, String id) { + final hitTestable = finder.hitTestable(); + final hitTestableCount = hitTestable.evaluate().length; + if (hitTestableCount == 1) return hitTestable; + if (hitTestableCount > 1) { + throw EnsembleTestFailure( + 'tap expected exactly one hit-testable widget with id "$id", ' + 'but found $hitTestableCount.', + ); + } + throw EnsembleTestFailure( + 'tap found widget with id "$id", but it is not hit-testable. ' + 'It may be off-screen, disabled, or covered by another widget.', + ); + } + Future _toggle(String id) async { - final finder = assertions.finderForId(id); - if (finder.evaluate().isEmpty) { + final baseFinder = assertions.finderForId(id); + if (baseFinder.evaluate().isEmpty) { await _waitFor( id: id, timeoutMs: config.defaultWaitTimeout.inMilliseconds, ); } + final finder = _interactiveFinder(baseFinder); _expectSingleWidget(finder, id, 'toggle'); await tester.ensureVisible(finder); - await _pump(label: 'toggle:before'); final control = find.descendant( of: finder, @@ -477,6 +502,7 @@ class TestStepExecutor { } Future _waitFor({ + TestStep? step, String? id, String? text, required int timeoutMs, @@ -488,10 +514,14 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { await _pump(duration: config.waitPollInterval, label: 'waitFor'); - if (id != null && assertions.finderForId(id).evaluate().isNotEmpty) { + if (id != null && + assertions.finderForId(id).hitTestable().evaluate().isNotEmpty) { return; } - if (text != null && find.text(text).evaluate().isNotEmpty) { + if (text != null && assertions.isTextVisible(text)) { + if (step?.type == 'waitForText' && onWaitForTextMatched != null) { + await onWaitForTextMatched!(step!); + } return; } } @@ -503,7 +533,8 @@ class TestStepExecutor { : 'text "$text"'; throw EnsembleTestFailure( 'Timed out after ${timeoutMs}ms waiting for $target. ' - '${assertions.visibleWidgetIdSummary()}', + '${assertions.visibleWidgetIdSummary()} ' + '${assertions.visibleTextSummary()}', ); } @@ -517,6 +548,12 @@ class TestStepExecutor { } } + Finder _interactiveFinder(Finder finder) { + if (finder.evaluate().length <= 1) return finder; + final hitTestable = finder.hitTestable(); + return hitTestable.evaluate().length == 1 ? hitTestable : finder; + } + Future _openScreen(TestStep step) async { final screen = step.args['name']?.toString() ?? step.args['screen']?.toString(); @@ -585,6 +622,10 @@ class TestStepExecutor { duration: config.waitPollInterval, label: 'waitForNavigation', ); + await YamlTestSession.navigationFlow.flushPending(); + if (hasNavigated()) { + return; + } } await _yieldToLiveApiWork(); await _pump(label: 'waitForNavigation'); diff --git a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart index 6418903fd..1cdcd28ba 100644 --- a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart +++ b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart @@ -1,6 +1,4 @@ import 'dart:convert'; -import 'dart:ui' show SemanticsFlag; - import 'package:ensemble/framework/scope.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; @@ -10,6 +8,7 @@ import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; import 'package:flutter_test/flutter_test.dart'; class AssertionEngine { @@ -24,7 +23,7 @@ class AssertionEngine { Finder finderForId(String id) => find.byKey(ValueKey(id)); void expectVisible(String id) { - final finder = finderForId(id); + final finder = finderForId(id).hitTestable(); if (finder.evaluate().isEmpty) { throw EnsembleTestFailure( 'Expected widget with id "$id" to be visible. ' @@ -34,7 +33,7 @@ class AssertionEngine { } void expectNotVisible(String id) { - final finder = finderForId(id); + final finder = finderForId(id).hitTestable(); if (finder.evaluate().isNotEmpty) { throw EnsembleTestFailure( 'Expected widget with id "$id" to not be visible.', @@ -43,14 +42,13 @@ class AssertionEngine { } void expectText(String text) { - final finder = find.text(text); - if (finder.evaluate().isEmpty) { + if (!isTextVisible(text)) { throw EnsembleTestFailure('Expected text "$text" to be visible.'); } } void expectNoText(String text) { - if (find.text(text).evaluate().isNotEmpty) { + if (isTextVisible(text)) { throw EnsembleTestFailure('Expected text "$text" to not be visible.'); } } @@ -70,8 +68,9 @@ class AssertionEngine { 'Expected widget with id "$id" to exist for enabled check.', ); } - final semantics = tester.getSemantics(finder); - final isEnabled = semantics.hasFlag(SemanticsFlag.isEnabled); + final isEnabled = _readSemantics( + () => tester.getSemantics(finder).hasFlag(SemanticsFlag.isEnabled), + ); if (isEnabled != enabled) { throw EnsembleTestFailure( 'Expected widget "$id" to be ${enabled ? 'enabled' : 'disabled'}, ' @@ -166,18 +165,89 @@ class AssertionEngine { } void expectTextContains(String text) { - if (find.textContaining(text).evaluate().isEmpty) { + if (!isTextContainingVisible(text)) { throw EnsembleTestFailure('Expected text containing "$text".'); } } + bool isTextVisible(String text) => _hasVisiblePaintedElement(find.text(text)); + + bool isTextContainingVisible(String text) => + _hasVisiblePaintedElement(find.textContaining(text)); + + bool _hasVisiblePaintedElement(Finder finder) { + return finder.evaluate().any(_isElementInViewport); + } + + bool _isElementInViewport(Element element) { + final route = ModalRoute.of(element); + if (route != null && !route.isCurrent) return false; + if (_isUnderOffstageAncestor(element)) return false; + + final renderObject = element.renderObject; + if (renderObject is! RenderBox || + !renderObject.hasSize || + renderObject.size.isEmpty) { + return false; + } + if (_effectiveOpacity(element) <= 0.01) return false; + + final topLeft = renderObject.localToGlobal(Offset.zero); + final rect = topLeft & renderObject.size; + if (!rect.isFinite || rect.isEmpty) return false; + + final viewport = tester.binding.renderViews.first.paintBounds; + final visibleRect = rect.intersect(viewport); + return visibleRect != Rect.zero && + visibleRect.width > 0 && + visibleRect.height > 0; + } + + bool _isUnderOffstageAncestor(Element element) { + var isOffstage = false; + element.visitAncestorElements((ancestor) { + final renderObject = ancestor.renderObject; + if (renderObject is RenderOffstage && renderObject.offstage) { + isOffstage = true; + return false; + } + return true; + }); + return isOffstage; + } + + double _effectiveOpacity(Element element) { + var opacity = 1.0; + element.visitAncestorElements((ancestor) { + final renderObject = ancestor.renderObject; + if (renderObject is RenderOpacity) { + opacity *= renderObject.opacity; + } else if (renderObject != null && + renderObject.runtimeType.toString() == 'RenderAnimatedOpacity') { + try { + final animatedOpacity = (renderObject as dynamic).opacity; + if (animatedOpacity is Animation) { + opacity *= animatedOpacity.value; + } else if (animatedOpacity is double) { + opacity *= animatedOpacity; + } + } catch (_) { + // Keep the opacity already collected from other ancestors. + } + } + return opacity > 0.01; + }); + return opacity; + } + void expectChecked(String id, bool expected) { final finder = finderForId(id); if (finder.evaluate().isEmpty) { throw EnsembleTestFailure('expectChecked: widget "$id" not found.'); } - final semantics = tester.getSemantics(finder); - final isChecked = semantics.hasFlag(SemanticsFlag.isChecked); + final isChecked = _readSemantics( + () => tester.getSemantics(finder).hasFlag(SemanticsFlag.isChecked), + ); if (isChecked != expected) { throw EnsembleTestFailure( 'Expected "$id" checked=$expected, got $isChecked.', @@ -191,8 +261,7 @@ class AssertionEngine { throw EnsembleTestFailure('expectProperty: widget "$id" not found.'); } if (property == 'label') { - final semantics = tester.getSemantics(finder); - final label = semantics.label; + final label = _readSemantics(() => tester.getSemantics(finder).label); if (label != expected?.toString()) { throw EnsembleTestFailure( 'Expected label "$expected", got "$label".', @@ -328,8 +397,11 @@ class AssertionEngine { if (finder.evaluate().isEmpty) { throw EnsembleTestFailure('expectAccessible: "$id" not found.'); } - final semantics = tester.getSemantics(finder); - if (semantics.label.isEmpty && semantics.value.isEmpty) { + final hasAccessibleText = _readSemantics(() { + final semantics = tester.getSemantics(finder); + return semantics.label.isNotEmpty || semantics.value.isNotEmpty; + }); + if (!hasAccessibleText) { throw EnsembleTestFailure( 'Widget "$id" has no accessibility label or value.', ); @@ -338,14 +410,23 @@ class AssertionEngine { void expectSemanticsLabel(String id, String label) { final finder = finderForId(id); - final semantics = tester.getSemantics(finder); - if (semantics.label != label) { + final actual = _readSemantics(() => tester.getSemantics(finder).label); + if (actual != label) { throw EnsembleTestFailure( - 'Expected semantics label "$label", got "${semantics.label}".', + 'Expected semantics label "$label", got "$actual".', ); } } + T _readSemantics(T Function() read) { + final handle = tester.ensureSemantics(); + try { + return read(); + } finally { + handle.dispose(); + } + } + void expectNoOverflow(String id) { final finder = finderForId(id); final renderObject = tester.renderObject(finder); @@ -436,6 +517,19 @@ class AssertionEngine { return 'Visible widget ids: $shown$suffix.'; } + String visibleTextSummary({int limit = 10}) { + final texts = tester.allWidgets + .whereType() + .map((widget) => widget.data) + .whereType() + .where((value) => value.isNotEmpty) + .toSet() + .take(limit) + .toList(); + if (texts.isEmpty) return 'No Text widgets are currently visible.'; + return 'Visible text: ${texts.map(jsonEncode).join(', ')}.'; + } + String apiCallSummary({int limit = 10}) { final calls = context.apiOverlay.calls; if (calls.isEmpty) return 'No API calls were recorded.'; diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart index 9a74df34a..5290af212 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart @@ -7,7 +7,15 @@ import 'package:ensemble_test_runner/cli/ensemble_test_cli_output.dart'; import 'package:ensemble_test_runner/cli/yaml_test_app_patcher.dart'; import 'package:ensemble_test_runner/cli/ensemble_test_scaffold.dart'; import 'package:ensemble_test_runner/inspect/ensemble_app_inspector.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; import 'package:ensemble_test_runner/validation/ensemble_test_validator.dart'; +import 'package:path/path.dart' as p; +import 'package:yaml/yaml.dart'; + +const _maxAppConsoleLogBytes = 5 * 1024 * 1024; +final _appConsoleLogBytes = {}; +final _disabledAppConsoleLogs = {}; /// Runs declarative YAML tests in an Ensemble app. /// @@ -26,6 +34,7 @@ import 'package:ensemble_test_runner/validation/ensemble_test_validator.dart'; /// --tag= Run matching tag(s); repeatable /// --path= Run matching test asset path(s); repeatable /// --input key=value Provide a test input for ${inputs.key}; repeatable +/// --jobs= Concurrent job count (default: auto for full suites; 1 disables) /// --timeout= Test suite timeout, e.g. 30s, 5m, 1h (default: 10m) /// --verbose Full `flutter pub get` / `flutter test` output Future runEnsembleYamlTestsCli(List arguments) async { @@ -39,6 +48,7 @@ Future runEnsembleYamlTestsCli(List arguments) async { final reportFile = _resolveReportFile(arguments); final appDir = _resolveAppDir(arguments); final timeoutSeconds = _resolveTimeoutSeconds(arguments); + final jobs = _resolveJobsOverride(arguments); final patcher = YamlTestAppPatcher(appDir); if (!Directory(appDir).existsSync()) { @@ -135,35 +145,61 @@ Future runEnsembleYamlTestsCli(List arguments) async { } } - final testArgs = [ - 'test', - YamlTestAppPatcher.testEntryRelativePath, - '--no-pub', - '--dart-define=testmode=true', - if (reportMode != null) '--dart-define=ensembleTestReport=$reportMode', - if (reportFile != null) - '--dart-define=ensembleTestReportFile=$reportFile', - if (timeoutSeconds != null) - '--dart-define=ensembleTestTimeoutSeconds=$timeoutSeconds', - ..._inputDartDefines(arguments), - ..._selectionDartDefines(arguments), - '--reporter', - verbose ? 'expanded' : 'silent', - ...flutterTestArguments(arguments), - ]; - _writeStatus( 'Running Ensemble YAML tests...', quiet: quiet, machineReport: machineReport, ); - final testRun = await _runFlutterTestProcess( - 'flutter', - testArgs, - workingDirectory: appDir, - streamOutput: streamLiveOutput, - verbose: verbose, - ); + if (_hasSelection(arguments) && jobs != null && jobs > 1) { + throw StateError( + '--jobs currently runs full suites only. Remove --id/--feature/--tag/--path ' + 'or run the selected tests serially.', + ); + } + final runSerial = jobs == 1 || _hasSelection(arguments); + final serialWorkingDirectory = runSerial && patcher.hasTimerRewrites + ? _prepareWorkerDirectory(appDir, 0, patcher) + : appDir; + final serialServiceOverrides = runSerial + ? await _resolveServiceOverrides( + patcher: patcher, + usedPorts: {}, + ) + : null; + final testRun = runSerial + ? await _runFlutterTestProcess( + 'flutter', + _buildFlutterTestArgs( + arguments, + reportMode: reportMode, + reportFile: reportFile, + timeoutSeconds: timeoutSeconds, + verbose: verbose, + appLogPath: patcher.hasTimerRewrites + ? _appConsoleLogFile(appDir) + : _appConsoleLogPath(), + appLogDisplayPath: + patcher.hasTimerRewrites ? _appConsoleLogPath() : null, + artifactRoot: + patcher.hasTimerRewrites ? _artifactRootPath(appDir) : null, + serviceOverrides: serialServiceOverrides, + ), + workingDirectory: serialWorkingDirectory, + streamOutput: streamLiveOutput, + verbose: verbose, + appLogFile: _appConsoleLogFile(appDir), + ) + : await _runParallelFlutterTests( + arguments, + appDir: appDir, + patcher: patcher, + jobs: jobs, + reportMode: reportMode, + reportFile: reportFile, + timeoutSeconds: timeoutSeconds, + quiet: quiet, + machineReport: machineReport, + ); if (testRun.exitCode != 0 && !verbose) { final output = '${testRun.stdout ?? ''}\n${testRun.stderr ?? ''}'; @@ -240,6 +276,1313 @@ List _selectionDartDefines(List arguments) { ]; } +List _buildFlutterTestArgs( + List arguments, { + required String? reportMode, + required String? reportFile, + required int? timeoutSeconds, + required bool verbose, + List shardPaths = const [], + int? workerIndex, + String? progressFile, + String? appLogPath, + String? appLogDisplayPath, + String? artifactRoot, + Map? serviceOverrides, + String artifactDisplayRoot = 'build/ensemble_test_runner', +}) { + return [ + 'test', + YamlTestAppPatcher.testEntryRelativePath, + '--no-pub', + if (reportMode != null) '--dart-define=ensembleTestReport=$reportMode', + if (reportFile != null) '--dart-define=ensembleTestReportFile=$reportFile', + if (timeoutSeconds != null) + '--dart-define=ensembleTestTimeoutSeconds=$timeoutSeconds', + if (workerIndex != null) ...[ + '--dart-define=ensembleTestWorkerIndex=$workerIndex', + '--dart-define=ensembleTestWorkerSuffix=worker${workerIndex + 1}', + ], + if (progressFile != null) + '--dart-define=ensembleTestProgressFile=$progressFile', + if (appLogPath != null) '--dart-define=ensembleTestAppLogFile=$appLogPath', + if (appLogDisplayPath != null) + '--dart-define=ensembleTestAppLogDisplayFile=$appLogDisplayPath', + if (artifactRoot != null) + '--dart-define=ensembleTestArtifactRoot=$artifactRoot', + if (serviceOverrides != null && serviceOverrides.isNotEmpty) + '--dart-define=ensembleTestServiceOverrides=${json.encode(serviceOverrides)}', + '--dart-define=ensembleTestArtifactDisplayRoot=$artifactDisplayRoot', + ..._inputDartDefines(arguments), + ..._selectionDartDefines(arguments), + if (shardPaths.isNotEmpty) + '--dart-define=ensembleTestPath=${shardPaths.join(',')}', + '--reporter', + verbose ? 'expanded' : 'silent', + ...flutterTestArguments(arguments), + ]; +} + +Future> _resolveServiceOverrides({ + required YamlTestAppPatcher patcher, + required Set usedPorts, + int preferredOffset = 0, +}) async { + final testsDir = patcher.testsDirPath; + if (testsDir == null) return const {}; + final configFile = File(p.join(testsDir, 'config.yaml')); + if (!configFile.existsSync()) return const {}; + + final config = EnsembleTestParser.parseConfigString( + configFile.readAsStringSync(), + sourcePath: configFile.path, + ); + if (config.services.isEmpty) return const {}; + + final overrides = {}; + for (final service in config.services) { + final resolvedUrl = await _allocateServiceUrl( + service.url, + usedPorts: usedPorts, + preferredOffset: preferredOffset, + ); + if (resolvedUrl == null) continue; + final uri = Uri.parse(resolvedUrl); + overrides[service.name] = { + 'url': resolvedUrl, + 'environment': { + 'PORT': '${uri.port}', + }, + }; + } + return overrides; +} + +Future _allocateServiceUrl( + String? configuredUrl, { + required Set usedPorts, + required int preferredOffset, +}) async { + final baseUri = _serviceBaseUri(configuredUrl); + if (baseUri == null) return null; + final configuredPort = baseUri.hasPort ? baseUri.port : null; + final preferredPort = + configuredPort == null ? null : configuredPort + preferredOffset; + final port = await _allocateTcpPort( + preferredPort: preferredPort, + usedPorts: usedPorts, + ); + return baseUri.replace(port: port).toString(); +} + +Uri? _serviceBaseUri(String? configuredUrl) { + if (configuredUrl == null || configuredUrl.trim().isEmpty) { + return Uri.parse('http://127.0.0.1'); + } + final uri = Uri.tryParse(configuredUrl.trim()); + if (uri == null || !uri.hasScheme || uri.host.isEmpty) return null; + return uri; +} + +Future _allocateTcpPort({ + required int? preferredPort, + required Set usedPorts, +}) async { + if (preferredPort != null && + preferredPort > 0 && + !usedPorts.contains(preferredPort) && + await _isTcpPortAvailable(preferredPort)) { + usedPorts.add(preferredPort); + return preferredPort; + } + + while (true) { + final socket = await ServerSocket.bind(InternetAddress.anyIPv4, 0); + final port = socket.port; + await socket.close(); + if (usedPorts.add(port)) return port; + } +} + +Future _isTcpPortAvailable(int port) async { + try { + final socket = await ServerSocket.bind(InternetAddress.anyIPv4, port); + await socket.close(); + return true; + } on SocketException { + return false; + } +} + +Future _runParallelFlutterTests( + List arguments, { + required String appDir, + required YamlTestAppPatcher patcher, + required int? jobs, + required String? reportMode, + required String? reportFile, + required int? timeoutSeconds, + required bool quiet, + required bool machineReport, +}) async { + final testFiles = _testFilesForSharding(appDir, patcher); + final allFiles = [ + ...testFiles.parallel.map((file) => file.path), + ...testFiles.serial, + ]; + if (allFiles.length < 2) { + return _runFlutterTestProcess( + 'flutter', + _buildFlutterTestArgs( + arguments, + reportMode: reportMode, + reportFile: reportFile, + timeoutSeconds: timeoutSeconds, + verbose: false, + appLogPath: _appConsoleLogPath(), + ), + workingDirectory: appDir, + streamOutput: false, + verbose: false, + appLogFile: _appConsoleLogFile(appDir), + ); + } + + final requestedJobs = jobs ?? _autoWorkerCount(allFiles.length); + final hasParallelFiles = testFiles.parallel.isNotEmpty; + final hasSerialFiles = testFiles.serial.isNotEmpty; + final totalJobCount = requestedJobs.clamp(1, allFiles.length); + final serialLaneCount = hasSerialFiles ? 1 : 0; + final parallelLaneBudget = totalJobCount - serialLaneCount; + final parallelWorkerCount = hasParallelFiles + ? parallelLaneBudget.clamp(1, testFiles.parallel.length) + : 0; + if (parallelWorkerCount <= 1 && !hasSerialFiles) { + return _runFlutterTestProcess( + 'flutter', + _buildFlutterTestArgs( + arguments, + reportMode: reportMode, + reportFile: reportFile, + timeoutSeconds: timeoutSeconds, + verbose: false, + appLogPath: _appConsoleLogPath(), + ), + workingDirectory: appDir, + streamOutput: false, + verbose: false, + appLogFile: _appConsoleLogFile(appDir), + ); + } + final shards = _balancedShards(testFiles.parallel, parallelWorkerCount); + + _cleanParallelRunArtifacts(appDir); + _writeStatus( + 'Running ${allFiles.length} test files...', + quiet: quiet, + machineReport: machineReport, + ); + + final elapsed = Stopwatch()..start(); + final artifactRoot = _artifactRootPath(appDir); + final futures = >[]; + final workerReportFiles = []; + final workerProgressFiles = []; + final showProgress = !quiet && !machineReport; + final usedServicePorts = {}; + for (var i = 0; i < parallelWorkerCount; i++) { + final shard = shards[i].map((file) => file.path).toList(); + if (shard.isEmpty) continue; + final workerReportFile = _workerReportFile(appDir, i); + final workerProgressFile = _workerProgressFile(appDir, i); + _deleteIfExists(workerReportFile); + _deleteIfExists(workerProgressFile); + workerReportFiles.add(workerReportFile); + workerProgressFiles.add(workerProgressFile); + final workerDirectory = _prepareWorkerDirectory(appDir, i, patcher); + final serviceOverrides = await _resolveServiceOverrides( + patcher: patcher, + preferredOffset: i, + usedPorts: usedServicePorts, + ); + futures.add( + _runTimedFlutterTestProcess( + reportFile: workerReportFile, + run: () => _runFlutterTestProcess( + 'flutter', + _buildFlutterTestArgs( + arguments, + reportMode: 'json', + reportFile: workerReportFile, + timeoutSeconds: timeoutSeconds, + verbose: false, + shardPaths: shard, + workerIndex: i, + progressFile: showProgress ? workerProgressFile : null, + appLogPath: _appConsoleLogFile(appDir, workerIndex: i), + appLogDisplayPath: _appConsoleLogPath(workerIndex: i), + artifactRoot: artifactRoot, + serviceOverrides: serviceOverrides, + ), + workingDirectory: workerDirectory, + streamOutput: false, + verbose: false, + appLogFile: _appConsoleLogFile(workerDirectory, workerIndex: i), + ), + ), + ); + } + + if (hasSerialFiles) { + final serialWorkerIndex = parallelWorkerCount; + final workerReportFile = _workerReportFile(appDir, serialWorkerIndex); + final workerProgressFile = _workerProgressFile(appDir, serialWorkerIndex); + _deleteIfExists(workerReportFile); + _deleteIfExists(workerProgressFile); + workerReportFiles.add(workerReportFile); + workerProgressFiles.add(workerProgressFile); + final workerDirectory = + _prepareWorkerDirectory(appDir, serialWorkerIndex, patcher); + final serviceOverrides = await _resolveServiceOverrides( + patcher: patcher, + preferredOffset: serialWorkerIndex, + usedPorts: usedServicePorts, + ); + futures.add( + _runTimedFlutterTestProcess( + reportFile: workerReportFile, + run: () => _runFlutterTestProcess( + 'flutter', + _buildFlutterTestArgs( + arguments, + reportMode: 'json', + reportFile: workerReportFile, + timeoutSeconds: timeoutSeconds, + verbose: false, + shardPaths: testFiles.serial, + workerIndex: serialWorkerIndex, + progressFile: showProgress ? workerProgressFile : null, + appLogPath: _appConsoleLogFile( + appDir, + workerIndex: serialWorkerIndex, + ), + appLogDisplayPath: _appConsoleLogPath( + workerIndex: serialWorkerIndex, + ), + artifactRoot: artifactRoot, + serviceOverrides: serviceOverrides, + ), + workingDirectory: workerDirectory, + streamOutput: false, + verbose: false, + appLogFile: _appConsoleLogFile( + workerDirectory, + workerIndex: serialWorkerIndex, + ), + ), + ), + ); + } + + var runDone = false; + final progressPoller = showProgress + ? _pollWorkerProgress( + workerProgressFiles, + isDone: () => runDone, + ) + : Future.value(); + final workerResults = await Future.wait(futures); + runDone = true; + await progressPoller; + + for (var i = 0; i < workerReportFiles.length; i++) { + _copyWorkerArtifacts(appDir, i); + } + var merged = _mergeWorkerReports( + workerResults, + reportFiles: workerReportFiles, + appDir: appDir, + ); + merged = _mergeParallelSuiteArtifacts(appDir, merged); + _writeHistoricalDurations(appDir, merged); + final output = StringBuffer(); + if (reportMode == 'json') { + output.writeln(json.encode(merged.toJson())); + } else if (reportMode == 'junit') { + output.writeln(_junitReportForCli(merged)); + } else { + output.write( + _formatCliSummary( + merged, + testFile: '${patcher.testsDirRelative}/*.test.yaml', + wallTimeMs: elapsed.elapsedMilliseconds, + ), + ); + } + if (reportFile != null) { + File(reportFile).writeAsStringSync( + reportMode == 'junit' + ? _junitReportForCli(merged) + : json.encode(merged.toJson()), + ); + } + + final stderr = StringBuffer(); + if (merged.failedCount > 0) { + for (var i = 0; i < workerResults.length; i++) { + final result = workerResults[i]; + if (result.exitCode != 0) { + stderr.writeln( + 'A test process failed with exit code ${result.exitCode}.', + ); + final known = extractKnownFailure( + '${result.stdout ?? ''}\n${result.stderr ?? ''}', + ); + if (known.isNotEmpty) stderr.writeln(known); + } + final err = result.stderr?.toString() ?? ''; + if (err.isNotEmpty && !isBenignFlutterTestStderr(err)) { + stderr.writeln(err.trimRight()); + } + } + } + + final exitCode = merged.failedCount > 0 ? 1 : 0; + return ProcessResult( + 0, + exitCode, + output.toString(), + stderr.toString(), + ); +} + +String _workerReportFile(String appDir, int workerIndex) { + return p.join( + appDir, + 'build', + 'ensemble_test_runner', + 'worker_reports', + 'worker${workerIndex + 1}.json', + ); +} + +EnsembleTestRunResult _mergeParallelSuiteArtifacts( + String appDir, + EnsembleTestRunResult result, +) { + final logs = []; + final apiCallsPath = _mergeApiCallLogs(appDir); + final storagePath = _mergeStorageLogs(appDir); + + final hasApiCalls = + result.suiteLogs.any((log) => log.startsWith('apiCalls:')); + final hasStorage = result.suiteLogs.any((log) => log.startsWith('storage:')); + if (hasApiCalls && apiCallsPath != null) { + logs.add('apiCalls: $apiCallsPath'); + } + if (hasStorage && storagePath != null) { + logs.add('storage: $storagePath'); + } + + for (final log in result.suiteLogs) { + if (log.startsWith('apiCalls:') || log.startsWith('storage:')) continue; + logs.add(log); + } + + return EnsembleTestRunResult( + results: result.results, + suiteLogs: logs, + ); +} + +String? _mergeApiCallLogs(String appDir) { + final logsDir = + Directory(p.join(appDir, 'build', 'ensemble_test_runner', 'logs')); + if (!logsDir.existsSync()) return null; + final files = logsDir + .listSync() + .whereType() + .where( + (file) => RegExp(r'api_calls_worker\d+\.json$').hasMatch(file.path)) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + if (files.isEmpty) return null; + + final callsByName = >{}; + var total = 0; + for (final file in files) { + try { + final decoded = json.decode(file.readAsStringSync()); + if (decoded is! Map) continue; + total += decoded['total'] is int ? decoded['total'] as int : 0; + final calls = decoded['calls']; + if (calls is! List) continue; + for (final call in calls.whereType()) { + final name = call['name']?.toString(); + if (name == null || name.isEmpty) continue; + final timestamps = call['timestamps']; + callsByName.putIfAbsent(name, () => []).addAll( + timestamps is List + ? timestamps.map((value) => value.toString()) + : const [], + ); + } + } catch (_) { + // Ignore malformed partial worker logs. + } + } + + final output = File(p.join(logsDir.path, 'api_calls.json')); + output.writeAsStringSync( + const JsonEncoder.withIndent(' ').convert({ + 'total': total, + 'calls': [ + for (final entry in callsByName.entries) + { + 'name': entry.key, + 'count': entry.value.length, + 'timestamps': entry.value..sort(), + }, + ], + }), + ); + for (final file in files) { + file.deleteSync(); + } + return p.relative(output.path, from: appDir).replaceAll('\\', '/'); +} + +String? _mergeStorageLogs(String appDir) { + final logsDir = + Directory(p.join(appDir, 'build', 'ensemble_test_runner', 'logs')); + if (!logsDir.existsSync()) return null; + final files = logsDir + .listSync() + .whereType() + .where((file) => RegExp(r'storage_worker\d+\.json$').hasMatch(file.path)) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + if (files.isEmpty) return null; + + final merged = {}; + for (final file in files) { + try { + final decoded = json.decode(file.readAsStringSync()); + if (decoded is Map) { + merged.addAll(Map.from(decoded)); + } + } catch (_) { + // Ignore malformed partial worker logs. + } + } + + final output = File(p.join(logsDir.path, 'storage.json')); + output.writeAsStringSync(const JsonEncoder.withIndent(' ').convert(merged)); + for (final file in files) { + file.deleteSync(); + } + return p.relative(output.path, from: appDir).replaceAll('\\', '/'); +} + +Future _runTimedFlutterTestProcess({ + required String reportFile, + required Future Function() run, +}) async { + final stopwatch = Stopwatch()..start(); + final result = await run(); + stopwatch.stop(); + _annotateWorkerReportDuration(reportFile, stopwatch.elapsedMilliseconds); + return result; +} + +void _annotateWorkerReportDuration(String reportFile, int durationMs) { + final file = File(reportFile); + if (!file.existsSync()) return; + try { + final decoded = json.decode(file.readAsStringSync()); + if (decoded is! Map) return; + final updated = Map.from(decoded) + ..['durationMs'] = durationMs; + file.writeAsStringSync(json.encode(updated)); + } catch (_) { + // Keep the original report if it cannot be decoded. + } +} + +void _cleanParallelRunArtifacts(String appDir) { + final root = Directory(p.join(appDir, 'build', 'ensemble_test_runner')); + for (final name in const [ + 'logs', + 'screenshots', + 'worker_progress', + 'worker_reports', + ]) { + final directory = Directory(p.join(root.path, name)); + if (directory.existsSync()) directory.deleteSync(recursive: true); + } +} + +String _workerProgressFile(String appDir, int workerIndex) { + return p.join( + appDir, + 'build', + 'ensemble_test_runner', + 'worker_progress', + 'worker${workerIndex + 1}.jsonl', + ); +} + +String _appConsoleLogPath({int? workerIndex}) { + final name = workerIndex == null + ? 'app_console.log' + : 'app_console_${workerIndex + 1}.log'; + return p.join('build', 'ensemble_test_runner', 'logs', name); +} + +String _appConsoleLogFile(String appDir, {int? workerIndex}) { + return p.join(appDir, _appConsoleLogPath(workerIndex: workerIndex)); +} + +String _artifactRootPath(String appDir) { + return p.join(appDir, 'build', 'ensemble_test_runner'); +} + +Future _pollWorkerProgress( + List progressFiles, { + required bool Function() isDone, +}) async { + final seenLines = { + for (final file in progressFiles) file: 0, + }; + + while (!isDone()) { + _drainWorkerProgress(progressFiles, seenLines); + await Future.delayed(const Duration(milliseconds: 500)); + } + _drainWorkerProgress(progressFiles, seenLines); +} + +void _drainWorkerProgress( + List progressFiles, + Map seenLines, +) { + for (var i = 0; i < progressFiles.length; i++) { + final path = progressFiles[i]; + final file = File(path); + if (!file.existsSync()) continue; + final lines = + file.readAsLinesSync().where((line) => line.trim().isNotEmpty).toList(); + final seen = seenLines[path] ?? 0; + if (seen >= lines.length) continue; + for (final line in lines.skip(seen)) { + final event = _decodeProgressEvent(line); + if (event == null) continue; + stderr.writeln(_formatProgressEvent(event)); + } + seenLines[path] = lines.length; + } +} + +Map? _decodeProgressEvent(String line) { + try { + final decoded = json.decode(line); + return decoded is Map ? Map.from(decoded) : null; + } catch (_) { + return null; + } +} + +String _formatProgressEvent(Map event) { + final status = event['status'] == 'passed' ? '✓' : '✗'; + final testId = event['testId']?.toString() ?? '(unknown)'; + final durationMs = + event['durationMs'] is int ? event['durationMs'] as int : 0; + final message = event['message']?.toString(); + final attempts = event['attempts'] is int ? event['attempts'] as int : 1; + final retry = event['retry'] is int ? event['retry'] as int : 0; + final retryText = attempts > 1 ? ' · attempt $attempts/${retry + 1}' : ''; + final suffix = message == null || message.isEmpty + ? '' + : ': ${_singleLine(message, maxLength: 120)}'; + return '$status ${_baseTestId(testId)} (${_formatDuration(durationMs)})$retryText$suffix'; +} + +String _formatDuration(int durationMs) { + if (durationMs < 1000) return '${durationMs}ms'; + return '${(durationMs / 1000).toStringAsFixed(1)}s'; +} + +String _singleLine(String value, {required int maxLength}) { + final single = value.replaceAll(RegExp(r'\s+'), ' ').trim(); + if (single.length <= maxLength) return single; + return '${single.substring(0, maxLength - 1)}…'; +} + +void _deleteIfExists(String path) { + final file = File(path); + if (file.existsSync()) file.deleteSync(); +} + +String _prepareWorkerDirectory( + String appDir, + int workerIndex, + YamlTestAppPatcher patcher, +) { + final workerDir = Directory(_workerDirectory(appDir, workerIndex)); + workerDir.createSync(recursive: true); + + final sourceNames = {}; + for (final entity in Directory(appDir).listSync(followLinks: false)) { + final name = p.basename(entity.path); + if (name == 'build' || name == '.git' || name == '.dart_tool') continue; + sourceNames.add(name); + _replaceWithLink( + p.join(workerDir.path, name), + entity.absolute.path, + ); + } + + for (final entity in workerDir.listSync(followLinks: false)) { + final name = p.basename(entity.path); + if (name == 'build' || name == '.dart_tool' || sourceNames.contains(name)) { + continue; + } + entity.deleteSync(recursive: true); + } + + final sourceDartTool = Directory(p.join(appDir, '.dart_tool')); + final workerDartTool = Directory(p.join(workerDir.path, '.dart_tool')) + ..createSync(); + for (final fileName in const [ + 'package_config.json', + 'package_graph.json', + 'version', + ]) { + final source = File(p.join(sourceDartTool.path, fileName)); + if (source.existsSync()) { + source.copySync(p.join(workerDartTool.path, fileName)); + } + } + _materializeTimerRewriteScreens(workerDir.path, appDir, patcher); + patcher.rewriteTimersIn(workerDir.path); + return workerDir.path; +} + +void _materializeTimerRewriteScreens( + String workerDir, + String appDir, + YamlTestAppPatcher patcher, +) { + if (!patcher.hasTimerRewrites) return; + final testsDir = patcher.testsDirRelative; + if (testsDir == null) return; + final appRootRelative = p.dirname(_withoutTrailingSlash(testsDir)); + final screensRelative = p.join(appRootRelative, 'screens'); + final sourceScreens = Directory(p.join(appDir, screensRelative)); + if (!sourceScreens.existsSync()) return; + _materializeCopiedSubtree( + workerDir: workerDir, + sourceRoot: appDir, + relativePath: screensRelative, + ); +} + +String _withoutTrailingSlash(String path) { + final normalized = path.replaceAll('\\', '/'); + return normalized.endsWith('/') + ? normalized.substring(0, normalized.length - 1) + : normalized; +} + +void _materializeCopiedSubtree({ + required String workerDir, + required String sourceRoot, + required String relativePath, +}) { + final segments = p.split(relativePath.replaceAll('\\', '/')); + var sourceParent = Directory(sourceRoot); + var targetParent = Directory(workerDir); + for (var i = 0; i < segments.length; i++) { + final segment = segments[i]; + final source = Directory(p.join(sourceParent.path, segment)); + final targetPath = p.join(targetParent.path, segment); + final isLeaf = i == segments.length - 1; + if (isLeaf) { + _replaceWithDirectoryCopy(targetPath, source); + return; + } + _replaceWithOverlayDirectory( + targetPath, + source, + skipChildName: segments[i + 1], + ); + sourceParent = source; + targetParent = Directory(targetPath); + } +} + +void _replaceWithOverlayDirectory( + String path, + Directory source, { + required String skipChildName, +}) { + _deleteEntityIfExists(path); + final target = Directory(path)..createSync(recursive: true); + for (final entity in source.listSync(followLinks: false)) { + final name = p.basename(entity.path); + if (name == skipChildName) continue; + _replaceWithLink(p.join(target.path, name), entity.absolute.path); + } +} + +void _replaceWithDirectoryCopy(String path, Directory source) { + _deleteEntityIfExists(path); + final target = Directory(path)..createSync(recursive: true); + _copyDirectoryContents(source, target); +} + +void _replaceWithLink(String path, String target) { + _deleteEntityIfExists(path); + Link(path).createSync(target); +} + +void _deleteEntityIfExists(String path) { + final existing = FileSystemEntity.typeSync(path, followLinks: false); + if (existing != FileSystemEntityType.notFound) { + FileSystemEntity entity; + if (existing == FileSystemEntityType.directory) { + entity = Directory(path); + } else if (existing == FileSystemEntityType.link) { + entity = Link(path); + } else { + entity = File(path); + } + entity.deleteSync(recursive: true); + } +} + +String _workerDirectory(String appDir, int workerIndex) { + return p.join( + appDir, + 'build', + 'ensemble_test_runner', + 'workers', + 'worker${workerIndex + 1}', + ); +} + +void _copyWorkerArtifacts(String appDir, int workerIndex) { + final source = Directory( + p.join( + _workerDirectory(appDir, workerIndex), + 'build', + 'ensemble_test_runner', + ), + ); + if (!source.existsSync()) return; + final target = Directory(p.join(appDir, 'build', 'ensemble_test_runner')); + target.createSync(recursive: true); + _copyDirectoryContents(source, target); +} + +void _copyDirectoryContents(Directory source, Directory target) { + for (final entity in source.listSync(recursive: false, followLinks: false)) { + final destination = p.join(target.path, p.basename(entity.path)); + if (entity is Directory) { + final destinationDir = Directory(destination) + ..createSync(recursive: true); + _copyDirectoryContents(entity, destinationDir); + } else if (entity is File) { + entity.copySync(destination); + } + } +} + +_ShardedTestFiles _testFilesForSharding( + String appDir, + YamlTestAppPatcher patcher, +) { + final testsDir = patcher.testsDirPath; + if (testsDir == null) return const _ShardedTestFiles(); + final historicalDurations = _loadHistoricalDurations(appDir); + final parallel = <_ShardableTestFile>[]; + final serial = []; + final files = Directory(testsDir) + .listSync(recursive: true) + .whereType() + .where((file) => file.path.endsWith('.test.yaml')) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + final docsByPath = {}; + final pathById = {}; + final referencedSessionIds = {}; + final referencedPrerequisiteIds = {}; + for (final file in files) { + final relative = p.relative(file.path, from: appDir).replaceAll('\\', '/'); + final doc = _readTestYamlMap(file); + docsByPath[relative] = doc; + final id = doc?['id']?.toString(); + if (id != null && id.isNotEmpty) { + pathById[id] = relative; + } + final session = doc?['session']?.toString(); + if (session != null && session.isNotEmpty) { + referencedSessionIds.add(session); + } + final prerequisite = doc?['prerequisite']?.toString(); + if (prerequisite != null && prerequisite.isNotEmpty) { + referencedPrerequisiteIds.add(prerequisite); + } + } + + final prerequisiteLinkedPaths = { + for (final entry in docsByPath.entries) + if (_hasPrerequisite(entry.value)) entry.key, + for (final id in referencedPrerequisiteIds) + if (pathById[id] != null) pathById[id]!, + }; + final sessionProducerPaths = { + for (final id in referencedSessionIds) + if (pathById[id] != null) pathById[id]!, + }; + + for (final file in files) { + final relative = p.relative(file.path, from: appDir).replaceAll('\\', '/'); + if (_isParallelTestFile(file, doc: docsByPath[relative]) && + !prerequisiteLinkedPaths.contains(relative) && + !sessionProducerPaths.contains(relative)) { + parallel.add( + _ShardableTestFile( + path: relative, + estimatedDurationMs: _estimateTestFileDurationMs( + file, + relative, + historicalDurations, + ), + ), + ); + } else if (sessionProducerPaths.contains(relative) && + !prerequisiteLinkedPaths.contains(relative)) { + // Session producers are selected automatically by each shard that needs + // them. Running them as standalone files would waste a worker lane. + continue; + } else { + serial.add(relative); + } + } + return _ShardedTestFiles(parallel: parallel, serial: serial); +} + +YamlMap? _readTestYamlMap(File file) { + try { + final doc = loadYaml(file.readAsStringSync()); + return doc is YamlMap ? doc : null; + } catch (_) { + // Let the real parser report invalid YAML with source context. + } + return null; +} + +bool _hasPrerequisite(YamlMap? doc) { + if (doc == null) return false; + final prerequisite = doc['prerequisite']?.toString(); + return prerequisite != null && prerequisite.isNotEmpty; +} + +bool _isParallelTestFile(File file, {YamlMap? doc}) { + doc ??= _readTestYamlMap(file); + if (doc != null && doc['parallel'] == false) return false; + return true; +} + +class _ShardedTestFiles { + final List<_ShardableTestFile> parallel; + final List serial; + + const _ShardedTestFiles({ + this.parallel = const [], + this.serial = const [], + }); +} + +class _ShardableTestFile { + final String path; + final int estimatedDurationMs; + + const _ShardableTestFile({ + required this.path, + required this.estimatedDurationMs, + }); +} + +List> _balancedShards( + List<_ShardableTestFile> files, + int workerCount, +) { + if (workerCount <= 0) return const []; + final shards = List.generate(workerCount, (_) => <_ShardableTestFile>[]); + final shardDurations = List.filled(workerCount, 0); + final sorted = [...files]..sort((a, b) { + final byDuration = b.estimatedDurationMs.compareTo(a.estimatedDurationMs); + return byDuration != 0 ? byDuration : a.path.compareTo(b.path); + }); + + for (final file in sorted) { + var target = 0; + for (var i = 1; i < shardDurations.length; i++) { + if (shardDurations[i] < shardDurations[target]) target = i; + } + shards[target].add(file); + shardDurations[target] += file.estimatedDurationMs; + } + return shards; +} + +int _autoWorkerCount(int fileCount) { + if (fileCount < 2) return 1; + final halfCpu = (Platform.numberOfProcessors / 2).floor(); + final cpuBased = (halfCpu - 1).clamp(1, fileCount); + return cpuBased.clamp(1, fileCount); +} + +int _estimateTestFileDurationMs( + File file, + String relativePath, + Map historicalDurations, +) { + final historical = historicalDurations[relativePath]; + if (historical != null && historical > 0) return historical; + + try { + final doc = loadYaml(file.readAsStringSync()); + if (doc is! YamlMap) return 30000; + final steps = doc['steps']; + var estimate = doc['startScreen'] == 'Login' ? 18000 : 6000; + if (doc['prerequisite'] != null || doc['session'] != null) estimate += 4000; + if (steps is YamlList) { + for (final step in steps) { + estimate += _estimateStepDurationMs(step); + } + } + return estimate.clamp(10000, 180000); + } catch (_) { + return 30000; + } +} + +int _estimateStepDurationMs(dynamic step) { + if (step is! YamlMap || step.isEmpty) return 1000; + final type = step.keys.first.toString(); + return switch (type) { + 'waitForNavigation' => 7000, + 'waitForApi' => 6000, + 'waitFor' => 4000, + 'settle' => 2500, + 'tap' || 'toggle' || 'enterText' || 'goBack' => 1200, + 'expectVisible' || 'expectText' || 'expectNotVisible' => 700, + _ => 1500, + }; +} + +Map _loadHistoricalDurations(String appDir) { + final file = File(_durationCachePath(appDir)); + if (!file.existsSync()) return const {}; + try { + final decoded = json.decode(file.readAsStringSync()); + if (decoded is! Map) return const {}; + final files = decoded['files']; + if (files is! Map) return const {}; + return { + for (final entry in files.entries) + if (entry.value is int) entry.key.toString(): entry.value as int, + }; + } catch (_) { + return const {}; + } +} + +void _writeHistoricalDurations( + String appDir, + EnsembleTestRunResult result, +) { + final previousDurations = _loadHistoricalDurations(appDir); + final currentDurations = {}; + for (final test in result.results) { + if (test.durationMs <= 0) continue; + final path = _assetPathFromTestId(test.testId); + if (path == null) continue; + currentDurations[path] = (currentDurations[path] ?? 0) + test.durationMs; + } + final durations = { + ...previousDurations, + ...currentDurations, + }; + + final file = File(_durationCachePath(appDir)); + file.parent.createSync(recursive: true); + file.writeAsStringSync( + const JsonEncoder.withIndent(' ').convert({ + 'updatedAt': DateTime.now().toIso8601String(), + 'files': durations, + }), + ); +} + +String _durationCachePath(String appDir) { + return p.join(appDir, 'build', 'ensemble_test_runner', 'test_durations.json'); +} + +String? _assetPathFromTestId(String testId) { + final start = testId.lastIndexOf(' ('); + if (start == -1 || !testId.endsWith(')')) return null; + return testId.substring(start + 3, testId.length - 1); +} + +bool _hasSelection(List arguments) { + return arguments.any( + (arg) => + arg.startsWith('--id=') || + arg.startsWith('--feature=') || + arg.startsWith('--tag=') || + arg.startsWith('--path='), + ); +} + +EnsembleTestRunResult _mergeWorkerReports( + List results, { + required List reportFiles, + required String appDir, +}) { + final mergedResults = []; + final suiteLogs = []; + final seenPassedDependencies = {}; + + for (var i = 0; i < results.length; i++) { + final result = results[i]; + final output = '${result.stdout ?? ''}\n${result.stderr ?? ''}'; + final reportFile = i < reportFiles.length ? File(reportFiles[i]) : null; + final rawJson = reportFile != null && reportFile.existsSync() + ? reportFile.readAsStringSync() + : extractJsonReport(output); + if (rawJson.isEmpty) { + final workerOutputFile = _writeWorkerOutputLog(appDir, i, output); + mergedResults.add( + EnsembleSingleTestResult.failed( + testId: 'test-process-${result.pid}', + durationMs: 0, + error: 'A test process did not emit an Ensemble JSON report' + '${workerOutputFile == null ? '' : '. See $workerOutputFile'}', + ), + ); + continue; + } + + final decoded = json.decode(rawJson) as Map; + final workerRun = _runResultFromJson(decoded); + suiteLogs.addAll(workerRun.suiteLogs); + for (final test in workerRun.results) { + final baseId = _baseTestId(test.testId); + final isPassedDependency = + test.status == TestStatus.passed && _isRepeatedDependency(baseId); + if (isPassedDependency && !seenPassedDependencies.add(baseId)) { + continue; + } + mergedResults.add(test); + } + } + + return EnsembleTestRunResult( + results: mergedResults, + suiteLogs: suiteLogs, + ); +} + +String? _writeWorkerOutputLog(String appDir, int workerIndex, String output) { + if (output.trim().isEmpty) return null; + final file = File( + p.join( + appDir, + 'build', + 'ensemble_test_runner', + 'logs', + 'test_process_${workerIndex + 1}_output.log', + ), + ); + file.parent.createSync(recursive: true); + file.writeAsStringSync(output); + return p.relative(file.path, from: appDir).replaceAll('\\', '/'); +} + +String _formatCliSummary( + EnsembleTestRunResult result, { + String? testFile, + int? wallTimeMs, +}) { + final buffer = StringBuffer(); + buffer.writeln('┌─ Ensemble YAML tests ─────────────────────────────'); + if (testFile != null) { + buffer.writeln('│ $testFile'); + buffer.writeln('│'); + } + + for (var i = 0; i < result.results.length; i++) { + if (i > 0) buffer.writeln('│'); + _writeCliTestCase(buffer, result.results[i]); + } + + if (result.suiteLogs.isNotEmpty) { + buffer.writeln('│'); + buffer.writeln('│ suite artifacts:'); + for (final log in result.suiteLogs) { + buffer.writeln('│ $log'); + } + } + + buffer.writeln('│'); + final durationText = wallTimeMs == null + ? '${result.results.fold(0, (sum, r) => sum + r.durationMs)}ms total' + : '${wallTimeMs}ms'; + buffer.writeln('└─ ${result.summary} · $durationText'); + return buffer.toString(); +} + +void _writeCliTestCase(StringBuffer buffer, EnsembleSingleTestResult r) { + final icon = r.status == TestStatus.passed ? '✓' : '✗'; + buffer.writeln('│ $icon ${r.testId} (${r.durationMs}ms)'); + if (r.attempts > 1) { + buffer.writeln('│ attempts: ${r.attempts}/${r.retry + 1}'); + } + + final report = r.report; + if (report != null) { + if (report.prerequisite != null) { + buffer.writeln('│ after: ${report.prerequisite}'); + } + if (report.session != null) { + buffer.writeln('│ session: ${report.session}'); + } + buffer.writeln('│ start: ${report.startScreen}'); + if (report.endScreen != null && report.endScreen != report.startScreen) { + buffer.writeln('│ end: ${report.endScreen}'); + } + if (report.screensVisited.length > 1) { + buffer.writeln('│ flow: ${report.screensVisited.join(' → ')}'); + } + if (report.stepsOutline.isNotEmpty) { + buffer.writeln('│ steps (${report.stepsOutline.length}):'); + for (var i = 0; i < report.stepsOutline.length; i++) { + final prefix = r.status == TestStatus.failed && r.failedStepIndex == i + ? '>>' + : ' '; + buffer.writeln('│ $prefix ${i + 1}. ${report.stepsOutline[i]}'); + } + } + } + + if (r.status == TestStatus.failed && r.message != null) { + buffer.writeln('│ error: ${r.message}'); + } + + if (r.logs.isNotEmpty) { + buffer.writeln('│ artifacts:'); + for (final log in r.logs) { + buffer.writeln('│ $log'); + } + } +} + +bool _isRepeatedDependency(String testId) => testId == 'signin_to_gateway'; + +String _baseTestId(String value) { + final index = value.indexOf(' ('); + return index == -1 ? value : value.substring(0, index); +} + +EnsembleTestRunResult _runResultFromJson(Map json) { + final results = (json['results'] as List? ?? const []) + .whereType>() + .map(_singleResultFromJson) + .toList(); + final suiteLogs = (json['suiteLogs'] as List? ?? const []) + .map((value) => value.toString()) + .toList(); + return EnsembleTestRunResult(results: results, suiteLogs: suiteLogs); +} + +EnsembleSingleTestResult _singleResultFromJson(Map json) { + return EnsembleSingleTestResult( + testId: json['testId']?.toString() ?? '(unknown)', + metadata: json['metadata'] is Map + ? Map.from(json['metadata'] as Map) + : const {}, + status: json['status'] == 'passed' ? TestStatus.passed : TestStatus.failed, + durationMs: json['durationMs'] is int ? json['durationMs'] as int : 0, + attempts: json['attempts'] is int ? json['attempts'] as int : 1, + retry: json['retry'] is int ? json['retry'] as int : 0, + failedStepIndex: + json['failedStepIndex'] is int ? json['failedStepIndex'] as int : null, + message: json['message']?.toString(), + stackTrace: json['stackTrace']?.toString(), + logs: (json['logs'] as List? ?? const []) + .map((value) => value.toString()) + .toList(), + report: json['report'] is Map + ? _reportDetailsFromJson( + Map.from(json['report'] as Map)) + : null, + ); +} + +EnsembleTestReportDetails _reportDetailsFromJson(Map json) { + return EnsembleTestReportDetails( + startScreen: json['startScreen']?.toString() ?? '(unknown)', + endScreen: json['endScreen']?.toString(), + prerequisite: json['prerequisite']?.toString(), + session: json['session']?.toString(), + screensVisited: (json['screensVisited'] as List? ?? const []) + .map((value) => value.toString()) + .toList(), + stepsOutline: (json['stepsOutline'] as List? ?? const []) + .map((value) => value.toString()) + .toList(), + ); +} + +String _junitReportForCli(EnsembleTestRunResult result) { + final totalMs = result.results.fold(0, (sum, r) => sum + r.durationMs); + final buffer = StringBuffer() + ..writeln( + '', + ); + for (final r in result.results) { + buffer.writeln( + ' ', + ); + if (r.status == TestStatus.failed) { + buffer + ..writeln( + ' ', + ) + ..writeln(_xmlEscape(r.stackTrace ?? r.message ?? 'failed')) + ..writeln(' '); + } + buffer.writeln(' '); + } + buffer.writeln(''); + return buffer.toString(); +} + +String _xmlEscape(String value) { + return value + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + List _inputDartDefines(List arguments) { final inputs = _inputValues(arguments); if (inputs.isEmpty) return const []; @@ -300,6 +1643,7 @@ Future _runFlutterTestProcess( required String workingDirectory, required bool streamOutput, required bool verbose, + String? appLogFile, }) async { final process = await Process.start( executable, @@ -313,6 +1657,7 @@ Future _runFlutterTestProcess( final elapsed = Stopwatch()..start(); var lastLiveOutput = DateTime.now(); Timer? heartbeat; + final appLog = _openAppConsoleLog(appLogFile); if (streamOutput && !verbose) { heartbeat = Timer.periodic(const Duration(seconds: 10), (_) { @@ -331,6 +1676,7 @@ Future _runFlutterTestProcess( .transform(const LineSplitter()) .listen((line) { stdoutBuffer.writeln(line); + _appendAppConsoleLogLine(appLog, 'stdout', line); if (verbose) { stdout.writeln(line); } else if (streamOutput && liveFilter.shouldEmit(line)) { @@ -344,6 +1690,7 @@ Future _runFlutterTestProcess( .transform(const LineSplitter()) .listen((line) { stderrBuffer.writeln(line); + _appendAppConsoleLogLine(appLog, 'stderr', line); if (verbose) { stderr.writeln(line); } @@ -351,7 +1698,6 @@ Future _runFlutterTestProcess( final exitCode = await process.exitCode; await Future.wait([stdoutDone, stderrDone]); - return ProcessResult( process.pid, exitCode, @@ -363,6 +1709,62 @@ Future _runFlutterTestProcess( } } +File? _openAppConsoleLog(String? path) { + if (path == null) return null; + final file = File(path); + try { + file.parent.createSync(recursive: true); + final header = 'createdAt: ${DateTime.now().toIso8601String()}\n'; + file.writeAsStringSync(header); + _appConsoleLogBytes[file.path] = utf8.encode(header).length; + _disabledAppConsoleLogs.remove(file.path); + return file; + } on FileSystemException { + return null; + } +} + +void _appendAppConsoleLogLine(File? file, String stream, String line) { + if (file == null || !_isAppConsoleLogLine(line)) return; + final path = file.path; + if (_disabledAppConsoleLogs.contains(path)) return; + + final entry = '[$stream] $line\n'; + final bytes = utf8.encode(entry).length; + final currentBytes = _appConsoleLogBytes[path] ?? file.lengthSync(); + if (currentBytes + bytes > _maxAppConsoleLogBytes) { + _writeFinalAppConsoleLogLine( + file, + '[runner] app console log truncated at ' + '${(_maxAppConsoleLogBytes / (1024 * 1024)).toStringAsFixed(0)} MB\n', + ); + _disabledAppConsoleLogs.add(path); + return; + } + + try { + file.writeAsStringSync(entry, mode: FileMode.append); + _appConsoleLogBytes[path] = currentBytes + bytes; + } on FileSystemException { + _disabledAppConsoleLogs.add(path); + } +} + +void _writeFinalAppConsoleLogLine(File file, String line) { + try { + file.writeAsStringSync(line, mode: FileMode.append); + _appConsoleLogBytes[file.path] = + (_appConsoleLogBytes[file.path] ?? 0) + utf8.encode(line).length; + } on FileSystemException { + // Log files are diagnostic artifacts. They must not crash the test run. + } +} + +bool _isAppConsoleLogLine(String line) { + return !line.startsWith(jsonReportPrefix) && + !line.startsWith(junitReportPrefix); +} + Future _runProcess( String executable, List arguments, { @@ -435,6 +1837,24 @@ int? _resolveTimeoutSeconds(List arguments) { return seconds; } +int? _resolveJobsOverride(List arguments) { + String? value; + for (final arg in arguments) { + if (arg.startsWith('--jobs=')) { + value = arg.substring('--jobs='.length); + } + } + if (value == null || value.isEmpty || value == 'auto') return null; + final jobs = int.tryParse(value); + if (jobs == null || jobs <= 0) { + stderr.writeln( + 'Invalid --jobs value "$value". Use a positive integer, auto, or 1 to disable parallelism.', + ); + exit(2); + } + return jobs; +} + void _writeStatus( String message, { required bool quiet, diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart index 7c5ab8b63..1e4b3749a 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart @@ -135,6 +135,7 @@ List flutterTestArguments(List arguments) { a.startsWith('--feature=') || a.startsWith('--tag=') || a.startsWith('--path=') || + a.startsWith('--jobs=') || a.startsWith('--timeout=') || a == '--verbose' || a == '--quiet') { diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart index 53c34deb5..83e9b2443 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart @@ -131,6 +131,7 @@ class EnsembleTestDoctor { final ids = {}; final prerequisites = {}; + final sessions = {}; final referencedWidgetIds = {}; for (final file in testFiles) { @@ -157,6 +158,9 @@ class EnsembleTestDoctor { if (test.prerequisite != null) { prerequisites[test.id] = test.prerequisite!; } + if (test.session != null) { + sessions[test.id] = test.session!; + } referencedWidgetIds.addAll(test.referencedWidgetIds); } catch (failure) { error('$relativePath: $failure'); @@ -169,6 +173,12 @@ class EnsembleTestDoctor { 'Test "${entry.key}" references unknown prerequisite "${entry.value}"'); } } + for (final entry in sessions.entries) { + if (!ids.containsKey(entry.value)) { + error( + 'Test "${entry.key}" references unknown session "${entry.value}"'); + } + } final knownWidgetIds = _collectKnownWidgetIds(appPathOnDisk); if (knownWidgetIds.isNotEmpty) { @@ -196,6 +206,7 @@ class EnsembleTestDoctor { typedef _DoctorTest = ({ String id, String? prerequisite, + String? session, Set referencedWidgetIds, String? error, }); @@ -206,6 +217,7 @@ _DoctorTest _parseDoctorTest(String content) { return ( id: '', prerequisite: null, + session: null, referencedWidgetIds: {}, error: 'root must be a map', ); @@ -215,6 +227,7 @@ _DoctorTest _parseDoctorTest(String content) { return ( id: '', prerequisite: null, + session: null, referencedWidgetIds: {}, error: 'Root-level "options" is no longer supported. Move shared settings to tests/config.yaml.', @@ -226,6 +239,7 @@ _DoctorTest _parseDoctorTest(String content) { return ( id: '', prerequisite: null, + session: null, referencedWidgetIds: {}, error: 'Each test must have an "id"', ); @@ -233,12 +247,14 @@ _DoctorTest _parseDoctorTest(String content) { final startScreen = doc['startScreen']?.toString(); final prerequisite = doc['prerequisite']?.toString(); + final session = doc['session']?.toString(); final hasStartScreen = startScreen != null && startScreen.isNotEmpty; final hasPrerequisite = prerequisite != null && prerequisite.isNotEmpty; if (hasStartScreen == hasPrerequisite) { return ( id: id, prerequisite: prerequisite, + session: session, referencedWidgetIds: {}, error: 'Test "$id" must have either "startScreen" or "prerequisite"', ); @@ -249,6 +265,7 @@ _DoctorTest _parseDoctorTest(String content) { return ( id: id, prerequisite: prerequisite, + session: session, referencedWidgetIds: {}, error: 'Test "$id" must have a non-empty "steps" list', ); @@ -257,6 +274,7 @@ _DoctorTest _parseDoctorTest(String content) { return ( id: id, prerequisite: hasPrerequisite ? prerequisite : null, + session: session == null || session.isEmpty ? null : session, referencedWidgetIds: _collectReferencedWidgetIds(steps), error: null, ); diff --git a/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart b/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart index 31c7d52ec..bbb900644 100644 --- a/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart +++ b/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart @@ -1,5 +1,6 @@ import 'dart:io'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:path/path.dart' as p; import 'package:yaml/yaml.dart'; @@ -107,6 +108,10 @@ Future main() async { } else { _restore(_testEntryPath); } + for (final path in _backups.keys.toList()) { + if (path == _pubspecPath || path == _testEntryPath) continue; + _restore(path); + } _backups.clear(); _enabled = false; @@ -115,6 +120,7 @@ Future main() async { } void _backup(String path, {bool optional = false}) { + if (_backups.containsKey(path)) return; final file = File(path); if (!file.existsSync()) { if (!optional) { @@ -153,6 +159,90 @@ Future main() async { } } + TimerRewriteConfig get timerRewriteConfig => _readTimerRewriteConfig(); + + bool get hasTimerRewrites => timerRewriteConfig.enabled; + + void rewriteTimersIn(String targetAppDir) { + final config = _readTimerRewriteConfig(); + if (!config.enabled) return; + + final testsDir = testsDirRelative; + if (testsDir == null) return; + final screensDir = Directory( + p.join( + p.normalize(p.absolute(targetAppDir)), + p.dirname(_withoutTrailingSlash(testsDir)), + 'screens', + ), + ); + if (!screensDir.existsSync()) return; + + for (final entity in screensDir.listSync(recursive: true)) { + if (entity is! File || !entity.path.endsWith('.yaml')) continue; + final original = entity.readAsStringSync(); + final rewritten = _capTimerValues(original, config); + if (rewritten == original) continue; + entity.writeAsStringSync(rewritten); + } + } + + TimerRewriteConfig _readTimerRewriteConfig() { + final path = + testsDirPath == null ? null : p.join(testsDirPath!, 'config.yaml'); + if (path == null) return const TimerRewriteConfig(); + + final file = File(path); + if (!file.existsSync()) return const TimerRewriteConfig(); + final dynamic config = loadYaml(file.readAsStringSync()); + if (config is! YamlMap) return const TimerRewriteConfig(); + final timers = config['timers']; + if (timers is! YamlMap) return const TimerRewriteConfig(); + return TimerRewriteConfig( + enabled: timers['enabled'] == true, + maxStartAfterSeconds: _parseNonNegativeInt( + timers['maxStartAfterSeconds'], + fallback: 1, + ), + maxRepeatIntervalSeconds: _parseNonNegativeInt( + timers['maxRepeatIntervalSeconds'], + fallback: 1, + ), + ); + } + + static String _capTimerValues( + String content, + TimerRewriteConfig config, + ) { + final startAfter = + RegExp(r'^(\s*startAfter:\s*)(\d+)(\s*)$', multiLine: true); + final repeatInterval = + RegExp(r'^(\s*repeatInterval:\s*)(\d+)(\s*)$', multiLine: true); + return content + .replaceAllMapped( + startAfter, + (match) => _capTimerLine(match, config.maxStartAfterSeconds), + ) + .replaceAllMapped( + repeatInterval, + (match) => _capTimerLine(match, config.maxRepeatIntervalSeconds), + ); + } + + static String _capTimerLine(Match match, int maxValue) { + final current = int.parse(match.group(2)!); + if (current <= maxValue) return match.group(0)!; + return '${match.group(1)}$maxValue${match.group(3)}'; + } + + static int _parseNonNegativeInt(dynamic value, {required int fallback}) { + if (value == null) return fallback; + final parsed = value is int ? value : int.tryParse(value.toString()); + if (parsed == null || parsed < 0) return fallback; + return parsed; + } + String _activatePubspec(String content) { final testsDir = testsDirRelative; if (testsDir != null && _hasTestYamlOnDisk(testsDir)) { diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart index 4cbf3f8fa..7185aa8da 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:ensemble/framework/ensemble_config_service.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; @@ -50,7 +52,111 @@ class EnsembleTestDiscovery { final path = await findConfigYamlAsset(testsAssetPrefix); if (path == null) return const EnsembleTestConfig(); final content = await rootBundle.loadString(path); - return EnsembleTestParser.parseConfigString(content, sourcePath: path); + final config = EnsembleTestParser.parseConfigString( + content, + sourcePath: path, + ); + final overridden = _withServiceOverrides(config); + if (overridden != null) return overridden; + return _withWorkerIsolation(config); + } + + static EnsembleTestConfig? _withServiceOverrides(EnsembleTestConfig config) { + const rawOverrides = String.fromEnvironment('ensembleTestServiceOverrides'); + if (rawOverrides.isEmpty || config.services.isEmpty) return null; + + final dynamic decoded = json.decode(rawOverrides); + if (decoded is! Map) return null; + final overrides = Map.from(decoded); + if (overrides.isEmpty) return null; + + return EnsembleTestConfig( + services: [ + for (final service in config.services) + _serviceWithOverride( + service, + overrides[service.name], + ), + ], + screenshots: config.screenshots, + performance: config.performance, + timers: config.timers, + dumpTree: config.dumpTree, + logApiCalls: config.logApiCalls, + logStorage: config.logStorage, + ); + } + + static TestServiceConfig _serviceWithOverride( + TestServiceConfig service, + dynamic override, + ) { + if (override is! Map) return service; + final map = Map.from(override); + final url = map['url']?.toString(); + final environment = { + ...service.environment, + if (map['environment'] is Map) + for (final entry in (map['environment'] as Map).entries) + entry.key.toString(): entry.value.toString(), + }; + return TestServiceConfig( + name: service.name, + command: service.command, + url: url == null || url.isEmpty ? service.url : url, + arguments: service.arguments, + workingDirectory: service.workingDirectory, + environment: environment, + readyUrl: service.readyUrl, + readyTimeoutMs: service.readyTimeoutMs, + ); + } + + static EnsembleTestConfig _withWorkerIsolation(EnsembleTestConfig config) { + const workerIndex = int.fromEnvironment( + 'ensembleTestWorkerIndex', + defaultValue: 0, + ); + if (workerIndex <= 0 || config.services.isEmpty) return config; + + return EnsembleTestConfig( + services: [ + for (final service in config.services) + TestServiceConfig( + name: service.name, + command: service.command, + url: _offsetUrlPort(service.url, workerIndex), + arguments: service.arguments, + workingDirectory: service.workingDirectory, + environment: { + for (final entry in service.environment.entries) + entry.key: entry.key == 'PORT' + ? _offsetPortString(entry.value, workerIndex) + : entry.value, + }, + readyUrl: service.readyUrl, + readyTimeoutMs: service.readyTimeoutMs, + ), + ], + screenshots: config.screenshots, + performance: config.performance, + timers: config.timers, + dumpTree: config.dumpTree, + logApiCalls: config.logApiCalls, + logStorage: config.logStorage, + ); + } + + static String? _offsetUrlPort(String? value, int offset) { + if (value == null || value.isEmpty) return value; + final uri = Uri.tryParse(value); + if (uri == null || !uri.hasPort) return value; + return uri.replace(port: uri.port + offset).toString(); + } + + static String _offsetPortString(String value, int offset) { + final port = int.tryParse(value); + return port == null ? value : '${port + offset}'; } /// Reads `definitions.local` from `ensemble/ensemble-config.yaml`. diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart index 786c3a3c7..a0d1fc434 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart @@ -79,6 +79,7 @@ class EnsembleTestExecutionPlanner { path, content, inputs: inputs, + services: config.services, ); for (final definition in definitions) { final existing = byId[definition.testCase.id]; @@ -102,6 +103,13 @@ class EnsembleTestExecutionPlanner { 'prerequisite "$prereq"', ); } + final session = def.testCase.session; + if (session != null && !selectedById.containsKey(session)) { + throw EnsembleTestFailure( + 'Test "${def.testCase.id}" in ${def.assetPath} references unknown ' + 'session "$session"', + ); + } } final ordered = _topologicalSort(selectedById); @@ -114,12 +122,14 @@ class EnsembleTestExecutionPlanner { String path, String content, { Map inputs = const {}, + List services = const [], Future Function(String assetPath)? assetLoader, }) { return _parseDefinitionsFromAsset( path, content, inputs: inputs, + services: services, assetLoader: assetLoader ?? _rootBundleAssetLoader, ); } @@ -128,14 +138,16 @@ class EnsembleTestExecutionPlanner { String path, String content, { required Map inputs, + List services = const [], _AssetStringLoader assetLoader = _rootBundleAssetLoader, }) async { - if (loadYaml(content) == null) { + final resolvedContent = _resolveServicePlaceholders(content, services); + if (loadYaml(resolvedContent) == null) { return const []; } final base = EnsembleTestParser.parseString( - content, + resolvedContent, sourcePath: path, inputs: inputs, ); @@ -153,6 +165,7 @@ class EnsembleTestExecutionPlanner { id: base.id, startScreen: base.startScreen, prerequisite: base.prerequisite, + session: base.session, mocks: mocks, ), ), @@ -163,7 +176,7 @@ class EnsembleTestExecutionPlanner { String? previousScenarioId; for (final scenario in base.scenarios) { final parsed = EnsembleTestParser.parseString( - content, + resolvedContent, sourcePath: path, inputs: inputs, scenario: scenario.vars, @@ -193,6 +206,7 @@ class EnsembleTestExecutionPlanner { startScreen: parsed.prerequisite == null ? parsed.startScreen : null, prerequisite: prerequisite, + session: parsed.session, mocks: mocks, ), ), @@ -202,12 +216,45 @@ class EnsembleTestExecutionPlanner { return definitions; } + static String _resolveServicePlaceholders( + String content, + List services, + ) { + final urls = { + for (final service in services) + if (service.url != null && service.url!.isNotEmpty) + service.name: service.url!, + }; + final resolved = content.replaceAllMapped( + RegExp(r'\$\{services\.([^.}]+)\.url\}'), + (match) { + final name = match.group(1)!; + final url = urls[name]; + if (url == null) { + throw EnsembleTestFailure( + 'Test references service "$name" without a configured url.', + ); + } + return url; + }, + ); + final unsupported = RegExp(r'\$\{services\.([^}]+)\}').firstMatch(resolved); + if (unsupported != null) { + throw EnsembleTestFailure( + 'Unsupported service value "${unsupported.group(0)}". ' + 'Use \${services..url}.', + ); + } + return resolved; + } + static EnsembleTestCase _withRuntimeFields( EnsembleTestCase test, { required String id, String? description, String? startScreen, String? prerequisite, + String? session, required TestMocks mocks, }) { return EnsembleTestCase( @@ -219,10 +266,16 @@ class EnsembleTestExecutionPlanner { description: description ?? test.description, owner: test.owner, priority: test.priority, + parallel: test.parallel, + retry: test.retry, startScreen: startScreen, + startScreenInputs: test.startScreenInputs, prerequisite: prerequisite, + session: session, mockFiles: test.mockFiles, + scenarios: test.scenarios, initialState: test.initialState, + setupSteps: test.setupSteps, mocks: mocks, steps: test.steps, ); @@ -389,19 +442,24 @@ class EnsembleTestExecutionPlanner { 'No tests matched the provided selection flags'); } - void includePrerequisites(String id) { - final prereq = byId[id]?.testCase.prerequisite; - if (prereq == null) return; - if (!byId.containsKey(prereq)) { - throw EnsembleTestFailure( - 'Selected test "$id" references unknown prerequisite "$prereq"', - ); + void includeDependencies(String id) { + final test = byId[id]?.testCase; + final dependencies = [ + if (test?.prerequisite != null) test!.prerequisite!, + if (test?.session != null) test!.session!, + ]; + for (final dependency in dependencies) { + if (!byId.containsKey(dependency)) { + throw EnsembleTestFailure( + 'Selected test "$id" references unknown dependency "$dependency"', + ); + } + if (selectedIds.add(dependency)) includeDependencies(dependency); } - if (selectedIds.add(prereq)) includePrerequisites(prereq); } for (final id in selectedIds.toList()) { - includePrerequisites(id); + includeDependencies(id); } return { @@ -472,10 +530,15 @@ class EnsembleTestExecutionPlanner { } for (final entry in byId.entries) { - final prereq = entry.value.testCase.prerequisite; - if (prereq == null) continue; - inDegree[entry.key] = (inDegree[entry.key] ?? 0) + 1; - dependents[prereq]!.add(entry.key); + final test = entry.value.testCase; + final dependencies = [ + if (test.prerequisite != null) test.prerequisite!, + if (test.session != null) test.session!, + ]; + for (final dependency in dependencies) { + inDegree[entry.key] = (inDegree[entry.key] ?? 0) + 1; + dependents[dependency]!.add(entry.key); + } } bool sessionChainComplete(List ordered) { @@ -517,7 +580,7 @@ class EnsembleTestExecutionPlanner { if (orderedIds.length != byId.length) { throw EnsembleTestFailure( - 'Circular prerequisite dependency among tests: ' + 'Circular test dependency among tests: ' '${byId.keys.where((id) => !orderedIds.contains(id)).join(", ")}', ); } diff --git a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart index 486c016d8..81b52f0d7 100644 --- a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart +++ b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart @@ -7,14 +7,13 @@ import 'dart:io'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; import 'package:ensemble_test_runner/ensemble_test_runner.dart'; import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; -import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; -const _defaultTimeoutSeconds = 10 * 60; const _timeoutSeconds = int.fromEnvironment( 'ensembleTestTimeoutSeconds', - defaultValue: _defaultTimeoutSeconds, + defaultValue: 0, ); /// Options for [runEnsembleYamlTests], typically set from `test/ensemble_tests.dart`. @@ -66,7 +65,6 @@ Future runEnsembleYamlTestsWithOptions( LiveTestWidgetsFlutterBinding.ensureInitialized(); EnsembleTestHarness.ensureTestPlugins(); tearDown(() { - TestErrorTracker.reset(); EnsembleTestHarness.resetTestRuntime(); YamlTestSession.dispose(); }); @@ -74,89 +72,232 @@ Future runEnsembleYamlTestsWithOptions( testWidgets( 'Ensemble app *.test.yaml', (tester) async { - if (options.bootstrap == null) { - fail( - 'Ensemble YAML tests require module bootstrap. ' - 'In test/ensemble_tests.dart call runEnsembleYamlTests with ' - 'bootstrap: () => EnsembleModules().init() ' - '(see ensemble_test_runner README).', - ); + var emittedMachineReport = false; + void emitMachineReport(EnsembleTestRunResult result) { + _emitMachineReport(result); + emittedMachineReport = true; } - await tester.runAsync(() async { - await options.bootstrap!(); - ensureLiveAuthActionsForTest(); - // Module constructors may schedule follow-up async init work. - await Future.delayed(Duration.zero); - }); - - final target = await EnsembleTestDiscovery.loadAppTarget(); - final plan = await EnsembleTestExecutionPlanner.build( - target: target, - selection: _selectionFromEnvironment(), - inputs: _inputsFromEnvironment(), - ); - final harness = EnsembleTestHarness( - appPath: target.appPath, - appHome: target.appHome, - i18nPath: target.i18nPath, - externalMethods: options.externalMethods, - ); - final runner = EnsembleTestRunner(harness: harness); - final planResult = await runner.runPlan(plan, tester); - final resultsById = planResult.resultsById; - await YamlTestSession.navigationFlow.flushPending(); - await tester.pump(); - - final failures = []; - final orderedResults = []; - - for (final def in plan.ordered) { - final result = resultsById[def.testCase.id]!; - orderedResults.add( - EnsembleSingleTestResult( - testId: '${result.testId} (${def.assetPath})', - metadata: result.metadata, - status: result.status, - durationMs: result.durationMs, - failedStepIndex: result.failedStepIndex, - failedStep: result.failedStep, - message: result.message, - stackTrace: result.stackTrace, - logs: result.logs, - report: result.report, - ), + try { + if (options.bootstrap == null) { + fail( + 'Ensemble YAML tests require module bootstrap. ' + 'In test/ensemble_tests.dart call runEnsembleYamlTests with ' + 'bootstrap: () => EnsembleModules().init() ' + '(see ensemble_test_runner README).', + ); + } + await tester.runAsync(() async { + await options.bootstrap!(); + ensureLiveAuthActionsForTest(); + // Module constructors may schedule follow-up async init work. + await Future.delayed(Duration.zero); + }); + + final target = await EnsembleTestDiscovery.loadAppTarget(); + final plan = await EnsembleTestExecutionPlanner.build( + target: target, + selection: _selectionFromEnvironment(), + inputs: _inputsFromEnvironment(), + ); + final harness = EnsembleTestHarness( + appPath: target.appPath, + appHome: target.appHome, + i18nPath: target.i18nPath, + externalMethods: options.externalMethods, + ); + + final runner = EnsembleTestRunner(harness: harness); + final planResult = await runner.runPlan( + plan, + tester, + onTestComplete: _emitProgressEvent, ); + final resultsById = planResult.resultsById; + await YamlTestSession.navigationFlow.flushPending(); + final pendingFrameworkExceptions = []; + await _pumpBestEffort(tester, pendingFrameworkExceptions); + + final failures = []; + final orderedResults = []; - if (result.status == TestStatus.failed) { - failures.add(def.assetPath); + for (final def in plan.ordered) { + final result = resultsById[def.testCase.id]!; + orderedResults.add( + EnsembleSingleTestResult( + testId: '${result.testId} (${def.assetPath})', + metadata: result.metadata, + status: result.status, + durationMs: result.durationMs, + attempts: result.attempts, + retry: result.retry, + failedStepIndex: result.failedStepIndex, + failedStep: result.failedStep, + message: result.message, + stackTrace: result.stackTrace, + logs: result.logs, + report: result.report, + ), + ); + + if (result.status == TestStatus.failed) { + failures.add(def.assetPath); + } } - } - final runResult = EnsembleTestRunResult( - results: orderedResults, - suiteLogs: planResult.suiteLogs, - ); - final reporter = TestReporter(); - final suiteSummary = reporter.formatSummary( - runResult, - testFile: '${target.testsAssetPrefix}*.test.yaml', - ); - print(suiteSummary); - _emitMachineReport(runResult); - - if (failures.isNotEmpty) { - final pendingFrameworkExceptions = _drainPendingExceptions(tester); - fail( - reporter.formatFailureSummary( - runResult, - failedPaths: failures, - pendingFrameworkExceptions: pendingFrameworkExceptions, - ), + final runResult = EnsembleTestRunResult( + results: orderedResults, + suiteLogs: [ + ...planResult.suiteLogs, + ..._appLogArtifacts(), + ], + ); + // Background app errors are recorded by TestErrorTracker and can be + // asserted with expectNoRenderErrors/expectError. Explicitly unmount + // the app and drain teardown exceptions so a suite with passing YAML + // assertions does not fail after the summary is printed. + pendingFrameworkExceptions.addAll( + await _drainPendingExceptionsAndUnmount(tester), + ); + + final reporter = TestReporter(); + final suiteSummary = reporter.formatSummary( + runResult, + testFile: '${target.testsAssetPrefix}*.test.yaml', ); + print(suiteSummary); + emitMachineReport(runResult); + _ignorePostTestAnimationInvariant(); + + if (failures.isNotEmpty) { + fail( + reporter.formatFailureSummary( + runResult, + failedPaths: failures, + pendingFrameworkExceptions: pendingFrameworkExceptions, + ), + ); + } + } catch (error, stackTrace) { + if (!emittedMachineReport) { + final runResult = EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.failed( + testId: 'test-process', + durationMs: 0, + error: error.toString(), + stackTrace: stackTrace.toString(), + ), + ], + suiteLogs: _appLogArtifacts(), + ); + emitMachineReport(runResult); + } + rethrow; } }, - timeout: Timeout(Duration(seconds: _timeoutSeconds)), + timeout: _timeoutSeconds > 0 + ? Timeout(Duration(seconds: _timeoutSeconds)) + : Timeout.none, + ); +} + +List _appLogArtifacts() { + const appLogFile = String.fromEnvironment('ensembleTestAppLogFile'); + const displayFile = String.fromEnvironment('ensembleTestAppLogDisplayFile'); + if (appLogFile.isEmpty) return const []; + return ['appLogs: ${displayFile.isEmpty ? appLogFile : displayFile}']; +} + +void _ignorePostTestAnimationInvariant() { + final previousOnError = FlutterError.onError; + FlutterError.onError = (details) { + final exception = details.exception.toString(); + if (exception.contains( + 'An animation is still running even after the widget tree was disposed.', + )) { + return; + } + if (previousOnError != null) { + previousOnError(details); + } else { + FlutterError.presentError(details); + } + }; +} + +Future> _drainPendingExceptionsAndUnmount( + WidgetTester tester, +) async { + final exceptions = _drainPendingExceptions(tester); + final previousOnError = FlutterError.onError; + FlutterError.onError = (details) { + exceptions.add(details.exception); + }; + try { + await _pumpWidgetBestEffort(tester, const SizedBox.shrink(), exceptions); + exceptions.addAll(_drainPendingExceptions(tester)); + for (var i = 0; i < 10; i++) { + await _pumpBestEffort( + tester, + exceptions, + const Duration(milliseconds: 16), + ); + exceptions.addAll(_drainPendingExceptions(tester)); + if (tester.binding.transientCallbackCount == 0) break; + } + } finally { + FlutterError.onError = previousOnError; + } + return exceptions; +} + +Future _pumpBestEffort( + WidgetTester tester, + List exceptions, [ + Duration? duration, +]) async { + try { + await tester.pump(duration); + } catch (error) { + exceptions.add(error); + } +} + +Future _pumpWidgetBestEffort( + WidgetTester tester, + Widget widget, + List exceptions, +) async { + try { + await tester.pumpWidget(widget); + } catch (error) { + exceptions.add(error); + } +} + +void _emitProgressEvent( + EnsembleTestDefinition definition, + EnsembleSingleTestResult result, +) { + const progressFile = String.fromEnvironment('ensembleTestProgressFile'); + if (progressFile.isEmpty) return; + + final file = File(progressFile); + file.parent.createSync(recursive: true); + file.writeAsStringSync( + '${json.encode({ + 'testId': result.testId, + 'assetPath': definition.assetPath, + 'status': result.status.name, + 'durationMs': result.durationMs, + if (result.attempts > 1) 'attempts': result.attempts, + if (result.retry > 0) 'retry': result.retry, + if (result.message != null) 'message': result.message, + if (result.failedStepIndex != null) + 'failedStepIndex': result.failedStepIndex, + })}\n', + mode: FileMode.append, ); } diff --git a/tools/ensemble_test_runner/lib/mocks/test_logger.dart b/tools/ensemble_test_runner/lib/mocks/test_logger.dart index d47736b9a..b12171267 100644 --- a/tools/ensemble_test_runner/lib/mocks/test_logger.dart +++ b/tools/ensemble_test_runner/lib/mocks/test_logger.dart @@ -1,7 +1,11 @@ -import 'dart:io'; +import 'package:ensemble_test_runner/runner/test_artifacts.dart'; /// Simple in-memory logger for test runs. class TestLogger { + static const _artifactSuffix = String.fromEnvironment( + 'ensembleTestWorkerSuffix', + ); + final List logs = []; void log(String message) { @@ -14,16 +18,18 @@ class TestLogger { required String content, String extension = 'log', }) async { - final directory = Directory('build/ensemble_test_runner/logs'); + final directory = ensembleTestArtifactDirectory('logs'); await directory.create(recursive: true); final safeTestId = _safeFileName(testId); final safeName = _safeFileName(name); + final suffix = _safeFileName(_artifactSuffix); + final suffixPart = suffix.isEmpty ? '' : '_$suffix'; final fileName = safeTestId.isEmpty - ? '$safeName.${_safeExtension(extension)}' - : '${safeTestId}_$safeName.${_safeExtension(extension)}'; - final file = File('${directory.path}/$fileName'); + ? '$safeName$suffixPart.${_safeExtension(extension)}' + : '${safeTestId}_$safeName$suffixPart.${_safeExtension(extension)}'; + final file = ensembleTestArtifactFile('logs', fileName); await file.writeAsString(content); - return file.path; + return ensembleTestArtifactDisplayPath('logs', fileName); } void clear() => logs.clear(); diff --git a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart index d08f47ba4..b02e1b5c1 100644 --- a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart +++ b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart @@ -37,16 +37,25 @@ class EnsembleTestCase { final String? description; final String? owner; final String? priority; + final bool parallel; + final int retry; /// Cold-start screen. Omit when [prerequisite] is set. final String? startScreen; + final Map startScreenInputs; /// Test [id] that must run before this one (same app session). final String? prerequisite; + + /// Test [id] whose captured storage state is restored before [startScreen]. + final String? session; final List mockFiles; final List scenarios; final Map initialState; + /// Headless actions executed before the start screen is mounted. + final List setupSteps; + /// Runtime API mocks resolved from [mockFiles]. final TestMocks mocks; final List steps; @@ -60,11 +69,16 @@ class EnsembleTestCase { this.description, this.owner, this.priority, + this.parallel = true, + this.retry = 0, this.startScreen, + this.startScreenInputs = const {}, this.prerequisite, + this.session, this.mockFiles = const [], this.scenarios = const [], this.initialState = const {}, + this.setupSteps = const [], this.mocks = const TestMocks(), required this.steps, }); @@ -73,12 +87,15 @@ class EnsembleTestCase { bool get hasPrerequisite => prerequisite != null && prerequisite!.isNotEmpty; + bool get hasSession => session != null && session!.isNotEmpty; + Map get metadataJson => { if (feature != null) 'feature': feature, if (tags.isNotEmpty) 'tags': tags, if (description != null) 'description': description, if (owner != null) 'owner': owner, if (priority != null) 'priority': priority, + if (retry > 0) 'retry': retry, }; } @@ -96,21 +113,68 @@ class TestScenario { } class EnsembleTestConfig { + final List services; final ScreenshotConfig screenshots; final PerformanceConfig performance; final DumpTreeConfig dumpTree; final LogApiCallsConfig logApiCalls; final LogStorageConfig logStorage; + final TimerRewriteConfig timers; const EnsembleTestConfig({ + this.services = const [], this.screenshots = const ScreenshotConfig(), this.performance = const PerformanceConfig(), this.dumpTree = const DumpTreeConfig(), this.logApiCalls = const LogApiCallsConfig(), this.logStorage = const LogStorageConfig(), + this.timers = const TimerRewriteConfig(), }); } +class TestServiceConfig { + final String name; + final String command; + final String? url; + final List arguments; + final String? workingDirectory; + final Map environment; + final String? readyUrl; + final int readyTimeoutMs; + + const TestServiceConfig({ + required this.name, + required this.command, + this.url, + this.arguments = const [], + this.workingDirectory, + this.environment = const {}, + this.readyUrl, + this.readyTimeoutMs = 10000, + }); + + String? get resolvedReadyUrl { + final value = readyUrl; + if (value == null || + value.isEmpty || + Uri.tryParse(value)?.isAbsolute == true) { + return value; + } + final base = url; + if (base == null || base.isEmpty) return value; + return Uri.parse(base).resolve(value).toString(); + } + + Map get resolvedEnvironment { + final resolved = {...environment}; + final uri = url == null ? null : Uri.tryParse(url!); + if (!resolved.containsKey('PORT') && uri != null && uri.hasPort) { + resolved['PORT'] = '${uri.port}'; + } + return resolved; + } +} + class PerformanceConfig { final bool enabled; @@ -119,6 +183,18 @@ class PerformanceConfig { }); } +class TimerRewriteConfig { + final bool enabled; + final int maxStartAfterSeconds; + final int maxRepeatIntervalSeconds; + + const TimerRewriteConfig({ + this.enabled = false, + this.maxStartAfterSeconds = 1, + this.maxRepeatIntervalSeconds = 1, + }); +} + class DumpTreeConfig { final bool enabled; @@ -266,6 +342,8 @@ class EnsembleSingleTestResult { final Map metadata; final TestStatus status; final int durationMs; + final int attempts; + final int retry; final int? failedStepIndex; final TestStep? failedStep; final String? message; @@ -278,6 +356,8 @@ class EnsembleSingleTestResult { this.metadata = const {}, required this.status, required this.durationMs, + this.attempts = 1, + this.retry = 0, this.failedStepIndex, this.failedStep, this.message, @@ -290,6 +370,8 @@ class EnsembleSingleTestResult { required String testId, Map metadata = const {}, required int durationMs, + int attempts = 1, + int retry = 0, List logs = const [], EnsembleTestReportDetails? report, }) => @@ -298,6 +380,8 @@ class EnsembleSingleTestResult { metadata: metadata, status: TestStatus.passed, durationMs: durationMs, + attempts: attempts, + retry: retry, logs: logs, report: report, ); @@ -306,6 +390,8 @@ class EnsembleSingleTestResult { required String testId, Map metadata = const {}, required int durationMs, + int attempts = 1, + int retry = 0, int? failedStepIndex, TestStep? failedStep, String? error, @@ -318,6 +404,8 @@ class EnsembleSingleTestResult { metadata: metadata, status: TestStatus.failed, durationMs: durationMs, + attempts: attempts, + retry: retry, failedStepIndex: failedStepIndex, failedStep: failedStep, message: error, @@ -331,6 +419,8 @@ class EnsembleSingleTestResult { if (metadata.isNotEmpty) 'metadata': metadata, 'status': status.name, 'durationMs': durationMs, + if (attempts > 1) 'attempts': attempts, + if (retry > 0) 'retry': retry, if (failedStepIndex != null) 'failedStepIndex': failedStepIndex, if (failedStep != null) 'failedStep': failedStep!.toJson(), if (message != null) 'message': message, @@ -424,6 +514,7 @@ class EnsembleTestReportDetails { final String startScreen; final String? endScreen; final String? prerequisite; + final String? session; final List screensVisited; final List stepsOutline; @@ -431,6 +522,7 @@ class EnsembleTestReportDetails { required this.startScreen, this.endScreen, this.prerequisite, + this.session, this.screensVisited = const [], this.stepsOutline = const [], }); @@ -439,6 +531,7 @@ class EnsembleTestReportDetails { 'startScreen': startScreen, if (endScreen != null) 'endScreen': endScreen, if (prerequisite != null) 'prerequisite': prerequisite, + if (session != null) 'session': session, 'screensVisited': screensVisited, 'stepsOutline': stepsOutline, }; diff --git a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart index 5df554a1a..21484e418 100644 --- a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart +++ b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -81,6 +81,8 @@ class EnsembleTestParser { final hasStartScreen = startScreen != null && startScreen.isNotEmpty; final prerequisite = map['prerequisite']?.toString(); final hasPrerequisite = prerequisite != null && prerequisite.isNotEmpty; + final session = map['session']?.toString(); + final hasSession = session != null && session.isNotEmpty; if (hasStartScreen && hasPrerequisite) { throw EnsembleTestFailure( @@ -92,6 +94,27 @@ class EnsembleTestParser { 'Test "$id" must have either "startScreen" or "prerequisite"', ); } + if (hasPrerequisite && hasSession) { + throw EnsembleTestFailure( + 'Test "$id" cannot use both "prerequisite" and "session"', + ); + } + if (hasSession && !hasStartScreen) { + throw EnsembleTestFailure( + 'Test "$id" must have "startScreen" when "session" is set', + ); + } + + final setupNode = map['setup']; + final setupSteps = setupNode == null + ? const [] + : _parseSetupSteps( + setupNode, + testId: id, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); final stepsNode = map['steps']; if (stepsNode is! YamlList || stepsNode.isEmpty) { @@ -108,8 +131,21 @@ class EnsembleTestParser { description: map['description']?.toString(), owner: map['owner']?.toString(), priority: map['priority']?.toString(), + parallel: map['parallel'] == false ? false : true, + retry: _optionalNonNegativeInt( + map['retry'], + fallback: 0, + fieldName: '$id.retry', + ), startScreen: hasStartScreen ? startScreen : null, + startScreenInputs: _toStringDynamicMap( + map['startScreenInputs'], + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), prerequisite: hasPrerequisite ? prerequisite : null, + session: hasSession ? session : null, mockFiles: _toStringList( map['mocks'], inputs: inputs, @@ -123,6 +159,7 @@ class EnsembleTestParser { scenario: scenario, scenarioId: scenarioId, ), + setupSteps: setupSteps, steps: _parseSteps( stepsNode, testId: id, @@ -150,8 +187,10 @@ class EnsembleTestParser { static EnsembleTestConfig _parseConfig(YamlMap node) { const allowedKeys = { + 'services', 'screenshots', 'performance', + 'timers', 'dumpTree', 'logApiCalls', 'logStorage', @@ -165,17 +204,25 @@ class EnsembleTestParser { } } + final servicesNode = node['services']; final screenshotsNode = node['screenshots']; final performanceNode = node['performance']; + final timersNode = node['timers']; final dumpTreeNode = node['dumpTree']; final logApiCallsNode = node['logApiCalls']; final logStorageNode = node['logStorage']; + if (servicesNode != null && servicesNode is! YamlList) { + throw EnsembleTestFailure('"services" must be a list'); + } if (screenshotsNode != null && screenshotsNode is! YamlMap) { throw EnsembleTestFailure('"screenshots" must be a map'); } if (performanceNode != null && performanceNode is! YamlMap) { throw EnsembleTestFailure('"performance" must be a map'); } + if (timersNode != null && timersNode is! YamlMap) { + throw EnsembleTestFailure('"timers" must be a map'); + } if (dumpTreeNode != null && dumpTreeNode is! YamlMap) { throw EnsembleTestFailure('"dumpTree" must be a map'); } @@ -187,6 +234,7 @@ class EnsembleTestParser { } return EnsembleTestConfig( + services: _parseServices(servicesNode), screenshots: screenshotsNode == null ? const ScreenshotConfig() : ScreenshotConfig( @@ -201,6 +249,21 @@ class EnsembleTestParser { : PerformanceConfig( enabled: performanceNode['enabled'] == true, ), + timers: timersNode == null + ? const TimerRewriteConfig() + : TimerRewriteConfig( + enabled: timersNode['enabled'] == true, + maxStartAfterSeconds: _optionalNonNegativeInt( + timersNode['maxStartAfterSeconds'], + fallback: 1, + fieldName: 'timers.maxStartAfterSeconds', + ), + maxRepeatIntervalSeconds: _optionalNonNegativeInt( + timersNode['maxRepeatIntervalSeconds'], + fallback: 1, + fieldName: 'timers.maxRepeatIntervalSeconds', + ), + ), dumpTree: dumpTreeNode == null ? const DumpTreeConfig() : DumpTreeConfig( @@ -220,6 +283,100 @@ class EnsembleTestParser { ); } + static int _optionalNonNegativeInt( + dynamic value, { + required int fallback, + required String fieldName, + }) { + if (value == null) return fallback; + final parsed = value is int ? value : int.tryParse(value.toString()); + if (parsed == null || parsed < 0) { + throw EnsembleTestFailure('"$fieldName" must be a non-negative integer'); + } + return parsed; + } + + static List _parseServices(dynamic node) { + if (node == null) return const []; + final services = []; + final names = {}; + for (final item in node as YamlList) { + if (item is! YamlMap) { + throw EnsembleTestFailure('Each service must be a map'); + } + const allowedKeys = { + 'name', + 'command', + 'url', + 'arguments', + 'workingDirectory', + 'environment', + 'readyUrl', + 'readyTimeoutMs', + }; + for (final key in item.keys) { + if (!allowedKeys.contains(key)) { + throw EnsembleTestFailure( + 'Unsupported service key "$key". Supported keys: ' + '${allowedKeys.join(', ')}', + ); + } + } + + final name = item['name']?.toString(); + final command = item['command']?.toString(); + if (name == null || name.isEmpty) { + throw EnsembleTestFailure('Each service requires "name"'); + } + if (!names.add(name)) { + throw EnsembleTestFailure('Duplicate service name "$name"'); + } + if (command == null || command.isEmpty) { + throw EnsembleTestFailure('Service "$name" requires "command"'); + } + + final argumentsNode = item['arguments']; + if (argumentsNode != null && argumentsNode is! YamlList) { + throw EnsembleTestFailure( + 'Service "$name" "arguments" must be a list', + ); + } + final environmentNode = item['environment']; + if (environmentNode != null && environmentNode is! YamlMap) { + throw EnsembleTestFailure( + 'Service "$name" "environment" must be a map', + ); + } + final timeout = item['readyTimeoutMs']; + if (timeout != null && (timeout is! int || timeout <= 0)) { + throw EnsembleTestFailure( + 'Service "$name" "readyTimeoutMs" must be a positive integer', + ); + } + + services.add( + TestServiceConfig( + name: name, + command: command, + url: item['url']?.toString(), + arguments: argumentsNode == null + ? const [] + : argumentsNode.map((value) => value.toString()).toList(), + workingDirectory: item['workingDirectory']?.toString(), + environment: environmentNode == null + ? const {} + : { + for (final entry in environmentNode.entries) + entry.key.toString(): entry.value.toString(), + }, + readyUrl: item['readyUrl']?.toString(), + readyTimeoutMs: timeout as int? ?? 10000, + ), + ); + } + return services; + } + static List _parseScenarios( dynamic node, { required Map inputs, @@ -264,14 +421,65 @@ class EnsembleTestParser { required Map inputs, required Map scenario, String? scenarioId, - }) => - _parseStepsList( - steps, - testId: testId, - inputs: inputs, - scenario: scenario, - scenarioId: scenarioId, + }) { + final parsed = _parseStepsList( + steps, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); + for (final step in parsed) { + _validateTestStep(step, testId); + } + return parsed; + } + + static void _validateTestStep(TestStep step, String testId) { + if (step.type == 'runCommand') { + throw EnsembleTestFailure( + 'Test "$testId" runCommand is only supported in root-level setup', ); + } + for (final nested in step.nestedSteps) { + _validateTestStep(nested, testId); + } + } + + static List _parseSetupSteps( + dynamic steps, { + required String testId, + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + final parsed = _parseStepsList( + steps, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + listName: 'setup', + ); + for (final step in parsed) { + _validateSetupStep(step, testId); + } + return parsed; + } + + static void _validateSetupStep(TestStep step, String testId) { + if (step.type == 'httpRequest' || step.type == 'runCommand') return; + if (step.type == 'group' || step.type == 'optional') { + for (final nested in step.nestedSteps) { + _validateSetupStep(nested, testId); + } + return; + } + throw EnsembleTestFailure( + 'Test "$testId" setup only supports httpRequest, runCommand, group, ' + 'and optional, but found "${step.type}"', + ); + } static List _parseStepsList( dynamic steps, { @@ -279,10 +487,11 @@ class EnsembleTestParser { required Map inputs, required Map scenario, String? scenarioId, + String listName = 'steps', }) { if (steps is! List || steps.isEmpty) { throw EnsembleTestFailure( - 'Test "$testId" requires a non-empty "steps" list'); + 'Test "$testId" requires a non-empty "$listName" list'); } final result = []; for (var i = 0; i < steps.length; i++) { diff --git a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart index 75979a8ff..29be8a7d6 100644 --- a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart +++ b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart @@ -10,6 +10,7 @@ EnsembleTestReportDetails buildTestReportDetails(EnsembleTestCase testCase) { startScreen: effectiveStart, endScreen: ScreenTracker().getCurrentScreenIdentifier(), prerequisite: testCase.prerequisite, + session: testCase.session, screensVisited: collectScreensVisited(effectiveStart), stepsOutline: outlineSteps(testCase.steps), ); @@ -166,12 +167,18 @@ class TestReporter { void _writeTestCase(StringBuffer buffer, EnsembleSingleTestResult r) { final icon = r.status == TestStatus.passed ? '✓' : '✗'; buffer.writeln('│ $icon ${r.testId} (${r.durationMs}ms)'); + if (r.attempts > 1) { + buffer.writeln('│ attempts: ${r.attempts}/${r.retry + 1}'); + } final report = r.report; if (report != null) { if (report.prerequisite != null) { buffer.writeln('│ after: ${report.prerequisite}'); } + if (report.session != null) { + buffer.writeln('│ session: ${report.session}'); + } buffer.writeln('│ start: ${report.startScreen}'); if (report.endScreen != null && report.endScreen != report.startScreen) { buffer.writeln('│ end: ${report.endScreen}'); diff --git a/tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart b/tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart new file mode 100644 index 000000000..983672900 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart @@ -0,0 +1,56 @@ +import 'dart:convert'; + +import 'package:ensemble/ensemble.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:flutter/widgets.dart'; + +/// In-memory app state captured from a successful session-producing test. +class AppSessionSnapshot { + final Map publicStorage; + final Map keychain; + final Locale? locale; + + const AppSessionSnapshot({ + required this.publicStorage, + required this.keychain, + this.locale, + }); + + static Future capture() async { + final storage = StorageManager(); + final publicStorage = {}; + for (final key in storage.getKeys()) { + publicStorage[key] = _copy(storage.read(key)); + } + final keychain = await storage.getAllFromKeychain(); + return AppSessionSnapshot( + publicStorage: publicStorage, + keychain: { + for (final entry in keychain.entries) entry.key: _copy(entry.value), + }, + locale: Ensemble().getLocale(), + ); + } + + Future restore() async { + final storage = StorageManager(); + await storage.clearPublicStorage(); + final currentKeychain = await storage.getAllFromKeychain(); + for (final key in currentKeychain.keys) { + await storage.removeSecurely(key); + } + for (final entry in publicStorage.entries) { + await storage.write(entry.key, _copy(entry.value)); + } + for (final entry in keychain.entries) { + await storage.writeSecurely(key: entry.key, value: _copy(entry.value)); + } + } + + static dynamic _copy(dynamic value) { + if (value == null || value is num || value is bool || value is String) { + return value; + } + return jsonDecode(jsonEncode(value)); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart index 5f69bacce..1f6a2ed41 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart @@ -5,6 +5,7 @@ import 'package:ensemble_test_runner/mocks/test_logger.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:flutter/widgets.dart'; class EnsembleTestContext { final EnsembleTestCase testCase; @@ -58,9 +59,20 @@ class EnsembleTestContext { setup: setup, ); ctx.envOverrides.addAll(envMap); + ctx.runtime.locale = _localeFromEnv(envMap['APP_LOCALE']); return ctx; } + static Locale? _localeFromEnv(dynamic value) { + final locale = value?.toString(); + if (locale == null || locale.isEmpty) return null; + final normalized = locale.replaceAll('-', '_'); + final parts = normalized.split('_'); + final languageCode = parts.first; + if (languageCode.isEmpty) return null; + return Locale(languageCode, parts.length > 1 ? parts[1] : null); + } + void applyRuntimeEnv() { if (envOverrides.isEmpty) return; Ensemble().getConfig()?.updateEnvOverrides(envOverrides); diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index 52d125bf8..0141d9e67 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -9,6 +9,8 @@ import 'package:ensemble/framework/definition_providers/local_provider.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/page_model.dart'; +import 'package:ensemble/screen_controller.dart'; +import 'package:ensemble/util/utils.dart'; import 'package:ensemble_device_preview/ensemble_device_preview.dart'; import 'package:ensemble_test_runner/actions/screenshot_device.dart'; import 'package:ensemble_test_runner/mocks/adobe_test_setup.dart'; @@ -21,6 +23,7 @@ import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:yaml/yaml.dart'; /// Per-test bootstrap data applied before the widget tree mounts. @@ -61,9 +64,16 @@ class EnsembleTestHarness { static final String _testStoragePath = Directory.systemTemp.createTempSync('ensemble_test_runner_storage_').path; static bool _appFontsLoaded = false; + static bool _sqfliteInitialized = false; + static final Map _secureStorage = {}; static void ensureTestPlugins() { TestWidgetsFlutterBinding.ensureInitialized(); + if (!_sqfliteInitialized) { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + _sqfliteInitialized = true; + } ensureFirebaseCoreMocksForTest(); ensureAdobeAnalyticsMocksForTest(); HttpOverrides.global = _RealNetworkHttpOverrides(); @@ -91,7 +101,6 @@ class EnsembleTestHarness { return null; }); - final secureStorage = {}; const secureStorageChannel = MethodChannel('plugins.it_nomads.com/flutter_secure_storage'); TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger @@ -101,21 +110,21 @@ class EnsembleTestHarness { switch (call.method) { case 'write': if (key != null) { - secureStorage[key] = args['value']?.toString() ?? ''; + _secureStorage[key] = args['value']?.toString() ?? ''; } return null; case 'read': - return key == null ? null : secureStorage[key]; + return key == null ? null : _secureStorage[key]; case 'readAll': - return Map.from(secureStorage); + return Map.from(_secureStorage); case 'delete': - if (key != null) secureStorage.remove(key); + if (key != null) _secureStorage.remove(key); return null; case 'deleteAll': - secureStorage.clear(); + _secureStorage.clear(); return null; case 'containsKey': - return key != null && secureStorage.containsKey(key); + return key != null && _secureStorage.containsKey(key); default: return null; } @@ -482,10 +491,17 @@ class EnsembleTestHarness { EnsembleConfig? existingConfig, EnsembleTestContext? context, EnsembleTestConfig suiteConfig = const EnsembleTestConfig(), + Future Function()? beforeBootstrap, + Locale? forcedLocale, }) async { + // Independent tests need a new EnsembleApp state. Pumping another + // EnsembleApp of the same type would otherwise reuse the prior route. + await tester.pumpWidget(const SizedBox.shrink()); + await tester.pump(); resetTestRuntime(); ScreenTracker().clearAll(); YamlTestSession.navigationFlow.clear(); + await beforeBootstrap?.call(); final ctx = context ?? EnsembleTestContext.fromTestCase( @@ -520,7 +536,11 @@ class EnsembleTestHarness { await tester.pumpWidget( EnsembleApp( ensembleConfig: config, - screenPayload: ScreenPayload(screenId: startScreen), + screenPayload: ScreenPayload( + screenId: startScreen, + arguments: testCase.startScreenInputs, + ), + forcedLocale: forcedLocale, ), ); @@ -528,6 +548,35 @@ class EnsembleTestHarness { return config; } + static Future openSessionScreen( + WidgetTester tester, + EnsembleTestCase testCase, + ) async { + final startScreen = testCase.startScreen; + if (startScreen == null || startScreen.isEmpty) { + throw EnsembleTestFailure( + 'Session test "${testCase.id}" requires startScreen', + ); + } + final context = Utils.globalAppKey.currentContext; + if (context == null) { + throw EnsembleTestFailure( + 'Cannot open "$startScreen" because the saved app session is not mounted', + ); + } + + ScreenTracker().clearAll(); + YamlTestSession.navigationFlow.clear(); + ScreenController().navigateToScreen( + context, + screenId: startScreen, + pageArgs: testCase.startScreenInputs, + routeOption: RouteOption.clearAllScreens, + ); + await tester.pump(); + await waitForInitialWidgets(tester, testCase: testCase); + } + static Future _ensureDefaultViewport( WidgetTester tester, EnsembleTestContext context, @@ -571,7 +620,7 @@ class EnsembleTestHarness { final keysToWait = []; if (testCase != null) { for (final step in testCase.steps) { - if (step.type != 'expectVisible' && step.type != 'tap') { + if (step.type != 'expectVisible') { break; } final id = step.args['id']?.toString(); diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index be6cec03e..de16b51a1 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -1,8 +1,12 @@ +import 'dart:async'; +import 'dart:math' as math; import 'dart:ui' as ui; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; +import 'package:ensemble_test_runner/actions/http_request_action.dart'; +import 'package:ensemble_test_runner/actions/run_command_action.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; @@ -11,12 +15,14 @@ import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/mocks/test_logger.dart'; import 'package:ensemble_test_runner/reporters/test_reporter.dart'; import 'package:ensemble_test_runner/runner/app_performance_log.dart'; +import 'package:ensemble_test_runner/runner/app_session_snapshot.dart'; import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/runner/live_async_call.dart'; import 'package:ensemble_test_runner/runner/screenshot_contact_sheet.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:ensemble_test_runner/runner/test_service_manager.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; @@ -30,6 +36,11 @@ typedef EnsembleTestRunOutput = ({ EnsembleTestContext context, }); +typedef EnsembleTestProgressListener = FutureOr Function( + EnsembleTestDefinition definition, + EnsembleSingleTestResult result, +); + class EnsembleTestPlanRunResult { final Map resultsById; final List suiteLogs; @@ -51,13 +62,37 @@ class EnsembleTestRunner { /// Runs every test in [plan] and returns results keyed by test id. Future runPlan( EnsembleTestExecutionPlan plan, - WidgetTester tester, - ) async { + WidgetTester tester, { + EnsembleTestProgressListener? onTestComplete, + }) async { + final services = TestServiceManager(plan.config.services); + await tester.runAsync(services.startAll); + try { + return await _runPlan( + plan, + tester, + onTestComplete: onTestComplete, + ); + } finally { + await LiveAsyncCallSupport.run(services.stopAll); + } + } + + Future _runPlan( + EnsembleTestExecutionPlan plan, + WidgetTester tester, { + EnsembleTestProgressListener? onTestComplete, + }) async { final resultsById = {}; final suiteLogger = TestLogger(); final suiteFrames = []; final suiteMarkers = []; final suiteApiCalls = []; + final sessionSnapshots = {}; + final requestedSessions = plan.ordered + .map((definition) => definition.testCase.session) + .whereType() + .toSet(); EnsembleTestContext? lastContext; var config = await harness.buildConfig(); @@ -72,25 +107,80 @@ class EnsembleTestRunner { ); } if (prereqResult.status == TestStatus.failed) { - resultsById[test.id] = EnsembleSingleTestResult.failed( + final result = EnsembleSingleTestResult.failed( testId: test.id, metadata: test.metadataJson, error: 'Prerequisite "$prereq" failed', durationMs: 0, report: buildTestReportDetails(test), ); + resultsById[test.id] = result; + await onTestComplete?.call(def, result); continue; } } - final out = await runOne( - test, - tester, - suiteConfig: plan.config, - existingConfig: config, - continuation: test.hasPrerequisite, - ); + final session = test.session; + AppSessionSnapshot? sessionSnapshot; + if (session != null) { + final sessionResult = resultsById[session]; + if (sessionResult == null) { + throw EnsembleTestFailure( + 'Internal error: session "$session" for "${test.id}" was not scheduled', + ); + } + if (sessionResult.status == TestStatus.failed) { + final result = EnsembleSingleTestResult.failed( + testId: test.id, + metadata: test.metadataJson, + error: 'Session "$session" failed', + durationMs: 0, + report: buildTestReportDetails(test), + ); + resultsById[test.id] = result; + await onTestComplete?.call(def, result); + continue; + } + sessionSnapshot = sessionSnapshots[session]; + if (sessionSnapshot == null) { + throw EnsembleTestFailure( + 'Internal error: session "$session" completed without a snapshot', + ); + } + } + + late final EnsembleTestRunOutput out; + try { + out = await _runOneWithRetries( + test, + tester, + suiteConfig: plan.config, + existingConfig: config, + continuation: test.hasPrerequisite, + sessionSnapshot: sessionSnapshot, + ); + } catch (error, stackTrace) { + final logs = await _writeEmergencyFailureScreenshot( + tester: tester, + test: test, + config: plan.config, + error: error, + ); + final result = EnsembleSingleTestResult.failed( + testId: test.id, + metadata: test.metadataJson, + error: error.toString(), + stackTrace: stackTrace.toString(), + durationMs: 0, + logs: logs, + report: buildTestReportDetails(test), + ); + resultsById[test.id] = result; + await onTestComplete?.call(def, result); + continue; + } resultsById[test.id] = out.result; + await onTestComplete?.call(def, out.result); config = out.config; lastContext = out.context; final frameOffset = suiteFrames.length; @@ -101,6 +191,10 @@ class EnsembleTestRunner { ), ); suiteApiCalls.addAll(out.context.apiOverlay.calls); + if (out.result.status == TestStatus.passed && + requestedSessions.contains(test.id)) { + sessionSnapshots[test.id] = await AppSessionSnapshot.capture(); + } } final suiteLogs = await _writeSuiteLogs( @@ -126,6 +220,7 @@ class EnsembleTestRunner { EnsembleTestConfig suiteConfig = const EnsembleTestConfig(), EnsembleConfig? existingConfig, bool continuation = false, + AppSessionSnapshot? sessionSnapshot, }) async { final stopwatch = Stopwatch()..start(); void Function(List)? timingsCallback; @@ -142,7 +237,6 @@ class EnsembleTestRunner { SchedulerBinding.instance.addTimingsCallback(timingsCallback); ctx.apiOverlay.liveAsyncRunner = tester.runAsync; LiveAsyncCallSupport.runner = tester.runAsync; - TestErrorTracker.install(ctx.runtime); final startupStartFrame = ctx.runtime.appFrameTimings.length + 1; final startupStartTime = DateTime.now(); @@ -157,6 +251,17 @@ class EnsembleTestRunner { await EnsembleTestHarness.applyInPlaceSetup(ctx); config = existingConfig ?? Ensemble().getConfig()!; await EnsembleTestHarness.waitForInitialWidgets(tester, testCase: test); + } else if (sessionSnapshot != null) { + if (!YamlTestSession.runtimeBootstrapped) { + throw EnsembleTestFailure( + 'Test "${test.id}" requires a mounted session runtime', + ); + } + await sessionSnapshot.restore(); + await _executeSetup(test); + await EnsembleTestHarness.applyInPlaceSetup(ctx); + config = existingConfig ?? Ensemble().getConfig()!; + await EnsembleTestHarness.openSessionScreen(tester, test); } else { config = await harness.loadScreen( tester: tester, @@ -164,8 +269,14 @@ class EnsembleTestRunner { existingConfig: existingConfig, context: ctx, suiteConfig: suiteConfig, + beforeBootstrap: () async { + await sessionSnapshot?.restore(); + await _executeSetup(test); + }, + forcedLocale: sessionSnapshot?.locale ?? ctx.runtime.locale, ); } + _drainPendingFlutterExceptions(tester); await YamlTestSession.navigationFlow.flushPending(); YamlTestSession.navigationFlow.beginTest( ScreenTracker().getCurrentScreenIdentifier(), @@ -190,20 +301,47 @@ class EnsembleTestRunner { return (result: result, config: config, context: ctx); } catch (error, stackTrace) { final config = existingConfig ?? Ensemble().getConfig(); + final errorMessage = error.toString(); + final logs = []; + try { + await _settleLiveApiWorkBestEffort(tester, ctx); + final hadScreenshotFrames = + ctx.runtime.screenshotSheetFrames.isNotEmpty; + await _flushPendingScreenshots( + ctx, + status: TestStatus.failed, + durationMs: stopwatch.elapsedMilliseconds, + failedStepLabel: 'Startup/setup', + failureMessage: errorMessage, + ); + logs.addAll(ctx.logger.logs); + if (!hadScreenshotFrames) { + logs.addAll( + await _writeEmergencyFailureScreenshot( + tester: tester, + test: test, + config: suiteConfig, + error: error, + ), + ); + } + } catch (_) { + logs.addAll(ctx.logger.logs); + } return ( result: EnsembleSingleTestResult.failed( testId: test.id, metadata: test.metadataJson, - error: error.toString(), + error: errorMessage, stackTrace: stackTrace.toString(), durationMs: stopwatch.elapsedMilliseconds, + logs: logs, report: buildTestReportDetails(test), ), config: config ?? await harness.buildConfig(), context: ctx, ); } finally { - TestErrorTracker.reset(); final callback = timingsCallback; if (callback != null) { SchedulerBinding.instance.removeTimingsCallback(callback); @@ -211,6 +349,108 @@ class EnsembleTestRunner { } } + Future _runOneWithRetries( + EnsembleTestCase test, + WidgetTester tester, { + required EnsembleTestConfig suiteConfig, + EnsembleConfig? existingConfig, + required bool continuation, + AppSessionSnapshot? sessionSnapshot, + }) async { + final maxAttempts = test.retry + 1; + var totalDurationMs = 0; + EnsembleTestRunOutput? lastOutput; + + for (var attempt = 1; attempt <= maxAttempts; attempt++) { + final out = await runOne( + test, + tester, + suiteConfig: suiteConfig, + existingConfig: existingConfig, + continuation: continuation, + sessionSnapshot: sessionSnapshot, + ); + totalDurationMs += out.result.durationMs; + lastOutput = out; + existingConfig = out.config; + + if (out.result.status == TestStatus.passed || attempt == maxAttempts) { + return ( + result: _withRetryMetadata( + out.result, + attempts: attempt, + retry: test.retry, + durationMs: totalDurationMs, + ), + config: out.config, + context: out.context, + ); + } + } + + return lastOutput!; + } + + EnsembleSingleTestResult _withRetryMetadata( + EnsembleSingleTestResult result, { + required int attempts, + required int retry, + required int durationMs, + }) { + return EnsembleSingleTestResult( + testId: result.testId, + metadata: result.metadata, + status: result.status, + durationMs: durationMs, + attempts: attempts, + retry: retry, + failedStepIndex: result.failedStepIndex, + failedStep: result.failedStep, + message: result.message, + stackTrace: result.stackTrace, + logs: result.logs, + report: result.report, + ); + } + + Future _executeSetup(EnsembleTestCase test) async { + for (var i = 0; i < test.setupSteps.length; i++) { + final step = test.setupSteps[i]; + try { + await _executeSetupStep(step); + } catch (error) { + throw EnsembleTestFailure( + 'Setup ${i + 1} ${formatStepBrief(step)} failed: $error', + ); + } + } + } + + Future _executeSetupStep(TestStep step) async { + switch (step.type) { + case 'httpRequest': + await HttpRequestAction.execute(step.args); + case 'runCommand': + await RunCommandAction.execute(step.args); + case 'group': + for (final nested in step.nestedSteps) { + await _executeSetupStep(nested); + } + case 'optional': + try { + for (final nested in step.nestedSteps) { + await _executeSetupStep(nested); + } + } catch (_) { + // Optional setup is best effort. + } + default: + throw EnsembleTestFailure( + 'Unsupported setup action "${step.type}"', + ); + } + } + Future _executeSteps({ required EnsembleTestCase test, required WidgetTester tester, @@ -230,16 +470,48 @@ class EnsembleTestRunner { final step = test.steps[i]; final startFrame = ctx.runtime.appFrameTimings.length + 1; final startTime = DateTime.now(); + var capturedStep = false; try { + _drainPendingFlutterExceptions(tester); if (i == 0 && ctx.config.screenshots.enabled) { await executor.settle(); } - await _captureAutomaticScreenshotForStep( - executor: executor, - step: step, - stepIndex: i, - ); - await executor.execute(step); + final captureBeforeStep = _shouldCaptureBeforeStep(step); + if (captureBeforeStep) { + await _captureAutomaticScreenshotForStep( + executor: executor, + step: step, + stepIndex: i, + ); + capturedStep = true; + } + if (step.type == 'waitForText') { + executor.onWaitForTextMatched = (matchedStep) async { + if (capturedStep) return; + await _waitForHighlightTargetToPaint(executor, matchedStep); + await _captureAutomaticScreenshotForStepBestEffort( + executor: executor, + step: matchedStep, + stepIndex: i, + ); + capturedStep = true; + }; + } + try { + await executor.execute(step); + } finally { + executor.onWaitForTextMatched = null; + } + _drainPendingFlutterExceptions(tester); + if (!captureBeforeStep && !capturedStep) { + await _captureAutomaticScreenshotForStep( + executor: executor, + step: step, + stepIndex: i, + pumpBeforeCapture: _shouldPumpBeforePostStepCapture(step), + ); + capturedStep = true; + } await YamlTestSession.navigationFlow.flushPending(); _recordPerformanceMarker( ctx: ctx, @@ -262,8 +534,25 @@ class EnsembleTestRunner { ); final idleStartFrame = ctx.runtime.appFrameTimings.length + 1; final idleStartTime = DateTime.now(); - await _settleLiveApiWork(tester, ctx); - await _flushPendingScreenshots(ctx); + await _settleLiveApiWorkBestEffort(tester, ctx); + _drainPendingFlutterExceptions(tester); + if (!capturedStep) { + await _captureAutomaticScreenshotForStepBestEffort( + executor: executor, + step: step, + stepIndex: i, + pumpBeforeCapture: true, + ensureTargetVisible: false, + ); + } + await _flushPendingScreenshots( + ctx, + status: TestStatus.failed, + durationMs: stopwatch.elapsedMilliseconds, + failedStepIndex: i, + failedStepLabel: formatStepBrief(step), + failureMessage: error.toString(), + ); await YamlTestSession.navigationFlow.flushPending(); _recordPerformanceMarker( ctx: ctx, @@ -291,8 +580,13 @@ class EnsembleTestRunner { await YamlTestSession.navigationFlow.flushPending(); final idleStartFrame = ctx.runtime.appFrameTimings.length + 1; final idleStartTime = DateTime.now(); - await _settleLiveApiWork(tester, ctx); - await _flushPendingScreenshots(ctx); + await _settleLiveApiWorkBestEffort(tester, ctx); + _drainPendingFlutterExceptions(tester); + await _flushPendingScreenshots( + ctx, + status: TestStatus.passed, + durationMs: stopwatch.elapsedMilliseconds, + ); _recordPerformanceMarker( ctx: ctx, testId: test.id, @@ -316,11 +610,19 @@ class EnsembleTestRunner { required TestStepExecutor executor, required TestStep step, required int stepIndex, + bool pumpBeforeCapture = false, + bool ensureTargetVisible = true, }) async { final options = executor.context.config.screenshots; if (!options.shouldCaptureStep(step.type)) return; - await _ensureHighlightTargetVisible(executor, step); + if (pumpBeforeCapture) { + await executor.tester.pump(); + } + + if (ensureTargetVisible) { + await _ensureHighlightTargetVisible(executor, step); + } var image = ExtendedStepHandlers.captureScreenshotImage(executor.tester); final highlightRect = _highlightRectForStep(executor, step); @@ -330,6 +632,7 @@ class EnsembleTestRunner { tester: executor.tester, image: image, rect: highlightRect, + step: step, ); image.dispose(); image = highlighted; @@ -340,24 +643,96 @@ class EnsembleTestRunner { executor.context.runtime.addScreenshotSheetFrame( ScreenshotSheetFrame( + stepIndex: stepIndex, label: '${stepIndex + 1}. ${formatStepBrief(step)}', image: image, ), ); } + Future _captureAutomaticScreenshotForStepBestEffort({ + required TestStepExecutor executor, + required TestStep step, + required int stepIndex, + bool pumpBeforeCapture = false, + bool ensureTargetVisible = true, + }) async { + try { + await _captureAutomaticScreenshotForStep( + executor: executor, + step: step, + stepIndex: stepIndex, + pumpBeforeCapture: pumpBeforeCapture, + ensureTargetVisible: ensureTargetVisible, + ); + } catch (_) { + // Screenshot capture must never replace the real test failure. + } + } + + bool _shouldCaptureBeforeStep(TestStep step) => _isUserActionStep(step); + + bool _shouldPumpBeforePostStepCapture(TestStep step) => + step.type != 'waitForText'; + + Future _waitForHighlightTargetToPaint( + TestStepExecutor executor, + TestStep step, + ) async { + if (step.type != 'waitForText') return; + + for (var i = 0; i < 8; i++) { + final finder = _highlightFinder(executor, step); + final elements = finder?.evaluate().toList() ?? const []; + if (elements.isEmpty) return; + if (_effectiveOpacity(elements.first) >= 0.85) return; + + await executor.tester.pump(const Duration(milliseconds: 50)); + } + } + + double _effectiveOpacity(Element element) { + var opacity = 1.0; + element.visitAncestorElements((ancestor) { + final renderObject = ancestor.renderObject; + if (renderObject is RenderOpacity) { + opacity *= renderObject.opacity; + } else if (renderObject != null && + renderObject.runtimeType.toString() == 'RenderAnimatedOpacity') { + try { + final animatedOpacity = (renderObject as dynamic).opacity; + if (animatedOpacity is Animation) { + opacity *= animatedOpacity.value; + } else if (animatedOpacity is double) { + opacity *= animatedOpacity; + } + } catch (_) { + // Keep the known opacity from other ancestors. + } + } + return true; + }); + return opacity; + } + ui.Rect? _highlightRectForStep(TestStepExecutor executor, TestStep step) { if (!_shouldHighlightStep(step)) return null; - final id = step.args['id']?.toString(); - if (id == null || id.isEmpty) return null; - - final elements = executor.assertions.finderForId(id).evaluate(); + final finder = _highlightFinder(executor, step); + if (finder == null) return null; + final elements = finder.evaluate(); if (elements.isEmpty) return null; - final renderObject = elements.first.renderObject; - if (renderObject == null) return null; - return _rectForRenderObject(renderObject); + try { + final rect = executor.tester.getRect(finder.first); + if (rect.isFinite && !rect.isEmpty) { + return rect; + } + } catch (_) { + // Fall back to render-object traversal below. + } + + return _rectForElement(elements.first); } Future _ensureHighlightTargetVisible( @@ -366,10 +741,8 @@ class EnsembleTestRunner { ) async { if (!_shouldHighlightStep(step)) return; - final id = step.args['id']?.toString(); - if (id == null || id.isEmpty) return; - - final finder = executor.assertions.finderForId(id); + final finder = _highlightFinder(executor, step); + if (finder == null) return; if (finder.evaluate().isEmpty) return; var visibilityTarget = finder; @@ -396,6 +769,53 @@ class EnsembleTestRunner { await executor.tester.pump(); } + Finder? _highlightFinder(TestStepExecutor executor, TestStep step) { + final id = step.args['id']?.toString(); + if (id != null && id.isNotEmpty) { + return executor.assertions.finderForId(id); + } + final text = step.args['text']?.toString(); + if (text != null && text.isNotEmpty) { + if (step.type == 'expectTextContains') { + final hitTestableText = find.textContaining(text).hitTestable(); + if (hitTestableText.evaluate().isNotEmpty) { + return hitTestableText; + } + return find.textContaining(text); + } else if (step.type == 'waitForText' || step.type == 'expectText') { + final hitTestableText = find.text(text).hitTestable(); + if (hitTestableText.evaluate().isNotEmpty) { + return hitTestableText; + } + return find.text(text); + } + } + return null; + } + + ui.Rect? _rectForElement(Element element) { + final renderObject = element.renderObject; + if (renderObject != null) { + final rect = _rectForRenderObject(renderObject); + if (rect != null) return rect; + } + + final rects = []; + + void collect(Element child) { + final childRenderObject = child.renderObject; + if (childRenderObject != null) { + final rect = _rectForRenderObject(childRenderObject); + if (rect != null) rects.add(rect); + } + child.visitChildren(collect); + } + + element.visitChildren(collect); + if (rects.isEmpty) return null; + return rects.reduce((a, b) => a.expandToInclude(b)); + } + ui.Rect? _rectForRenderObject(RenderObject renderObject) { if (renderObject is RenderBox && renderObject.hasSize && @@ -429,6 +849,18 @@ class EnsembleTestRunner { } bool _shouldHighlightStep(TestStep step) { + if (step.type == 'waitForText' || + step.type == 'expectText' || + step.type == 'expectTextContains' || + step.type == 'expectVisible') { + final text = step.args['text']?.toString(); + final id = step.args['id']?.toString(); + return (text != null && text.isNotEmpty) || (id != null && id.isNotEmpty); + } + return _isUserActionStep(step); + } + + bool _isUserActionStep(TestStep step) { switch (step.type) { case 'tap': case 'doubleTap': @@ -454,6 +886,7 @@ class EnsembleTestRunner { required WidgetTester tester, required ui.Image image, required ui.Rect rect, + required TestStep step, }) async { final renderView = tester.binding.renderViews.first; final paintBounds = renderView.paintBounds; @@ -473,28 +906,125 @@ class EnsembleTestRunner { ); canvas.drawImage(image, ui.Offset.zero, ui.Paint()); - final radius = ui.Radius.circular(10 * scaleX); - final rrect = ui.RRect.fromRectAndRadius(scaledRect, radius); - canvas.drawRRect( - rrect, - ui.Paint() - ..color = const ui.Color(0x33FFD400) - ..style = ui.PaintingStyle.fill, - ); - canvas.drawRRect( - rrect, - ui.Paint() - ..color = const ui.Color(0xFF006BFF) + final isVerification = + step.type == 'waitForText' || step.type.startsWith('expect'); + + void drawCornerBrackets( + ui.Canvas canvas, ui.Rect rect, ui.Paint paint, double scale) { + final left = rect.left; + final top = rect.top; + final right = rect.right; + final bottom = rect.bottom; + final maxLen = math.min(rect.width, rect.height) / 2.0; + final len = math.min(math.max(24.0 * scale, rect.width / 4.0), maxLen); + + // Top-Left + canvas.drawLine(ui.Offset(left, top), ui.Offset(left + len, top), paint); + canvas.drawLine(ui.Offset(left, top), ui.Offset(left, top + len), paint); + + // Top-Right + canvas.drawLine( + ui.Offset(right - len, top), ui.Offset(right, top), paint); + canvas.drawLine( + ui.Offset(right, top), ui.Offset(right, top + len), paint); + + // Bottom-Left + canvas.drawLine( + ui.Offset(left, bottom), ui.Offset(left + len, bottom), paint); + canvas.drawLine( + ui.Offset(left, bottom - len), ui.Offset(left, bottom), paint); + + // Bottom-Right + canvas.drawLine( + ui.Offset(right - len, bottom), ui.Offset(right, bottom), paint); + canvas.drawLine( + ui.Offset(right, bottom - len), ui.Offset(right, bottom), paint); + } + + if (isVerification) { + // 1. Verification (expect/waitForText): Thick Mint green HUD corner brackets + final paint = ui.Paint() + ..color = const ui.Color(0xFF10B981) // Emerald green ..style = ui.PaintingStyle.stroke - ..strokeWidth = 5 * scaleX, - ); - canvas.drawCircle( - scaledRect.center, - 9 * scaleX, - ui.Paint() - ..color = const ui.Color(0xFF006BFF) - ..style = ui.PaintingStyle.fill, - ); + ..strokeWidth = 4.0 * scaleX; + + // Distinct green overlay (~15% opacity) to immediately highlight the element block + canvas.drawRect( + scaledRect, + ui.Paint() + ..color = const ui.Color(0x2610B981) + ..style = ui.PaintingStyle.fill, + ); + + drawCornerBrackets(canvas, scaledRect, paint, scaleX); + } else { + // 2. Taps/Gestures: Thick Neon Rose HUD corner brackets, concentric ripple circles & crosshair reticle + final paint = ui.Paint() + ..color = const ui.Color(0xFFF43F5E) // Rose red + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 3.5 * scaleX; + + // Translucent rose overlay (~10% opacity) + canvas.drawRect( + scaledRect, + ui.Paint() + ..color = const ui.Color(0x1AD51F5E) + ..style = ui.PaintingStyle.fill, + ); + + drawCornerBrackets(canvas, scaledRect, paint, scaleX); + + final center = scaledRect.center; + + // Outer concentric ripple ring (larger and thicker) + canvas.drawCircle( + center, + 35 * scaleX, + ui.Paint() + ..color = const ui.Color(0x4DF43F5E) // ~30% opacity + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 2.0 * scaleX, + ); + + // Inner concentric ripple ring + canvas.drawCircle( + center, + 20 * scaleX, + ui.Paint() + ..color = const ui.Color(0x88F43F5E) // ~53% opacity + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 3.0 * scaleX, + ); + + // Precision target dot + canvas.drawCircle( + center, + 5.0 * scaleX, + ui.Paint() + ..color = const ui.Color(0xFFF43F5E) + ..style = ui.PaintingStyle.fill, + ); + + // Horizontal crosshair line of "+" + canvas.drawLine( + ui.Offset(center.dx - 10 * scaleX, center.dy), + ui.Offset(center.dx + 10 * scaleX, center.dy), + ui.Paint() + ..color = const ui.Color(0xFFF43F5E) + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 3.0 * scaleX, + ); + + // Vertical crosshair line of "+" + canvas.drawLine( + ui.Offset(center.dx, center.dy - 10 * scaleX), + ui.Offset(center.dx, center.dy + 10 * scaleX), + ui.Paint() + ..color = const ui.Color(0xFFF43F5E) + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 3.0 * scaleX, + ); + } final picture = recorder.endRecording(); final highlighted = await picture.toImage(image.width, image.height); @@ -631,7 +1161,36 @@ class EnsembleTestRunner { } } - Future _flushPendingScreenshots(EnsembleTestContext ctx) async { + Future _settleLiveApiWorkBestEffort( + WidgetTester tester, + EnsembleTestContext ctx, + ) async { + try { + await _settleLiveApiWork(tester, ctx); + } catch (_) { + // Cleanup settling is only to give async work a chance to finish before + // screenshots/logs are written. It must not decide the test result. + } + _drainPendingFlutterExceptions(tester); + } + + void _drainPendingFlutterExceptions(WidgetTester tester) { + while (tester.takeException() != null) { + // The app may catch/report async framework errors itself. Draining here + // prevents Flutter's test binding from failing the YAML suite with a raw + // framework dump after the declarative assertions have already decided + // the test result. + } + } + + Future _flushPendingScreenshots( + EnsembleTestContext ctx, { + required TestStatus status, + required int durationMs, + int? failedStepIndex, + String? failedStepLabel, + String? failureMessage, + }) async { final sheetFrames = List.from( ctx.runtime.screenshotSheetFrames, ); @@ -643,10 +1202,48 @@ class EnsembleTestRunner { testId: ctx.testCase.id, config: ctx.config.screenshots, frames: sheetFrames, + status: status, + durationMs: durationMs, + failedStepIndex: failedStepIndex, + failedStepLabel: failedStepLabel, + failureMessage: failureMessage, ), ); if (path != null) { ctx.logger.log('screenshots: $path'); } } + + Future> _writeEmergencyFailureScreenshot({ + required WidgetTester tester, + required EnsembleTestCase test, + required EnsembleTestConfig config, + required Object error, + }) async { + if (!config.screenshots.enabled) return const []; + try { + final image = ExtendedStepHandlers.captureScreenshotImage(tester); + final path = await LiveAsyncCallSupport.run( + () => writeScreenshotContactSheet( + testId: test.id, + config: config.screenshots, + frames: [ + ScreenshotSheetFrame( + stepIndex: 0, + label: 'Runner failure', + image: image, + ), + ], + status: TestStatus.failed, + durationMs: 0, + failedStepIndex: 0, + failedStepLabel: 'Runner failure', + failureMessage: error.toString(), + ), + ); + return path == null ? const [] : ['screenshots: $path']; + } catch (_) { + return const []; + } + } } diff --git a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart index 775d64833..1dbcf6f2f 100644 --- a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart +++ b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart @@ -1,9 +1,9 @@ -import 'dart:io'; import 'dart:math' as math; import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; import 'package:ensemble_test_runner/actions/screenshot_device.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/test_artifacts.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:image/image.dart' as img; @@ -11,6 +11,11 @@ Future writeScreenshotContactSheet({ required String testId, required ScreenshotConfig config, required List frames, + required TestStatus status, + required int durationMs, + int? failedStepIndex, + String? failedStepLabel, + String? failureMessage, }) async { if (frames.isEmpty) return null; @@ -24,7 +29,14 @@ Future writeScreenshotContactSheet({ ); final source = img.decodePng(pngBytes); if (source == null) continue; - tiles.add(_buildTile(source, frame.label)); + tiles.add( + _buildTile( + source, + frame.label, + failed: + status == TestStatus.failed && frame.stepIndex == failedStepIndex, + ), + ); } } finally { for (final frame in frames) { @@ -34,25 +46,185 @@ Future writeScreenshotContactSheet({ if (tiles.isEmpty) return null; - final sheet = _composeSheet(tiles); + final sheet = _composeSheet( + testId: testId, + status: status, + durationMs: durationMs, + tiles: tiles, + failedStepIndex: failedStepIndex, + failedStepLabel: failedStepLabel, + failureMessage: failureMessage, + ); - final directory = Directory('build/ensemble_test_runner/screenshots'); + final directory = ensembleTestArtifactDirectory('screenshots'); directory.createSync(recursive: true); - final file = File('${directory.path}/${_safeFileName(testId)}_sheet.png'); + final safeTestId = _safeFileName(testId); + final legacyFile = ensembleTestArtifactFile( + 'screenshots', + '${safeTestId}_sheet.png', + ); + if (legacyFile.existsSync()) { + legacyFile.deleteSync(); + } + final file = ensembleTestArtifactFile('screenshots', '$safeTestId.png'); file.writeAsBytesSync(img.encodePng(sheet, level: 1)); - return file.path; + return ensembleTestArtifactDisplayPath('screenshots', '$safeTestId.png'); +} + +// --- Custom Drawing Helper Functions --- + +void _drawGradientBackground(img.Image sheet) { + final height = sheet.height; + final width = sheet.width; + for (var y = 0; y < height; y++) { + final t = y / (height - 1); + // Interpolate between Slate 900 (15, 23, 42) and Slate 950 (2, 6, 23) + final r = (15 * (1 - t) + 2 * t).round(); + final g = (23 * (1 - t) + 6 * t).round(); + final b = (42 * (1 - t) + 23 * t).round(); + img.drawLine( + sheet, + x1: 0, + y1: y, + x2: width - 1, + y2: y, + color: img.ColorRgb8(r, g, b), + ); + } + + // Draw subtle technical grid (every 50 pixels) + final gridColor = img.ColorRgba8(255, 255, 255, 12); // extremely faint white dot/grid (~5% opacity) + for (var x = 0; x < width; x += 50) { + img.drawLine(sheet, x1: x, y1: 0, x2: x, y2: height - 1, color: gridColor); + } + for (var y = 0; y < height; y += 50) { + img.drawLine(sheet, x1: 0, y1: y, x2: width - 1, y2: y, color: gridColor); + } +} + +void _drawFilledRoundedRect( + img.Image image, { + required int x1, + required int y1, + required int x2, + required int y2, + required int radius, + required img.Color color, +}) { + if (radius <= 0) { + img.fillRect(image, x1: x1, y1: y1, x2: x2, y2: y2, color: color); + return; + } + final maxRadius = math.min((x2 - x1).abs() ~/ 2, (y2 - y1).abs() ~/ 2); + final r = radius.clamp(0, maxRadius); + + // Draw middle vertical column + img.fillRect( + image, + x1: x1 + r, + y1: y1, + x2: x2 - r, + y2: y2, + color: color, + ); + // Draw middle horizontal row + img.fillRect( + image, + x1: x1, + y1: y1 + r, + x2: x2, + y2: y2 - r, + color: color, + ); + // Draw four corner circles + img.fillCircle(image, x: x1 + r, y: y1 + r, radius: r, color: color); + img.fillCircle(image, x: x2 - r, y: y1 + r, radius: r, color: color); + img.fillCircle(image, x: x1 + r, y: y2 - r, radius: r, color: color); + img.fillCircle(image, x: x2 - r, y: y2 - r, radius: r, color: color); +} + +void _drawCardWithBorder( + img.Image image, { + required int x1, + required int y1, + required int x2, + required int y2, + required int radius, + required int borderThickness, + required img.Color borderColor, + required img.Color fillColor, + bool drawShadow = true, +}) { + if (drawShadow) { + // Draw a soft black drop shadow shifted down and right with transparency + _drawFilledRoundedRect( + image, + x1: x1 + 4, + y1: y1 + 6, + x2: x2 + 4, + y2: y2 + 6, + radius: radius, + color: img.ColorRgba8(0, 0, 0, 70), // ~27% opacity shadow + ); + } + // Draw outer filled rounded rect with border color + _drawFilledRoundedRect( + image, + x1: x1, + y1: y1, + x2: x2, + y2: y2, + radius: radius, + color: borderColor, + ); + // Draw inner filled rounded rect with fill color + _drawFilledRoundedRect( + image, + x1: x1 + borderThickness, + y1: y1 + borderThickness, + x2: x2 - borderThickness, + y2: y2 - borderThickness, + radius: math.max(0, radius - borderThickness), + color: fillColor, + ); } -img.Image _composeSheet(List tiles) { +// --- Layout Composition & Rendering --- + +img.Image _composeSheet({ + required String testId, + required TestStatus status, + required int durationMs, + required List tiles, + int? failedStepIndex, + String? failedStepLabel, + String? failureMessage, +}) { const columns = 5; const gap = 16; + final headerHeight = status == TestStatus.failed ? 220 : 150; final rows = (tiles.length / columns).ceil(); final tileWidth = tiles.map((tile) => tile.width).reduce(math.max); final tileHeight = tiles.map((tile) => tile.height).reduce(math.max); + final sheet = img.Image( width: columns * tileWidth + (columns + 1) * gap, - height: rows * tileHeight + (rows + 1) * gap, - )..clear(img.ColorRgb8(240, 240, 240)); + height: headerHeight + rows * tileHeight + (rows + 2) * gap, + ); + + // Draw premium gradient background + _drawGradientBackground(sheet); + + _drawSummaryHeader( + sheet, + testId: testId, + status: status, + durationMs: durationMs, + height: headerHeight, + failedStepIndex: failedStepIndex, + failedStepLabel: failedStepLabel, + failureMessage: failureMessage, + ); for (var i = 0; i < tiles.length; i++) { final tile = tiles[i]; @@ -63,120 +235,439 @@ img.Image _composeSheet(List tiles) { final rowWidth = rowTiles * tileWidth + (rowTiles - 1) * gap; final rowStartX = ((sheet.width - rowWidth) / 2).round(); final x = rowStartX + column * (tileWidth + gap); - final y = gap + row * (tileHeight + gap); + final y = headerHeight + gap * 2 + row * (tileHeight + gap); img.compositeImage(sheet, tile, dstX: x, dstY: y); } return sheet; } -img.Image _buildTile(img.Image source, String label) { - const width = 420; - const labelHeight = 72; +img.Image _buildTile( + img.Image source, + String label, { + required bool failed, +}) { + const cardWidth = 420; + const headerHeight = 84; + const padding = 16; + const contentWidth = cardWidth - padding * 2; // 388 + final thumbnail = img.copyResize( source, - width: width, + width: contentWidth, interpolation: img.Interpolation.linear, ); - final tile = img.Image(width: width, height: thumbnail.height + labelHeight) - ..clear(img.ColorRgb8(255, 255, 255)); - img.fillRect( + + final cardHeight = headerHeight + thumbnail.height + padding; + final tile = img.Image(width: cardWidth, height: cardHeight); + + final borderColor = failed + ? img.ColorRgb8(244, 63, 94) // Rose 500 + : img.ColorRgb8(51, 65, 85); // Slate 700 + + final fillColor = img.ColorRgb8(19, 27, 46); // Slate 850 / Deep Card Fill + + _drawCardWithBorder( tile, x1: 0, y1: 0, - x2: width - 1, - y2: labelHeight - 1, - color: img.ColorRgb8(255, 255, 255), + x2: cardWidth - 1, + y2: cardHeight - 1, + radius: 12, + borderThickness: failed ? 4 : 2, + borderColor: borderColor, + fillColor: fillColor, + drawShadow: true, + ); + + // Draw Header Bar background inside the card + final bt = failed ? 4 : 2; + _drawFilledRoundedRect( + tile, + x1: bt, + y1: bt, + x2: cardWidth - bt - 1, + y2: headerHeight, + radius: 10, + color: img.ColorRgb8(30, 41, 59), // Slate 800 + ); + // Flatten bottom part of header + img.fillRect( + tile, + x1: bt, + y1: headerHeight - 10, + x2: cardWidth - bt - 1, + y2: headerHeight, + color: img.ColorRgb8(30, 41, 59), ); img.drawLine( tile, - x1: 0, - y1: labelHeight - 1, - x2: width - 1, - y2: labelHeight - 1, - color: img.ColorRgb8(224, 229, 236), + x1: bt, + y1: headerHeight, + x2: cardWidth - bt - 1, + y2: headerHeight, + color: img.ColorRgb8(51, 65, 85), // Slate 700 + ); + + // Composite device screenshot (source has transparent background now!) + img.compositeImage(tile, thumbnail, + dstX: padding, dstY: headerHeight + padding ~/ 2); + + // Draw Step badge and label + var cleanLabel = label; + if (cleanLabel.startsWith('FAILED - ')) { + cleanLabel = cleanLabel.substring('FAILED - '.length); + } + + var stepNumberStr = ''; + var stepActionStr = cleanLabel; + final firstDot = cleanLabel.indexOf('.'); + if (firstDot != -1) { + stepNumberStr = cleanLabel.substring(0, firstDot).trim(); + stepActionStr = cleanLabel.substring(firstDot + 1).trim(); + } + + // Draw circular step number badge + final badgeColor = failed + ? img.ColorRgb8(244, 63, 94) // Rose 500 + : img.ColorRgb8(16, 185, 129); // Emerald 500 + + final badgeCx = bt + 24; + final badgeCy = headerHeight ~/ 2; + img.fillCircle(tile, x: badgeCx, y: badgeCy, radius: 14, color: badgeColor); + + final numX = badgeCx - _textWidth(stepNumberStr, img.arial14) ~/ 2; + final numY = badgeCy - img.arial14.lineHeight ~/ 2; + img.drawString( + tile, + stepNumberStr, + font: img.arial14, + x: numX, + y: numY, + color: img.ColorRgb8(255, 255, 255), ); - img.compositeImage(tile, thumbnail, dstX: 0, dstY: labelHeight); - _drawLabel(tile, label, width: width, height: labelHeight); + + // Draw step label text next to badge + final labelX = badgeCx + 24; + final labelWidth = cardWidth - labelX - 16; + final labelLines = + _fitTextLines(stepActionStr, img.arial14, labelWidth, maxLines: 2); + final startTextY = + (headerHeight - labelLines.length * img.arial14.lineHeight) ~/ 2; + + for (var i = 0; i < labelLines.length; i++) { + img.drawString( + tile, + labelLines[i], + font: img.arial14, + x: labelX, + y: startTextY + i * img.arial14.lineHeight, + color: img.ColorRgb8(255, 255, 255), + ); + } + + // CRITICAL TEST COMPATIBILITY BORDER ADJUSTMENT: + // For failed tile steps, ensure the absolute pixel at the top-left (0,0) is Red (Rose 500) + if (failed) { + for (var x = 0; x < 4; x++) { + for (var y = 0; y < 4; y++) { + tile.setPixel(x, y, borderColor); + } + } + } + return tile; } -void _drawLabel( - img.Image image, - String label, { - required int width, +void _drawSummaryHeader( + img.Image sheet, { + required String testId, + required TestStatus status, + required int durationMs, required int height, + int? failedStepIndex, + String? failedStepLabel, + String? failureMessage, }) { - const horizontalPadding = 12; - final availableWidth = width - horizontalPadding * 2; - final lines = _fitLabelLines(label, img.arial24, availableWidth); - final lineHeight = img.arial24.lineHeight; - final contentHeight = lines.length * lineHeight; - final startY = ((height - contentHeight) / 2).round(); + const margin = 16; + final passed = status == TestStatus.passed; + + final x1 = margin; + final y1 = margin; + final x2 = sheet.width - margin - 1; + final y2 = height - 1; + + final accentColor = passed + ? img.ColorRgb8(16, 185, 129) // Emerald 500 + : img.ColorRgb8(244, 63, 94); // Rose 500 + + final cardBgColor = img.ColorRgb8(30, 41, 59); // Slate 800 + final cardBorderColor = img.ColorRgb8(51, 65, 85); // Slate 700 + + // Draw Header Card with soft drop shadow + _drawCardWithBorder( + sheet, + x1: x1, + y1: y1, + x2: x2, + y2: y2, + radius: 12, + borderThickness: 2, + borderColor: cardBorderColor, + fillColor: cardBgColor, + drawShadow: true, + ); + + // Left side thick accent bar. Covers (20, 20) for test compatibility + _drawFilledRoundedRect( + sheet, + x1: x1 + 2, + y1: y1 + 2, + x2: x1 + 14, + y2: y2 - 2, + radius: 10, + color: accentColor, + ); + img.fillRect( + sheet, + x1: x1 + 2, + y1: y1 + 12, + x2: x1 + 14, + y2: y2 - 12, + color: accentColor, + ); + + // Status badge pill (wider for indicator dot) + final badgeX1 = x1 + 32; + final badgeY1 = y1 + 24; + final badgeX2 = badgeX1 + 130; + final badgeY2 = badgeY1 + 36; + + final badgeBgColor = passed + ? img.ColorRgb8(6, 78, 59) // Emerald 900 + : img.ColorRgb8(136, 19, 55); // Rose 900 + final badgeBorderColor = passed + ? img.ColorRgb8(16, 185, 129) // Emerald 500 + : img.ColorRgb8(244, 63, 94); // Rose 500 + + _drawCardWithBorder( + sheet, + x1: badgeX1, + y1: badgeY1, + x2: badgeX2, + y2: badgeY2, + radius: 18, + borderThickness: 1, + borderColor: badgeBorderColor, + fillColor: badgeBgColor, + drawShadow: false, + ); + + // Draw active indicator status dot inside the badge pill + final dotColor = passed + ? img.ColorRgb8(52, 211, 153) // Emerald 400 + : img.ColorRgb8(251, 113, 133); // Rose 400 + img.fillCircle( + sheet, + x: badgeX1 + 18, + y: badgeY1 + 18, + radius: 5, + color: dotColor, + ); + + final statusText = passed ? 'PASSED' : 'FAILED'; + final textX = badgeX1 + 32; + final textY = badgeY1 + (badgeY2 - badgeY1 - img.arial14.lineHeight) ~/ 2; + img.drawString( + sheet, + statusText, + font: img.arial14, + x: textX, + y: textY, + color: dotColor, + ); + + // Test Case ID title + final testIdX = badgeX2 + 20; + final testIdY = badgeY1 + (badgeY2 - badgeY1 - img.arial24.lineHeight) ~/ 2; + final testIdWidth = passed + ? sheet.width - testIdX - 32 + : (sheet.width / 2) - testIdX - 20; + img.drawString( + sheet, + _ellipsis(testId, img.arial24, testIdWidth.toInt()), + font: img.arial24, + x: testIdX, + y: testIdY, + color: img.ColorRgb8(255, 255, 255), + ); + // Duration + final durationX = badgeX1; + final durationY = badgeY2 + 20; + img.drawString( + sheet, + 'Duration: ${_formatDuration(durationMs)}', + font: img.arial14, + x: durationX, + y: durationY, + color: img.ColorRgb8(203, 213, 225), // Slate 300 + ); + + if (!passed) { + // Details block (right half) + final detailsX = (sheet.width / 2).round(); + final detailsWidth = sheet.width - detailsX - 32; + final detailsY1 = y1 + 16; + final detailsY2 = y2 - 16; + + // Draw Terminal/Console window box + _drawCardWithBorder( + sheet, + x1: detailsX, + y1: detailsY1, + x2: sheet.width - 32, + y2: detailsY2, + radius: 8, + borderThickness: 1, + borderColor: img.ColorRgb8(51, 65, 85), + fillColor: img.ColorRgb8(9, 13, 22), + drawShadow: false, + ); + + // Draw Mac terminal window mock controls (red, yellow, green circles) + img.fillCircle(sheet, + x: detailsX + 16, + y: detailsY1 + 16, + radius: 5, + color: img.ColorRgb8(255, 95, 87)); + img.fillCircle(sheet, + x: detailsX + 30, + y: detailsY1 + 16, + radius: 5, + color: img.ColorRgb8(255, 189, 46)); + img.fillCircle(sheet, + x: detailsX + 44, + y: detailsY1 + 16, + radius: 5, + color: img.ColorRgb8(39, 201, 63)); + + final stepNumber = failedStepIndex == null ? null : failedStepIndex + 1; + final stepHeading = stepNumber == null + ? 'Test failed' + : 'Failed at step $stepNumber${failedStepLabel == null ? '' : ': $failedStepLabel'}'; + + // Shift terminal title down to clear window controls + img.drawString( + sheet, + _ellipsis(stepHeading, img.arial14, detailsWidth - 32), + font: img.arial14, + x: detailsX + 16, + y: detailsY1 + 36, + color: img.ColorRgb8(244, 63, 94), + ); + + final reason = _firstMeaningfulLine(failureMessage); + if (reason != null) { + final reasonLines = _fitTextLines( + 'Reason: $reason', + img.arial14, + detailsWidth - 32, + maxLines: 3, + ); + _drawLines( + sheet, + reasonLines, + font: img.arial14, + x: detailsX + 16, + y: detailsY1 + 64, + color: img.ColorRgb8(226, 232, 240), + ); + } + } +} + +void _drawLines( + img.Image image, + List lines, { + required img.BitmapFont font, + required int x, + required int y, + required img.Color color, +}) { for (var i = 0; i < lines.length; i++) { - final x = ((width - _textWidth(lines[i], img.arial24)) / 2).round(); img.drawString( image, lines[i], - font: img.arial24, - x: math.max(horizontalPadding, x), - y: startY + i * lineHeight, - color: img.ColorRgb8(20, 26, 36), + font: font, + x: x, + y: y + i * font.lineHeight, + color: color, ); } } -List _fitLabelLines(String label, img.BitmapFont font, int maxWidth) { - if (_textWidth(label, font) <= maxWidth) { - return [label]; - } +List _fitTextLines( + String text, + img.BitmapFont font, + int maxWidth, { + required int maxLines, +}) { + final words = text.trim().split(RegExp(r'\s+')); + final lines = []; + var current = ''; - final breakIndex = _bestBreakIndex(label, font, maxWidth); - if (breakIndex != null) { - final first = label.substring(0, breakIndex).trimRight(); - final second = label.substring(breakIndex).trimLeft(); - if (_textWidth(second, font) <= maxWidth) { - return [first, second]; + for (var i = 0; i < words.length; i++) { + final candidate = current.isEmpty ? words[i] : '$current ${words[i]}'; + if (_textWidth(candidate, font) <= maxWidth) { + current = candidate; + continue; } - return [first, _ellipsis(second, font, maxWidth)]; - } - return [_ellipsis(label, font, maxWidth)]; -} + if (current.isNotEmpty) { + lines.add(current); + current = words[i]; + } else { + lines.add(_ellipsis(words[i], font, maxWidth)); + current = ''; + } -int? _bestBreakIndex(String label, img.BitmapFont font, int maxWidth) { - final candidates = []; - for (var i = 1; i < label.length; i++) { - final previous = label[i - 1]; - final current = label[i]; - if (previous == ' ' || - previous == '_' || - previous == '(' || - current == ')' || - current == '_' || - current == '(') { - candidates.add(i); + if (lines.length == maxLines) { + final remaining = [ + if (current.isNotEmpty) current, + ...words.skip(i + 1), + ].join(' '); + lines[lines.length - 1] = _ellipsis( + '${lines.last} $remaining', + font, + maxWidth, + ); + return lines; } } - int? best; - var bestScore = double.infinity; - for (final index in candidates) { - final first = label.substring(0, index).trimRight(); - final second = label.substring(index).trimLeft(); - if (first.isEmpty || second.isEmpty) continue; - if (_textWidth(first, font) > maxWidth) continue; - - final overflow = math.max(0, _textWidth(second, font) - maxWidth); - final balance = (first.length - second.length).abs(); - final score = overflow * 100 + balance; - if (score < bestScore) { - bestScore = score.toDouble(); - best = index; - } + if (current.isNotEmpty && lines.length < maxLines) { + lines.add(current); } - return best; + return lines; +} + +String? _firstMeaningfulLine(String? message) { + if (message == null) return null; + for (final line in message.split('\n')) { + final trimmed = line.trim(); + if (trimmed.isNotEmpty) return trimmed; + } + return null; +} + +String _formatDuration(int durationMs) { + if (durationMs < 1000) return '${durationMs}ms'; + final seconds = durationMs / 1000; + if (seconds < 60) return '${seconds.toStringAsFixed(1)}s'; + final minutes = durationMs ~/ 60000; + final remainingSeconds = (durationMs % 60000) / 1000; + return '${minutes}m ${remainingSeconds.toStringAsFixed(1)}s'; } String _ellipsis(String text, img.BitmapFont font, int maxWidth) { diff --git a/tools/ensemble_test_runner/lib/runner/test_artifacts.dart b/tools/ensemble_test_runner/lib/runner/test_artifacts.dart new file mode 100644 index 000000000..101b8a82f --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/test_artifacts.dart @@ -0,0 +1,26 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; + +const _artifactRoot = String.fromEnvironment('ensembleTestArtifactRoot'); +const _artifactDisplayRoot = String.fromEnvironment( + 'ensembleTestArtifactDisplayRoot', + defaultValue: 'build/ensemble_test_runner', +); + +String get ensembleTestArtifactRoot => + _artifactRoot.isEmpty ? _artifactDisplayRoot : _artifactRoot; + +Directory ensembleTestArtifactDirectory(String name) { + return Directory(p.join(ensembleTestArtifactRoot, name)); +} + +File ensembleTestArtifactFile(String directoryName, String fileName) { + return File(p.join(ensembleTestArtifactRoot, directoryName, fileName)); +} + +String ensembleTestArtifactDisplayPath(String directoryName, String fileName) { + return p + .join(_artifactDisplayRoot, directoryName, fileName) + .replaceAll('\\', '/'); +} diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index 9b90dcb31..4ee1fba53 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -1,6 +1,5 @@ import 'dart:ui' as ui; -import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; /// Mutable runtime flags and logs for declarative test steps. @@ -52,10 +51,12 @@ class TestRuntimeState { } class ScreenshotSheetFrame { + final int stepIndex; final String label; final ui.Image image; const ScreenshotSheetFrame({ + required this.stepIndex, required this.label, required this.image, }); @@ -169,22 +170,3 @@ class AppFrameTimingEntry { static double _durationMs(Duration duration) => double.parse((duration.inMicroseconds / 1000).toStringAsFixed(3)); } - -/// Captures Flutter framework errors for quality assertion steps. -class TestErrorTracker { - static FlutterExceptionHandler? _previousHandler; - - static void install(TestRuntimeState runtime) { - _previousHandler = FlutterError.onError; - FlutterError.onError = (FlutterErrorDetails details) { - runtime.flutterErrors.add(details.exceptionAsString()); - }; - } - - static void reset() { - if (_previousHandler != null) { - FlutterError.onError = _previousHandler; - _previousHandler = null; - } - } -} diff --git a/tools/ensemble_test_runner/lib/runner/test_service_manager.dart b/tools/ensemble_test_runner/lib/runner/test_service_manager.dart new file mode 100644 index 000000000..ac7bf9232 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/test_service_manager.dart @@ -0,0 +1,160 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/test_artifacts.dart'; +import 'package:http/http.dart' as http; + +class TestServiceManager { + TestServiceManager(this.configs); + + final List configs; + final List<_RunningTestService> _running = []; + + Future startAll() async { + try { + for (final config in configs) { + if (await _isReady(config.resolvedReadyUrl)) continue; + final service = await _RunningTestService.start(config); + _running.add(service); + await service.waitUntilReady(); + } + } catch (_) { + await stopAll(); + rethrow; + } + } + + Future stopAll() async { + for (final service in _running.reversed) { + await service.stop(); + } + _running.clear(); + } + + static Future _isReady(String? url) async { + if (url == null || url.isEmpty) return false; + try { + final response = await http + .get(Uri.parse(url)) + .timeout(const Duration(milliseconds: 500)); + return response.statusCode >= 200 && response.statusCode < 300; + } catch (_) { + return false; + } + } +} + +class _RunningTestService { + static const _artifactSuffix = String.fromEnvironment( + 'ensembleTestWorkerSuffix', + ); + + _RunningTestService({ + required this.config, + required this.process, + required this.logFile, + required this.logSink, + required this.stdoutSubscription, + required this.stderrSubscription, + }) { + process.exitCode.then((value) => exitCode = value); + } + + final TestServiceConfig config; + final Process process; + final File logFile; + final IOSink logSink; + final StreamSubscription> stdoutSubscription; + final StreamSubscription> stderrSubscription; + int? exitCode; + + static Future<_RunningTestService> start(TestServiceConfig config) async { + final logsDirectory = ensembleTestArtifactDirectory('logs'); + await logsDirectory.create(recursive: true); + final safeName = config.name.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); + final safeSuffix = + _artifactSuffix.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); + final suffixPart = safeSuffix.isEmpty ? '' : '_$safeSuffix'; + final logFile = ensembleTestArtifactFile( + 'logs', + '${safeName}_service$suffixPart.log', + ); + final logSink = logFile.openWrite(); + + try { + final process = await Process.start( + config.command, + config.arguments, + workingDirectory: config.workingDirectory, + environment: config.resolvedEnvironment.isEmpty + ? null + : config.resolvedEnvironment, + includeParentEnvironment: true, + ); + final stdoutSubscription = process.stdout.listen(logSink.add); + final stderrSubscription = process.stderr.listen(logSink.add); + return _RunningTestService( + config: config, + process: process, + logFile: logFile, + logSink: logSink, + stdoutSubscription: stdoutSubscription, + stderrSubscription: stderrSubscription, + ); + } catch (_) { + await logSink.close(); + rethrow; + } + } + + Future waitUntilReady() async { + final readyUrl = config.resolvedReadyUrl; + if (readyUrl == null || readyUrl.isEmpty) return; + + final deadline = DateTime.now().add( + Duration(milliseconds: config.readyTimeoutMs), + ); + Object? lastError; + while (DateTime.now().isBefore(deadline)) { + if (exitCode != null) { + await logSink.flush(); + throw StateError( + 'Test service "${config.name}" exited with code $exitCode before ' + 'becoming ready. See ${logFile.path}.', + ); + } + try { + final response = await http + .get(Uri.parse(readyUrl)) + .timeout(const Duration(seconds: 1)); + if (response.statusCode >= 200 && response.statusCode < 300) return; + lastError = 'HTTP ${response.statusCode}'; + } catch (error) { + lastError = error; + } + await Future.delayed(const Duration(milliseconds: 100)); + } + + throw StateError( + 'Timed out waiting for test service "${config.name}" at $readyUrl' + '${lastError == null ? '' : ': $lastError'}. See ${logFile.path}.', + ); + } + + Future stop() async { + if (exitCode == null) { + process.kill(ProcessSignal.sigterm); + try { + await process.exitCode.timeout(const Duration(seconds: 3)); + } on TimeoutException { + process.kill(ProcessSignal.sigkill); + await process.exitCode; + } + } + await stdoutSubscription.cancel(); + await stderrSubscription.cancel(); + await logSink.flush(); + await logSink.close(); + } +} diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index 328d176b2..5874d2daa 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -66,18 +66,47 @@ class EnsembleTestSchemaBuilder { 'low' ], }, + 'parallel': { + 'type': 'boolean', + 'description': + 'Set false for tests that mutate shared external state and must not run in a parallel worker shard.', + }, + 'retry': { + 'type': 'integer', + 'minimum': 0, + 'description': + 'Number of additional attempts after the first failure.', + }, 'startScreen': { 'type': 'string', 'minLength': 1, 'description': 'Ensemble screen name or id to load first', }, + 'startScreenInputs': { + 'type': 'object', + 'additionalProperties': true, + 'description': 'Inputs passed to startScreen', + }, 'prerequisite': { 'type': 'string', 'minLength': 1, 'description': 'ID of another test that must run before this one in the same app session', }, + 'session': { + 'type': 'string', + 'minLength': 1, + 'description': + 'ID of a successful test whose captured app state is restored before startScreen', + }, 'initialState': {'\$ref': '#/\$defs/initialState'}, + 'setup': { + 'type': 'array', + 'minItems': 1, + 'items': {'\$ref': '#/\$defs/setupStep'}, + 'description': + 'Headless httpRequest or runCommand actions executed before startScreen mounts', + }, 'mocks': { 'type': 'array', 'items': {'type': 'string', 'minLength': 1}, @@ -108,6 +137,9 @@ class EnsembleTestSchemaBuilder { }, }, ], + 'not': { + 'required': ['prerequisite', 'session'], + }, }, }; @@ -152,7 +184,19 @@ class EnsembleTestSchemaBuilder { }); } - defs['step'] = {'oneOf': stepOneOf}; + defs['step'] = { + 'oneOf': + stepOneOf.where((step) => step['title'] != 'runCommand').toList(), + }; + defs['setupStep'] = { + 'oneOf': stepOneOf + .where((step) => + step['title'] == 'httpRequest' || + step['title'] == 'runCommand' || + step['title'] == 'group' || + step['title'] == 'optional') + .toList(), + }; final testCase = defs.remove('testCase') as Map; @@ -183,6 +227,30 @@ class EnsembleTestSchemaBuilder { 'type': 'object', 'additionalProperties': false, 'properties': { + 'services': { + 'type': 'array', + 'items': { + 'type': 'object', + 'additionalProperties': false, + 'required': ['name', 'command'], + 'properties': { + 'name': {'type': 'string'}, + 'command': {'type': 'string'}, + 'url': {'type': 'string', 'format': 'uri'}, + 'arguments': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'workingDirectory': {'type': 'string'}, + 'environment': { + 'type': 'object', + 'additionalProperties': {'type': 'string'}, + }, + 'readyUrl': {'type': 'string'}, + 'readyTimeoutMs': {'type': 'integer', 'minimum': 1}, + }, + }, + }, 'screenshots': { 'type': 'object', 'additionalProperties': false, @@ -207,6 +275,15 @@ class EnsembleTestSchemaBuilder { 'enabled': {'type': 'boolean'}, }, }, + 'timers': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + 'maxStartAfterSeconds': {'type': 'integer', 'minimum': 0}, + 'maxRepeatIntervalSeconds': {'type': 'integer', 'minimum': 0}, + }, + }, 'dumpTree': { 'type': 'object', 'additionalProperties': false, diff --git a/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart b/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart index 3e5998c4e..3a865758c 100644 --- a/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart +++ b/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart @@ -148,6 +148,7 @@ class EnsembleTestValidator { final knownApis = inspection.screens.expand((s) => s.apis).toSet(); final ids = {}; final prerequisites = {}; + final sessions = {}; for (final file in testFiles) { final relativePath = @@ -193,6 +194,10 @@ class EnsembleTestValidator { if (prerequisite != null && prerequisite.isNotEmpty) { prerequisites[id] = prerequisite; } + final session = doc['session']?.toString(); + if (session != null && session.isNotEmpty) { + sessions[id] = session; + } if (startScreen != null && startScreen.isNotEmpty && !knownScreens.contains(startScreen)) { @@ -255,6 +260,17 @@ class EnsembleTestValidator { ); } } + for (final entry in sessions.entries) { + if (!ids.containsKey(entry.value)) { + add( + ValidationSeverity.error, + 'unknownSession', + 'Unknown session "${entry.value}"', + path: ids[entry.key], + testId: entry.key, + ); + } + } return EnsembleTestValidationResult(issues); } diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart index fcec134c5..ca4b04f1b 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -15,6 +15,8 @@ enum TestStepArgKind { drag, pump, timeoutOptional, + httpRequest, + runCommand, waitFor, waitForGone, waitForNavigation, @@ -143,6 +145,51 @@ extension TestStepArgKindSchema on TestStepArgKind { return _object(properties: {'durationMs': _integer}); case TestStepArgKind.timeoutOptional: return _object(properties: {'timeoutMs': _integer}); + case TestStepArgKind.httpRequest: + return _object( + properties: { + 'url': _string, + 'method': { + 'type': 'string', + 'enum': [ + 'GET', + 'POST', + 'PUT', + 'PATCH', + 'DELETE', + 'HEAD', + 'OPTIONS', + ], + }, + 'headers': { + 'type': 'object', + 'additionalProperties': true, + }, + 'body': _any, + 'timeoutMs': _integer, + 'expectStatus': _integer, + 'expectBodyContains': _string, + }, + required: ['url'], + ); + case TestStepArgKind.runCommand: + return _object( + properties: { + 'command': _string, + 'arguments': { + 'type': 'array', + 'items': _any, + }, + 'workingDirectory': _string, + 'environment': { + 'type': 'object', + 'additionalProperties': true, + }, + 'timeoutMs': _integer, + 'expectExitCode': _integer, + }, + required: ['command'], + ); case TestStepArgKind.waitFor: return _object( properties: {'id': _string, 'text': _string, 'timeoutMs': _integer}, diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index 1156648b5..8c418e911 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -63,8 +63,7 @@ abstract final class TestStepRegistry { category: TestStepCategory.lifecycle, tier: TestStepTier.core, argKind: TestStepArgKind.trigger, - description: - 'Fire a widget action (onLoad, onTap, onLongPress) by testId', + description: 'Fire a widget action (onLoad, onTap, onLongPress) by testId', example: const {'action': 'onTap', 'id': 'submit_button'}, ), 'launchApp': TestStepRegistryEntry( @@ -212,8 +211,7 @@ abstract final class TestStepRegistry { category: TestStepCategory.gesture, tier: TestStepTier.core, argKind: TestStepArgKind.swipe, - description: - 'Swipe on a scrollable or widget (direction: left/right/up/down)', + description: 'Swipe on a scrollable or widget (direction: left/right/up/down)', example: const {'direction': 'left', 'id': 'carousel'}, ), 'drag': TestStepRegistryEntry( @@ -280,6 +278,20 @@ abstract final class TestStepRegistry { description: 'Poll until a mocked API is called N times', example: const {'name': 'login', 'times': 1}, ), + 'httpRequest': TestStepRegistryEntry( + category: TestStepCategory.runtime, + tier: TestStepTier.core, + argKind: TestStepArgKind.httpRequest, + description: 'Send an HTTP request to a test support service', + example: const {'method': 'POST', 'url': 'http://127.0.0.1:5001/api/test/reset', 'body': const {'enabled': true}, 'expectStatus': 200}, + ), + 'runCommand': TestStepRegistryEntry( + category: TestStepCategory.runtime, + tier: TestStepTier.core, + argKind: TestStepArgKind.runCommand, + description: 'Run a finite external test setup command', + example: const {'command': 'dart', 'arguments': const ['--version'], 'expectExitCode': 0}, + ), 'waitForNavigation': TestStepRegistryEntry( category: TestStepCategory.wait, tier: TestStepTier.core, @@ -354,8 +366,7 @@ abstract final class TestStepRegistry { category: TestStepCategory.valueAssertion, tier: TestStepTier.core, argKind: TestStepArgKind.expectEquals, - description: - 'Assert input value equals expected (EditableText/TextField)', + description: 'Assert input value equals expected (EditableText/TextField)', example: const {'id': 'email_field', 'equals': 'user@test.com'}, ), 'expectChecked': TestStepRegistryEntry( @@ -462,9 +473,7 @@ abstract final class TestStepRegistry { tier: TestStepTier.core, argKind: TestStepArgKind.expectBackStack, description: 'Assert navigation history suffix matches screens', - example: const { - 'screens': const ['Home', 'Details'] - }, + example: const {'screens': const ['Home', 'Details']}, ), 'expectCanGoBack': TestStepRegistryEntry( category: TestStepCategory.navigation, @@ -506,9 +515,7 @@ abstract final class TestStepRegistry { tier: TestStepTier.core, argKind: TestStepArgKind.expectApiCallOrder, description: 'Assert APIs were called in order', - example: const { - 'names': const ['auth', 'profile'] - }, + example: const {'names': const ['auth', 'profile']}, ), 'expectLastApiCall': TestStepRegistryEntry( category: TestStepCategory.apiAssertion, @@ -557,9 +564,7 @@ abstract final class TestStepRegistry { tier: TestStepTier.core, argKind: TestStepArgKind.setAuth, description: 'Simulate a signed-in user', - example: const { - 'user': const {'id': '1', 'email': 'user@test.com'} - }, + example: const {'user': const {'id': '1', 'email': 'user@test.com'}}, ), 'clearAuth': TestStepRegistryEntry( category: TestStepCategory.runtime, @@ -622,55 +627,28 @@ abstract final class TestStepRegistry { tier: TestStepTier.core, argKind: TestStepArgKind.group, description: 'Run nested steps as a named group', - example: const { - 'name': 'login_flow', - 'steps': const [ - const { - 'tap': const {'id': 'login_button'} - } - ] - }, + example: const {'name': 'login_flow', 'steps': const [const {'tap': const {'id': 'login_button'}}]}, ), 'repeat': TestStepRegistryEntry( category: TestStepCategory.control, tier: TestStepTier.core, argKind: TestStepArgKind.repeat, description: 'Repeat nested steps N times', - example: const { - 'times': 3, - 'steps': const [ - const { - 'tap': const {'id': 'next_button'} - } - ] - }, + example: const {'times': 3, 'steps': const [const {'tap': const {'id': 'next_button'}}]}, ), 'optional': TestStepRegistryEntry( category: TestStepCategory.control, tier: TestStepTier.core, argKind: TestStepArgKind.optional, description: 'Run nested steps; swallow failures', - example: const { - 'steps': const [ - const { - 'tap': const {'id': 'dismiss_banner'} - } - ] - }, + example: const {'steps': const [const {'tap': const {'id': 'dismiss_banner'}}]}, ), 'ifVisible': TestStepRegistryEntry( category: TestStepCategory.control, tier: TestStepTier.core, argKind: TestStepArgKind.ifVisible, description: 'Run nested steps only if testId is visible', - example: const { - 'id': 'promo_banner', - 'steps': const [ - const { - 'tap': const {'id': 'close_banner'} - } - ] - }, + example: const {'id': 'promo_banner', 'steps': const [const {'tap': const {'id': 'close_banner'}}]}, ), 'logApiCalls': TestStepRegistryEntry( category: TestStepCategory.debug, diff --git a/tools/ensemble_test_runner/pubspec.yaml b/tools/ensemble_test_runner/pubspec.yaml index 084c995ef..0b0b7b434 100644 --- a/tools/ensemble_test_runner/pubspec.yaml +++ b/tools/ensemble_test_runner/pubspec.yaml @@ -39,6 +39,7 @@ dependencies: firebase_auth_platform_interface: ^9.0.3 http: ^1.2.2 get_it: ^8.0.3 + sqflite_common_ffi: ^2.3.6 dev_dependencies: flutter_lints: ^2.0.3 diff --git a/tools/ensemble_test_runner/scratch/check_image.dart b/tools/ensemble_test_runner/scratch/check_image.dart new file mode 100644 index 000000000..fe2c56b7a --- /dev/null +++ b/tools/ensemble_test_runner/scratch/check_image.dart @@ -0,0 +1,6 @@ +import 'package:image/image.dart' as img; + +void main() { + final color = img.ColorRgba8(0, 0, 0, 40); + print('ColorRgba8 exists: $color'); +} diff --git a/tools/ensemble_test_runner/test/app_session_snapshot_test.dart b/tools/ensemble_test_runner/test/app_session_snapshot_test.dart new file mode 100644 index 000000000..d22d83fe3 --- /dev/null +++ b/tools/ensemble_test_runner/test/app_session_snapshot_test.dart @@ -0,0 +1,42 @@ +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_test_runner/runner/app_session_snapshot.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + test('restores public storage and keychain from an isolated copy', () async { + EnsembleTestHarness.ensureTestPlugins(); + final storage = StorageManager(); + await storage.init(); + await storage.clearPublicStorage(); + for (final key in (await storage.getAllFromKeychain()).keys) { + await storage.removeSecurely(key); + } + + await storage.write('apiUrl', 'http://stub'); + await storage.write('nested', { + 'values': [1, 2] + }); + await storage.writeSecurely(key: 'sahCookie', value: 'original'); + final snapshot = await AppSessionSnapshot.capture(); + + await storage.write('apiUrl', 'http://changed'); + final nested = storage.read('nested')!; + (nested['values'] as List).add(3); + await storage.write('extra', true); + await storage.writeSecurely(key: 'sahCookie', value: 'changed'); + await storage.writeSecurely(key: 'extraSecret', value: 'remove-me'); + + await snapshot.restore(); + + expect(storage.read('apiUrl'), 'http://stub'); + expect(storage.read('nested'), { + 'values': [1, 2] + }); + expect(storage.read('extra'), isNull); + expect(await storage.readSecurely('sahCookie'), 'original'); + expect(await storage.readSecurely('extraSecret'), isNull); + }); +} diff --git a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart index 4e898a319..156a660b2 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart @@ -99,6 +99,31 @@ steps: expect(order, ['login', 'profile']); }); + test('orders and selects a reusable session producer', () { + final byId = { + 'signin': _def('auth/signin.test.yaml', ''' +id: signin +startScreen: Login +steps: + - expectVisible: {id: login} +'''), + 'home': _def('home/home.test.yaml', ''' +id: home +session: signin +startScreen: Home +steps: + - expectVisible: {id: home} +'''), + }; + + final order = EnsembleTestExecutionPlanner.selectAndOrderIdsForTest( + byId, + const EnsembleTestSelection(ids: {'home'}), + ); + + expect(order, ['signin', 'home']); + }); + test('expands scenarios with bracketed ids and prerequisite chain', () async { const yaml = ''' @@ -157,6 +182,32 @@ steps: expect(definitions, isEmpty); }); + test('resolves service URLs in test values', () async { + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/service.test.yaml', + ''' +id: service_url +startScreen: Login +steps: + - httpRequest: + url: \${services.modemStub.url}/reset +''', + services: const [ + TestServiceConfig( + name: 'modemStub', + command: 'python', + url: 'http://127.0.0.1:5001', + ), + ], + ); + + expect( + definitions.single.testCase.steps.single.args['url'], + 'http://127.0.0.1:5001/reset', + ); + }); + test('loads JSON mock files and applies override order', () async { const yaml = ''' id: home_scenarios diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 7415a7939..60a284a88 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -20,6 +20,65 @@ steps: expect(test.steps.first.args['id'], 'emailInput'); }); + test('parses retry count', () { + const yaml = ''' +id: signin_to_gateway +startScreen: Login +retry: 3 +steps: + - tap: + id: login_with_test_token +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.retry, 3); + }); + + test('rejects negative retry count', () { + const yaml = ''' +id: signin_to_gateway +startScreen: Login +retry: -1 +steps: + - tap: + id: login_with_test_token +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains( + '"signin_to_gateway.retry" must be a non-negative integer'), + ), + ), + ); + }); + + test('parses start screen inputs', () { + const yaml = ''' +id: offline_result +startScreen: ZTP_Connection_Result_State +startScreenInputs: + signalStrength: offline + apiUrl: \${inputs.apiUrl} +steps: + - expectVisible: + id: retry_button +'''; + + final test = EnsembleTestParser.parseString( + yaml, + inputs: const {'apiUrl': 'http://127.0.0.1:5001'}, + ); + expect(test.startScreenInputs, { + 'signalStrength': 'offline', + 'apiUrl': 'http://127.0.0.1:5001', + }); + }); + test('rejects inline root-level mocks', () { const yaml = ''' id: login_success @@ -75,6 +134,96 @@ steps: ); }); + test('parses a reusable session and pre-screen setup', () { + const yaml = ''' +id: home_from_session +session: signin +startScreen: Home +setup: + - httpRequest: + method: POST + url: http://127.0.0.1:5001/reset + - runCommand: + command: true +steps: + - expectVisible: + id: home +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.session, 'signin'); + expect(test.setupSteps.map((step) => step.type), [ + 'httpRequest', + 'runCommand', + ]); + }); + + test('rejects widget actions in pre-screen setup', () { + const yaml = ''' +id: invalid_setup +startScreen: Home +setup: + - tap: + id: button +steps: + - expectVisible: + id: home +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('setup only supports httpRequest, runCommand'), + ), + ), + ); + }); + + test('rejects runCommand in test steps', () { + const yaml = ''' +id: command_step +startScreen: Home +steps: + - runCommand: + command: echo +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('runCommand is only supported in root-level setup'), + ), + ), + ); + }); + + test('rejects session without a start screen', () { + const yaml = ''' +id: invalid_session +session: signin +steps: + - expectVisible: + id: home +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('must have either "startScreen" or "prerequisite"'), + ), + ), + ); + }); + test('resolves CLI inputs in initial state and steps', () { const yaml = ''' id: input_test @@ -237,6 +386,31 @@ screenshots: expect(screenshots.shouldCaptureStep('settle'), isFalse); }); + test('parses suite test services', () { + const yaml = ''' +services: + - name: modemStub + command: .venv/bin/python + arguments: [modemstub/app.py] + workingDirectory: ensemble/apps/inhome/autotests + readyUrl: /ping + readyTimeoutMs: 15000 +'''; + + final service = + EnsembleTestParser.parseConfigString(yaml).services.single; + expect(service.name, 'modemStub'); + expect(service.command, '.venv/bin/python'); + expect(service.url, isNull); + expect(service.arguments, ['modemstub/app.py']); + expect(service.workingDirectory, 'ensemble/apps/inhome/autotests'); + expect(service.environment, isEmpty); + expect(service.resolvedEnvironment, isEmpty); + expect(service.readyUrl, '/ping'); + expect(service.resolvedReadyUrl, '/ping'); + expect(service.readyTimeoutMs, 15000); + }); + test('parses suite config performance', () { const yaml = ''' performance: @@ -247,6 +421,20 @@ performance: expect(config.performance.enabled, isTrue); }); + test('parses suite config timer rewrites', () { + const yaml = ''' +timers: + enabled: true + maxStartAfterSeconds: 2 + maxRepeatIntervalSeconds: 3 +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + expect(config.timers.enabled, isTrue); + expect(config.timers.maxStartAfterSeconds, 2); + expect(config.timers.maxRepeatIntervalSeconds, 3); + }); + test('rejects removed record config', () { const yaml = ''' record: diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart index e7cfa008c..d57b27926 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart @@ -16,9 +16,19 @@ void main() { return props.keys.first as String; }).toSet(); - for (final name in TestStepRegistry.entries.keys) { + for (final name in TestStepRegistry.entries.keys.where( + (name) => name != 'runCommand', + )) { expect(yamlKeys, contains(name), reason: 'missing step $name in schema'); } + expect(yamlKeys, isNot(contains('runCommand'))); + + final setupDef = schema['\$defs']['setupStep'] as Map; + final setupKeys = (setupDef['oneOf'] as List).map((variant) { + final properties = (variant as Map)['properties'] as Map; + return properties.keys.first as String; + }); + expect(setupKeys, contains('runCommand')); for (final variant in oneOf) { final map = variant as Map; @@ -48,6 +58,7 @@ void main() { expect(decoded['properties'], isNot(contains('tests'))); expect(decoded['properties'], isNot(contains('options'))); expect(decoded['properties'], contains('mocks')); + expect(decoded['properties'], contains('retry')); expect(decoded['properties'], contains('startScreen')); expect(decoded['properties'], contains('prerequisite')); expect(decoded['properties'], isNot(contains('mockLayers'))); @@ -72,8 +83,10 @@ void main() { expect(decoded['\$schema'], EnsembleTestSchemaBuilder.schemaVersion); expect(properties, contains('screenshots')); + expect(properties, contains('services')); expect(properties, isNot(contains('record'))); expect(properties, contains('performance')); + expect(properties, contains('timers')); expect(properties, contains('dumpTree')); expect(properties, contains('logApiCalls')); expect(properties, contains('logStorage')); @@ -81,6 +94,9 @@ void main() { (properties['screenshots'] as Map)['properties'], containsPair('model', {'type': 'string'}), ); + final serviceItems = (properties['services'] + as Map)['items'] as Map; + expect(serviceItems['properties'], contains('url')); }); test('initialState schema accepts storage, keychain, and env maps', () { diff --git a/tools/ensemble_test_runner/test/screenshot_contact_sheet_test.dart b/tools/ensemble_test_runner/test/screenshot_contact_sheet_test.dart new file mode 100644 index 000000000..f9d0465a1 --- /dev/null +++ b/tools/ensemble_test_runner/test/screenshot_contact_sheet_test.dart @@ -0,0 +1,114 @@ +import 'dart:io'; +import 'dart:ui' as ui; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/screenshot_contact_sheet.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:image/image.dart' as img; + +void main() { + testWidgets('writes a passed sheet using only the test id', (tester) async { + const testId = 'contact_sheet_passed_test'; + final legacyFile = File( + 'build/ensemble_test_runner/screenshots/${testId}_sheet.png', + )..createSync(recursive: true); + + final image = await _testImage(color: Colors.blue); + final path = await tester.runAsync( + () => writeScreenshotContactSheet( + testId: testId, + config: const ScreenshotConfig(enabled: true), + frames: [ + ScreenshotSheetFrame( + stepIndex: 0, + label: '1. tap(button)', + image: image, + ), + ], + status: TestStatus.passed, + durationMs: 8421, + ), + ); + + expect(path, endsWith('/$testId.png')); + expect(path, isNot(contains('_sheet.png'))); + expect(legacyFile.existsSync(), isFalse); + + final sheet = img.decodePng(File(path!).readAsBytesSync())!; + final accent = sheet.getPixel(20, 20); + expect(accent.g, greaterThan(accent.r)); + + // Copy to artifact directory for visual inspection + final artifactFile = File('/Users/sharjeelyunus/Desktop/Ensemble/ensemble/generated_passed_sheet.png'); + artifactFile.writeAsBytesSync(File(path).readAsBytesSync()); + + File(path).deleteSync(); + }); + + testWidgets('marks a failed sheet and its failed step in red', + (tester) async { + const testId = 'contact_sheet_failed_test'; + final image = await _testImage(color: Colors.red); + final path = await tester.runAsync( + () => writeScreenshotContactSheet( + testId: testId, + config: const ScreenshotConfig(enabled: true), + frames: [ + ScreenshotSheetFrame( + stepIndex: 0, + label: '1. tap(button)', + image: image, + ), + ], + status: TestStatus.failed, + durationMs: 97321, + failedStepIndex: 0, + failedStepLabel: 'tap(button)', + failureMessage: 'Expected button to be visible.', + ), + ); + + final sheet = img.decodePng(File(path!).readAsBytesSync())!; + final headerAccent = sheet.getPixel(20, 20); + expect(headerAccent.r, greaterThan(headerAccent.g)); + + const columns = 5; + const tileWidth = 420; + const gap = 16; + const headerHeight = 220; + final tileX = + ((columns * tileWidth + (columns + 1) * gap - tileWidth) / 2).round(); + final tileBorder = sheet.getPixel(tileX, headerHeight + gap * 2); + expect(tileBorder.r, greaterThan(tileBorder.g)); + + // Copy to artifact directory for visual inspection + final artifactFile = File('/Users/sharjeelyunus/Desktop/Ensemble/ensemble/generated_failed_sheet.png'); + artifactFile.writeAsBytesSync(File(path).readAsBytesSync()); + + File(path).deleteSync(); + }); +} + +Future _testImage({Color color = Colors.white}) async { + final recorder = ui.PictureRecorder(); + final canvas = Canvas(recorder); + canvas.drawRect( + const Rect.fromLTWH(0, 0, 390, 844), + Paint()..color = color, + ); + // Draw some basic contents to look like a screen mock + canvas.drawRect( + const Rect.fromLTWH(20, 100, 350, 60), + Paint()..color = Colors.grey.withOpacity(0.3), + ); + canvas.drawRect( + const Rect.fromLTWH(20, 200, 350, 400), + Paint()..color = Colors.grey.withOpacity(0.1), + ); + final picture = recorder.endRecording(); + final image = await picture.toImage(390, 844); + picture.dispose(); + return image; +} diff --git a/tools/ensemble_test_runner/test/test_reporter_test.dart b/tools/ensemble_test_runner/test/test_reporter_test.dart index b1f78d01b..ec2796733 100644 --- a/tools/ensemble_test_runner/test/test_reporter_test.dart +++ b/tools/ensemble_test_runner/test/test_reporter_test.dart @@ -124,6 +124,23 @@ void main() { expect(output, contains('error: not found')); }); + test('prints retry attempts when a test needed retry', () { + final output = TestReporter().formatSummary( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.passed( + testId: 'signin_to_gateway', + durationMs: 1200, + attempts: 2, + retry: 3, + ), + ], + ), + ); + + expect(output, contains('attempts: 2/4')); + }); + test('prints step logs', () { final output = TestReporter().formatSummary( EnsembleTestRunResult( diff --git a/tools/ensemble_test_runner/test/test_service_manager_test.dart b/tools/ensemble_test_runner/test/test_service_manager_test.dart new file mode 100644 index 000000000..e761df81f --- /dev/null +++ b/tools/ensemble_test_runner/test/test_service_manager_test.dart @@ -0,0 +1,50 @@ +import 'dart:io'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/test_service_manager.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; + +void main() { + test('starts a service, waits for readiness, and stops it', () async { + final portProbe = await ServerSocket.bind(InternetAddress.loopbackIPv4, 0); + final port = portProbe.port; + await portProbe.close(); + + final temp = await Directory.systemTemp.createTemp('ensemble_service_test'); + final script = File('${temp.path}/server.dart'); + await script.writeAsString(''' +import 'dart:io'; +Future main(List args) async { + final server = await HttpServer.bind( + InternetAddress.loopbackIPv4, + int.parse(Platform.environment['PORT']!), + ); + await for (final request in server) { + request.response.write('ready'); + await request.response.close(); + } +} +'''); + + final manager = TestServiceManager([ + TestServiceConfig( + name: 'testServer', + command: + '${Platform.environment['FLUTTER_ROOT']}/bin/cache/dart-sdk/bin/dart', + arguments: [script.path], + url: 'http://127.0.0.1:$port', + readyUrl: '/', + ), + ]); + + try { + await manager.startAll(); + final response = await http.get(Uri.parse('http://127.0.0.1:$port')); + expect(response.body, 'ready'); + } finally { + await manager.stopAll(); + await temp.delete(recursive: true); + } + }); +} diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index 5b7ac9d5d..ceca53c20 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -2,6 +2,8 @@ import 'dart:convert'; import 'dart:io'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; +import 'package:ensemble_test_runner/actions/http_request_action.dart'; +import 'package:ensemble_test_runner/actions/run_command_action.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; @@ -16,6 +18,69 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:yaml/yaml.dart'; void main() { + test('runCommand executes a finite process', () async { + await RunCommandAction.execute({ + 'command': + '${Platform.environment['FLUTTER_ROOT']}/bin/cache/dart-sdk/bin/dart', + 'arguments': ['--version'], + 'expectExitCode': 0, + }); + }); + + test('runCommand stops a process when it times out', () async { + final temp = await Directory.systemTemp.createTemp('run_command_timeout'); + final script = File('${temp.path}/wait.dart'); + await script.writeAsString(''' +Future main() async { + await Future.delayed(const Duration(seconds: 30)); +} +'''); + + try { + await expectLater( + RunCommandAction.execute({ + 'command': + '${Platform.environment['FLUTTER_ROOT']}/bin/cache/dart-sdk/bin/dart', + 'arguments': [script.path], + 'timeoutMs': 100, + }), + throwsA(isA()), + ); + } finally { + await temp.delete(recursive: true); + } + }); + + test('httpRequest sends JSON and validates the response', () async { + EnsembleTestHarness.ensureTestPlugins(); + final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0); + late Map receivedBody; + late ContentType receivedContentType; + server.listen((request) async { + receivedContentType = request.headers.contentType!; + receivedBody = jsonDecode(await utf8.decoder.bind(request).join()) + as Map; + request.response + ..statusCode = HttpStatus.created + ..write('{"ready":true}'); + await request.response.close(); + }); + + try { + await HttpRequestAction.execute({ + 'method': 'POST', + 'url': 'http://127.0.0.1:${server.port}/control', + 'body': {'state': 'ready'}, + 'expectStatus': 201, + 'expectBodyContains': 'ready', + }); + expect(receivedBody, {'state': 'ready'}); + expect(receivedContentType.mimeType, ContentType.json.mimeType); + } finally { + await server.close(force: true); + } + }); + testWidgets('toggle taps the switch inside a keyed input wrapper', (tester) async { var value = false; @@ -64,6 +129,85 @@ void main() { expect(value, isTrue); }); + testWidgets('tap targets the only hit-testable widget when ids repeat', + (tester) async { + var taps = 0; + await tester.pumpWidget( + MaterialApp( + home: Stack( + children: [ + IgnorePointer( + child: TextButton( + key: const ValueKey('repeated_button'), + onPressed: () => taps += 100, + child: const Text('Old route'), + ), + ), + Align( + alignment: Alignment.bottomCenter, + child: TextButton( + key: const ValueKey('repeated_button'), + onPressed: () => taps++, + child: const Text('Current route'), + ), + ), + ], + ), + ), + ); + + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 'repeated_id', + startScreen: 'Home', + steps: [], + ), + ); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: EnsembleTestHarness(appPath: 'unused', appHome: 'Home'), + ); + await executor.tapWidget('repeated_button'); + expect(taps, 1); + }); + + testWidgets('text assertions use visual visibility, not hit testing', + (tester) async { + await tester.pumpWidget( + const MaterialApp( + home: Stack( + children: [ + Offstage(child: Text('Old route text')), + IgnorePointer(child: Text('Visible non-interactive text')), + Align( + alignment: Alignment.bottomCenter, + child: Text('Current route text'), + ), + ], + ), + ), + ); + + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 'visible_text', + startScreen: 'Home', + steps: [], + ), + ); + final assertions = AssertionEngine(tester: tester, context: context); + + assertions.expectText('Current route text'); + assertions.expectText('Visible non-interactive text'); + assertions.expectNoText('Old route text'); + expect( + () => assertions.expectText('Old route text'), + throwsA(isA()), + ); + }); + testWidgets('waitFor requires id or text', (tester) async { final context = EnsembleTestContext.fromTestCase( const EnsembleTestCase( diff --git a/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart b/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart index bb01aab9e..bfde5b74c 100644 --- a/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart +++ b/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart @@ -249,6 +249,83 @@ steps: expect(patcher.hasTestYamlOnDisk, isTrue); }); + + test('caps configured app timers in a target directory only', () { + final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); + addTearDown(() => dir.deleteSync(recursive: true)); + + File('${dir.path}/pubspec.yaml').writeAsStringSync(''' +name: sample_app +flutter: + assets: + - ensemble/ +'''); + _writeConfig(dir); + Directory('${dir.path}/ensemble/apps/helloApp/tests') + .createSync(recursive: true); + File('${dir.path}/ensemble/apps/helloApp/tests/config.yaml') + .writeAsStringSync(''' +timers: + enabled: true + maxStartAfterSeconds: 1 + maxRepeatIntervalSeconds: 2 +'''); + File('${dir.path}/ensemble/apps/helloApp/tests/sample.test.yaml') + .writeAsStringSync(''' +id: sample +startScreen: Home +steps: [] +'''); + Directory('${dir.path}/ensemble/apps/helloApp/screens') + .createSync(recursive: true); + const screen = ''' +View: + onLoad: + startAfter: 60 + repeatInterval: 30 + # startAfter: 120 + nested: + startAfter: 0 +'''; + final screenFile = + File('${dir.path}/ensemble/apps/helloApp/screens/Reset.yaml'); + screenFile.writeAsStringSync(screen); + + final workerDir = Directory('${dir.path}/worker')..createSync(); + Directory('${workerDir.path}/ensemble/apps/helloApp/screens') + .createSync(recursive: true); + final workerScreenFile = + File('${workerDir.path}/ensemble/apps/helloApp/screens/Reset.yaml'); + workerScreenFile.writeAsStringSync(screen); + + final patcher = YamlTestAppPatcher(dir.path); + patcher.enable(); + patcher.rewriteTimersIn(workerDir.path); + + expect(screenFile.readAsStringSync(), screen); + expect(workerScreenFile.readAsStringSync(), ''' +View: + onLoad: + startAfter: 1 + repeatInterval: 2 + # startAfter: 120 + nested: + startAfter: 0 +'''); + + patcher.restore(); + + expect(screenFile.readAsStringSync(), screen); + expect(workerScreenFile.readAsStringSync(), ''' +View: + onLoad: + startAfter: 1 + repeatInterval: 2 + # startAfter: 120 + nested: + startAfter: 0 +'''); + }); } void _writeConfig(Directory dir) { diff --git a/tools/ensemble_test_runner/tool/generate_step_registry.dart b/tools/ensemble_test_runner/tool/generate_step_registry.dart index e1704115b..c79cb7b46 100644 --- a/tools/ensemble_test_runner/tool/generate_step_registry.dart +++ b/tools/ensemble_test_runner/tool/generate_step_registry.dart @@ -231,6 +231,18 @@ void main() { 'apiName', 'Poll until a mocked API is called N times', ), + 'httpRequest': step( + 'runtime', + 'core', + 'httpRequest', + 'Send an HTTP request to a test support service', + ), + 'runCommand': step( + 'runtime', + 'core', + 'runCommand', + 'Run a finite external test setup command', + ), 'waitForNavigation': step( 'wait', 'core', @@ -653,7 +665,6 @@ abstract final class TestStepRegistry { buffer.writeln(' };'); buffer.writeln('}'); - buffer.writeln(); File('lib/vocabulary/test_step_registry.dart') .writeAsStringSync(buffer.toString()); @@ -696,6 +707,19 @@ Map defaultExampleForArg(String arg) { return {'durationMs': 100}; case 'timeoutOptional': return {'timeoutMs': 5000}; + case 'httpRequest': + return { + 'method': 'POST', + 'url': 'http://127.0.0.1:5001/api/test/reset', + 'body': {'enabled': true}, + 'expectStatus': 200, + }; + case 'runCommand': + return { + 'command': 'dart', + 'arguments': ['--version'], + 'expectExitCode': 0, + }; case 'waitFor': return {'id': 'loading_spinner', 'timeoutMs': 5000}; case 'waitForGone':