diff --git a/modules/ensemble/lib/framework/apiproviders/api_provider.dart b/modules/ensemble/lib/framework/apiproviders/api_provider.dart index 2dd658927..234078b31 100644 --- a/modules/ensemble/lib/framework/apiproviders/api_provider.dart +++ b/modules/ensemble/lib/framework/apiproviders/api_provider.dart @@ -29,7 +29,7 @@ class APIProviders extends InheritedWidget { if (provider == null) { return httpProvider; } else if (provider == 'firebaseFunction') { - return FirebaseFunctionsAPIProvider(); + return providers[provider] ?? FirebaseFunctionsAPIProvider(); } else if (provider == 'sse') { return providers[provider] ?? SSEAPIProvider(); } else { diff --git a/modules/ensemble/lib/framework/widget/view_util.dart b/modules/ensemble/lib/framework/widget/view_util.dart index dc15e4809..f03fc447a 100644 --- a/modules/ensemble/lib/framework/widget/view_util.dart +++ b/modules/ensemble/lib/framework/widget/view_util.dart @@ -171,6 +171,9 @@ class ViewUtil { if (callerPayload?['id'] != null) { props["id"] = callerPayload?['id']; } + if (callerPayload?['testId'] != null) { + props["testId"] = callerPayload?['testId']; + } Map inputPayload = {}; if (callerPayload?['inputs'] is Map) { diff --git a/modules/ensemble/lib/layout/ensemble_page_route.dart b/modules/ensemble/lib/layout/ensemble_page_route.dart index 199f8eb75..d5d9a9b93 100644 --- a/modules/ensemble/lib/layout/ensemble_page_route.dart +++ b/modules/ensemble/lib/layout/ensemble_page_route.dart @@ -78,8 +78,12 @@ extension PageTransitionTypeX on PageTransitionType { /// Page route builder that presents a screen without transition animation. class EnsemblePageRouteNoTransitionBuilder extends PageRouteBuilder { /// Creates a [EnsemblePageRouteNoTransitionBuilder] object. - EnsemblePageRouteNoTransitionBuilder({required Widget screenWidget}) + EnsemblePageRouteNoTransitionBuilder({ + required Widget screenWidget, + RouteSettings? settings, + }) : super( + settings: settings, pageBuilder: (context, animation, secondaryAnimation) => screenWidget, ); } diff --git a/modules/ensemble/lib/screen_controller.dart b/modules/ensemble/lib/screen_controller.dart index cd7f02fd1..5cb63a9c9 100644 --- a/modules/ensemble/lib/screen_controller.dart +++ b/modules/ensemble/lib/screen_controller.dart @@ -279,7 +279,7 @@ class ScreenController { (isRepeat ? repeatInterval! : 0); // we always execute at least once, delayed by startAfter and fallback to repeatInterval (or immediate if startAfter is 0) - Timer(Duration(seconds: delay), () { + final timer = Timer(Duration(seconds: delay), () { // execute the action executeActionWithScope(context, scopeManager, action.onTimer); @@ -298,7 +298,7 @@ class ScreenController { int? repeatCount = maxTimes != null ? maxTimes - 1 : null; if (repeatCount != 0) { int counter = 0; - final timer = + final periodicTimer = Timer.periodic(Duration(seconds: repeatInterval), (timer) { // execute the action executeActionWithScope(context, scopeManager, action.onTimer); @@ -317,10 +317,11 @@ class ScreenController { // save our timer to our PageData since user may want to cancel at anytime // and also when we navigate away from the page - scopeManager.addTimer(action, timer); + scopeManager.addTimer(action, periodicTimer); } } }); + scopeManager.addTimer(action, timer); } else if (action is StopTimerAction) { try { scopeManager.removeTimer(action.id); @@ -584,12 +585,24 @@ class ScreenController { defaultTransitionOptions[_pageType]?['duration'], fallback: 250); + final routeSettings = RouteSettings( + name: screenName ?? screenId, + arguments: ScreenPayload( + screenId: screenId, + screenName: screenName, + pageType: pageType, + arguments: pageArgs, + isExternal: isExternal, + ), + ); + PageRouteBuilder route = getScreenBuilder( screenWidget, pageType: pageType, transitionType: transitionType, alignment: alignment, duration: duration, + settings: routeSettings, ); // push the new route and remove all existing screens. This is suitable for logging out. if (routeOption == RouteOption.clearAllScreens) { @@ -651,12 +664,14 @@ class ScreenController { PageTransitionType? transitionType, Alignment? alignment, int? duration, + RouteSettings? settings, }) { const enableTransition = bool.fromEnvironment('transitions', defaultValue: true); if (!enableTransition) { - return EnsemblePageRouteNoTransitionBuilder(screenWidget: screenWidget); + return EnsemblePageRouteNoTransitionBuilder( + screenWidget: screenWidget, settings: settings); } if (pageType == PageType.modal) { @@ -669,6 +684,7 @@ class ScreenController { duration: Duration(milliseconds: duration ?? 250), barrierDismissible: true, barrierColor: Colors.black54, + settings: settings, ); } else { return EnsemblePageRouteBuilder( @@ -676,6 +692,7 @@ class ScreenController { transitionType: transitionType ?? PageTransitionType.fade, alignment: alignment ?? Alignment.center, duration: Duration(milliseconds: duration ?? 250), + settings: settings, ); } } diff --git a/modules/ensemble/test/custom_widget_test.dart b/modules/ensemble/test/custom_widget_test.dart new file mode 100644 index 000000000..c3fb31fe7 --- /dev/null +++ b/modules/ensemble/test/custom_widget_test.dart @@ -0,0 +1,27 @@ +import 'package:ensemble/framework/widget/view_util.dart'; +import 'package:ensemble/widget/custom_widget/custom_widget_model.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + test('custom widget call preserves explicit testId without requiring id', () { + final customWidgets = { + 'MiniCard': loadYaml(''' +body: + Text: + text: Devices +''') as YamlMap, + }; + + final model = ViewUtil.buildModel( + loadYaml(''' +MiniCard: + testId: devices_mini_card +'''), + customWidgets, + ) as CustomWidgetModel; + + expect(model.props['testId'], 'devices_mini_card'); + expect(model.props.containsKey('id'), isFalse); + }); +} diff --git a/tools/ensemble_test_runner/README.md b/tools/ensemble_test_runner/README.md index 90df23580..d34fd436a 100644 --- a/tools/ensemble_test_runner/README.md +++ b/tools/ensemble_test_runner/README.md @@ -29,7 +29,7 @@ Widget YAML must set `testId` (or `id`, which maps to the same `ValueKey`). ### Step vocabulary -The full official catalog (lifecycle, gestures, API mocks, fixtures, debug, etc.) is in **[STEP_VOCABULARY.md](STEP_VOCABULARY.md)**. +The full official catalog (lifecycle, gestures, API assertions, debug, etc.) is in **[STEP_VOCABULARY.md](STEP_VOCABULARY.md)**. Machine-readable registry (single source): `lib/vocabulary/test_step_registry.dart`. @@ -47,6 +47,28 @@ Or per file at the top of a test: # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json ``` +Suite-wide runner config lives in `tests/config.yaml`. The schema is hosted at +`https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json`: + +```yaml +# yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json +screenshots: + enabled: true + platform: ios + model: iPhone 15 Pro + includeSteps: [] + excludeSteps: [] + +performance: + enabled: true +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true +``` + ## App setup 1. Add `*.test.yaml` files under `definitions.local.path/tests/`, for example @@ -75,6 +97,24 @@ By default, output is quiet: no `pub get` package list, no Flutter test progress Optional: `--app-dir=` when not running from the app root. +Pass test-runner inputs with repeatable `--input key=value` flags. Tests can +reference them as `${inputs.key}` in `initialState`, mocks, and steps: + +```bash +dart run ensemble_test_runner:ensemble_test \ + --input adminPassword='s4C>M7U6t~' \ + --input expectedDeviceCount=2 +``` + +```yaml +initialState: + keychain: + adminPassword: ${inputs.adminPassword} +steps: + - expectText: + text: ${inputs.expectedDeviceCount} +``` + The default suite timeout is 10 minutes. Override it when a flow should fail faster or when a long chain needs more time: @@ -117,7 +157,7 @@ Create a starter test under `definitions.local.path/tests/`: dart run ensemble_test_runner:ensemble_test --scaffold-test=login_valid --feature=login --tag=smoke --screen=Login ``` -See [`docs/TEST_AUTHORING.md`](docs/TEST_AUTHORING.md) for the test authoring workflow, fixture conventions, validation rules, and repair-loop output. +See [`docs/TEST_AUTHORING.md`](docs/TEST_AUTHORING.md) for the test authoring workflow, mock file conventions, validation rules, and repair-loop output. ### CI output @@ -157,13 +197,9 @@ On success the console prints one consolidated boxed report for the suite: each # yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json id: login_flow startScreen: Login +mocks: + - mocks/login_success.mock.json steps: - - mockApi: - name: login - response: - statusCode: 200 - body: - token: test-token - enterText: id: email_field value: user@test.com diff --git a/tools/ensemble_test_runner/STEP_VOCABULARY.md b/tools/ensemble_test_runner/STEP_VOCABULARY.md index 5c3e1a910..90e0715c1 100644 --- a/tools/ensemble_test_runner/STEP_VOCABULARY.md +++ b/tools/ensemble_test_runner/STEP_VOCABULARY.md @@ -22,7 +22,7 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense `scroll`, `scrollUntilVisible`, `swipe`, `drag`, `pullToRefresh` ### Wait / sync -`wait` (alias `pump`), `pump`, `settle`, `waitFor`, `waitForText`, `waitForGone`, `waitForApi`, `waitForNavigation`, `waitUntil` +`wait` (alias `pump`), `pump`, `settle`, `waitFor`, `waitForText`, `waitForGone`, `waitForApi`, `waitForNavigation` ### UI assertions `expectVisible`, `expectNotVisible`, `expectExists`, `expectNotExists`, `expectText`, `expectNoText`, `expectTextContains`, `expectEnabled`, `expectDisabled` @@ -33,14 +33,14 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense ### Navigation `expectScreen` (alias), `expectNavigateTo`, `expectVisited`, `expectNotVisited`, `expectBackStack`, `expectCanGoBack`, `goBack` -### API mock / assert -`mockApi`, `mockApiError`, `mockApiFromFixture`, `mockApiException`, `mockTimeout`, `mockNetworkOffline`, `mockNetworkOnline`, `resetApiCalls`, `clearApiMocks`, `expectApiCalled`, `expectApiNotCalled`, `expectApiRequest`, `expectApiRequestContains`, `expectApiHeader`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` +### API assert / logs +`resetApiCalls`, `expectApiCalled`, `expectApiNotCalled`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` -### State / storage / runtime -`setState`, `expectState`, `expectStateContains`, `expectStateExists`, `expectStateNotExists`, `resetState`, `setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` +### Storage / runtime +`setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` -### Scripts / fixtures / debug / quality -`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `loadFixture`, `setStateFromFixture`, `expectMatchesFixture`, `logState`, `logStorage`, `screenshot`, `dumpTree`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` +### Scripts / debug / quality +`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` ### Control flow `group`, `repeat`, `optional`, `ifVisible` @@ -51,12 +51,6 @@ Official step catalog for app-local `tests/*.test.yaml` files, for example `ense id: login_flow startScreen: Login steps: - - mockApi: - name: login - response: - statusCode: 200 - body: - token: test - enterText: id: email_field value: user@test.com @@ -72,6 +66,27 @@ steps: Editor validation: `https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json` (committed copy at [`assets/schema/ensemble_tests_schema.json`](assets/schema/ensemble_tests_schema.json), regenerate with `dart run tool/generate_schema.dart`). Arg shapes come from [`TestStepArgKind`](lib/vocabulary/test_step_arg_kind.dart) on each [`TestStepRegistryEntry`](lib/vocabulary/test_step_registry.dart). +Performance logging writes Flutter app frame timing metrics to +`build/ensemble_test_runner/logs/app_performance.json`. Enable it once for the +full suite in `tests/config.yaml`: + +```yaml +performance: + enabled: true +``` + +`dumpTree`, `logApiCalls`, and `logStorage` are also suite-level config +artifacts: + +```yaml +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true +``` + Before running a new suite, use `dart run ensemble_test_runner:ensemble_test --doctor` from the Flutter wrapper app root to validate config, test discovery, duplicate IDs, prerequisites, schema comments, and obvious widget IDs. CI can request JSON diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json new file mode 100644 index 000000000..2a2089756 --- /dev/null +++ b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json @@ -0,0 +1,76 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json", + "title": "Ensemble declarative test config", + "description": "Suite-wide config for app-local tests/config.yaml", + "type": "object", + "additionalProperties": false, + "properties": { + "screenshots": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "platform": { + "type": "string" + }, + "model": { + "type": "string" + }, + "includeSteps": { + "type": "array", + "items": { + "type": "string" + } + }, + "excludeSteps": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "performance": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "dumpTree": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "logApiCalls": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "logStorage": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "key": { + "type": "string" + } + } + } + } +} diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 04dff0fd4..555f7467c 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -59,7 +59,18 @@ "$ref": "#/$defs/initialState" }, "mocks": { - "$ref": "#/$defs/mocks" + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "scenarios": { + "type": "array", + "items": { + "$ref": "#/$defs/scenario" + }, + "minItems": 1 }, "steps": { "type": "array", @@ -96,40 +107,15 @@ } ], "$defs": { - "mockResponse": { + "initialState": { "type": "object", "additionalProperties": false, "properties": { - "statusCode": { - "type": "integer" - }, - "body": true, - "headers": { + "storage": { "type": "object", "additionalProperties": true - } - } - }, - "mockApiEntry": { - "type": "object", - "additionalProperties": false, - "properties": { - "response": { - "$ref": "#/$defs/mockResponse" }, - "delayMs": { - "type": "integer" - } - }, - "required": [ - "response" - ] - }, - "initialState": { - "type": "object", - "additionalProperties": false, - "properties": { - "storage": { + "keychain": { "type": "object", "additionalProperties": true }, @@ -139,17 +125,25 @@ } } }, - "mocks": { + "scenario": { "type": "object", "additionalProperties": false, "properties": { - "apis": { + "id": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "vars": { "type": "object", - "additionalProperties": { - "$ref": "#/$defs/mockApiEntry" - } + "additionalProperties": true } - } + }, + "required": [ + "id" + ] }, "args_openScreen": { "type": "object", @@ -693,7 +687,7 @@ }, "additionalProperties": false, "title": "wait", - "description": "Alias for pump — advance frame clock by durationMs", + "description": "Real-time delay using runAsync, followed by a frame pump", "examples": [ { "durationMs": 100 @@ -847,37 +841,6 @@ } ] }, - "args_waitUntil": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "equals": true, - "state": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "equals": true - }, - "additionalProperties": false - }, - "timeoutMs": { - "type": "integer" - } - }, - "additionalProperties": false, - "title": "waitUntil", - "description": "Poll until app state at path equals expected value", - "examples": [ - { - "path": "user.name", - "equals": "Jane" - } - ] - }, "args_expectVisible": { "type": "object", "properties": { @@ -1433,49 +1396,45 @@ {} ] }, - "args_mockApi": { + "args_resetApiCalls": { + "type": "object", + "additionalProperties": false, + "title": "resetApiCalls", + "description": "Clear recorded API call history", + "examples": [ + {} + ] + }, + "args_expectApiCalled": { "type": "object", "properties": { "name": { "type": "string" }, - "response": { - "$ref": "#/$defs/mockResponse" - }, - "delayMs": { + "times": { "type": "integer" } }, "required": [ - "name", - "response" + "name" ], "additionalProperties": false, - "title": "mockApi", - "description": "Register a mock HTTP API response by API name", + "title": "expectApiCalled", + "description": "Assert an API was called an exact number of times", "examples": [ { "name": "login", - "response": { - "statusCode": 200, - "body": { - "token": "test-token" - } - } + "times": 1 } ] }, - "args_mockApiError": { + "args_expectApiNotCalled": { "type": "object", "properties": { "name": { "type": "string" }, - "statusCode": { - "type": "integer" - }, - "body": true, - "delayMs": { + "times": { "type": "integer" } }, @@ -1483,1606 +1442,588 @@ "name" ], "additionalProperties": false, - "title": "mockApiError", - "description": "Mock an API to return an error status/body", + "title": "expectApiNotCalled", + "description": "Assert an API was never called", "examples": [ { "name": "login", - "statusCode": 401, - "body": { - "error": "Unauthorized" - } + "times": 1 } ] }, - "args_mockApiFromFixture": { + "args_expectApiCallOrder": { "type": "object", "properties": { - "name": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statusCode": { - "type": "integer" + "names": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 } }, "required": [ - "name", - "fixture" + "names" ], "additionalProperties": false, - "title": "mockApiFromFixture", - "description": "Load mock response body from a JSON fixture asset", + "title": "expectApiCallOrder", + "description": "Assert APIs were called in order", "examples": [ { - "name": "users", - "fixture": "users.json" + "names": [ + "auth", + "profile" + ] } ] }, - "args_mockApiException": { + "args_expectLastApiCall": { "type": "object", "properties": { "name": { "type": "string" }, - "message": { - "type": "string" + "times": { + "type": "integer" } }, "required": [ "name" ], "additionalProperties": false, - "title": "mockApiException", - "description": "Force an API call to throw an exception", + "title": "expectLastApiCall", + "description": "Assert the most recent API call name", "examples": [ { "name": "login", - "message": "Network error" + "times": 1 } ] }, - "args_mockTimeout": { + "args_setStorage": { "type": "object", "properties": { - "name": { + "key": { "type": "string" }, - "delayMs": { - "type": "integer" - } + "equals": true, + "value": true }, "required": [ - "name" + "key" ], "additionalProperties": false, - "title": "mockTimeout", - "description": "Mock an API with a long delay (simulate timeout)", + "title": "setStorage", + "description": "Write a value to public GetStorage by key", "examples": [ { - "name": "slow_api", - "delayMs": 60000 + "key": "onboarding_done", + "value": true } ] }, - "args_mockNetworkOffline": { - "type": "object", - "additionalProperties": false, - "title": "mockNetworkOffline", - "description": "Simulate offline network for API calls", - "examples": [ - {} - ] - }, - "args_mockNetworkOnline": { + "args_expectStorage": { "type": "object", + "properties": { + "key": { + "type": "string" + }, + "equals": true, + "value": true + }, + "required": [ + "key" + ], "additionalProperties": false, - "title": "mockNetworkOnline", - "description": "Restore online network for API calls", + "title": "expectStorage", + "description": "Assert public storage key equals expected", "examples": [ - {} + { + "key": "onboarding_done", + "value": true + } ] }, - "args_resetApiCalls": { + "args_removeStorage": { "type": "object", + "properties": { + "key": { + "type": "string" + }, + "equals": true, + "value": true + }, + "required": [ + "key" + ], "additionalProperties": false, - "title": "resetApiCalls", - "description": "Clear recorded API call history", + "title": "removeStorage", + "description": "Remove a key from public storage", "examples": [ - {} + { + "key": "onboarding_done", + "value": true + } ] }, - "args_clearApiMocks": { + "args_clearStorage": { "type": "object", "additionalProperties": false, - "title": "clearApiMocks", - "description": "Remove all registered API mocks", + "title": "clearStorage", + "description": "Clear all non-encrypted public storage keys", "examples": [ {} ] }, - "args_expectApiCalled": { + "args_setEnv": { "type": "object", "properties": { - "name": { + "key": { "type": "string" }, - "times": { - "type": "integer" - } + "equals": true, + "value": true }, "required": [ - "name" + "key" ], "additionalProperties": false, - "title": "expectApiCalled", - "description": "Assert an API was called an exact number of times", + "title": "setEnv", + "description": "Override an environment variable for the test", "examples": [ { - "name": "login", - "times": 1 + "key": "onboarding_done", + "value": true } ] }, - "args_expectApiNotCalled": { + "args_setAuth": { "type": "object", "properties": { - "name": { - "type": "string" - }, - "times": { - "type": "integer" + "user": { + "type": "object", + "additionalProperties": true } }, "required": [ - "name" + "user" ], "additionalProperties": false, - "title": "expectApiNotCalled", - "description": "Assert an API was never called", + "title": "setAuth", + "description": "Simulate a signed-in user", "examples": [ { - "name": "login", - "times": 1 + "user": { + "id": "1", + "email": "user@test.com" + } } ] }, - "args_expectApiRequest": { + "args_clearAuth": { + "type": "object", + "additionalProperties": false, + "title": "clearAuth", + "description": "Clear the signed-in user", + "examples": [ + {} + ] + }, + "args_setPermission": { "type": "object", "properties": { "name": { "type": "string" }, - "body": true, - "query": true, - "headers": true, - "times": { - "type": "integer" + "value": { + "type": "string" } }, "required": [ "name" ], "additionalProperties": false, - "title": "expectApiRequest", - "description": "Assert last API request body/query/headers match", + "title": "setPermission", + "description": "Set a permission flag for the test runtime", "examples": [ { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } + "name": "camera", + "value": "granted" } ] }, - "args_expectApiRequestContains": { + "args_setDevice": { "type": "object", "properties": { - "name": { - "type": "string" + "width": { + "type": "number" }, - "body": true, - "query": true, - "headers": true, - "times": { - "type": "integer" + "height": { + "type": "number" } }, - "required": [ - "name" - ], "additionalProperties": false, - "title": "expectApiRequestContains", - "description": "Assert API request contains partial body/query", + "title": "setDevice", + "description": "Override viewport physical size (width/height)", "examples": [ { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } + "width": 390, + "height": 844 } ] }, - "args_expectApiHeader": { + "args_setLocale": { "type": "object", "properties": { - "name": { - "type": "string" - }, - "header": { + "locale": { "type": "string" - }, - "equals": true, - "times": { - "type": "integer" - } - }, - "required": [ - "name", - "header", - "equals" - ], - "additionalProperties": false, - "title": "expectApiHeader", - "description": "Assert an API request header equals expected", - "examples": [ - { - "name": "login", - "header": "Authorization", - "equals": "Bearer test-token" - } - ] - }, - "args_expectApiCallOrder": { - "type": "object", - "properties": { - "names": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1 } }, - "required": [ - "names" - ], "additionalProperties": false, - "title": "expectApiCallOrder", - "description": "Assert APIs were called in order", + "title": "setLocale", + "description": "Set APP_LOCALE environment override", "examples": [ { - "names": [ - "auth", - "profile" - ] + "locale": "en_US" } ] }, - "args_expectLastApiCall": { + "args_setTheme": { "type": "object", "properties": { - "name": { + "mode": { "type": "string" }, - "times": { - "type": "integer" + "theme": { + "type": "string" } }, - "required": [ - "name" - ], "additionalProperties": false, - "title": "expectLastApiCall", - "description": "Assert the most recent API call name", + "title": "setTheme", + "description": "Set APP_THEME / theme mode override", "examples": [ { - "name": "login", - "times": 1 + "mode": "dark" } ] }, - "args_setState": { + "args_runScript": { "type": "object", "properties": { + "script": { + "type": "string" + }, "path": { "type": "string" }, - "value": true + "equals": true }, - "required": [ - "path" - ], "additionalProperties": false, - "title": "setState", - "description": "Set app data-context state at path to value", + "title": "runScript", + "description": "Evaluate a script expression in the data context", "examples": [ { - "path": "user.name", - "value": "Jane" + "script": "1 + 1", + "equals": 2 } ] }, - "args_expectState": { + "args_expectScriptResult": { "type": "object", "properties": { + "script": { + "type": "string" + }, "path": { "type": "string" }, - "equals": true, - "contains": true + "equals": true }, - "required": [ - "path" - ], "additionalProperties": false, - "title": "expectState", - "description": "Assert app state at path equals expected", + "title": "expectScriptResult", + "description": "Evaluate script and assert result equals expected", "examples": [ { - "path": "user.name", - "equals": "Jane" + "script": "1 + 1", + "equals": 2 } ] }, - "args_expectStateContains": { + "args_expectConsoleLog": { "type": "object", "properties": { - "path": { + "contains": { "type": "string" - }, - "equals": true, - "contains": true + } }, "required": [ - "path" + "contains" ], "additionalProperties": false, - "title": "expectStateContains", - "description": "Assert app state at path contains subset", + "title": "expectConsoleLog", + "description": "Assert a console log line contains text", "examples": [ { - "path": "user.name", - "equals": "Jane" + "contains": "Screen loaded" } ] }, - "args_expectStateExists": { + "args_group": { "type": "object", "properties": { - "path": { + "name": { "type": "string" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/$defs/step" + }, + "minItems": 1 } }, "required": [ - "path" + "steps" ], "additionalProperties": false, - "title": "expectStateExists", - "description": "Assert state path resolves without error", + "title": "group", + "description": "Run nested steps as a named group", "examples": [ { - "path": "user.id" + "name": "login_flow", + "steps": [ + { + "tap": { + "id": "login_button" + } + } + ] } ] }, - "args_expectStateNotExists": { + "args_repeat": { "type": "object", "properties": { - "path": { - "type": "string" + "times": { + "type": "integer" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/$defs/step" + }, + "minItems": 1 } }, "required": [ - "path" + "times", + "steps" ], "additionalProperties": false, - "title": "expectStateNotExists", - "description": "Assert state path is null or absent", + "title": "repeat", + "description": "Repeat nested steps N times", "examples": [ { - "path": "user.id" + "times": 3, + "steps": [ + { + "tap": { + "id": "next_button" + } + } + ] } ] }, - "args_resetState": { + "args_optional": { "type": "object", "properties": { - "path": { - "type": "string" + "step": { + "$ref": "#/$defs/step" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/$defs/step" + }, + "minItems": 1 } }, "additionalProperties": false, - "title": "resetState", - "description": "Clear state at path (set to null)", + "title": "optional", + "description": "Run nested steps; swallow failures", "examples": [ { - "path": "cart" + "steps": [ + { + "tap": { + "id": "dismiss_banner" + } + } + ] } ] }, - "args_setStorage": { + "args_ifVisible": { "type": "object", "properties": { - "key": { + "id": { "type": "string" }, - "equals": true, - "value": true + "step": { + "$ref": "#/$defs/step" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/$defs/step" + }, + "minItems": 1 + } }, "required": [ - "key" + "id" ], "additionalProperties": false, - "title": "setStorage", - "description": "Write a value to public GetStorage by key", + "title": "ifVisible", + "description": "Run nested steps only if testId is visible", "examples": [ { - "key": "onboarding_done", - "value": true + "id": "promo_banner", + "steps": [ + { + "tap": { + "id": "close_banner" + } + } + ] } ] }, - "args_expectStorage": { + "args_logApiCalls": { "type": "object", - "properties": { - "key": { - "type": "string" - }, - "equals": true, - "value": true - }, - "required": [ - "key" - ], "additionalProperties": false, - "title": "expectStorage", - "description": "Assert public storage key equals expected", + "title": "logApiCalls", + "description": "Log all recorded API calls to the test log", "examples": [ - { - "key": "onboarding_done", - "value": true - } + {} ] }, - "args_removeStorage": { + "args_expectNoConsoleErrors": { "type": "object", - "properties": { - "key": { - "type": "string" - }, - "equals": true, - "value": true - }, - "required": [ - "key" - ], "additionalProperties": false, - "title": "removeStorage", - "description": "Remove a key from public storage", + "title": "expectNoConsoleErrors", + "description": "Assert no console errors were recorded", "examples": [ - { - "key": "onboarding_done", - "value": true - } + {} ] }, - "args_clearStorage": { + "args_expectNoRenderErrors": { "type": "object", "additionalProperties": false, - "title": "clearStorage", - "description": "Clear all non-encrypted public storage keys", + "title": "expectNoRenderErrors", + "description": "Assert no Flutter render errors were recorded", "examples": [ {} ] }, - "args_setEnv": { + "args_expectError": { "type": "object", "properties": { - "key": { + "contains": { "type": "string" - }, - "equals": true, - "value": true + } }, - "required": [ - "key" - ], "additionalProperties": false, - "title": "setEnv", - "description": "Override an environment variable for the test", + "title": "expectError", + "description": "Assert a Flutter error was recorded (optional filter)", "examples": [ { - "key": "onboarding_done", - "value": true + "contains": "overflow" } ] }, - "args_setAuth": { - "type": "object", - "properties": { - "user": { - "type": "object", - "additionalProperties": true - } - }, - "required": [ - "user" - ], - "additionalProperties": false, - "title": "setAuth", - "description": "Simulate a signed-in user", - "examples": [ - { - "user": { - "id": "1", - "email": "user@test.com" - } - } - ] - }, - "args_clearAuth": { - "type": "object", - "additionalProperties": false, - "title": "clearAuth", - "description": "Clear the signed-in user", - "examples": [ - {} - ] - }, - "args_setPermission": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "required": [ - "name" - ], - "additionalProperties": false, - "title": "setPermission", - "description": "Set a permission flag for the test runtime", - "examples": [ - { - "name": "camera", - "value": "granted" - } - ] - }, - "args_setDevice": { - "type": "object", - "properties": { - "width": { - "type": "number" - }, - "height": { - "type": "number" - } - }, - "additionalProperties": false, - "title": "setDevice", - "description": "Override viewport physical size (width/height)", - "examples": [ - { - "width": 390, - "height": 844 - } - ] - }, - "args_setLocale": { - "type": "object", - "properties": { - "locale": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "setLocale", - "description": "Set APP_LOCALE environment override", - "examples": [ - { - "locale": "en_US" - } - ] - }, - "args_setTheme": { - "type": "object", - "properties": { - "mode": { - "type": "string" - }, - "theme": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "setTheme", - "description": "Set APP_THEME / theme mode override", - "examples": [ - { - "mode": "dark" - } - ] - }, - "args_runScript": { - "type": "object", - "properties": { - "script": { - "type": "string" - }, - "path": { - "type": "string" - }, - "equals": true - }, - "additionalProperties": false, - "title": "runScript", - "description": "Evaluate a script expression in the data context", - "examples": [ - { - "script": "1 + 1", - "equals": 2 - } - ] - }, - "args_expectScriptResult": { - "type": "object", - "properties": { - "script": { - "type": "string" - }, - "path": { - "type": "string" - }, - "equals": true - }, - "additionalProperties": false, - "title": "expectScriptResult", - "description": "Evaluate script and assert result equals expected", - "examples": [ - { - "script": "1 + 1", - "equals": 2 - } - ] - }, - "args_expectConsoleLog": { - "type": "object", - "properties": { - "contains": { - "type": "string" - } - }, - "required": [ - "contains" - ], - "additionalProperties": false, - "title": "expectConsoleLog", - "description": "Assert a console log line contains text", - "examples": [ - { - "contains": "Screen loaded" - } - ] - }, - "args_group": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "steps": { - "type": "array", - "items": { - "$ref": "#/$defs/step" - }, - "minItems": 1 - } - }, - "required": [ - "steps" - ], - "additionalProperties": false, - "title": "group", - "description": "Run nested steps as a named group", - "examples": [ - { - "name": "login_flow", - "steps": [ - { - "tap": { - "id": "login_button" - } - } - ] - } - ] - }, - "args_repeat": { - "type": "object", - "properties": { - "times": { - "type": "integer" - }, - "steps": { - "type": "array", - "items": { - "$ref": "#/$defs/step" - }, - "minItems": 1 - } - }, - "required": [ - "times", - "steps" - ], - "additionalProperties": false, - "title": "repeat", - "description": "Repeat nested steps N times", - "examples": [ - { - "times": 3, - "steps": [ - { - "tap": { - "id": "next_button" - } - } - ] - } - ] - }, - "args_optional": { - "type": "object", - "properties": { - "step": { - "$ref": "#/$defs/step" - }, - "steps": { - "type": "array", - "items": { - "$ref": "#/$defs/step" - }, - "minItems": 1 - } - }, - "additionalProperties": false, - "title": "optional", - "description": "Run nested steps; swallow failures", - "examples": [ - { - "steps": [ - { - "tap": { - "id": "dismiss_banner" - } - } - ] - } - ] - }, - "args_ifVisible": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "step": { - "$ref": "#/$defs/step" - }, - "steps": { - "type": "array", - "items": { - "$ref": "#/$defs/step" - }, - "minItems": 1 - } - }, - "required": [ - "id" - ], - "additionalProperties": false, - "title": "ifVisible", - "description": "Run nested steps only if testId is visible", - "examples": [ - { - "id": "promo_banner", - "steps": [ - { - "tap": { - "id": "close_banner" - } - } - ] - } - ] - }, - "args_logApiCalls": { - "type": "object", - "additionalProperties": false, - "title": "logApiCalls", - "description": "Log all recorded API calls to the test log", - "examples": [ - {} - ] - }, - "args_screenshot": { - "type": "object", - "properties": { - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "screenshot", - "description": "Capture golden or dump widget tree for debugging", - "examples": [ - { - "name": "home_screen" - } - ] - }, - "args_dumpTree": { - "type": "object", - "additionalProperties": false, - "title": "dumpTree", - "description": "Print the widget tree to the debug console", - "examples": [ - {} - ] - }, - "args_logState": { - "type": "object", - "properties": { - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "additionalProperties": false, - "title": "logState", - "description": "Log resolved state at path", - "examples": [ - { - "path": "user.id" - } - ] - }, - "args_logStorage": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "equals": true, - "value": true - }, - "required": [ - "key" - ], - "additionalProperties": false, - "title": "logStorage", - "description": "Log public storage value for key", - "examples": [ - { - "key": "onboarding_done", - "value": true - } - ] - }, - "args_expectNoConsoleErrors": { - "type": "object", - "additionalProperties": false, - "title": "expectNoConsoleErrors", - "description": "Assert no console errors were recorded", - "examples": [ - {} - ] - }, - "args_expectNoRenderErrors": { - "type": "object", - "additionalProperties": false, - "title": "expectNoRenderErrors", - "description": "Assert no Flutter render errors were recorded", - "examples": [ - {} - ] - }, - "args_expectError": { - "type": "object", - "properties": { - "contains": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "expectError", - "description": "Assert a Flutter error was recorded (optional filter)", - "examples": [ - { - "contains": "overflow" - } - ] - }, - "args_expectNoErrors": { - "type": "object", - "additionalProperties": false, - "title": "expectNoErrors", - "description": "Alias for expectNoRenderErrors", - "examples": [ - {} - ] - }, - "args_expectAccessible": { - "type": "object", - "properties": { - "id": { - "type": "string" - } - }, - "required": [ - "id" - ], - "additionalProperties": false, - "title": "expectAccessible", - "description": "Assert widget has accessibility label or value", - "examples": [ - { - "id": "my_widget" - } - ] - }, - "args_expectSemanticsLabel": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "required": [ - "id", - "label" - ], - "additionalProperties": false, - "title": "expectSemanticsLabel", - "description": "Assert semantics label equals expected", - "examples": [ - { - "id": "submit_button", - "label": "Submit" - } - ] - }, - "args_expectNoOverflow": { - "type": "object", - "properties": { - "id": { - "type": "string" - } - }, - "required": [ - "id" - ], - "additionalProperties": false, - "title": "expectNoOverflow", - "description": "Assert widget renders without overflow issues", - "examples": [ - { - "id": "my_widget" - } - ] - }, - "args_loadFixture": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "path": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statePath": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "loadFixture", - "description": "Load a JSON fixture into the test fixture map", - "examples": [ - { - "fixture": "user.json" - } - ] - }, - "args_setStateFromFixture": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "path": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statePath": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "setStateFromFixture", - "description": "Apply all keys from a JSON fixture to state", - "examples": [ - { - "fixture": "user.json" - } - ] - }, - "args_expectMatchesFixture": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "path": { - "type": "string" - }, - "fixture": { - "type": "string" - }, - "statePath": { - "type": "string" - } - }, - "additionalProperties": false, - "title": "expectMatchesFixture", - "description": "Assert state or path matches a JSON fixture", - "examples": [ - { - "fixture": "user.json" - } - ] - }, - "step": { - "oneOf": [ - { - "type": "object", - "title": "openScreen", - "description": "Navigate to another screen by name or id mid-test", - "examples": [ - { - "openScreen": { - "screen": "Home" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "openScreen": { - "$ref": "#/$defs/args_openScreen", - "description": "Navigate to another screen by name or id mid-test", - "examples": [ - { - "screen": "Home" - } - ] - } - }, - "required": [ - "openScreen" - ] - }, - { - "type": "object", - "title": "reloadScreen", - "description": "Reload the current screen (same as re-opening it)", - "examples": [ - { - "reloadScreen": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "reloadScreen": { - "$ref": "#/$defs/args_reloadScreen", - "description": "Reload the current screen (same as re-opening it)", - "examples": [ - {} - ] - } - }, - "required": [ - "reloadScreen" - ] - }, - { - "type": "object", - "title": "restartApp", - "description": "Reset runtime and reopen the test case start screen", - "examples": [ - { - "restartApp": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "restartApp": { - "$ref": "#/$defs/args_restartApp", - "description": "Reset runtime and reopen the test case start screen", - "examples": [ - {} - ] - } - }, - "required": [ - "restartApp" - ] - }, - { - "type": "object", - "title": "resetAppState", - "description": "Clear screen tracker, API call log, and public storage", - "examples": [ - { - "resetAppState": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "resetAppState": { - "$ref": "#/$defs/args_resetAppState", - "description": "Clear screen tracker, API call log, and public storage", - "examples": [ - {} - ] - } - }, - "required": [ - "resetAppState" - ] - }, - { - "type": "object", - "title": "trigger", - "description": "Fire a widget action (onLoad, onTap, onLongPress) by testId", - "examples": [ - { - "trigger": { - "action": "onTap", - "id": "submit_button" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "trigger": { - "$ref": "#/$defs/args_trigger", - "description": "Fire a widget action (onLoad, onTap, onLongPress) by testId", - "examples": [ - { - "action": "onTap", - "id": "submit_button" - } - ] - } - }, - "required": [ - "trigger" - ] - }, - { - "type": "object", - "title": "launchApp", - "description": "Alias for restartApp — bootstrap from startScreen again", - "examples": [ - { - "launchApp": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "launchApp": { - "$ref": "#/$defs/args_launchApp", - "description": "Alias for restartApp — bootstrap from startScreen again", - "examples": [ - {} - ] - } - }, - "required": [ - "launchApp" - ] - }, - { - "type": "object", - "title": "tap", - "description": "Tap a widget by testId (ValueKey)", - "examples": [ - { - "tap": { - "id": "my_widget" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "tap": { - "$ref": "#/$defs/args_tap", - "description": "Tap a widget by testId (ValueKey)", - "examples": [ - { - "id": "my_widget" - } - ] - } - }, - "required": [ - "tap" - ] - }, - { - "type": "object", - "title": "doubleTap", - "description": "Double-tap a widget by testId", - "examples": [ - { - "doubleTap": { - "id": "my_widget" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "doubleTap": { - "$ref": "#/$defs/args_doubleTap", - "description": "Double-tap a widget by testId", - "examples": [ - { - "id": "my_widget" - } - ] - } - }, - "required": [ - "doubleTap" - ] - }, - { - "type": "object", - "title": "longPress", - "description": "Long-press a widget by testId", - "examples": [ - { - "longPress": { - "id": "my_widget" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "longPress": { - "$ref": "#/$defs/args_longPress", - "description": "Long-press a widget by testId", - "examples": [ - { - "id": "my_widget" - } - ] - } - }, - "required": [ - "longPress" - ] - }, - { - "type": "object", - "title": "enterText", - "description": "Type text into an input field by testId", - "examples": [ - { - "enterText": { - "id": "email_field", - "value": "user@test.com" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "enterText": { - "$ref": "#/$defs/args_enterText", - "description": "Type text into an input field by testId", - "examples": [ - { - "id": "email_field", - "value": "user@test.com" - } - ] - } - }, - "required": [ - "enterText" - ] - }, - { - "type": "object", - "title": "clearText", - "description": "Clear text in an input field by testId", - "examples": [ - { - "clearText": { - "id": "my_widget" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "clearText": { - "$ref": "#/$defs/args_clearText", - "description": "Clear text in an input field by testId", - "examples": [ - { - "id": "my_widget" - } - ] - } - }, - "required": [ - "clearText" - ] - }, - { - "type": "object", - "title": "replaceText", - "description": "Replace the full contents of an input field", - "examples": [ - { - "replaceText": { - "id": "email_field", - "value": "user@test.com" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "replaceText": { - "$ref": "#/$defs/args_replaceText", - "description": "Replace the full contents of an input field", - "examples": [ - { - "id": "email_field", - "value": "user@test.com" - } - ] - } - }, - "required": [ - "replaceText" - ] - }, - { - "type": "object", - "title": "submitText", - "description": "Submit an input field (TextInputAction.done)", - "examples": [ - { - "submitText": { - "id": "my_widget" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "submitText": { - "$ref": "#/$defs/args_submitText", - "description": "Submit an input field (TextInputAction.done)", - "examples": [ - { - "id": "my_widget" - } - ] - } - }, - "required": [ - "submitText" - ] - }, - { - "type": "object", - "title": "focus", - "description": "Focus an input field by testId", - "examples": [ - { - "focus": { - "id": "my_widget" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "focus": { - "$ref": "#/$defs/args_focus", - "description": "Focus an input field by testId", - "examples": [ - { - "id": "my_widget" - } - ] - } - }, - "required": [ - "focus" - ] - }, - { - "type": "object", - "title": "unfocus", - "description": "Remove focus from the current field", - "examples": [ - { - "unfocus": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "unfocus": { - "$ref": "#/$defs/args_unfocus", - "description": "Remove focus from the current field", - "examples": [ - {} - ] - } - }, - "required": [ - "unfocus" - ] - }, + "args_expectNoErrors": { + "type": "object", + "additionalProperties": false, + "title": "expectNoErrors", + "description": "Alias for expectNoRenderErrors", + "examples": [ + {} + ] + }, + "args_expectAccessible": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectAccessible", + "description": "Assert widget has accessibility label or value", + "examples": [ { - "type": "object", - "title": "select", - "description": "Open a dropdown and choose an option by visible label", - "examples": [ - { - "select": { - "id": "country_dropdown", - "value": "USA" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "select": { - "$ref": "#/$defs/args_select", - "description": "Open a dropdown and choose an option by visible label", - "examples": [ - { - "id": "country_dropdown", - "value": "USA" - } - ] - } - }, - "required": [ - "select" - ] + "id": "my_widget" + } + ] + }, + "args_expectSemanticsLabel": { + "type": "object", + "properties": { + "id": { + "type": "string" }, + "label": { + "type": "string" + } + }, + "required": [ + "id", + "label" + ], + "additionalProperties": false, + "title": "expectSemanticsLabel", + "description": "Assert semantics label equals expected", + "examples": [ { - "type": "object", - "title": "selectIndex", - "description": "Open a dropdown and choose the option at index", - "examples": [ - { - "selectIndex": { - "id": "country_dropdown", - "index": 0 - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "selectIndex": { - "$ref": "#/$defs/args_selectIndex", - "description": "Open a dropdown and choose the option at index", - "examples": [ - { - "id": "country_dropdown", - "index": 0 - } - ] - } - }, - "required": [ - "selectIndex" - ] - }, + "id": "submit_button", + "label": "Submit" + } + ] + }, + "args_expectNoOverflow": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectNoOverflow", + "description": "Assert widget renders without overflow issues", + "examples": [ { - "type": "object", - "title": "check", - "description": "Check a checkbox or toggle by testId", - "examples": [ - { - "check": { - "id": "my_widget" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "check": { - "$ref": "#/$defs/args_check", - "description": "Check a checkbox or toggle by testId", - "examples": [ - { - "id": "my_widget" - } - ] - } - }, - "required": [ - "check" - ] - }, + "id": "my_widget" + } + ] + }, + "step": { + "oneOf": [ { "type": "object", - "title": "uncheck", - "description": "Uncheck a checkbox by testId if currently checked", + "title": "openScreen", + "description": "Navigate to another screen by name or id mid-test", "examples": [ { - "uncheck": { - "id": "my_widget" + "openScreen": { + "screen": "Home" } } ], @@ -3090,120 +2031,104 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "uncheck": { - "$ref": "#/$defs/args_uncheck", - "description": "Uncheck a checkbox by testId if currently checked", + "openScreen": { + "$ref": "#/$defs/args_openScreen", + "description": "Navigate to another screen by name or id mid-test", "examples": [ { - "id": "my_widget" + "screen": "Home" } ] } }, "required": [ - "uncheck" + "openScreen" ] }, { "type": "object", - "title": "toggle", - "description": "Tap to toggle a switch or checkbox by testId", + "title": "reloadScreen", + "description": "Reload the current screen (same as re-opening it)", "examples": [ { - "toggle": { - "id": "my_widget" - } + "reloadScreen": {} } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "toggle": { - "$ref": "#/$defs/args_toggle", - "description": "Tap to toggle a switch or checkbox by testId", + "reloadScreen": { + "$ref": "#/$defs/args_reloadScreen", + "description": "Reload the current screen (same as re-opening it)", "examples": [ - { - "id": "my_widget" - } + {} ] } }, "required": [ - "toggle" + "reloadScreen" ] }, { "type": "object", - "title": "setSlider", - "description": "Move a slider under testId to a normalized value (0–1)", + "title": "restartApp", + "description": "Reset runtime and reopen the test case start screen", "examples": [ { - "setSlider": { - "id": "volume_slider", - "value": 0.5 - } + "restartApp": {} } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "setSlider": { - "$ref": "#/$defs/args_setSlider", - "description": "Move a slider under testId to a normalized value (0–1)", + "restartApp": { + "$ref": "#/$defs/args_restartApp", + "description": "Reset runtime and reopen the test case start screen", "examples": [ - { - "id": "volume_slider", - "value": 0.5 - } + {} ] } }, "required": [ - "setSlider" + "restartApp" ] }, { "type": "object", - "title": "chooseDate", - "description": "Set a date field by testId to the given value string", + "title": "resetAppState", + "description": "Clear screen tracker, API call log, and public storage", "examples": [ { - "chooseDate": { - "id": "birth_date", - "value": "2024-01-15" - } + "resetAppState": {} } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "chooseDate": { - "$ref": "#/$defs/args_chooseDate", - "description": "Set a date field by testId to the given value string", + "resetAppState": { + "$ref": "#/$defs/args_resetAppState", + "description": "Clear screen tracker, API call log, and public storage", "examples": [ - { - "id": "birth_date", - "value": "2024-01-15" - } + {} ] } }, "required": [ - "chooseDate" + "resetAppState" ] }, { "type": "object", - "title": "chooseTime", - "description": "Set a time field by testId to the given value string", + "title": "trigger", + "description": "Fire a widget action (onLoad, onTap, onLongPress) by testId", "examples": [ { - "chooseTime": { - "id": "birth_date", - "value": "2024-01-15" + "trigger": { + "action": "onTap", + "id": "submit_button" } } ], @@ -3211,57 +2136,53 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "chooseTime": { - "$ref": "#/$defs/args_chooseTime", - "description": "Set a time field by testId to the given value string", + "trigger": { + "$ref": "#/$defs/args_trigger", + "description": "Fire a widget action (onLoad, onTap, onLongPress) by testId", "examples": [ { - "id": "birth_date", - "value": "2024-01-15" + "action": "onTap", + "id": "submit_button" } ] } }, "required": [ - "chooseTime" + "trigger" ] }, { "type": "object", - "title": "scroll", - "description": "Drag the first Scrollable by delta pixels", + "title": "launchApp", + "description": "Alias for restartApp — bootstrap from startScreen again", "examples": [ { - "scroll": { - "delta": 300 - } + "launchApp": {} } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "scroll": { - "$ref": "#/$defs/args_scroll", - "description": "Drag the first Scrollable by delta pixels", + "launchApp": { + "$ref": "#/$defs/args_launchApp", + "description": "Alias for restartApp — bootstrap from startScreen again", "examples": [ - { - "delta": 300 - } + {} ] } }, "required": [ - "scroll" + "launchApp" ] }, { "type": "object", - "title": "scrollUntilVisible", - "description": "Scroll until a widget with testId is visible", + "title": "tap", + "description": "Tap a widget by testId (ValueKey)", "examples": [ { - "scrollUntilVisible": { + "tap": { "id": "my_widget" } } @@ -3270,9 +2191,9 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "scrollUntilVisible": { - "$ref": "#/$defs/args_scrollUntilVisible", - "description": "Scroll until a widget with testId is visible", + "tap": { + "$ref": "#/$defs/args_tap", + "description": "Tap a widget by testId (ValueKey)", "examples": [ { "id": "my_widget" @@ -3281,18 +2202,17 @@ } }, "required": [ - "scrollUntilVisible" + "tap" ] }, { "type": "object", - "title": "swipe", - "description": "Swipe on a scrollable or widget (direction: left/right/up/down)", + "title": "doubleTap", + "description": "Double-tap a widget by testId", "examples": [ { - "swipe": { - "direction": "left", - "id": "carousel" + "doubleTap": { + "id": "my_widget" } } ], @@ -3300,31 +2220,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "swipe": { - "$ref": "#/$defs/args_swipe", - "description": "Swipe on a scrollable or widget (direction: left/right/up/down)", + "doubleTap": { + "$ref": "#/$defs/args_doubleTap", + "description": "Double-tap a widget by testId", "examples": [ { - "direction": "left", - "id": "carousel" + "id": "my_widget" } ] } }, "required": [ - "swipe" + "doubleTap" ] }, { "type": "object", - "title": "drag", - "description": "Drag a widget by testId by dx/dy offset", + "title": "longPress", + "description": "Long-press a widget by testId", "examples": [ { - "drag": { - "id": "handle", - "dx": 50, - "dy": 0 + "longPress": { + "id": "my_widget" } } ], @@ -3332,30 +2249,29 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "drag": { - "$ref": "#/$defs/args_drag", - "description": "Drag a widget by testId by dx/dy offset", + "longPress": { + "$ref": "#/$defs/args_longPress", + "description": "Long-press a widget by testId", "examples": [ { - "id": "handle", - "dx": 50, - "dy": 0 + "id": "my_widget" } ] } }, "required": [ - "drag" + "longPress" ] }, { "type": "object", - "title": "pullToRefresh", - "description": "Pull down on a scrollable to trigger refresh", + "title": "enterText", + "description": "Type text into an input field by testId", "examples": [ { - "pullToRefresh": { - "id": "scroll_view" + "enterText": { + "id": "email_field", + "value": "user@test.com" } } ], @@ -3363,28 +2279,29 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "pullToRefresh": { - "$ref": "#/$defs/args_pullToRefresh", - "description": "Pull down on a scrollable to trigger refresh", + "enterText": { + "$ref": "#/$defs/args_enterText", + "description": "Type text into an input field by testId", "examples": [ { - "id": "scroll_view" + "id": "email_field", + "value": "user@test.com" } ] } }, "required": [ - "pullToRefresh" + "enterText" ] }, { "type": "object", - "title": "wait", - "description": "Alias for pump — advance frame clock by durationMs", + "title": "clearText", + "description": "Clear text in an input field by testId", "examples": [ { - "wait": { - "durationMs": 100 + "clearText": { + "id": "my_widget" } } ], @@ -3392,28 +2309,29 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "wait": { - "$ref": "#/$defs/args_wait", - "description": "Alias for pump — advance frame clock by durationMs", + "clearText": { + "$ref": "#/$defs/args_clearText", + "description": "Clear text in an input field by testId", "examples": [ { - "durationMs": 100 + "id": "my_widget" } ] } }, "required": [ - "wait" + "clearText" ] }, { "type": "object", - "title": "pump", - "description": "Advance the Flutter frame clock by durationMs", + "title": "replaceText", + "description": "Replace the full contents of an input field", "examples": [ { - "pump": { - "durationMs": 100 + "replaceText": { + "id": "email_field", + "value": "user@test.com" } } ], @@ -3421,28 +2339,29 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "pump": { - "$ref": "#/$defs/args_pump", - "description": "Advance the Flutter frame clock by durationMs", + "replaceText": { + "$ref": "#/$defs/args_replaceText", + "description": "Replace the full contents of an input field", "examples": [ { - "durationMs": 100 + "id": "email_field", + "value": "user@test.com" } ] } }, "required": [ - "pump" + "replaceText" ] }, { "type": "object", - "title": "settle", - "description": "Run pumpAndSettle until idle or timeout", + "title": "submitText", + "description": "Submit an input field (TextInputAction.done)", "examples": [ { - "settle": { - "timeoutMs": 5000 + "submitText": { + "id": "my_widget" } } ], @@ -3450,29 +2369,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "settle": { - "$ref": "#/$defs/args_settle", - "description": "Run pumpAndSettle until idle or timeout", + "submitText": { + "$ref": "#/$defs/args_submitText", + "description": "Submit an input field (TextInputAction.done)", "examples": [ { - "timeoutMs": 5000 + "id": "my_widget" } ] } }, "required": [ - "settle" + "submitText" ] }, { "type": "object", - "title": "waitFor", - "description": "Poll until a widget id and/or text appears", + "title": "focus", + "description": "Focus an input field by testId", "examples": [ { - "waitFor": { - "id": "loading_spinner", - "timeoutMs": 5000 + "focus": { + "id": "my_widget" } } ], @@ -3480,61 +2398,54 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "waitFor": { - "$ref": "#/$defs/args_waitFor", - "description": "Poll until a widget id and/or text appears", + "focus": { + "$ref": "#/$defs/args_focus", + "description": "Focus an input field by testId", "examples": [ { - "id": "loading_spinner", - "timeoutMs": 5000 + "id": "my_widget" } ] } }, "required": [ - "waitFor" + "focus" ] }, { "type": "object", - "title": "waitForText", - "description": "Poll until the given text appears on screen", + "title": "unfocus", + "description": "Remove focus from the current field", "examples": [ { - "waitForText": { - "id": "loading_spinner", - "timeoutMs": 5000 - } + "unfocus": {} } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "waitForText": { - "$ref": "#/$defs/args_waitForText", - "description": "Poll until the given text appears on screen", + "unfocus": { + "$ref": "#/$defs/args_unfocus", + "description": "Remove focus from the current field", "examples": [ - { - "id": "loading_spinner", - "timeoutMs": 5000 - } + {} ] } }, "required": [ - "waitForText" + "unfocus" ] }, { "type": "object", - "title": "waitForGone", - "description": "Poll until a widget with testId is removed from the tree", + "title": "select", + "description": "Open a dropdown and choose an option by visible label", "examples": [ { - "waitForGone": { - "id": "loading_spinner", - "timeoutMs": 5000 + "select": { + "id": "country_dropdown", + "value": "USA" } } ], @@ -3542,30 +2453,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "waitForGone": { - "$ref": "#/$defs/args_waitForGone", - "description": "Poll until a widget with testId is removed from the tree", + "select": { + "$ref": "#/$defs/args_select", + "description": "Open a dropdown and choose an option by visible label", "examples": [ { - "id": "loading_spinner", - "timeoutMs": 5000 + "id": "country_dropdown", + "value": "USA" } ] } }, "required": [ - "waitForGone" + "select" ] }, { "type": "object", - "title": "waitForApi", - "description": "Poll until a mocked API is called N times", + "title": "selectIndex", + "description": "Open a dropdown and choose the option at index", "examples": [ { - "waitForApi": { - "name": "login", - "times": 1 + "selectIndex": { + "id": "country_dropdown", + "index": 0 } } ], @@ -3573,30 +2484,29 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "waitForApi": { - "$ref": "#/$defs/args_waitForApi", - "description": "Poll until a mocked API is called N times", + "selectIndex": { + "$ref": "#/$defs/args_selectIndex", + "description": "Open a dropdown and choose the option at index", "examples": [ { - "name": "login", - "times": 1 + "id": "country_dropdown", + "index": 0 } ] } }, "required": [ - "waitForApi" + "selectIndex" ] }, { "type": "object", - "title": "waitForNavigation", - "description": "Poll until the given screen is visible", + "title": "check", + "description": "Check a checkbox or toggle by testId", "examples": [ { - "waitForNavigation": { - "screen": "Home", - "timeoutMs": 5000 + "check": { + "id": "my_widget" } } ], @@ -3604,30 +2514,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "waitForNavigation": { - "$ref": "#/$defs/args_waitForNavigation", - "description": "Poll until the given screen is visible", + "check": { + "$ref": "#/$defs/args_check", + "description": "Check a checkbox or toggle by testId", "examples": [ { - "screen": "Home", - "timeoutMs": 5000 + "id": "my_widget" } ] } }, "required": [ - "waitForNavigation" + "check" ] }, { "type": "object", - "title": "waitUntil", - "description": "Poll until app state at path equals expected value", + "title": "uncheck", + "description": "Uncheck a checkbox by testId if currently checked", "examples": [ { - "waitUntil": { - "path": "user.name", - "equals": "Jane" + "uncheck": { + "id": "my_widget" } } ], @@ -3635,28 +2543,27 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "waitUntil": { - "$ref": "#/$defs/args_waitUntil", - "description": "Poll until app state at path equals expected value", + "uncheck": { + "$ref": "#/$defs/args_uncheck", + "description": "Uncheck a checkbox by testId if currently checked", "examples": [ { - "path": "user.name", - "equals": "Jane" + "id": "my_widget" } ] } }, "required": [ - "waitUntil" + "uncheck" ] }, { "type": "object", - "title": "expectVisible", - "description": "Assert a widget with testId is visible", + "title": "toggle", + "description": "Tap to toggle a switch or checkbox by testId", "examples": [ { - "expectVisible": { + "toggle": { "id": "my_widget" } } @@ -3665,9 +2572,9 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectVisible": { - "$ref": "#/$defs/args_expectVisible", - "description": "Assert a widget with testId is visible", + "toggle": { + "$ref": "#/$defs/args_toggle", + "description": "Tap to toggle a switch or checkbox by testId", "examples": [ { "id": "my_widget" @@ -3676,17 +2583,18 @@ } }, "required": [ - "expectVisible" + "toggle" ] }, { "type": "object", - "title": "expectNotVisible", - "description": "Assert a widget with testId is not visible", + "title": "setSlider", + "description": "Move a slider under testId to a normalized value (0–1)", "examples": [ { - "expectNotVisible": { - "id": "my_widget" + "setSlider": { + "id": "volume_slider", + "value": 0.5 } } ], @@ -3694,28 +2602,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectNotVisible": { - "$ref": "#/$defs/args_expectNotVisible", - "description": "Assert a widget with testId is not visible", + "setSlider": { + "$ref": "#/$defs/args_setSlider", + "description": "Move a slider under testId to a normalized value (0–1)", "examples": [ { - "id": "my_widget" + "id": "volume_slider", + "value": 0.5 } ] } }, "required": [ - "expectNotVisible" + "setSlider" ] }, { "type": "object", - "title": "expectExists", - "description": "Assert a widget with testId exists in the tree", + "title": "chooseDate", + "description": "Set a date field by testId to the given value string", "examples": [ { - "expectExists": { - "id": "my_widget" + "chooseDate": { + "id": "birth_date", + "value": "2024-01-15" } } ], @@ -3723,28 +2633,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectExists": { - "$ref": "#/$defs/args_expectExists", - "description": "Assert a widget with testId exists in the tree", + "chooseDate": { + "$ref": "#/$defs/args_chooseDate", + "description": "Set a date field by testId to the given value string", "examples": [ { - "id": "my_widget" + "id": "birth_date", + "value": "2024-01-15" } ] } }, "required": [ - "expectExists" + "chooseDate" ] }, { "type": "object", - "title": "expectNotExists", - "description": "Assert no widget with testId exists", + "title": "chooseTime", + "description": "Set a time field by testId to the given value string", "examples": [ { - "expectNotExists": { - "id": "my_widget" + "chooseTime": { + "id": "birth_date", + "value": "2024-01-15" } } ], @@ -3752,28 +2664,29 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectNotExists": { - "$ref": "#/$defs/args_expectNotExists", - "description": "Assert no widget with testId exists", + "chooseTime": { + "$ref": "#/$defs/args_chooseTime", + "description": "Set a time field by testId to the given value string", "examples": [ { - "id": "my_widget" + "id": "birth_date", + "value": "2024-01-15" } ] } }, "required": [ - "expectNotExists" + "chooseTime" ] }, { "type": "object", - "title": "expectText", - "description": "Assert exact text is shown", + "title": "scroll", + "description": "Drag the first Scrollable by delta pixels", "examples": [ { - "expectText": { - "text": "Welcome" + "scroll": { + "delta": 300 } } ], @@ -3781,28 +2694,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectText": { - "$ref": "#/$defs/args_expectText", - "description": "Assert exact text is shown", + "scroll": { + "$ref": "#/$defs/args_scroll", + "description": "Drag the first Scrollable by delta pixels", "examples": [ { - "text": "Welcome" + "delta": 300 } ] } }, "required": [ - "expectText" + "scroll" ] }, { "type": "object", - "title": "expectNoText", - "description": "Assert text is not shown", + "title": "scrollUntilVisible", + "description": "Scroll until a widget with testId is visible", "examples": [ { - "expectNoText": { - "text": "Welcome" + "scrollUntilVisible": { + "id": "my_widget" } } ], @@ -3810,28 +2723,29 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectNoText": { - "$ref": "#/$defs/args_expectNoText", - "description": "Assert text is not shown", + "scrollUntilVisible": { + "$ref": "#/$defs/args_scrollUntilVisible", + "description": "Scroll until a widget with testId is visible", "examples": [ { - "text": "Welcome" + "id": "my_widget" } ] } }, "required": [ - "expectNoText" + "scrollUntilVisible" ] }, { "type": "object", - "title": "expectTextContains", - "description": "Assert some text containing the given substring", + "title": "swipe", + "description": "Swipe on a scrollable or widget (direction: left/right/up/down)", "examples": [ { - "expectTextContains": { - "text": "Welcome" + "swipe": { + "direction": "left", + "id": "carousel" } } ], @@ -3839,28 +2753,31 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectTextContains": { - "$ref": "#/$defs/args_expectTextContains", - "description": "Assert some text containing the given substring", + "swipe": { + "$ref": "#/$defs/args_swipe", + "description": "Swipe on a scrollable or widget (direction: left/right/up/down)", "examples": [ { - "text": "Welcome" + "direction": "left", + "id": "carousel" } ] } }, "required": [ - "expectTextContains" + "swipe" ] }, { "type": "object", - "title": "expectEnabled", - "description": "Assert widget semantics report enabled", + "title": "drag", + "description": "Drag a widget by testId by dx/dy offset", "examples": [ { - "expectEnabled": { - "id": "my_widget" + "drag": { + "id": "handle", + "dx": 50, + "dy": 0 } } ], @@ -3868,28 +2785,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectEnabled": { - "$ref": "#/$defs/args_expectEnabled", - "description": "Assert widget semantics report enabled", + "drag": { + "$ref": "#/$defs/args_drag", + "description": "Drag a widget by testId by dx/dy offset", "examples": [ { - "id": "my_widget" + "id": "handle", + "dx": 50, + "dy": 0 } ] } }, "required": [ - "expectEnabled" + "drag" ] }, { "type": "object", - "title": "expectDisabled", - "description": "Assert widget semantics report disabled", + "title": "pullToRefresh", + "description": "Pull down on a scrollable to trigger refresh", "examples": [ { - "expectDisabled": { - "id": "my_widget" + "pullToRefresh": { + "id": "scroll_view" } } ], @@ -3897,29 +2816,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectDisabled": { - "$ref": "#/$defs/args_expectDisabled", - "description": "Assert widget semantics report disabled", + "pullToRefresh": { + "$ref": "#/$defs/args_pullToRefresh", + "description": "Pull down on a scrollable to trigger refresh", "examples": [ { - "id": "my_widget" + "id": "scroll_view" } ] } }, "required": [ - "expectDisabled" + "pullToRefresh" ] }, { "type": "object", - "title": "expectValue", - "description": "Assert input value equals expected (EditableText/TextField)", + "title": "wait", + "description": "Real-time delay using runAsync, followed by a frame pump", "examples": [ { - "expectValue": { - "id": "email_field", - "equals": "user@test.com" + "wait": { + "durationMs": 100 } } ], @@ -3927,30 +2845,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectValue": { - "$ref": "#/$defs/args_expectValue", - "description": "Assert input value equals expected (EditableText/TextField)", + "wait": { + "$ref": "#/$defs/args_wait", + "description": "Real-time delay using runAsync, followed by a frame pump", "examples": [ { - "id": "email_field", - "equals": "user@test.com" + "durationMs": 100 } ] } }, "required": [ - "expectValue" + "wait" ] }, { "type": "object", - "title": "expectChecked", - "description": "Assert checkbox checked state matches equals", + "title": "pump", + "description": "Advance the Flutter frame clock by durationMs", "examples": [ { - "expectChecked": { - "id": "terms_checkbox", - "equals": true + "pump": { + "durationMs": 100 } } ], @@ -3958,31 +2874,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectChecked": { - "$ref": "#/$defs/args_expectChecked", - "description": "Assert checkbox checked state matches equals", + "pump": { + "$ref": "#/$defs/args_pump", + "description": "Advance the Flutter frame clock by durationMs", "examples": [ { - "id": "terms_checkbox", - "equals": true + "durationMs": 100 } ] } }, "required": [ - "expectChecked" + "pump" ] }, { "type": "object", - "title": "expectProperty", - "description": "Assert a widget property (e.g. label) equals expected", + "title": "settle", + "description": "Run pumpAndSettle until idle or timeout", "examples": [ { - "expectProperty": { - "id": "title", - "property": "label", - "equals": "Hello" + "settle": { + "timeoutMs": 5000 } } ], @@ -3990,32 +2903,29 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectProperty": { - "$ref": "#/$defs/args_expectProperty", - "description": "Assert a widget property (e.g. label) equals expected", - "examples": [ - { - "id": "title", - "property": "label", - "equals": "Hello" + "settle": { + "$ref": "#/$defs/args_settle", + "description": "Run pumpAndSettle until idle or timeout", + "examples": [ + { + "timeoutMs": 5000 } ] } }, "required": [ - "expectProperty" + "settle" ] }, { "type": "object", - "title": "expectStyle", - "description": "Assert style-related property equals expected", + "title": "waitFor", + "description": "Poll until a widget id and/or text appears", "examples": [ { - "expectStyle": { - "id": "title", - "property": "label", - "equals": "Hello" + "waitFor": { + "id": "loading_spinner", + "timeoutMs": 5000 } } ], @@ -4023,31 +2933,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectStyle": { - "$ref": "#/$defs/args_expectStyle", - "description": "Assert style-related property equals expected", + "waitFor": { + "$ref": "#/$defs/args_waitFor", + "description": "Poll until a widget id and/or text appears", "examples": [ { - "id": "title", - "property": "label", - "equals": "Hello" + "id": "loading_spinner", + "timeoutMs": 5000 } ] } }, "required": [ - "expectStyle" + "waitFor" ] }, { "type": "object", - "title": "expectSelected", - "description": "Assert selected/checked state matches equals", + "title": "waitForText", + "description": "Poll until the given text appears on screen", "examples": [ { - "expectSelected": { - "id": "terms_checkbox", - "equals": true + "waitForText": { + "id": "loading_spinner", + "timeoutMs": 5000 } } ], @@ -4055,30 +2964,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectSelected": { - "$ref": "#/$defs/args_expectSelected", - "description": "Assert selected/checked state matches equals", + "waitForText": { + "$ref": "#/$defs/args_waitForText", + "description": "Poll until the given text appears on screen", "examples": [ { - "id": "terms_checkbox", - "equals": true + "id": "loading_spinner", + "timeoutMs": 5000 } ] } }, "required": [ - "expectSelected" + "waitForText" ] }, { "type": "object", - "title": "expectCount", - "description": "Assert count of widgets with the same testId", + "title": "waitForGone", + "description": "Poll until a widget with testId is removed from the tree", "examples": [ { - "expectCount": { - "id": "badge", - "equals": 2 + "waitForGone": { + "id": "loading_spinner", + "timeoutMs": 5000 } } ], @@ -4086,30 +2995,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectCount": { - "$ref": "#/$defs/args_expectCount", - "description": "Assert count of widgets with the same testId", + "waitForGone": { + "$ref": "#/$defs/args_waitForGone", + "description": "Poll until a widget with testId is removed from the tree", "examples": [ { - "id": "badge", - "equals": 2 + "id": "loading_spinner", + "timeoutMs": 5000 } ] } }, "required": [ - "expectCount" + "waitForGone" ] }, { "type": "object", - "title": "expectListCount", - "description": "Assert number of list items under a list testId", + "title": "waitForApi", + "description": "Poll until a mocked API is called N times", "examples": [ { - "expectListCount": { - "id": "items_list", - "equals": 3 + "waitForApi": { + "name": "login", + "times": 1 } } ], @@ -4117,30 +3026,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectListCount": { - "$ref": "#/$defs/args_expectListCount", - "description": "Assert number of list items under a list testId", + "waitForApi": { + "$ref": "#/$defs/args_waitForApi", + "description": "Poll until a mocked API is called N times", "examples": [ { - "id": "items_list", - "equals": 3 + "name": "login", + "times": 1 } ] } }, "required": [ - "expectListCount" + "waitForApi" ] }, { "type": "object", - "title": "expectListContains", - "description": "Assert list contains text", + "title": "waitForNavigation", + "description": "Poll until the given screen is visible", "examples": [ { - "expectListContains": { - "id": "items_list", - "text": "Item 1" + "waitForNavigation": { + "screen": "Home", + "timeoutMs": 5000 } } ], @@ -4148,29 +3057,29 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectListContains": { - "$ref": "#/$defs/args_expectListContains", - "description": "Assert list contains text", + "waitForNavigation": { + "$ref": "#/$defs/args_waitForNavigation", + "description": "Poll until the given screen is visible", "examples": [ { - "id": "items_list", - "text": "Item 1" + "screen": "Home", + "timeoutMs": 5000 } ] } }, "required": [ - "expectListContains" + "waitForNavigation" ] }, { "type": "object", - "title": "expectListItem", - "description": "Assert a list item widget with itemId is visible", + "title": "expectVisible", + "description": "Assert a widget with testId is visible", "examples": [ { - "expectListItem": { - "itemId": "row_0" + "expectVisible": { + "id": "my_widget" } } ], @@ -4178,27 +3087,27 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectListItem": { - "$ref": "#/$defs/args_expectListItem", - "description": "Assert a list item widget with itemId is visible", + "expectVisible": { + "$ref": "#/$defs/args_expectVisible", + "description": "Assert a widget with testId is visible", "examples": [ { - "itemId": "row_0" + "id": "my_widget" } ] } }, "required": [ - "expectListItem" + "expectVisible" ] }, { "type": "object", - "title": "expectEmpty", - "description": "Assert a list has zero items", + "title": "expectNotVisible", + "description": "Assert a widget with testId is not visible", "examples": [ { - "expectEmpty": { + "expectNotVisible": { "id": "my_widget" } } @@ -4207,9 +3116,9 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectEmpty": { - "$ref": "#/$defs/args_expectEmpty", - "description": "Assert a list has zero items", + "expectNotVisible": { + "$ref": "#/$defs/args_expectNotVisible", + "description": "Assert a widget with testId is not visible", "examples": [ { "id": "my_widget" @@ -4218,16 +3127,16 @@ } }, "required": [ - "expectEmpty" + "expectNotVisible" ] }, { "type": "object", - "title": "expectNotEmpty", - "description": "Assert a list has at least one item", + "title": "expectExists", + "description": "Assert a widget with testId exists in the tree", "examples": [ { - "expectNotEmpty": { + "expectExists": { "id": "my_widget" } } @@ -4236,9 +3145,9 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectNotEmpty": { - "$ref": "#/$defs/args_expectNotEmpty", - "description": "Assert a list has at least one item", + "expectExists": { + "$ref": "#/$defs/args_expectExists", + "description": "Assert a widget with testId exists in the tree", "examples": [ { "id": "my_widget" @@ -4247,17 +3156,17 @@ } }, "required": [ - "expectNotEmpty" + "expectExists" ] }, { "type": "object", - "title": "expectScreen", - "description": "Alias for expectNavigateTo — assert current screen", + "title": "expectNotExists", + "description": "Assert no widget with testId exists", "examples": [ { - "expectScreen": { - "screen": "Home" + "expectNotExists": { + "id": "my_widget" } } ], @@ -4265,28 +3174,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectScreen": { - "$ref": "#/$defs/args_expectScreen", - "description": "Alias for expectNavigateTo — assert current screen", + "expectNotExists": { + "$ref": "#/$defs/args_expectNotExists", + "description": "Assert no widget with testId exists", "examples": [ { - "screen": "Home" + "id": "my_widget" } ] } }, "required": [ - "expectScreen" + "expectNotExists" ] }, { "type": "object", - "title": "expectNavigateTo", - "description": "Assert the current visible screen name/id", + "title": "expectText", + "description": "Assert exact text is shown", "examples": [ { - "expectNavigateTo": { - "screen": "Home" + "expectText": { + "text": "Welcome" } } ], @@ -4294,28 +3203,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectNavigateTo": { - "$ref": "#/$defs/args_expectNavigateTo", - "description": "Assert the current visible screen name/id", + "expectText": { + "$ref": "#/$defs/args_expectText", + "description": "Assert exact text is shown", "examples": [ { - "screen": "Home" + "text": "Welcome" } ] } }, "required": [ - "expectNavigateTo" + "expectText" ] }, { "type": "object", - "title": "expectVisited", - "description": "Assert a screen appears in navigation history", + "title": "expectNoText", + "description": "Assert text is not shown", "examples": [ { - "expectVisited": { - "screen": "Login" + "expectNoText": { + "text": "Welcome" } } ], @@ -4323,28 +3232,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectVisited": { - "$ref": "#/$defs/args_expectVisited", - "description": "Assert a screen appears in navigation history", + "expectNoText": { + "$ref": "#/$defs/args_expectNoText", + "description": "Assert text is not shown", "examples": [ { - "screen": "Login" + "text": "Welcome" } ] } }, "required": [ - "expectVisited" + "expectNoText" ] }, { "type": "object", - "title": "expectNotVisited", - "description": "Assert a screen was never visited", + "title": "expectTextContains", + "description": "Assert some text containing the given substring", "examples": [ { - "expectNotVisited": { - "screen": "Login" + "expectTextContains": { + "text": "Welcome" } } ], @@ -4352,31 +3261,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectNotVisited": { - "$ref": "#/$defs/args_expectNotVisited", - "description": "Assert a screen was never visited", + "expectTextContains": { + "$ref": "#/$defs/args_expectTextContains", + "description": "Assert some text containing the given substring", "examples": [ { - "screen": "Login" + "text": "Welcome" } ] } }, "required": [ - "expectNotVisited" + "expectTextContains" ] }, { "type": "object", - "title": "expectBackStack", - "description": "Assert navigation history suffix matches screens", + "title": "expectEnabled", + "description": "Assert widget semantics report enabled", "examples": [ { - "expectBackStack": { - "screens": [ - "Home", - "Details" - ] + "expectEnabled": { + "id": "my_widget" } } ], @@ -4384,31 +3290,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectBackStack": { - "$ref": "#/$defs/args_expectBackStack", - "description": "Assert navigation history suffix matches screens", + "expectEnabled": { + "$ref": "#/$defs/args_expectEnabled", + "description": "Assert widget semantics report enabled", "examples": [ { - "screens": [ - "Home", - "Details" - ] + "id": "my_widget" } ] } }, "required": [ - "expectBackStack" + "expectEnabled" ] }, { "type": "object", - "title": "expectCanGoBack", - "description": "Assert whether back navigation is possible", + "title": "expectDisabled", + "description": "Assert widget semantics report disabled", "examples": [ { - "expectCanGoBack": { - "equals": true + "expectDisabled": { + "id": "my_widget" } } ], @@ -4416,59 +3319,60 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectCanGoBack": { - "$ref": "#/$defs/args_expectCanGoBack", - "description": "Assert whether back navigation is possible", + "expectDisabled": { + "$ref": "#/$defs/args_expectDisabled", + "description": "Assert widget semantics report disabled", "examples": [ { - "equals": true + "id": "my_widget" } ] } }, "required": [ - "expectCanGoBack" + "expectDisabled" ] }, { "type": "object", - "title": "goBack", - "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", + "title": "expectValue", + "description": "Assert input value equals expected (EditableText/TextField)", "examples": [ { - "goBack": {} + "expectValue": { + "id": "email_field", + "equals": "user@test.com" + } } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "goBack": { - "$ref": "#/$defs/args_goBack", - "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", + "expectValue": { + "$ref": "#/$defs/args_expectValue", + "description": "Assert input value equals expected (EditableText/TextField)", "examples": [ - {} + { + "id": "email_field", + "equals": "user@test.com" + } ] } }, "required": [ - "goBack" + "expectValue" ] }, { "type": "object", - "title": "mockApi", - "description": "Register a mock HTTP API response by API name", + "title": "expectChecked", + "description": "Assert checkbox checked state matches equals", "examples": [ { - "mockApi": { - "name": "login", - "response": { - "statusCode": 200, - "body": { - "token": "test-token" - } - } + "expectChecked": { + "id": "terms_checkbox", + "equals": true } } ], @@ -4476,38 +3380,31 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "mockApi": { - "$ref": "#/$defs/args_mockApi", - "description": "Register a mock HTTP API response by API name", + "expectChecked": { + "$ref": "#/$defs/args_expectChecked", + "description": "Assert checkbox checked state matches equals", "examples": [ { - "name": "login", - "response": { - "statusCode": 200, - "body": { - "token": "test-token" - } - } + "id": "terms_checkbox", + "equals": true } ] } }, "required": [ - "mockApi" + "expectChecked" ] }, { "type": "object", - "title": "mockApiError", - "description": "Mock an API to return an error status/body", + "title": "expectProperty", + "description": "Assert a widget property (e.g. label) equals expected", "examples": [ { - "mockApiError": { - "name": "login", - "statusCode": 401, - "body": { - "error": "Unauthorized" - } + "expectProperty": { + "id": "title", + "property": "label", + "equals": "Hello" } } ], @@ -4515,33 +3412,32 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "mockApiError": { - "$ref": "#/$defs/args_mockApiError", - "description": "Mock an API to return an error status/body", + "expectProperty": { + "$ref": "#/$defs/args_expectProperty", + "description": "Assert a widget property (e.g. label) equals expected", "examples": [ { - "name": "login", - "statusCode": 401, - "body": { - "error": "Unauthorized" - } + "id": "title", + "property": "label", + "equals": "Hello" } ] } }, "required": [ - "mockApiError" + "expectProperty" ] }, { "type": "object", - "title": "mockApiFromFixture", - "description": "Load mock response body from a JSON fixture asset", + "title": "expectStyle", + "description": "Assert style-related property equals expected", "examples": [ { - "mockApiFromFixture": { - "name": "users", - "fixture": "users.json" + "expectStyle": { + "id": "title", + "property": "label", + "equals": "Hello" } } ], @@ -4549,30 +3445,31 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "mockApiFromFixture": { - "$ref": "#/$defs/args_mockApiFromFixture", - "description": "Load mock response body from a JSON fixture asset", + "expectStyle": { + "$ref": "#/$defs/args_expectStyle", + "description": "Assert style-related property equals expected", "examples": [ { - "name": "users", - "fixture": "users.json" + "id": "title", + "property": "label", + "equals": "Hello" } ] } }, "required": [ - "mockApiFromFixture" + "expectStyle" ] }, { "type": "object", - "title": "mockApiException", - "description": "Force an API call to throw an exception", + "title": "expectSelected", + "description": "Assert selected/checked state matches equals", "examples": [ { - "mockApiException": { - "name": "login", - "message": "Network error" + "expectSelected": { + "id": "terms_checkbox", + "equals": true } } ], @@ -4580,30 +3477,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "mockApiException": { - "$ref": "#/$defs/args_mockApiException", - "description": "Force an API call to throw an exception", + "expectSelected": { + "$ref": "#/$defs/args_expectSelected", + "description": "Assert selected/checked state matches equals", "examples": [ { - "name": "login", - "message": "Network error" + "id": "terms_checkbox", + "equals": true } ] } }, "required": [ - "mockApiException" + "expectSelected" ] }, { "type": "object", - "title": "mockTimeout", - "description": "Mock an API with a long delay (simulate timeout)", + "title": "expectCount", + "description": "Assert count of widgets with the same testId", "examples": [ { - "mockTimeout": { - "name": "slow_api", - "delayMs": 60000 + "expectCount": { + "id": "badge", + "equals": 2 } } ], @@ -4611,130 +3508,149 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "mockTimeout": { - "$ref": "#/$defs/args_mockTimeout", - "description": "Mock an API with a long delay (simulate timeout)", + "expectCount": { + "$ref": "#/$defs/args_expectCount", + "description": "Assert count of widgets with the same testId", "examples": [ { - "name": "slow_api", - "delayMs": 60000 + "id": "badge", + "equals": 2 } ] } }, "required": [ - "mockTimeout" + "expectCount" ] }, { "type": "object", - "title": "mockNetworkOffline", - "description": "Simulate offline network for API calls", + "title": "expectListCount", + "description": "Assert number of list items under a list testId", "examples": [ { - "mockNetworkOffline": {} + "expectListCount": { + "id": "items_list", + "equals": 3 + } } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "mockNetworkOffline": { - "$ref": "#/$defs/args_mockNetworkOffline", - "description": "Simulate offline network for API calls", + "expectListCount": { + "$ref": "#/$defs/args_expectListCount", + "description": "Assert number of list items under a list testId", "examples": [ - {} + { + "id": "items_list", + "equals": 3 + } ] } }, "required": [ - "mockNetworkOffline" + "expectListCount" ] }, { "type": "object", - "title": "mockNetworkOnline", - "description": "Restore online network for API calls", + "title": "expectListContains", + "description": "Assert list contains text", "examples": [ { - "mockNetworkOnline": {} + "expectListContains": { + "id": "items_list", + "text": "Item 1" + } } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "mockNetworkOnline": { - "$ref": "#/$defs/args_mockNetworkOnline", - "description": "Restore online network for API calls", + "expectListContains": { + "$ref": "#/$defs/args_expectListContains", + "description": "Assert list contains text", "examples": [ - {} + { + "id": "items_list", + "text": "Item 1" + } ] } }, "required": [ - "mockNetworkOnline" + "expectListContains" ] }, { "type": "object", - "title": "resetApiCalls", - "description": "Clear recorded API call history", + "title": "expectListItem", + "description": "Assert a list item widget with itemId is visible", "examples": [ { - "resetApiCalls": {} + "expectListItem": { + "itemId": "row_0" + } } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "resetApiCalls": { - "$ref": "#/$defs/args_resetApiCalls", - "description": "Clear recorded API call history", + "expectListItem": { + "$ref": "#/$defs/args_expectListItem", + "description": "Assert a list item widget with itemId is visible", "examples": [ - {} + { + "itemId": "row_0" + } ] } }, "required": [ - "resetApiCalls" + "expectListItem" ] }, { "type": "object", - "title": "clearApiMocks", - "description": "Remove all registered API mocks", + "title": "expectEmpty", + "description": "Assert a list has zero items", "examples": [ { - "clearApiMocks": {} + "expectEmpty": { + "id": "my_widget" + } } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "clearApiMocks": { - "$ref": "#/$defs/args_clearApiMocks", - "description": "Remove all registered API mocks", + "expectEmpty": { + "$ref": "#/$defs/args_expectEmpty", + "description": "Assert a list has zero items", "examples": [ - {} + { + "id": "my_widget" + } ] } }, "required": [ - "clearApiMocks" + "expectEmpty" ] }, { "type": "object", - "title": "expectApiCalled", - "description": "Assert an API was called an exact number of times", + "title": "expectNotEmpty", + "description": "Assert a list has at least one item", "examples": [ { - "expectApiCalled": { - "name": "login", - "times": 1 + "expectNotEmpty": { + "id": "my_widget" } } ], @@ -4742,30 +3658,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectApiCalled": { - "$ref": "#/$defs/args_expectApiCalled", - "description": "Assert an API was called an exact number of times", + "expectNotEmpty": { + "$ref": "#/$defs/args_expectNotEmpty", + "description": "Assert a list has at least one item", "examples": [ { - "name": "login", - "times": 1 + "id": "my_widget" } ] } }, "required": [ - "expectApiCalled" + "expectNotEmpty" ] }, { "type": "object", - "title": "expectApiNotCalled", - "description": "Assert an API was never called", + "title": "expectScreen", + "description": "Alias for expectNavigateTo — assert current screen", "examples": [ { - "expectApiNotCalled": { - "name": "login", - "times": 1 + "expectScreen": { + "screen": "Home" } } ], @@ -4773,33 +3687,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectApiNotCalled": { - "$ref": "#/$defs/args_expectApiNotCalled", - "description": "Assert an API was never called", + "expectScreen": { + "$ref": "#/$defs/args_expectScreen", + "description": "Alias for expectNavigateTo — assert current screen", "examples": [ { - "name": "login", - "times": 1 + "screen": "Home" } ] } }, "required": [ - "expectApiNotCalled" + "expectScreen" ] }, { "type": "object", - "title": "expectApiRequest", - "description": "Assert last API request body/query/headers match", + "title": "expectNavigateTo", + "description": "Assert the current visible screen name/id", "examples": [ { - "expectApiRequest": { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } + "expectNavigateTo": { + "screen": "Home" } } ], @@ -4807,36 +3716,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectApiRequest": { - "$ref": "#/$defs/args_expectApiRequest", - "description": "Assert last API request body/query/headers match", + "expectNavigateTo": { + "$ref": "#/$defs/args_expectNavigateTo", + "description": "Assert the current visible screen name/id", "examples": [ { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } + "screen": "Home" } ] } }, "required": [ - "expectApiRequest" + "expectNavigateTo" ] }, { "type": "object", - "title": "expectApiRequestContains", - "description": "Assert API request contains partial body/query", + "title": "expectVisited", + "description": "Assert a screen appears in navigation history", "examples": [ { - "expectApiRequestContains": { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } + "expectVisited": { + "screen": "Login" } } ], @@ -4844,34 +3745,28 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectApiRequestContains": { - "$ref": "#/$defs/args_expectApiRequestContains", - "description": "Assert API request contains partial body/query", + "expectVisited": { + "$ref": "#/$defs/args_expectVisited", + "description": "Assert a screen appears in navigation history", "examples": [ { - "name": "login", - "body": { - "email": "user@test.com", - "password": "secret" - } + "screen": "Login" } ] } }, "required": [ - "expectApiRequestContains" + "expectVisited" ] }, { "type": "object", - "title": "expectApiHeader", - "description": "Assert an API request header equals expected", + "title": "expectNotVisited", + "description": "Assert a screen was never visited", "examples": [ { - "expectApiHeader": { - "name": "login", - "header": "Authorization", - "equals": "Bearer test-token" + "expectNotVisited": { + "screen": "Login" } } ], @@ -4879,32 +3774,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectApiHeader": { - "$ref": "#/$defs/args_expectApiHeader", - "description": "Assert an API request header equals expected", + "expectNotVisited": { + "$ref": "#/$defs/args_expectNotVisited", + "description": "Assert a screen was never visited", "examples": [ { - "name": "login", - "header": "Authorization", - "equals": "Bearer test-token" + "screen": "Login" } ] } }, "required": [ - "expectApiHeader" + "expectNotVisited" ] }, { "type": "object", - "title": "expectApiCallOrder", - "description": "Assert APIs were called in order", + "title": "expectBackStack", + "description": "Assert navigation history suffix matches screens", "examples": [ { - "expectApiCallOrder": { - "names": [ - "auth", - "profile" + "expectBackStack": { + "screens": [ + "Home", + "Details" ] } } @@ -4913,32 +3806,31 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectApiCallOrder": { - "$ref": "#/$defs/args_expectApiCallOrder", - "description": "Assert APIs were called in order", + "expectBackStack": { + "$ref": "#/$defs/args_expectBackStack", + "description": "Assert navigation history suffix matches screens", "examples": [ { - "names": [ - "auth", - "profile" + "screens": [ + "Home", + "Details" ] } ] } }, "required": [ - "expectApiCallOrder" + "expectBackStack" ] }, { "type": "object", - "title": "expectLastApiCall", - "description": "Assert the most recent API call name", + "title": "expectCanGoBack", + "description": "Assert whether back navigation is possible", "examples": [ { - "expectLastApiCall": { - "name": "login", - "times": 1 + "expectCanGoBack": { + "equals": true } } ], @@ -4946,92 +3838,79 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectLastApiCall": { - "$ref": "#/$defs/args_expectLastApiCall", - "description": "Assert the most recent API call name", + "expectCanGoBack": { + "$ref": "#/$defs/args_expectCanGoBack", + "description": "Assert whether back navigation is possible", "examples": [ { - "name": "login", - "times": 1 + "equals": true } ] } }, "required": [ - "expectLastApiCall" + "expectCanGoBack" ] }, { "type": "object", - "title": "setState", - "description": "Set app data-context state at path to value", + "title": "goBack", + "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", "examples": [ { - "setState": { - "path": "user.name", - "value": "Jane" - } + "goBack": {} } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "setState": { - "$ref": "#/$defs/args_setState", - "description": "Set app data-context state at path to value", + "goBack": { + "$ref": "#/$defs/args_goBack", + "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", "examples": [ - { - "path": "user.name", - "value": "Jane" - } + {} ] } }, "required": [ - "setState" + "goBack" ] }, { "type": "object", - "title": "expectState", - "description": "Assert app state at path equals expected", + "title": "resetApiCalls", + "description": "Clear recorded API call history", "examples": [ { - "expectState": { - "path": "user.name", - "equals": "Jane" - } + "resetApiCalls": {} } ], "additionalProperties": false, "minProperties": 1, "maxProperties": 1, "properties": { - "expectState": { - "$ref": "#/$defs/args_expectState", - "description": "Assert app state at path equals expected", + "resetApiCalls": { + "$ref": "#/$defs/args_resetApiCalls", + "description": "Clear recorded API call history", "examples": [ - { - "path": "user.name", - "equals": "Jane" - } + {} ] } }, "required": [ - "expectState" + "resetApiCalls" ] }, { "type": "object", - "title": "expectStateContains", - "description": "Assert app state at path contains subset", + "title": "expectApiCalled", + "description": "Assert an API was called an exact number of times", "examples": [ { - "expectStateContains": { - "path": "user.name", - "equals": "Jane" + "expectApiCalled": { + "name": "login", + "times": 1 } } ], @@ -5039,29 +3918,30 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectStateContains": { - "$ref": "#/$defs/args_expectStateContains", - "description": "Assert app state at path contains subset", + "expectApiCalled": { + "$ref": "#/$defs/args_expectApiCalled", + "description": "Assert an API was called an exact number of times", "examples": [ { - "path": "user.name", - "equals": "Jane" + "name": "login", + "times": 1 } ] } }, "required": [ - "expectStateContains" + "expectApiCalled" ] }, { "type": "object", - "title": "expectStateExists", - "description": "Assert state path resolves without error", + "title": "expectApiNotCalled", + "description": "Assert an API was never called", "examples": [ { - "expectStateExists": { - "path": "user.id" + "expectApiNotCalled": { + "name": "login", + "times": 1 } } ], @@ -5069,28 +3949,32 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectStateExists": { - "$ref": "#/$defs/args_expectStateExists", - "description": "Assert state path resolves without error", + "expectApiNotCalled": { + "$ref": "#/$defs/args_expectApiNotCalled", + "description": "Assert an API was never called", "examples": [ { - "path": "user.id" + "name": "login", + "times": 1 } ] } }, "required": [ - "expectStateExists" + "expectApiNotCalled" ] }, { "type": "object", - "title": "expectStateNotExists", - "description": "Assert state path is null or absent", + "title": "expectApiCallOrder", + "description": "Assert APIs were called in order", "examples": [ { - "expectStateNotExists": { - "path": "user.id" + "expectApiCallOrder": { + "names": [ + "auth", + "profile" + ] } } ], @@ -5098,28 +3982,32 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "expectStateNotExists": { - "$ref": "#/$defs/args_expectStateNotExists", - "description": "Assert state path is null or absent", + "expectApiCallOrder": { + "$ref": "#/$defs/args_expectApiCallOrder", + "description": "Assert APIs were called in order", "examples": [ { - "path": "user.id" + "names": [ + "auth", + "profile" + ] } ] } }, "required": [ - "expectStateNotExists" + "expectApiCallOrder" ] }, { "type": "object", - "title": "resetState", - "description": "Clear state at path (set to null)", + "title": "expectLastApiCall", + "description": "Assert the most recent API call name", "examples": [ { - "resetState": { - "path": "cart" + "expectLastApiCall": { + "name": "login", + "times": 1 } } ], @@ -5127,18 +4015,19 @@ "minProperties": 1, "maxProperties": 1, "properties": { - "resetState": { - "$ref": "#/$defs/args_resetState", - "description": "Clear state at path (set to null)", + "expectLastApiCall": { + "$ref": "#/$defs/args_expectLastApiCall", + "description": "Assert the most recent API call name", "examples": [ { - "path": "cart" + "name": "login", + "times": 1 } ] } }, "required": [ - "resetState" + "expectLastApiCall" ] }, { @@ -5756,120 +4645,6 @@ "logApiCalls" ] }, - { - "type": "object", - "title": "screenshot", - "description": "Capture golden or dump widget tree for debugging", - "examples": [ - { - "screenshot": { - "name": "home_screen" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "screenshot": { - "$ref": "#/$defs/args_screenshot", - "description": "Capture golden or dump widget tree for debugging", - "examples": [ - { - "name": "home_screen" - } - ] - } - }, - "required": [ - "screenshot" - ] - }, - { - "type": "object", - "title": "dumpTree", - "description": "Print the widget tree to the debug console", - "examples": [ - { - "dumpTree": {} - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "dumpTree": { - "$ref": "#/$defs/args_dumpTree", - "description": "Print the widget tree to the debug console", - "examples": [ - {} - ] - } - }, - "required": [ - "dumpTree" - ] - }, - { - "type": "object", - "title": "logState", - "description": "Log resolved state at path", - "examples": [ - { - "logState": { - "path": "user.id" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "logState": { - "$ref": "#/$defs/args_logState", - "description": "Log resolved state at path", - "examples": [ - { - "path": "user.id" - } - ] - } - }, - "required": [ - "logState" - ] - }, - { - "type": "object", - "title": "logStorage", - "description": "Log public storage value for key", - "examples": [ - { - "logStorage": { - "key": "onboarding_done", - "value": true - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "logStorage": { - "$ref": "#/$defs/args_logStorage", - "description": "Log public storage value for key", - "examples": [ - { - "key": "onboarding_done", - "value": true - } - ] - } - }, - "required": [ - "logStorage" - ] - }, { "type": "object", "title": "expectNoConsoleErrors", @@ -6062,93 +4837,6 @@ "required": [ "expectNoOverflow" ] - }, - { - "type": "object", - "title": "loadFixture", - "description": "Load a JSON fixture into the test fixture map", - "examples": [ - { - "loadFixture": { - "fixture": "user.json" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "loadFixture": { - "$ref": "#/$defs/args_loadFixture", - "description": "Load a JSON fixture into the test fixture map", - "examples": [ - { - "fixture": "user.json" - } - ] - } - }, - "required": [ - "loadFixture" - ] - }, - { - "type": "object", - "title": "setStateFromFixture", - "description": "Apply all keys from a JSON fixture to state", - "examples": [ - { - "setStateFromFixture": { - "fixture": "user.json" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "setStateFromFixture": { - "$ref": "#/$defs/args_setStateFromFixture", - "description": "Apply all keys from a JSON fixture to state", - "examples": [ - { - "fixture": "user.json" - } - ] - } - }, - "required": [ - "setStateFromFixture" - ] - }, - { - "type": "object", - "title": "expectMatchesFixture", - "description": "Assert state or path matches a JSON fixture", - "examples": [ - { - "expectMatchesFixture": { - "fixture": "user.json" - } - } - ], - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "expectMatchesFixture": { - "$ref": "#/$defs/args_expectMatchesFixture", - "description": "Assert state or path matches a JSON fixture", - "examples": [ - { - "fixture": "user.json" - } - ] - } - }, - "required": [ - "expectMatchesFixture" - ] } ] } diff --git a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md index 4805af308..d2a66c560 100644 --- a/tools/ensemble_test_runner/doc/TEST_AUTHORING.md +++ b/tools/ensemble_test_runner/doc/TEST_AUTHORING.md @@ -35,45 +35,97 @@ steps: Use `startScreen` for a cold start. Use `prerequisite` when a test should continue from another test in the same app session. -## App Context +## Suite Config -`--inspect-app` emits JSON with screens, widget IDs, APIs, navigation targets, imports, storage/env references, and lifecycle hints. +Put shared runner settings in `tests/config.yaml`, next to the `*.test.yaml` +files: -## Fixtures And Mocks +```yaml +# yaml-language-server: $schema=https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json +screenshots: + enabled: true + platform: ios + model: iPhone 15 Pro + +performance: + enabled: true +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true +``` -Put JSON fixtures under: +When `screenshots.enabled` is true, the runner captures automatic step +screenshots into one contact sheet per test case under +`build/ensemble_test_runner/screenshots/`. -```text -ensemble/apps//tests/fixtures/.json -``` +## App Context + +`--inspect-app` emits JSON with screens, widget IDs, APIs, navigation targets, imports, storage/env references, and lifecycle hints. + +## Mocks -Reference fixtures by filename: +Use `mocks` for API responses. The value is an ordered list of `.mock.json` +files. Later files override earlier files. ```yaml -steps: - - mockApiFromFixture: - name: profile - fixture: profile_success.json +mocks: + - mocks/common/base.mock.json ``` -Use root `mocks.apis` for APIs that may run during `onLoad` or startup. Use step-level `mockApi` or `mockApiFromFixture` for APIs triggered after user actions. +For scenario-based suites, keep reusable API data in mock files. The runner does +not know what your scenario variables mean; it only substitutes `${scenario.*}` +values and merges mock files in order. ```yaml +id: home_scenarios +prerequisite: signin_to_gateway + mocks: - apis: - profile: - response: - body: {name: Jane} + - mocks/common/base.mock.json + - mocks/devices/${scenario.device}.mock.json + - mocks/behaviors/${scenario.behavior}.mock.json + +scenarios: + - id: v14_online + vars: + device: v14 + behavior: online + expectedDeviceCount: 2 + steps: - - mockApi: - name: login - response: - body: {token: test-token} + - expectText: + text: ${scenario.expectedDeviceCount} ``` +Each scenario expands to its own test id, for example +`home_scenarios[v14_online]`. When a scenario suite has a `prerequisite`, the +first scenario depends on that prerequisite and later scenarios run after the +previous scenario in declaration order. + +Mock files are JSON files. The root object maps API names to response +overrides: + +```json +{ + "getDevices": { + "statusCode": 200, + "delayMs": 300, + "body": { + "status": [] + } + } +} +``` + +Later files override earlier files. Use `delayMs` when a mock should stay +pending briefly before returning, matching the loading behavior of a real API. + ## Validation -`--validate-only` checks generated tests without running Flutter. It reports blocking errors for invalid YAML shape, missing tests, duplicate IDs, unknown prerequisites, unknown screens, and missing fixtures. It reports warnings for likely unknown widget IDs/APIs and mock placement issues. +`--validate-only` checks generated tests without running Flutter. It reports blocking errors for invalid YAML shape, missing tests, duplicate IDs, unknown prerequisites, and unknown screens. It reports warnings for likely unknown widget IDs/APIs. Warnings do not fail the command. Errors exit with code `2`. @@ -90,6 +142,28 @@ dart run ensemble_test_runner:ensemble_test --id=login_valid Prerequisites are included automatically when a selected test depends on them. +## CLI Inputs + +Use repeatable `--input key=value` flags for values that should come from the +command line: + +```sh +dart run ensemble_test_runner:ensemble_test \ + --input adminPassword='s4C>M7U6t~' \ + --input expectedDeviceCount=2 +``` + +Reference them in test YAML with `${inputs.key}`: + +```yaml +initialState: + keychain: + adminPassword: ${inputs.adminPassword} +steps: + - expectText: + text: ${inputs.expectedDeviceCount} +``` + ## CI Output Stable exit codes: diff --git a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart index 594200688..6dfb11635 100644 --- a/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart +++ b/tools/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -1,15 +1,15 @@ -import 'dart:convert'; +import 'dart:ui' as ui; import 'package:ensemble/action/navigation_action.dart'; -import 'package:ensemble/framework/bindings.dart'; import 'package:ensemble/screen_controller.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; -import 'package:ensemble_test_runner/actions/state_helper.dart'; +import 'package:ensemble_device_preview/ensemble_device_preview.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -170,32 +170,6 @@ class ExtendedStepHandlers { case 'goBack': await _goBack(executor); return true; - case 'mockApiFromFixture': - await _mockApiFromFixture(executor, step); - return true; - case 'clearApiMocks': - executor.context.mockApiProvider.clearMocks(); - return true; - case 'mockApiException': - await _mockApiException(executor, step); - return true; - case 'mockTimeout': - await _mockTimeout(executor, step); - return true; - case 'mockNetworkOffline': - _setNetworkOffline(executor, true); - return true; - case 'mockNetworkOnline': - _setNetworkOffline(executor, false); - return true; - case 'expectApiHeader': - executor.assertions.expectApiHeader( - step.args['name']?.toString() ?? '', - step.args['header']?.toString() ?? '', - step.args['equals'], - times: step.args['times'] as int?, - ); - return true; case 'expectApiCallOrder': executor.assertions.expectApiCallOrder( (step.args['names'] as List?)?.map((e) => e.toString()).toList() ?? @@ -215,24 +189,6 @@ class ExtendedStepHandlers { case 'clearStorage': await executor.context.clearStorage(); return true; - case 'expectStateContains': - executor.assertions.expectStateContains( - step.args['path']?.toString() ?? '', - step.args['contains'], - ); - return true; - case 'expectStateExists': - executor.assertions - .expectStateExists(step.args['path']?.toString() ?? ''); - return true; - case 'expectStateNotExists': - executor.assertions.expectStateNotExists( - step.args['path']?.toString() ?? '', - ); - return true; - case 'resetState': - await _resetState(executor, step); - return true; case 'setAuth': _setAuth(executor, step); return true; @@ -262,26 +218,6 @@ class ExtendedStepHandlers { executor.assertions .expectConsoleLog(step.args['contains']?.toString() ?? ''); return true; - case 'loadFixture': - await _loadFixture(executor, step); - return true; - case 'setStateFromFixture': - await _setStateFromFixture(executor, step); - return true; - case 'expectMatchesFixture': - await _expectMatchesFixture(executor, step); - return true; - case 'logState': - executor.context.logger.log( - 'state: ${executor.assertions.readState(step.args['path']?.toString() ?? '')}', - ); - return true; - case 'logStorage': - final key = step.args['key']?.toString(); - executor.context.logger.log( - 'storage[$key]=${StorageManager().read(key ?? '')}', - ); - return true; case 'expectAccessible': executor.assertions.expectAccessible(executor.requireId(step)); return true; @@ -302,13 +238,6 @@ class ExtendedStepHandlers { case 'expectNoErrors': executor.assertions.expectNoRenderErrors(); return true; - case 'screenshot': - await _screenshot(executor, step); - return true; - case 'dumpTree': - debugDumpApp(); - executor.context.logger.log('dumpTree: see debug console'); - return true; case 'expectNoConsoleErrors': executor.assertions.expectNoConsoleErrors(); return true; @@ -333,7 +262,7 @@ class ExtendedStepHandlers { static Future _restartApp(TestStepExecutor e) async { EnsembleTestHarness.resetTestRuntime(); e.context.runtime.clear(); - e.context.mockApiProvider.resetCalls(); + e.context.apiOverlay.resetCalls(); final screen = e.context.testCase.startScreen ?? ScreenTracker().getCurrentScreenIdentifier(); if (screen == null || screen.isEmpty) { @@ -346,7 +275,7 @@ class ExtendedStepHandlers { static Future _resetAppState(TestStepExecutor e) async { ScreenTracker().clearAll(); - e.context.mockApiProvider.resetCalls(); + e.context.apiOverlay.resetCalls(); e.context.runtime.clear(); await StorageManager().clearPublicStorage(); } @@ -477,54 +406,6 @@ class ExtendedStepHandlers { throw EnsembleTestFailure('goBack: no navigator or active scope'); } - static Future _mockApiFromFixture( - TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - final fixture = step.args['fixture']?.toString(); - if (name == null || fixture == null) { - throw EnsembleTestFailure( - 'mockApiFromFixture requires "name" and "fixture"'); - } - final body = await _readFixture(e, fixture); - e.context.mockApiProvider.setMock( - name, - MockAPIResponse( - statusCode: step.args['statusCode'] as int? ?? 200, - body: body, - ), - ); - } - - static Future _mockApiException( - TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - if (name == null) - throw EnsembleTestFailure('mockApiException requires "name"'); - e.context.mockApiProvider.setApiException( - name, - Exception(step.args['message']?.toString() ?? 'API exception (test)'), - ); - } - - static Future _mockTimeout(TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - if (name == null) throw EnsembleTestFailure('mockTimeout requires "name"'); - e.context.mockApiProvider.setMock( - name, - MockAPIResponse( - statusCode: 200, - body: {}, - delayMs: step.args['delayMs'] as int? ?? 60000, - ), - ); - } - - static void _setNetworkOffline(TestStepExecutor e, bool offline) { - e.context.mockApiProvider.simulateNetworkOffline = offline; - ConnectivityState().isOnline = !offline; - e.context.runtime.networkOffline = offline; - } - static void _setAuth(TestStepExecutor e, TestStep step) { final user = step.args['user']; if (user is Map) { @@ -552,9 +433,13 @@ class ExtendedStepHandlers { static Future _setDevice(TestStepExecutor e, TestStep step) async { final width = (step.args['width'] as num?)?.toDouble() ?? 390; final height = (step.args['height'] as num?)?.toDouble() ?? 844; - e.tester.view.physicalSize = Size(width, height); + final size = Size(width, height); + await e.tester.binding.setSurfaceSize(size); + e.tester.view.physicalSize = size; e.tester.view.devicePixelRatio = 1.0; - e.context.runtime.deviceSize = Size(width, height); + e.tester.view.padding = FakeViewPadding.zero; + e.tester.view.viewPadding = FakeViewPadding.zero; + e.context.runtime.deviceSize = size; await e.settle(); } @@ -598,117 +483,87 @@ class ExtendedStepHandlers { } } - static Future _readFixture(TestStepExecutor e, String path) async { - Object? lastError; - for (final candidate in _fixtureAssetCandidates(e, path)) { - try { - final raw = await rootBundle.loadString(candidate); - return json.decode(raw); - } catch (error) { - lastError = error; - } + static Future captureScreenshotBytes( + WidgetTester tester, { + required DeviceInfo device, + }) async { + final image = captureScreenshotImage(tester); + try { + return await encodeScreenshotImage(image, device); + } finally { + image.dispose(); } - throw EnsembleTestFailure( - 'Fixture not found: $path. Use tests/fixtures/.json. $lastError', - ); } - static List _fixtureAssetCandidates(TestStepExecutor e, String path) { - final sourcePath = e.context.testCase.sourcePath; - final candidates = []; - if (sourcePath != null && sourcePath.contains('/')) { - final testDir = sourcePath.substring(0, sourcePath.lastIndexOf('/')); - candidates.add( - path.startsWith('fixtures/') - ? '$testDir/$path' - : '$testDir/fixtures/$path', + static ui.Image captureScreenshotImage(WidgetTester tester) { + final renderView = tester.binding.renderViews.first; + final layer = renderView.debugLayer; + if (layer is! OffsetLayer) { + throw EnsembleTestFailure( + 'screenshot requires a painted render view.', ); } - candidates.add(path); - if (!path.startsWith('ensemble/')) candidates.add('ensemble/$path'); - return candidates.toSet().toList(); - } - - static Future _loadFixture(TestStepExecutor e, TestStep step) async { - final key = step.args['key']?.toString(); - final path = - step.args['path']?.toString() ?? step.args['fixture']?.toString(); - if (key == null || path == null) { - throw EnsembleTestFailure('loadFixture requires "key" and "path"'); - } - e.context.runtime.fixtures[key] = await _readFixture(e, path); - } - static Future _setStateFromFixture( - TestStepExecutor e, TestStep step) async { - final path = - step.args['fixture']?.toString() ?? step.args['path']?.toString(); - if (path == null) { - throw EnsembleTestFailure( - 'setStateFromFixture requires "fixture" or "path"'); - } - final data = await _readFixture(e, path); - if (data is! Map) { - throw EnsembleTestFailure('Fixture must be a JSON object'); - } - final scope = e.assertions.activeScope(); - if (scope == null) { - throw EnsembleTestFailure('setStateFromFixture requires active screen'); - } - for (final entry in data.entries) { - setStatePath(scope, entry.key.toString(), entry.value); - } - await e.settle(); + return layer.toImageSync( + renderView.paintBounds, + pixelRatio: renderView.flutterView.devicePixelRatio, + ); } - static Future _expectMatchesFixture( - TestStepExecutor e, - TestStep step, + static Future encodeScreenshotImage( + ui.Image image, + DeviceInfo device, ) async { - final path = - step.args['fixture']?.toString() ?? step.args['path']?.toString(); - final statePath = step.args['statePath']?.toString(); - if (path == null) { - throw EnsembleTestFailure( - 'expectMatchesFixture requires "fixture" or "path"'); - } - final expected = await _readFixture(e, path); - if (statePath != null) { - e.assertions.expectState(statePath, expected); - } else { - final actual = - e.assertions.readState(step.args['path']?.toString() ?? ''); - if (!_deepEquals(actual, expected)) { - throw EnsembleTestFailure( - 'State does not match fixture $path: expected $expected, got $actual', - ); - } + final byteData = await _addDeviceFrame(image, device); + if (byteData == null) { + throw EnsembleTestFailure('Failed to encode screenshot as PNG.'); } + return byteData.buffer.asUint8List(); } - static Future _resetState(TestStepExecutor e, TestStep step) async { - final path = step.args['path']?.toString(); - final scope = e.assertions.activeScope(); - if (scope == null) { - throw EnsembleTestFailure('resetState requires active screen'); - } - if (path != null) { - setStatePath(scope, path, null); - } - await e.settle(); - } + static Future _addDeviceFrame( + ui.Image screenImage, + DeviceInfo device, + ) async { + final padding = device.frameSize.shortestSide * 0.025; + final outputWidth = (device.frameSize.width + padding * 2).ceil(); + final outputHeight = (device.frameSize.height + padding * 2).ceil(); + + final recorder = ui.PictureRecorder(); + final canvas = Canvas( + recorder, + Rect.fromLTWH(0, 0, outputWidth.toDouble(), outputHeight.toDouble()), + ); - static Future _screenshot(TestStepExecutor e, TestStep step) async { - final name = step.args['name']?.toString(); - if (name != null && name.isNotEmpty) { - await expectLater( - find.byType(MaterialApp), - matchesGoldenFile('goldens/$name.png'), - ); - } else { - debugDumpApp(); - e.context.logger.log('screenshot: debug tree dumped (no golden name)'); - } + canvas.drawColor(const Color(0xFFE9EDF3), BlendMode.src); + canvas.save(); + canvas.translate(padding, padding); + device.framePainter.paint(canvas, device.frameSize); + + final screenPath = device.screenPath; + final screenRect = screenPath.getBounds(); + canvas.save(); + canvas.clipPath(screenPath); + canvas.drawImageRect( + screenImage, + Rect.fromLTWH( + 0, + 0, + screenImage.width.toDouble(), + screenImage.height.toDouble(), + ), + screenRect, + Paint()..filterQuality = FilterQuality.high, + ); + canvas.restore(); + canvas.restore(); + + final picture = recorder.endRecording(); + final image = await picture.toImage(outputWidth, outputHeight); + picture.dispose(); + final byteData = await image.toByteData(format: ui.ImageByteFormat.png); + image.dispose(); + return byteData; } static bool _deepEquals(dynamic a, dynamic b) { diff --git a/tools/ensemble_test_runner/lib/actions/screenshot_device.dart b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart new file mode 100644 index 000000000..d3172f6f7 --- /dev/null +++ b/tools/ensemble_test_runner/lib/actions/screenshot_device.dart @@ -0,0 +1,52 @@ +import 'package:ensemble_device_preview/ensemble_device_preview.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; + +DeviceInfo? screenshotDeviceForTestCase( + EnsembleTestCase _, + EnsembleTestConfig config, +) { + if (config.screenshots.enabled) { + return resolveScreenshotDevice( + config.screenshots.toScreenshotArgs(), + ); + } + return null; +} + +DeviceInfo resolveScreenshotDevice(Map args) { + final platform = args['platform']?.toString(); + final model = args['model']?.toString(); + final platformKey = normalizeScreenshotDeviceName(platform); + final candidates = switch (platformKey) { + 'android' => Devices.android.all, + 'ios' || 'iphone' || 'ipad' => Devices.ios.all, + _ => Devices.all, + }; + + if (model != null && model.trim().isNotEmpty) { + final modelKey = normalizeScreenshotDeviceName(model); + for (final device in candidates) { + if (normalizeScreenshotDeviceName(device.name) == modelKey || + normalizeScreenshotDeviceName(device.identifier.name) == modelKey) { + return device; + } + } + for (final device in candidates) { + if (normalizeScreenshotDeviceName(device.name).contains(modelKey) || + normalizeScreenshotDeviceName(device.identifier.name) + .contains(modelKey)) { + return device; + } + } + throw EnsembleTestFailure( + 'Unknown screenshot device model "$model". Available models: ' + '${candidates.map((device) => device.name).join(', ')}', + ); + } + + if (platformKey == 'android') return Devices.android.all.first; + return Devices.ios.iPhone15Pro; +} + +String normalizeScreenshotDeviceName(String? value) => + (value ?? '').toLowerCase().replaceAll(RegExp(r'[^a-z0-9]'), ''); diff --git a/tools/ensemble_test_runner/lib/actions/state_helper.dart b/tools/ensemble_test_runner/lib/actions/state_helper.dart deleted file mode 100644 index ed99e5a7e..000000000 --- a/tools/ensemble_test_runner/lib/actions/state_helper.dart +++ /dev/null @@ -1,34 +0,0 @@ -import 'package:ensemble/framework/scope.dart'; -import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; - -/// Writes a dot-path value into the active [ScopeManager] data context. -void setStatePath(ScopeManager scope, String path, dynamic value) { - final parts = path.split('.').where((p) => p.isNotEmpty).toList(); - if (parts.isEmpty) { - throw EnsembleTestFailure('setState path cannot be empty'); - } - if (parts.length == 1) { - scope.dataContext.addDataContextById(parts.first, value); - return; - } - - final root = parts.first; - final existing = scope.dataContext.getContextMap()[root]; - final Map rootMap = existing is Map - ? Map.from(existing) - : {}; - - var cursor = rootMap; - for (var i = 1; i < parts.length - 1; i++) { - final key = parts[i]; - final next = cursor[key]; - if (next is Map) { - cursor[key] = Map.from(next); - } else { - cursor[key] = {}; - } - cursor = cursor[key] as Map; - } - cursor[parts.last] = value; - scope.dataContext.addDataContextById(root, rootMap); -} diff --git a/tools/ensemble_test_runner/lib/actions/test_execution_config.dart b/tools/ensemble_test_runner/lib/actions/test_execution_config.dart index 7ecff1cfc..31c8a0f89 100644 --- a/tools/ensemble_test_runner/lib/actions/test_execution_config.dart +++ b/tools/ensemble_test_runner/lib/actions/test_execution_config.dart @@ -7,7 +7,7 @@ class TestExecutionConfig { const TestExecutionConfig({ this.settleStepDuration = const Duration(milliseconds: 100), - this.settleTimeout = const Duration(seconds: 10), + this.settleTimeout = const Duration(seconds: 2), this.waitPollInterval = const Duration(milliseconds: 100), this.defaultWaitTimeout = const Duration(seconds: 5), }); diff --git a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart index 371a4631e..2328619e3 100644 --- a/tools/ensemble_test_runner/lib/actions/test_step_executor.dart +++ b/tools/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -1,13 +1,17 @@ +import 'dart:async'; + import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; -import 'package:ensemble_test_runner/actions/state_helper.dart'; import 'package:ensemble_test_runner/actions/test_execution_config.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; +import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -71,11 +75,11 @@ class TestStepExecutor { switch (step.type) { case 'wait': - await tester.pump( - Duration( - milliseconds: step.args['durationMs'] as int? ?? 500, - ), - ); + final durationMs = step.args['durationMs'] as int? ?? 500; + await tester.runAsync(() async { + await Future.delayed(Duration(milliseconds: durationMs)); + }); + await _pump(label: 'wait'); return; case 'waitForText': await _waitFor( @@ -110,21 +114,6 @@ class TestStepExecutor { config.defaultWaitTimeout.inMilliseconds, ); return; - case 'waitUntil': - String? statePath = step.args['path']?.toString(); - dynamic expected = step.args['equals']; - final stateNode = step.args['state']; - if (stateNode is Map) { - statePath ??= stateNode['path']?.toString(); - expected ??= stateNode['equals']; - } - await _waitUntil( - path: statePath, - expected: expected, - timeoutMs: step.args['timeoutMs'] as int? ?? - config.defaultWaitTimeout.inMilliseconds, - ); - return; case 'expectScreen': final screen = step.args['name']?.toString() ?? step.args['screen']?.toString(); @@ -172,7 +161,7 @@ class TestStepExecutor { await _select(_requireId(step), step.args['value']?.toString()); break; case 'toggle': - await _tap(_requireId(step)); + await _toggle(_requireId(step)); break; case 'waitFor': await _waitFor( @@ -183,11 +172,12 @@ class TestStepExecutor { ); break; case 'pump': - await tester.pump( - Duration( + await _pump( + duration: Duration( milliseconds: step.args['durationMs'] as int? ?? config.waitPollInterval.inMilliseconds, ), + label: 'pump', ); break; case 'settle': @@ -243,32 +233,6 @@ class TestStepExecutor { } assertions.expectApiNotCalled(name); break; - case 'expectApiRequest': - final name = step.args['name']?.toString(); - if (name == null) { - throw EnsembleTestFailure('expectApiRequest requires "name"'); - } - assertions.expectApiRequest( - name, - body: step.args['body'], - query: step.args['query'], - headers: step.args['headers'], - times: step.args['times'] as int?, - ); - break; - case 'expectApiRequestContains': - final containsName = step.args['name']?.toString(); - if (containsName == null) { - throw EnsembleTestFailure('expectApiRequestContains requires "name"'); - } - assertions.expectApiRequestContains( - containsName, - body: step.args['body'], - query: step.args['query'], - headers: step.args['headers'], - times: step.args['times'] as int?, - ); - break; case 'expectCount': final expected = step.args['equals'] as int?; if (expected == null) { @@ -297,26 +261,6 @@ class TestStepExecutor { } assertions.expectStorage(key, step.args['equals']); break; - case 'expectState': - final path = step.args['path']?.toString(); - if (path == null) { - throw EnsembleTestFailure('expectState requires "path"'); - } - assertions.expectState(path, step.args['equals']); - break; - case 'setState': - final path = step.args['path']?.toString(); - if (path == null) { - throw EnsembleTestFailure('setState requires "path"'); - } - final scope = assertions.activeScope(); - if (scope == null) { - throw EnsembleTestFailure( - 'setState requires an active Ensemble screen.', - ); - } - setStatePath(scope, path, step.args['value']); - break; case 'setStorage': final key = step.args['key']?.toString(); if (key == null) { @@ -331,39 +275,14 @@ class TestStepExecutor { } context.setEnv(key, step.args['value']); break; - case 'mockApi': - final name = step.args['name']?.toString(); - if (name == null) { - throw EnsembleTestFailure('mockApi requires "name"'); - } - context.mockApiProvider.setMock( - name, - context.mockFromStepArgs(step.args), - ); - break; - case 'mockApiError': - final name = step.args['name']?.toString(); - if (name == null) { - throw EnsembleTestFailure('mockApiError requires "name"'); - } - context.mockApiProvider.setMock( - name, - MockAPIResponse( - statusCode: step.args['statusCode'] as int? ?? 500, - body: step.args['body'], - delayMs: step.args['delayMs'] as int?, - ), - ); - break; case 'resetApiCalls': - context.mockApiProvider.resetCalls(); + context.apiOverlay.resetCalls(); break; case 'logApiCalls': - for (final call in context.mockApiProvider.calls) { - context.logger.log( - 'API ${call.name} body=${call.body} query=${call.query}', - ); - } + final path = await tester.runAsync(() { + return writeApiCallsLog(context); + }); + context.logger.log('apiCalls: $path'); break; default: if (await ExtendedStepHandlers.tryExecute(this, step)) { @@ -410,6 +329,7 @@ class TestStepExecutor { testCase: EnsembleTestCase( id: tc.id, startScreen: screen, + mockFiles: tc.mockFiles, initialState: tc.initialState, mocks: tc.mocks, steps: const [], @@ -429,11 +349,44 @@ class TestStepExecutor { } Future _settle({Duration? timeout}) async { - await tester.pumpAndSettle( - config.settleStepDuration, - EnginePhase.sendSemanticsUpdate, - timeout ?? config.settleTimeout, - ); + try { + await tester.pumpAndSettle( + config.settleStepDuration, + EnginePhase.sendSemanticsUpdate, + timeout ?? config.settleTimeout, + ); + } catch (e) { + if (e.toString().contains('timed out') || + e.toString().contains('timeout')) { + // Swallow timeout error because background streams/listeners (e.g. Firestore) + // might keep the event loop active, but the UI itself has settled. + } else { + rethrow; + } + } + await _yieldToLiveApiWork(); + } + + /// Lets in-flight live HTTP (wrapped in [WidgetTester.runAsync]) finish and + /// pumps a frame so Ensemble can apply API state. Uses [Duration.zero] so + /// timers from departed screens are not advanced while draining live HTTP. + Future _yieldToLiveApiWork() async { + for (var i = 0; i < 200; i++) { + final hadPending = context.apiOverlay.hasPendingLiveCalls; + if (hadPending) { + try { + await context.apiOverlay + .waitForLiveCalls() + .timeout(config.waitPollInterval); + } on TimeoutException { + // Keep polling; HTTP may still be in flight inside runAsync. + } + } + await _pump(label: 'liveApi'); + if (!hadPending) { + return; + } + } } Future _tap(String id) async { @@ -445,10 +398,35 @@ class TestStepExecutor { ); } _expectSingleWidget(finder, id, 'tap'); + await tester.ensureVisible(finder); + await _pump(label: 'tap:before'); await tester.tap(finder); await _settle(); } + Future _toggle(String id) async { + final finder = assertions.finderForId(id); + if (finder.evaluate().isEmpty) { + await _waitFor( + id: id, + timeoutMs: config.defaultWaitTimeout.inMilliseconds, + ); + } + _expectSingleWidget(finder, id, 'toggle'); + await tester.ensureVisible(finder); + await _pump(label: 'toggle:before'); + + final control = find.descendant( + of: finder, + matching: find.byWidgetPredicate( + (widget) => + widget is Switch || widget is CupertinoSwitch || widget is Checkbox, + ), + ); + await tester.tap(control.evaluate().isNotEmpty ? control.first : finder); + await _settle(); + } + Future _enterText(String id, String value, {bool submit = false}) async { final finder = assertions.finderForId(id); @@ -509,7 +487,7 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); + await _pump(duration: config.waitPollInterval, label: 'waitFor'); if (id != null && assertions.finderForId(id).evaluate().isNotEmpty) { return; } @@ -551,6 +529,7 @@ class TestStepExecutor { testCase: EnsembleTestCase( id: tc.id, startScreen: screen, + mockFiles: tc.mockFiles, initialState: tc.initialState, mocks: tc.mocks, steps: const [], @@ -572,8 +551,10 @@ class TestStepExecutor { final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); - if (context.mockApiProvider.callCount(name) >= times) { + await _yieldToLiveApiWork(); + await _pump(duration: config.waitPollInterval, label: 'waitForApi'); + if (context.apiOverlay.callCount(name) >= times) { + await _yieldToLiveApiWork(); return; } } @@ -589,37 +570,30 @@ class TestStepExecutor { }) async { final stopwatch = Stopwatch()..start(); final tracker = ScreenTracker(); - while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); - if (tracker.isScreenVisible(screenName: screen) || - tracker.isScreenVisible(screenId: screen)) { - return; - } - } - throw EnsembleTestFailure( - 'Timed out after ${timeoutMs}ms waiting for navigation to "$screen"', - ); - } - - Future _waitUntil({ - String? path, - dynamic expected, - required int timeoutMs, - }) async { - if (path == null || path.isEmpty) { - throw EnsembleTestFailure('waitUntil requires "path"'); - } + bool hasNavigated() => + tracker.isScreenVisible(screenName: screen) || + tracker.isScreenVisible(screenId: screen) || + YamlTestSession.navigationFlow.flow.contains(screen); - final stopwatch = Stopwatch()..start(); while (stopwatch.elapsedMilliseconds < timeoutMs) { - await tester.pump(config.waitPollInterval); - if (assertions.matchesState(path, expected)) { + await YamlTestSession.navigationFlow.flushPending(); + if (hasNavigated()) { return; } + await _yieldToLiveApiWork(); + await _pump( + duration: config.waitPollInterval, + label: 'waitForNavigation', + ); + } + await _yieldToLiveApiWork(); + await _pump(label: 'waitForNavigation'); + await YamlTestSession.navigationFlow.flushPending(); + if (hasNavigated()) { + return; } throw EnsembleTestFailure( - 'Timed out after ${timeoutMs}ms waiting for state "$path" ' - 'to equal "$expected"', + 'Timed out after ${timeoutMs}ms waiting for navigation to "$screen"', ); } @@ -642,4 +616,12 @@ class TestStepExecutor { 'Timed out after ${timeoutMs}ms waiting for id "$id" to disappear', ); } + + Future _pump({ + Duration? duration, + EnginePhase phase = EnginePhase.sendSemanticsUpdate, + required String label, + }) async { + await tester.pump(duration, phase); + } } diff --git a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart index 1bf4dbfa7..6418903fd 100644 --- a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart +++ b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart @@ -6,7 +6,6 @@ import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/framework/view/data_scope_widget.dart'; import 'package:ensemble/framework/view/page_group.dart'; -import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; @@ -82,7 +81,7 @@ class AssertionEngine { } void expectApiNotCalled(String apiName) { - final count = context.mockApiProvider.callCount(apiName); + final count = context.apiOverlay.callCount(apiName); if (count != 0) { throw EnsembleTestFailure( 'Expected API "$apiName" not to be called, but it was called $count times.', @@ -134,7 +133,7 @@ class AssertionEngine { } void expectApiCalled(String apiName, int times) { - final actual = context.mockApiProvider.callCount(apiName); + final actual = context.apiOverlay.callCount(apiName); if (actual != times) { throw EnsembleTestFailure( 'Expected API "$apiName" to be called $times times, but it was called $actual times. ' @@ -143,58 +142,6 @@ class AssertionEngine { } } - void expectApiRequest( - String apiName, { - dynamic body, - dynamic query, - dynamic headers, - int? times, - }) { - final record = _selectApiCall(apiName, times: times); - - if (body != null && !_deepEquals(record.body, body)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" body $body, but got ${record.body}.', - ); - } - if (query != null && !_deepEquals(record.query, query)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" query $query, but got ${record.query}.', - ); - } - if (headers != null && !_deepEquals(record.headers, headers)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" headers $headers, but got ${record.headers}.', - ); - } - } - - void expectApiRequestContains( - String apiName, { - dynamic body, - dynamic query, - dynamic headers, - int? times, - }) { - final record = _selectApiCall(apiName, times: times); - - if (body != null && !_deepContains(record.body, body)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" body to contain $body, but got ${record.body}.', - ); - } - if (query != null && !_deepContains(record.query, query)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" query to contain $query, but got ${record.query}.', - ); - } - if (headers != null && !_deepContains(record.headers, headers)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" headers to contain $headers, but got ${record.headers}.', - ); - } - } - void expectCount(String id, int expected) { final count = finderForId(id).evaluate().length; if (count != expected) { @@ -341,24 +288,8 @@ class AssertionEngine { } } - void expectApiHeader( - String apiName, - String header, - dynamic expected, { - int? times, - }) { - final record = _selectApiCall(apiName, times: times); - final headers = record.headers ?? {}; - final actual = headers[header.toLowerCase()]; - if (!_deepEquals(actual, expected)) { - throw EnsembleTestFailure( - 'Expected API "$apiName" header "$header" = "$expected", got "$actual".', - ); - } - } - void expectApiCallOrder(List names) { - final actual = context.mockApiProvider.calls.map((c) => c.name).toList(); + final actual = context.apiOverlay.calls.map((c) => c.name).toList(); var index = 0; for (final name in names) { while (index < actual.length && actual[index] != name) { @@ -374,7 +305,7 @@ class AssertionEngine { } void expectLastApiCall(String apiName) { - final calls = context.mockApiProvider.calls; + final calls = context.apiOverlay.calls; if (calls.isEmpty || calls.last.name != apiName) { throw EnsembleTestFailure( 'Expected last API call to be "$apiName", ' @@ -383,34 +314,6 @@ class AssertionEngine { } } - void expectStateContains(String path, dynamic contains) { - final actual = readState(path); - if (!_deepContains(actual, contains)) { - throw EnsembleTestFailure( - 'Expected state "$path" to contain $contains, got $actual.', - ); - } - } - - void expectStateExists(String path) { - try { - readState(path); - } catch (_) { - throw EnsembleTestFailure('Expected state path "$path" to exist.'); - } - } - - void expectStateNotExists(String path) { - try { - final value = readState(path); - if (value != null) { - throw EnsembleTestFailure('Expected state path "$path" to be absent.'); - } - } on EnsembleTestFailure { - // expected - } - } - void expectConsoleLog(String contains) { final logs = context.runtime.consoleLogs; if (!logs.any((l) => l.contains(contains))) { @@ -480,45 +383,6 @@ class AssertionEngine { } } - APICallRecord _selectApiCall(String apiName, {int? times}) { - final calls = context.mockApiProvider.callsFor(apiName); - if (calls.isEmpty) { - throw EnsembleTestFailure('Expected API "$apiName" to be called.'); - } - if (times != null) { - if (times < 1 || times > calls.length) { - throw EnsembleTestFailure( - 'Expected API "$apiName" call #$times, but it was called ${calls.length} times.', - ); - } - return calls[times - 1]; - } - return calls.last; - } - - /// Reads a data-context path without asserting. - dynamic readState(String path) { - final scope = _activeScope(); - if (scope == null) { - throw EnsembleTestFailure( - 'readState requires an active Ensemble screen (no ScopeManager found).', - ); - } - if (path.contains(r'${') || path.contains(r'$(')) { - return scope.dataContext.eval(path); - } - return scope.dataContext.eval('\${$path}'); - } - - bool matchesState(String path, dynamic expected) { - try { - expectState(path, expected); - return true; - } on EnsembleTestFailure { - return false; - } - } - void expectStorage(String key, dynamic expected) { final actual = StorageManager().read(key); if (actual != expected) { @@ -530,28 +394,6 @@ class AssertionEngine { ScopeManager? activeScope() => _activeScope(); - void expectState(String path, dynamic expected) { - final scope = _activeScope(); - if (scope == null) { - throw EnsembleTestFailure( - 'expectState requires an active Ensemble screen (no ScopeManager found).', - ); - } - - dynamic actual; - if (path.contains(r'${') || path.contains(r'$(')) { - actual = scope.dataContext.eval(path); - } else { - actual = scope.dataContext.eval('\${$path}'); - } - - if (!_deepEquals(actual, expected)) { - throw EnsembleTestFailure( - 'Expected state "$path" to equal "$expected", but got "$actual".', - ); - } - } - void expectNavigateTo(String screenName) { final tracker = ScreenTracker(); if (!tracker.isScreenVisible(screenName: screenName) && @@ -595,7 +437,7 @@ class AssertionEngine { } String apiCallSummary({int limit = 10}) { - final calls = context.mockApiProvider.calls; + final calls = context.apiOverlay.calls; if (calls.isEmpty) return 'No API calls were recorded.'; final names = calls.take(limit).map((call) => call.name).join(', '); final suffix = calls.length > limit ? ', ... (${calls.length} total)' : ''; @@ -614,37 +456,6 @@ class AssertionEngine { return null; } - /// True when [subset] is contained in [value] (maps/lists recurse). - bool _deepContains(dynamic value, dynamic subset) { - if (subset == null) return true; - if (value == null) return false; - - final normalizedValue = _normalizeForCompare(value); - final normalizedSubset = _normalizeForCompare(subset); - - if (normalizedSubset is Map) { - if (normalizedValue is! Map) return false; - for (final entry in normalizedSubset.entries) { - if (!normalizedValue.containsKey(entry.key)) return false; - if (!_deepContains(normalizedValue[entry.key], entry.value)) { - return false; - } - } - return true; - } - - if (normalizedSubset is List) { - if (normalizedValue is! List) return false; - for (final item in normalizedSubset) { - final found = normalizedValue.any((v) => _deepContains(v, item)); - if (!found) return false; - } - return true; - } - - return normalizedValue == normalizedSubset; - } - bool _deepEquals(dynamic a, dynamic b) { if (a == b) return true; diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart index ec09e50fb..9a74df34a 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli.dart @@ -1,3 +1,5 @@ +import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'package:ensemble_test_runner/cli/ensemble_test_doctor.dart'; @@ -23,13 +25,17 @@ import 'package:ensemble_test_runner/validation/ensemble_test_validator.dart'; /// --feature= Run matching feature(s); repeatable /// --tag= Run matching tag(s); repeatable /// --path= Run matching test asset path(s); repeatable +/// --input key=value Provide a test input for ${inputs.key}; repeatable /// --timeout= Test suite timeout, e.g. 30s, 5m, 1h (default: 10m) /// --verbose Full `flutter pub get` / `flutter test` output Future runEnsembleYamlTestsCli(List arguments) async { final verbose = isVerboseCli(arguments); + final quiet = arguments.contains('--quiet'); final reportMode = _resolveReportMode(arguments); final jsonReport = reportMode == 'json'; final junitReport = reportMode == 'junit'; + final machineReport = jsonReport || junitReport; + final streamLiveOutput = !quiet && !machineReport; final reportFile = _resolveReportFile(arguments); final appDir = _resolveAppDir(arguments); final timeoutSeconds = _resolveTimeoutSeconds(arguments); @@ -102,9 +108,19 @@ Future runEnsembleYamlTestsCli(List arguments) async { var exitCode = 0; try { + _writeStatus( + 'Preparing Ensemble YAML tests...', + quiet: quiet, + machineReport: machineReport, + ); patcher.enable(); if (patcher.pubspecChanged) { + _writeStatus( + 'Resolving Flutter dependencies...', + quiet: quiet, + machineReport: machineReport, + ); final pubGet = await _runProcess( 'flutter', ['pub', 'get', '--suppress-analytics'], @@ -129,34 +145,53 @@ Future runEnsembleYamlTestsCli(List arguments) async { '--dart-define=ensembleTestReportFile=$reportFile', if (timeoutSeconds != null) '--dart-define=ensembleTestTimeoutSeconds=$timeoutSeconds', + ..._inputDartDefines(arguments), ..._selectionDartDefines(arguments), '--reporter', verbose ? 'expanded' : 'silent', ...flutterTestArguments(arguments), ]; - final testRun = await _runProcess( + _writeStatus( + 'Running Ensemble YAML tests...', + quiet: quiet, + machineReport: machineReport, + ); + final testRun = await _runFlutterTestProcess( 'flutter', testArgs, workingDirectory: appDir, + streamOutput: streamLiveOutput, + verbose: verbose, ); if (testRun.exitCode != 0 && !verbose) { final output = '${testRun.stdout ?? ''}\n${testRun.stderr ?? ''}'; final json = jsonReport ? extractJsonReport(output) : ''; final junit = junitReport ? extractJunitReport(output) : ''; + final report = machineReport + ? '' + : extractSuiteReport( + output, + includeScreenTracker: !streamLiveOutput, + ); final knownFailure = extractKnownFailure(output); if (junit.isNotEmpty) { stdout.writeln(junit); } else if (json.isNotEmpty) { stdout.writeln(json); + } else if (report.isNotEmpty) { + stdout.write(report); + if (!report.endsWith('\n')) stdout.writeln(); } else if (knownFailure.isNotEmpty) { stderr.writeln(knownFailure); } else { _writeProcessStreams(testRun); } - } else if (verbose || testRun.exitCode != 0) { + } else if (!verbose && testRun.exitCode != 0) { _writeProcessStreams(testRun); + } else if (verbose) { + // Output was already streamed live. } else { final out = testRun.stdout?.toString() ?? ''; final json = jsonReport ? extractJsonReport(out) : ''; @@ -165,7 +200,10 @@ Future runEnsembleYamlTestsCli(List arguments) async { ? junit : json.isNotEmpty ? json - : extractSuiteReport(out); + : extractSuiteReport( + out, + includeScreenTracker: !streamLiveOutput, + ); if (report.isNotEmpty) { stdout.write(report); if (!report.endsWith('\n')) stdout.writeln(); @@ -202,6 +240,52 @@ List _selectionDartDefines(List arguments) { ]; } +List _inputDartDefines(List arguments) { + final inputs = _inputValues(arguments); + if (inputs.isEmpty) return const []; + final encoded = base64Url.encode(utf8.encode(jsonEncode(inputs))); + return ['--dart-define=ensembleTestInputs=$encoded']; +} + +Map _inputValues(List arguments) { + final inputs = {}; + for (var i = 0; i < arguments.length; i++) { + final arg = arguments[i]; + String? raw; + if (arg == '--input') { + if (i + 1 >= arguments.length) { + throw StateError('--input requires key=value'); + } + raw = arguments[++i]; + } else if (arg.startsWith('--input=')) { + raw = arg.substring('--input='.length); + } + if (raw == null) continue; + + final separator = raw.indexOf('='); + if (separator <= 0) { + throw StateError('Invalid --input "$raw". Use --input key=value.'); + } + final key = raw.substring(0, separator).trim(); + if (key.isEmpty) { + throw StateError('Invalid --input "$raw". Input key cannot be empty.'); + } + inputs[key] = _parseInputValue(raw.substring(separator + 1)); + } + return inputs; +} + +dynamic _parseInputValue(String value) { + final trimmed = value.trim(); + if (trimmed == 'true') return true; + if (trimmed == 'false') return false; + final intValue = int.tryParse(trimmed); + if (intValue != null) return intValue; + final doubleValue = double.tryParse(trimmed); + if (doubleValue != null) return doubleValue; + return value; +} + List _optionValues(List arguments, String name) { return arguments .where((arg) => arg.startsWith('$name=')) @@ -210,6 +294,75 @@ List _optionValues(List arguments, String name) { .toList(); } +Future _runFlutterTestProcess( + String executable, + List arguments, { + required String workingDirectory, + required bool streamOutput, + required bool verbose, +}) async { + final process = await Process.start( + executable, + arguments, + workingDirectory: workingDirectory, + runInShell: false, + ); + final stdoutBuffer = StringBuffer(); + final stderrBuffer = StringBuffer(); + final liveFilter = LiveFlutterTestOutputFilter(); + final elapsed = Stopwatch()..start(); + var lastLiveOutput = DateTime.now(); + Timer? heartbeat; + + if (streamOutput && !verbose) { + heartbeat = Timer.periodic(const Duration(seconds: 10), (_) { + final idleFor = DateTime.now().difference(lastLiveOutput); + if (idleFor.inSeconds >= 10) { + stderr.writeln( + 'Still running Flutter tests (${elapsed.elapsed.inSeconds}s)...', + ); + } + }); + } + + try { + final stdoutDone = process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((line) { + stdoutBuffer.writeln(line); + if (verbose) { + stdout.writeln(line); + } else if (streamOutput && liveFilter.shouldEmit(line)) { + lastLiveOutput = DateTime.now(); + stderr.writeln(line); + } + }).asFuture(); + + final stderrDone = process.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((line) { + stderrBuffer.writeln(line); + if (verbose) { + stderr.writeln(line); + } + }).asFuture(); + + final exitCode = await process.exitCode; + await Future.wait([stdoutDone, stderrDone]); + + return ProcessResult( + process.pid, + exitCode, + stdoutBuffer.toString(), + stderrBuffer.toString(), + ); + } finally { + heartbeat?.cancel(); + } +} + Future _runProcess( String executable, List arguments, { @@ -282,6 +435,15 @@ int? _resolveTimeoutSeconds(List arguments) { return seconds; } +void _writeStatus( + String message, { + required bool quiet, + required bool machineReport, +}) { + if (quiet || machineReport) return; + stderr.writeln(message); +} + void _writeProcessStreams(ProcessResult result) { final out = result.stdout?.toString() ?? ''; final err = result.stderr?.toString() ?? ''; diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart index b4037e4ca..7c5ab8b63 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart @@ -6,15 +6,45 @@ const screenTrackerPrefix = 'SCREEN TRACKER:'; const noDeclarativeTestsPrefix = 'No declarative tests found.'; const jsonReportPrefix = 'ENSEMBLE_TEST_JSON_REPORT:'; const junitReportPrefix = 'ENSEMBLE_TEST_JUNIT_REPORT:'; +const flutterTestExceptionStart = + '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK'; +const flutterTakeExceptionHint = + '(The following exception is now available via WidgetTester.takeException:)'; + +/// Filters stdout from a running `flutter test` process for live CLI display. +/// +/// The full stdout is still captured and parsed at the end. This filter only +/// decides which lines are safe to show while the process is still running. +class LiveFlutterTestOutputFilter { + var _suppressRest = false; + + bool shouldEmit(String line) { + if (_suppressRest) return false; + if (line.startsWith(jsonReportPrefix) || + line.startsWith(junitReportPrefix) || + line.startsWith(flutterTakeExceptionHint)) { + return false; + } + if (line.startsWith(suiteReportStart) || + line.startsWith(flutterTestExceptionStart)) { + _suppressRest = true; + return false; + } + return line.trim().isNotEmpty; + } +} /// Strips Flutter test framework noise; keeps navigation logs and the suite report. -String extractSuiteReport(String output) { +String extractSuiteReport( + String output, { + bool includeScreenTracker = true, +}) { final lines = output.split('\n'); final kept = []; var inReport = false; for (final line in lines) { - if (line.startsWith(screenTrackerPrefix)) { + if (includeScreenTracker && line.startsWith(screenTrackerPrefix)) { kept.add(line); continue; } @@ -38,6 +68,14 @@ bool isVerboseCli(List arguments) => arguments.contains('--verbose'); /// Extracts known actionable failures from Flutter's framework error output. String extractKnownFailure(String output) { + final ensembleFailure = _extractFrameworkFailureMessage( + output, + 'The following EnsembleTestFailure was thrown running a test:', + ); + if (ensembleFailure.isNotEmpty) { + return ensembleFailure; + } + final noTests = RegExp( r'No declarative tests found\. Add \*\.test\.yaml files under [^\r\n]+', ).firstMatch(output); @@ -47,6 +85,16 @@ String extractKnownFailure(String output) { return ''; } +String _extractFrameworkFailureMessage(String output, String marker) { + final start = output.indexOf(marker); + if (start < 0) return ''; + final messageStart = start + marker.length; + final messageEnd = + output.indexOf('\n\nWhen the exception was thrown', messageStart); + if (messageEnd < 0) return ''; + return output.substring(messageStart, messageEnd).trim(); +} + String extractJsonReport(String output) { return _extractPrefixedReport(output, jsonReportPrefix); } @@ -67,26 +115,34 @@ String _extractPrefixedReport(String output, String prefix) { /// Arguments forwarded to `flutter test` (CLI-only flags removed). List flutterTestArguments(List arguments) { - return arguments - .where( - (a) => - !a.startsWith('--app-dir') && - !a.startsWith('--report') && - a != '--doctor' && - a != '--inspect-app' && - a != '--validate-only' && - a != '--scaffold-test' && - !a.startsWith('--scaffold-test=') && - !a.startsWith('--screen=') && - !a.startsWith('--id=') && - !a.startsWith('--feature=') && - !a.startsWith('--tag=') && - !a.startsWith('--path=') && - !a.startsWith('--timeout=') && - a != '--verbose' && - a != '--quiet', - ) - .toList(); + final forwarded = []; + for (var i = 0; i < arguments.length; i++) { + final a = arguments[i]; + if (a == '--input') { + i++; + continue; + } + if (a.startsWith('--input=')) continue; + if (a.startsWith('--app-dir') || + a.startsWith('--report') || + a == '--doctor' || + a == '--inspect-app' || + a == '--validate-only' || + a == '--scaffold-test' || + a.startsWith('--scaffold-test=') || + a.startsWith('--screen=') || + a.startsWith('--id=') || + a.startsWith('--feature=') || + a.startsWith('--tag=') || + a.startsWith('--path=') || + a.startsWith('--timeout=') || + a == '--verbose' || + a == '--quiet') { + continue; + } + forwarded.add(a); + } + return forwarded; } /// Flutter sometimes prints asset cleanup warnings after a successful run. diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart index 046957465..53c34deb5 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_doctor.dart @@ -1,10 +1,13 @@ import 'dart:io'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; import 'package:path/path.dart' as p; import 'package:yaml/yaml.dart'; const hostedSchemaUrl = 'https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json'; +const hostedConfigSchemaUrl = + 'https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json'; class EnsembleTestDoctorResult { final List lines; @@ -108,6 +111,24 @@ class EnsembleTestDoctor { } ok('Found ${testFiles.length} YAML test file(s)'); + final testConfigFile = File(p.join(testsDir.path, 'config.yaml')); + if (testConfigFile.existsSync()) { + final relativePath = p.relative(testConfigFile.path, from: appDir); + final content = testConfigFile.readAsStringSync(); + if (!content.contains(hostedConfigSchemaUrl)) { + warn('$relativePath does not reference the hosted config schema URL'); + } + try { + EnsembleTestParser.parseConfigString( + content, + sourcePath: relativePath, + ); + ok('Found tests/config.yaml'); + } catch (failure) { + error('$relativePath: $failure'); + } + } + final ids = {}; final prerequisites = {}; final referencedWidgetIds = {}; @@ -190,6 +211,16 @@ _DoctorTest _parseDoctorTest(String content) { ); } + if (doc.containsKey('options')) { + return ( + id: '', + prerequisite: null, + referencedWidgetIds: {}, + error: + 'Root-level "options" is no longer supported. Move shared settings to tests/config.yaml.', + ); + } + final id = doc['id']?.toString(); if (id == null || id.isEmpty) { return ( diff --git a/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart b/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart index f29a8b8ad..ef9942b7b 100644 --- a/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart +++ b/tools/ensemble_test_runner/lib/cli/ensemble_test_scaffold.dart @@ -32,7 +32,7 @@ class EnsembleTestScaffold { final testsDir = Directory(p.join(appDir, inspection.appPath, 'tests')); testsDir.createSync(recursive: true); - Directory(p.join(testsDir.path, 'fixtures')).createSync(recursive: true); + Directory(p.join(testsDir.path, 'mocks')).createSync(recursive: true); final file = File(p.join(testsDir.path, '$normalizedId.test.yaml')); if (file.existsSync()) { diff --git a/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart b/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart index 1d4b09e2d..31c7d52ec 100644 --- a/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart +++ b/tools/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart @@ -13,13 +13,30 @@ class YamlTestAppPatcher { static String testsAssetLineFor(String testsDirRelative) => ' - ${_withTrailingSlash(testsDirRelative)}'; - static const testEntryContents = ''' + /// Legacy CLI stub without module bootstrap. + static const legacyTestEntryContents = ''' // Generated by ensemble_test_runner — run `dart run ensemble_test_runner:ensemble_test`. import 'package:ensemble_test_runner/entry/ensemble_test_entry.dart'; void main() => runEnsembleYamlTests(); '''; + static String testEntryContentsFor(String packageName) => ''' +// Generated by ensemble_test_runner — customize bootstrap here. +import 'package:ensemble_test_runner/entry/ensemble_test_entry.dart'; +import 'package:$packageName/generated/ensemble_modules.dart'; + +Future main() async { + await runEnsembleYamlTests( + bootstrap: () => EnsembleModules().init(), + ); +} +'''; + + /// @deprecated Use [testEntryContentsFor]. + static String get testEntryContents => + testEntryContentsFor('your_app_package'); + final Map _backups = {}; bool _enabled = false; bool _removeTestEntryOnRestore = false; @@ -52,10 +69,6 @@ void main() => runEnsembleYamlTests(); _backup(_pubspecPath); _backup(_testEntryPath, optional: true); - final priorTestEntry = _backups[_testEntryPath] ?? ''; - _removeTestEntryOnRestore = - priorTestEntry.isEmpty || priorTestEntry == testEntryContents; - final pubspec = File(_pubspecPath).readAsStringSync(); final activated = _activatePubspec(pubspec); _pubspecChanged = activated != pubspec; @@ -63,11 +76,23 @@ void main() => runEnsembleYamlTests(); File(_pubspecPath).writeAsStringSync(activated); } - Directory(_testDirPath).createSync(recursive: true); final testEntry = File(_testEntryPath); - if (!testEntry.existsSync() || - testEntry.readAsStringSync() != testEntryContents) { - testEntry.writeAsStringSync(testEntryContents); + final testEntryExisted = testEntry.existsSync(); + _removeTestEntryOnRestore = !testEntryExisted; + + if (!testEntryExisted) { + Directory(_testDirPath).createSync(recursive: true); + testEntry.writeAsStringSync( + testEntryContentsFor(_readPackageName()), + ); + } else { + final content = testEntry.readAsStringSync(); + if (content.trim() == legacyTestEntryContents.trim()) { + final upgraded = testEntryContentsFor(_readPackageName()); + testEntry.writeAsStringSync(upgraded); + _backups[_testEntryPath] = upgraded; + _removeTestEntryOnRestore = false; + } } _enabled = true; @@ -107,6 +132,16 @@ void main() => runEnsembleYamlTests(); File(path).writeAsStringSync(backup); } + String _readPackageName() { + final content = File(_pubspecPath).readAsStringSync(); + final match = + RegExp(r'^name:\s*(\S+)', multiLine: true).firstMatch(content); + if (match == null) { + throw StateError('Could not read package name from pubspec.yaml'); + } + return match.group(1)!; + } + void _deleteTestEntry() { final file = File(_testEntryPath); if (file.existsSync()) { @@ -121,11 +156,31 @@ void main() => runEnsembleYamlTests(); String _activatePubspec(String content) { final testsDir = testsDirRelative; if (testsDir != null && _hasTestYamlOnDisk(testsDir)) { - return _activateTestAssets(content, testsDir); + return _activateTestAssets(content, testsDir, _testsAssetLines(testsDir)); } return content; } + List _testsAssetLines(String testsDirRelative) { + final testsDir = Directory(p.join(appDir, testsDirRelative)); + final dirs = {_withTrailingSlash(testsDirRelative)}; + if (testsDir.existsSync()) { + for (final entity in testsDir.listSync(recursive: true)) { + if (entity is! File) continue; + final dir = p.dirname(p.relative(entity.path, from: appDir)); + dirs.add(_withTrailingSlash(p.split(dir).join('/'))); + } + } + final sortedDirs = dirs.toList() + ..sort((a, b) { + final depthCompare = '/'.allMatches(a).length.compareTo( + '/'.allMatches(b).length, + ); + return depthCompare == 0 ? a.compareTo(b) : depthCompare; + }); + return sortedDirs.map(testsAssetLineFor).toList(); + } + String? get _testsDirRelative { final configFile = File(_ensembleConfigPath); if (!configFile.existsSync()) return null; @@ -153,20 +208,37 @@ void main() => runEnsembleYamlTests(); .any((entity) => entity.path.endsWith('.test.yaml')); } - static String _activateTestAssets(String content, String testsDirRelative) { + static String _activateTestAssets( + String content, + String testsDirRelative, + List testAssetLines, + ) { final normalizedTestsDir = _withTrailingSlash(testsDirRelative); final testsAssetLine = testsAssetLineFor(normalizedTestsDir); - if (content.contains(testsAssetLine) || - content.contains('- $normalizedTestsDir')) { + final missingLines = testAssetLines + .where( + (line) => + !content.contains(line) && !content.contains(line.trimLeft()), + ) + .toList(); + if (missingLines.isEmpty) { return content; } + final insertion = missingLines.join('\n'); + + if (content.contains(testsAssetLine)) { + return content.replaceFirst( + testsAssetLine, + '$testsAssetLine\n$insertion', + ); + } final localAppLine = ' - ${_withTrailingSlash(p.posix.dirname(_withoutTrailingSlash(normalizedTestsDir)))}\n'; if (content.contains(localAppLine)) { return content.replaceFirst( localAppLine, - '$localAppLine$testsAssetLine\n', + '$localAppLine$insertion\n', ); } @@ -174,7 +246,7 @@ void main() => runEnsembleYamlTests(); if (content.contains(ensembleDirLine)) { return content.replaceFirst( ensembleDirLine, - '$ensembleDirLine$testsAssetLine\n', + '$ensembleDirLine$insertion\n', ); } @@ -182,7 +254,7 @@ void main() => runEnsembleYamlTests(); if (content.contains(assetsMarker)) { return content.replaceFirst( assetsMarker, - '$assetsMarker$testsAssetLine\n', + '$assetsMarker$insertion\n', ); } diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart index 61aa97af0..4cbf3f8fa 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart @@ -1,5 +1,6 @@ import 'package:ensemble/framework/ensemble_config_service.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter/services.dart'; @@ -36,6 +37,22 @@ class EnsembleTestDiscovery { return files; } + /// Optional suite-level config bundled as `tests/config.yaml`. + static Future findConfigYamlAsset(String testsAssetPrefix) async { + final manifest = await AssetManifest.loadFromAssetBundle(rootBundle); + final path = '${testsAssetPrefix}config.yaml'; + return manifest.listAssets().contains(path) ? path : null; + } + + static Future loadTestConfig( + String testsAssetPrefix, + ) async { + final path = await findConfigYamlAsset(testsAssetPrefix); + if (path == null) return const EnsembleTestConfig(); + final content = await rootBundle.loadString(path); + return EnsembleTestParser.parseConfigString(content, sourcePath: path); + } + /// Reads `definitions.local` from `ensemble/ensemble-config.yaml`. static Future loadAppTarget() async { if (!EnsembleConfigService.isInitialized) { diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart index 54b6d0cbe..786c3a3c7 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart @@ -1,8 +1,14 @@ +import 'dart:convert'; + import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_discovery.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; +import 'package:yaml/yaml.dart'; + +typedef _AssetStringLoader = Future Function(String assetPath); /// A parsed `*.test.yaml` file with its asset path. class EnsembleTestDefinition { @@ -18,8 +24,12 @@ class EnsembleTestDefinition { /// Topologically sorted test run order (each test id appears once). class EnsembleTestExecutionPlan { final List ordered; + final EnsembleTestConfig config; - const EnsembleTestExecutionPlan({required this.ordered}); + const EnsembleTestExecutionPlan({ + required this.ordered, + this.config = const EnsembleTestConfig(), + }); } class EnsembleTestSelection { @@ -45,12 +55,16 @@ class EnsembleTestExecutionPlanner { static Future build({ EnsembleTestAppTarget? target, EnsembleTestSelection selection = const EnsembleTestSelection(), + Map inputs = const {}, }) async { final resolvedTarget = target ?? await EnsembleTestDiscovery.loadAppTarget(); final paths = await EnsembleTestDiscovery.findTestYamlAssets( resolvedTarget.testsAssetPrefix, ); + final config = await EnsembleTestDiscovery.loadTestConfig( + resolvedTarget.testsAssetPrefix, + ); if (paths.isEmpty) { throw EnsembleTestFailure( 'No declarative tests found. Add *.test.yaml files under ' @@ -60,17 +74,22 @@ class EnsembleTestExecutionPlanner { final byId = {}; for (final path in paths) { - final testCase = await EnsembleTestParser.parseFile(path); - final existing = byId[testCase.id]; - if (existing != null) { - throw EnsembleTestFailure( - 'Duplicate test id "${testCase.id}" in ${existing.assetPath} and $path', - ); - } - byId[testCase.id] = EnsembleTestDefinition( - assetPath: path, - testCase: testCase, + final content = await rootBundle.loadString(path); + final definitions = await _parseDefinitionsFromAsset( + path, + content, + inputs: inputs, ); + for (final definition in definitions) { + final existing = byId[definition.testCase.id]; + if (existing != null) { + throw EnsembleTestFailure( + 'Duplicate test id "${definition.testCase.id}" in ' + '${existing.assetPath} and $path', + ); + } + byId[definition.testCase.id] = definition; + } } final selectedById = _applySelection(byId, selection); @@ -86,7 +105,273 @@ class EnsembleTestExecutionPlanner { } final ordered = _topologicalSort(selectedById); - return EnsembleTestExecutionPlan(ordered: ordered); + return EnsembleTestExecutionPlan(ordered: ordered, config: config); + } + + /// Exposed for unit tests only. + @visibleForTesting + static Future> parseDefinitionsForTest( + String path, + String content, { + Map inputs = const {}, + Future Function(String assetPath)? assetLoader, + }) { + return _parseDefinitionsFromAsset( + path, + content, + inputs: inputs, + assetLoader: assetLoader ?? _rootBundleAssetLoader, + ); + } + + static Future> _parseDefinitionsFromAsset( + String path, + String content, { + required Map inputs, + _AssetStringLoader assetLoader = _rootBundleAssetLoader, + }) async { + if (loadYaml(content) == null) { + return const []; + } + + final base = EnsembleTestParser.parseString( + content, + sourcePath: path, + inputs: inputs, + ); + if (base.scenarios.isEmpty) { + final mocks = await _mergedMocksFor( + assetPath: path, + mockFiles: base.mockFiles, + assetLoader: assetLoader, + ); + return [ + EnsembleTestDefinition( + assetPath: path, + testCase: _withRuntimeFields( + base, + id: base.id, + startScreen: base.startScreen, + prerequisite: base.prerequisite, + mocks: mocks, + ), + ), + ]; + } + + final definitions = []; + String? previousScenarioId; + for (final scenario in base.scenarios) { + final parsed = EnsembleTestParser.parseString( + content, + sourcePath: path, + inputs: inputs, + scenario: scenario.vars, + scenarioId: scenario.id, + ); + final parsedScenario = parsed.scenarios.firstWhere( + (item) => item.id == scenario.id, + orElse: () => scenario, + ); + final id = '${base.id}[${scenario.id}]'; + final prerequisite = parsed.prerequisite == null + ? null + : previousScenarioId ?? parsed.prerequisite; + final mocks = await _mergedMocksFor( + assetPath: path, + mockFiles: parsed.mockFiles, + assetLoader: assetLoader, + ); + + definitions.add( + EnsembleTestDefinition( + assetPath: path, + testCase: _withRuntimeFields( + parsed, + id: id, + description: parsedScenario.description ?? parsed.description, + startScreen: + parsed.prerequisite == null ? parsed.startScreen : null, + prerequisite: prerequisite, + mocks: mocks, + ), + ), + ); + previousScenarioId = id; + } + return definitions; + } + + static EnsembleTestCase _withRuntimeFields( + EnsembleTestCase test, { + required String id, + String? description, + String? startScreen, + String? prerequisite, + required TestMocks mocks, + }) { + return EnsembleTestCase( + id: id, + sourcePath: test.sourcePath, + type: test.type, + feature: test.feature, + tags: test.tags, + description: description ?? test.description, + owner: test.owner, + priority: test.priority, + startScreen: startScreen, + prerequisite: prerequisite, + mockFiles: test.mockFiles, + initialState: test.initialState, + mocks: mocks, + steps: test.steps, + ); + } + + static Future _mergedMocksFor({ + required String assetPath, + required List mockFiles, + required _AssetStringLoader assetLoader, + }) async { + final apis = {}; + for (final file in mockFiles) { + final fileMocks = await _loadMockFile( + assetPath, + file, + assetLoader: assetLoader, + ); + apis.addAll(fileMocks.apis); + } + return TestMocks(apis: apis); + } + + static Future _loadMockFile( + String testAssetPath, + String mockFilePath, { + required _AssetStringLoader assetLoader, + }) async { + final assetPath = _resolveAssetPath(testAssetPath, mockFilePath); + final content = await _loadRequiredAsset( + assetPath, + 'Mock file "$mockFilePath" referenced by $testAssetPath was not found.', + assetLoader: assetLoader, + ); + if (!assetPath.endsWith('.mock.json')) { + throw EnsembleTestFailure( + 'Mock file "$mockFilePath" must be a .mock.json file.', + ); + } + + final dynamic doc = _parseMockFileDocument(content, assetPath); + if (doc == null) return const TestMocks(); + if (doc is! Map) { + throw EnsembleTestFailure('Mock file "$assetPath" root must be a map'); + } + + final apis = {}; + for (final entry in doc.entries) { + if (entry.value is! Map) { + throw EnsembleTestFailure( + 'Mock for API "${entry.key}" in "$assetPath" must be a map', + ); + } + apis[entry.key.toString()] = _parseLayerMockApiResponse( + Map.from(entry.value as Map), + layerAssetPath: assetPath, + apiName: entry.key.toString(), + ); + } + return TestMocks(apis: apis); + } + + static dynamic _parseMockFileDocument(String content, String assetPath) { + try { + return jsonDecode(content); + } on FormatException catch (error) { + throw EnsembleTestFailure( + 'Mock file "$assetPath" is not valid JSON: ${error.message}', + ); + } + } + + static MockAPIResponse _parseLayerMockApiResponse( + Map map, { + required String layerAssetPath, + required String apiName, + }) { + if (map.isEmpty) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$layerAssetPath" must include a response', + ); + } + final unknownKeys = map.keys + .map((key) => key.toString()) + .where( + (key) => + !const {'statusCode', 'body', 'headers', 'delayMs'}.contains(key), + ) + .toList(); + if (unknownKeys.isNotEmpty) { + throw EnsembleTestFailure( + 'API mock "$apiName" in "$layerAssetPath" has unsupported keys: ' + '${unknownKeys.join(", ")}. Use direct JSON shape: ' + '{"statusCode": 200, "body": ...}.', + ); + } + + return MockAPIResponse( + statusCode: map['statusCode'] as int? ?? 200, + body: map.containsKey('body') ? _unwrapJsonValue(map['body']) : null, + headers: map['headers'] is Map + ? Map.from( + _unwrapJsonValue(map['headers']) as Map, + ) + : null, + delayMs: map['delayMs'] as int?, + ); + } + + static dynamic _unwrapJsonValue(dynamic value) { + if (value is Map) { + return { + for (final entry in value.entries) + entry.key.toString(): _unwrapJsonValue(entry.value), + }; + } + if (value is List) { + return value.map(_unwrapJsonValue).toList(); + } + return value; + } + + static Future _loadRequiredAsset( + String assetPath, + String missingMessage, { + required _AssetStringLoader assetLoader, + }) async { + try { + return await assetLoader(assetPath); + } on FlutterError { + throw EnsembleTestFailure(missingMessage); + } + } + + static Future _rootBundleAssetLoader(String assetPath) { + return rootBundle.loadString(assetPath); + } + + static String _resolveAssetPath(String fromAssetPath, String relativePath) { + if (relativePath.startsWith('/')) return relativePath.substring(1); + final segments = fromAssetPath.split('/')..removeLast(); + for (final part in relativePath.split('/')) { + if (part.isEmpty || part == '.') continue; + if (part == '..') { + if (segments.isNotEmpty) segments.removeLast(); + } else { + segments.add(part); + } + } + return segments.join('/'); } static Map _applySelection( @@ -130,8 +415,10 @@ class EnsembleTestExecutionPlanner { EnsembleTestSelection selection, ) { final test = def.testCase; - final idMatches = - selection.ids.isNotEmpty && selection.ids.contains(test.id); + final idMatches = selection.ids.isNotEmpty && + selection.ids.any( + (id) => test.id == id || test.id.startsWith('$id['), + ); final featureMatches = selection.features.isNotEmpty && selection.features.contains(test.feature); final tagMatches = selection.tags.isNotEmpty && diff --git a/tools/ensemble_test_runner/lib/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/ensemble_test_runner.dart index defbc1215..59e182835 100644 --- a/tools/ensemble_test_runner/lib/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/ensemble_test_runner.dart @@ -8,7 +8,7 @@ export 'actions/test_execution_config.dart'; export 'actions/test_step_executor.dart'; export 'assertions/assertion_engine.dart'; export 'models/ensemble_test_models.dart'; -export 'mocks/mock_api_provider.dart'; +export 'mocks/test_api_provider_overlay.dart'; export 'mocks/test_logger.dart'; export 'parser/ensemble_test_parser.dart'; export 'schema/ensemble_test_schema_builder.dart'; diff --git a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart index ff7327502..486c016d8 100644 --- a/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart +++ b/tools/ensemble_test_runner/lib/entry/ensemble_test_entry.dart @@ -6,6 +6,7 @@ import 'dart:io'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; import 'package:ensemble_test_runner/ensemble_test_runner.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -16,8 +17,53 @@ const _timeoutSeconds = int.fromEnvironment( defaultValue: _defaultTimeoutSeconds, ); +/// Options for [runEnsembleYamlTests], typically set from `test/ensemble_tests.dart`. +class EnsembleYamlTestOptions { + /// App bootstrap, typically `() => EnsembleModules().init()` from + /// `lib/generated/ensemble_modules.dart` (same as `main.dart`). + final Future Function()? bootstrap; + + /// Host-app methods passed to [EnsembleApp], e.g. `captureCertificateForHost`. + final Map? externalMethods; + + const EnsembleYamlTestOptions({ + this.bootstrap, + this.externalMethods, + }); +} + /// Flutter test entry: discovers app-local `tests/*.test.yaml` and runs them. -void runEnsembleYamlTests() { +/// +/// Mirror `main.dart` from `test/ensemble_tests.dart`: +/// ```dart +/// import 'package:my_app/generated/ensemble_modules.dart'; +/// +/// Future main() async { +/// await runEnsembleYamlTests( +/// bootstrap: () => EnsembleModules().init(), +/// externalMethods: { +/// 'captureCertificateForHost': captureCertificateForHost, +/// }, +/// ); +/// } +/// ``` +Future runEnsembleYamlTests({ + Future Function()? bootstrap, + Map? externalMethods, +}) { + return runEnsembleYamlTestsWithOptions( + EnsembleYamlTestOptions( + bootstrap: bootstrap, + externalMethods: externalMethods, + ), + ); +} + +/// Same as [runEnsembleYamlTests] with an explicit options object. +Future runEnsembleYamlTestsWithOptions( + EnsembleYamlTestOptions options, +) async { + LiveTestWidgetsFlutterBinding.ensureInitialized(); EnsembleTestHarness.ensureTestPlugins(); tearDown(() { TestErrorTracker.reset(); @@ -28,19 +74,39 @@ void runEnsembleYamlTests() { testWidgets( 'Ensemble app *.test.yaml', (tester) async { + if (options.bootstrap == null) { + fail( + 'Ensemble YAML tests require module bootstrap. ' + 'In test/ensemble_tests.dart call runEnsembleYamlTests with ' + 'bootstrap: () => EnsembleModules().init() ' + '(see ensemble_test_runner README).', + ); + } + await tester.runAsync(() async { + await options.bootstrap!(); + ensureLiveAuthActionsForTest(); + // Module constructors may schedule follow-up async init work. + await Future.delayed(Duration.zero); + }); + final target = await EnsembleTestDiscovery.loadAppTarget(); final plan = await EnsembleTestExecutionPlanner.build( target: target, selection: _selectionFromEnvironment(), + inputs: _inputsFromEnvironment(), ); final harness = EnsembleTestHarness( appPath: target.appPath, appHome: target.appHome, i18nPath: target.i18nPath, + externalMethods: options.externalMethods, ); final runner = EnsembleTestRunner(harness: harness); - final resultsById = await runner.runPlan(plan, tester); + final planResult = await runner.runPlan(plan, tester); + final resultsById = planResult.resultsById; + await YamlTestSession.navigationFlow.flushPending(); + await tester.pump(); final failures = []; final orderedResults = []; @@ -67,8 +133,12 @@ void runEnsembleYamlTests() { } } - final runResult = EnsembleTestRunResult(results: orderedResults); - final suiteSummary = TestReporter().formatSummary( + final runResult = EnsembleTestRunResult( + results: orderedResults, + suiteLogs: planResult.suiteLogs, + ); + final reporter = TestReporter(); + final suiteSummary = reporter.formatSummary( runResult, testFile: '${target.testsAssetPrefix}*.test.yaml', ); @@ -76,10 +146,13 @@ void runEnsembleYamlTests() { _emitMachineReport(runResult); if (failures.isNotEmpty) { + final pendingFrameworkExceptions = _drainPendingExceptions(tester); fail( - 'Failed YAML tests:\n' - '${failures.map((p) => '- $p').join('\n')}\n\n' - '$suiteSummary', + reporter.formatFailureSummary( + runResult, + failedPaths: failures, + pendingFrameworkExceptions: pendingFrameworkExceptions, + ), ); } }, @@ -87,6 +160,15 @@ void runEnsembleYamlTests() { ); } +List _drainPendingExceptions(WidgetTester tester) { + final exceptions = []; + Object? exception; + while ((exception = tester.takeException()) != null) { + exceptions.add(exception); + } + return exceptions; +} + EnsembleTestSelection _selectionFromEnvironment() { return EnsembleTestSelection( ids: _csvSet(const String.fromEnvironment('ensembleTestId')), @@ -105,6 +187,21 @@ Set _csvSet(String value) { .toSet(); } +Map _inputsFromEnvironment() { + const encoded = String.fromEnvironment('ensembleTestInputs'); + if (encoded.isEmpty) return const {}; + try { + final decoded = utf8.decode(base64Url.decode(encoded)); + final value = json.decode(decoded); + if (value is Map) { + return value.map((key, value) => MapEntry(key.toString(), value)); + } + } catch (_) { + // Fall through to the explicit failure below. + } + throw EnsembleTestFailure('Invalid ensembleTestInputs dart-define payload.'); +} + void _emitMachineReport(EnsembleTestRunResult result) { const reportMode = String.fromEnvironment('ensembleTestReport'); const reportFile = String.fromEnvironment('ensembleTestReportFile'); diff --git a/tools/ensemble_test_runner/lib/mocks/adobe_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/adobe_test_setup.dart new file mode 100644 index 000000000..02b39f25c --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/adobe_test_setup.dart @@ -0,0 +1,53 @@ +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _adobeMocksInstalled = false; + +/// Installs no-op Adobe Experience Platform method channel handlers so +/// [AdobeAnalyticsImpl] can initialize under [flutter test] without native SDKs. +void ensureAdobeAnalyticsMocksForTest() { + if (_adobeMocksInstalled) return; + + const channelNames = [ + 'flutter_aepcore', + 'flutter_aepedge', + 'flutter_aepidentity', + 'flutter_aeplifecycle', + 'flutter_aepsignal', + 'flutter_aepedgeidentity', + 'flutter_aepedgeconsent', + 'flutter_aepassurance', + 'flutter_aepuserprofile', + ]; + + for (final name in channelNames) { + _installAdobeChannelMock(name); + } + + _adobeMocksInstalled = true; +} + +void _installAdobeChannelMock(String channelName) { + final channel = MethodChannel(channelName); + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(channel, (call) async { + switch (call.method) { + case 'sendEvent': + return []; + case 'extensionVersion': + return 'ensemble-test'; + case 'getExperienceCloudId': + case 'getLocationHint': + case 'getUrlVariables': + return null; + case 'getIdentities': + return {}; + case 'getUserAttributes': + return {}; + case 'getConsents': + return {}; + default: + return null; + } + }); +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart new file mode 100644 index 000000000..2b932e5f2 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_auth_test_setup.dart @@ -0,0 +1,163 @@ +import 'package:firebase_auth_platform_interface/src/pigeon/messages.pigeon.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _firebaseAuthBridgeInstalled = false; + +class _LiveAuthSession { + _LiveAuthSession({ + required this.idToken, + required this.refreshToken, + required this.uid, + required this.expiresAtMs, + this.email, + this.isEmailVerified = false, + }); + + String idToken; + final String refreshToken; + final String uid; + int expiresAtMs; + final String? email; + final bool isEmailVerified; +} + +final Map _sessionsByApp = {}; + +void recordLiveAuthSession({ + required String appName, + required String idToken, + required String refreshToken, + required String uid, + required int expiresAtMs, + String? email, + bool isEmailVerified = false, +}) { + _sessionsByApp[appName] = _LiveAuthSession( + idToken: idToken, + refreshToken: refreshToken, + uid: uid, + expiresAtMs: expiresAtMs, + email: email, + isEmailVerified: isEmailVerified, + ); +} + +bool hasLiveAuthSession(String appName) => _sessionsByApp.containsKey(appName); + +_LiveAuthSession? liveAuthSessionForApp(String appName) => + _sessionsByApp[appName]; + +/// Bridges Firebase Auth user token pigeon calls for code paths that still use +/// [FirebaseAuth] after [LiveSignInWithCustomToken] has established a session. +void ensureLiveFirebaseAuthForTest() { + if (_firebaseAuthBridgeInstalled) return; + TestWidgetsFlutterBinding.ensureInitialized(); + + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + final codec = FirebaseAuthHostApi.pigeonChannelCodec; + + void registerHandler( + String channelName, + Future> Function(List args) handler, + ) { + messenger.setMockDecodedMessageHandler( + BasicMessageChannel(channelName, codec), + (Object? message) async { + final args = (message! as List); + try { + return await handler(args); + } on PlatformException catch (error) { + return [error.code, error.message, error.details]; + } catch (error) { + return ['error', error.toString(), null]; + } + }, + ); + } + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthHostApi.registerIdTokenListener', + (args) async { + final app = _decodeApp(args[0]); + final channel = 'ensemble_test_runner/auth/id-token/${app.appName}'; + _mockEventChannel(channel); + return [channel]; + }, + ); + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthHostApi.registerAuthStateListener', + (args) async { + final app = _decodeApp(args[0]); + final channel = 'ensemble_test_runner/auth/auth-state/${app.appName}'; + _mockEventChannel(channel); + return [channel]; + }, + ); + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthHostApi.signOut', + (args) async { + final app = _decodeApp(args[0]); + _sessionsByApp.remove(app.appName); + return []; + }, + ); + + registerHandler( + 'dev.flutter.pigeon.firebase_auth_platform_interface.FirebaseAuthUserHostApi.getIdToken', + (args) async { + final app = _decodeApp(args[0]); + final session = _sessionForApp(app); + return [ + InternalIdTokenResult( + token: session.idToken, + expirationTimestamp: session.expiresAtMs, + authTimestamp: DateTime.now().millisecondsSinceEpoch, + issuedAtTimestamp: DateTime.now().millisecondsSinceEpoch, + signInProvider: 'custom', + ), + ]; + }, + ); + + _firebaseAuthBridgeInstalled = true; +} + +void _mockEventChannel(String channelName) { + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + messenger.setMockMethodCallHandler(MethodChannel(channelName), (call) async { + switch (call.method) { + case 'listen': + return 0; + case 'cancel': + return null; + default: + return null; + } + }); +} + +AuthPigeonFirebaseApp _decodeApp(Object? value) { + if (value is AuthPigeonFirebaseApp) { + return value; + } + if (value is List) { + return AuthPigeonFirebaseApp.decode(value); + } + throw ArgumentError('Unexpected Firebase app argument: $value'); +} + +_LiveAuthSession _sessionForApp(AuthPigeonFirebaseApp app) { + final session = _sessionsByApp[app.appName]; + if (session == null) { + throw PlatformException( + code: 'no-current-user', + message: 'No signed-in Firebase user for app ${app.appName}', + ); + } + return session; +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart new file mode 100644 index 000000000..2e4b8cb24 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_firestore_test_setup.dart @@ -0,0 +1,537 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:cloud_firestore_platform_interface/cloud_firestore_platform_interface.dart'; +import 'package:ensemble_test_runner/mocks/firebase_auth_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _firestoreBridgeInstalled = false; +int _snapshotListenerCounter = 0; + +/// Bridges Firestore pigeon calls to the real Firestore REST API during +/// [flutter test]. Native plugins are unavailable in the VM test binding. +void ensureLiveFirestoreForTest() { + if (_firestoreBridgeInstalled) return; + TestWidgetsFlutterBinding.ensureInitialized(); + + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + final codec = FirebaseFirestoreHostApi.pigeonChannelCodec; + + void register( + String method, + Future> Function(List args) handler, + ) { + messenger.setMockDecodedMessageHandler( + BasicMessageChannel( + 'dev.flutter.pigeon.cloud_firestore_platform_interface.FirebaseFirestoreHostApi.$method', + codec, + ), + (Object? message) async { + final args = (message! as List); + try { + return await handler(args); + } on PlatformException catch (error) { + return [error.code, error.message, error.details]; + } catch (error) { + return ['error', error.toString(), null]; + } + }, + ); + } + + Future> noop(List args) async => []; + + register('setLoggingEnabled', (_) async => []); + register('enableNetwork', noop); + register('disableNetwork', noop); + register('clearPersistence', noop); + register('terminate', noop); + register('waitForPendingWrites', noop); + register( + 'snapshotsInSyncSetup', + (_) async => [ + 'ensemble_test_runner/firestore/snapshots-in-sync', + ]); + register('persistenceCacheIndexManagerRequest', noop); + + register('documentReferenceSet', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + await _LiveFirestoreRestClient().documentSet(app, request); + return []; + }); + + register('documentReferenceUpdate', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + await _LiveFirestoreRestClient().documentUpdate(app, request); + return []; + }); + + register('documentReferenceDelete', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + await _LiveFirestoreRestClient().documentDelete(app, request); + return []; + }); + + register('documentReferenceGet', (args) async { + final app = _decodeApp(args[0]); + final request = _decodeDocumentRequest(args[1]); + final snapshot = await _LiveFirestoreRestClient().documentGet(app, request); + return [snapshot]; + }); + + register('queryGet', (args) async { + final app = _decodeApp(args[0]); + final path = args[1]! as String; + final isCollectionGroup = args[2]! as bool; + final parameters = args[3]! as InternalQueryParameters; + final options = args[4]! as InternalGetOptions; + final snapshot = await _LiveFirestoreRestClient().queryGet( + app, + path: path, + isCollectionGroup: isCollectionGroup, + parameters: parameters, + options: options, + ); + return [snapshot]; + }); + + register('documentReferenceSnapshot', (args) async { + final request = _decodeDocumentRequest(args[1]); + final listenerId = + 'ensemble_test_runner/firestore/document/${_snapshotListenerCounter++}/${request.path}'; + _mockFirestoreEventChannel('document', listenerId); + return [listenerId]; + }); + + register('querySnapshot', (args) async { + final path = args[1]! as String; + final listenerId = + 'ensemble_test_runner/firestore/query/${_snapshotListenerCounter++}/$path'; + _mockFirestoreEventChannel('query', listenerId); + return [listenerId]; + }); + + _firestoreBridgeInstalled = true; +} + +void _mockFirestoreEventChannel(String kind, String listenerId) { + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + messenger.setMockMethodCallHandler( + MethodChannel('plugins.flutter.io/firebase_firestore/$kind/$listenerId'), + (call) async { + switch (call.method) { + case 'listen': + return 0; + case 'cancel': + return null; + default: + return null; + } + }, + ); +} + +FirestorePigeonFirebaseApp _decodeApp(Object? value) { + if (value is FirestorePigeonFirebaseApp) return value; + if (value is List) return FirestorePigeonFirebaseApp.decode(value); + throw ArgumentError('Unexpected Firestore app argument: $value'); +} + +DocumentReferenceRequest _decodeDocumentRequest(Object? value) { + if (value is DocumentReferenceRequest) return value; + if (value is List) return DocumentReferenceRequest.decode(value); + throw ArgumentError('Unexpected document request argument: $value'); +} + +class _LiveFirestoreRestClient { + Future documentSet( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + final encoded = _encodeDocumentFields(request.data ?? {}); + final transforms = encoded.transforms; + final body = { + if (encoded.fields.isNotEmpty) 'fields': encoded.fields, + if (transforms.isNotEmpty) 'updateTransforms': transforms, + }; + + final option = request.option; + final updateMaskPaths = []; + if (option?.merge == true) { + updateMaskPaths.addAll(encoded.fieldPaths); + } else if (option?.mergeFields != null && option!.mergeFields!.isNotEmpty) { + updateMaskPaths.addAll( + option.mergeFields! + .map((components) => components?.whereType().join('.')) + .whereType() + .where((path) => path.isNotEmpty), + ); + } + + await _request( + app, + method: 'PATCH', + path: request.path, + updateMaskPaths: updateMaskPaths, + body: body, + ); + } + + Future documentUpdate( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + final encoded = _encodeDocumentFields(request.data ?? {}); + final body = { + if (encoded.fields.isNotEmpty) 'fields': encoded.fields, + if (encoded.transforms.isNotEmpty) 'updateTransforms': encoded.transforms, + }; + await _request( + app, + method: 'PATCH', + path: request.path, + updateMaskPaths: encoded.fieldPaths, + body: body, + ); + } + + Future documentDelete( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + await _request(app, method: 'DELETE', path: request.path); + } + + Future documentGet( + FirestorePigeonFirebaseApp app, + DocumentReferenceRequest request, + ) async { + final decoded = await _request(app, method: 'GET', path: request.path); + if (decoded == null) { + return InternalDocumentSnapshot( + path: request.path, + data: null, + metadata: InternalSnapshotMetadata( + hasPendingWrites: false, + isFromCache: false, + ), + ); + } + return _toPigeonDocumentSnapshot(request.path, decoded); + } + + Future queryGet( + FirestorePigeonFirebaseApp app, { + required String path, + required bool isCollectionGroup, + required InternalQueryParameters parameters, + required InternalGetOptions options, + }) async { + final structuredQuery = { + 'from': [ + if (isCollectionGroup) + {'collectionId': path, 'allDescendants': true} + else + {'collectionId': path.split('/').last}, + ], + if (parameters.limit != null) 'limit': parameters.limit, + }; + + final body = { + 'structuredQuery': structuredQuery, + }; + if (path.contains('/') && !isCollectionGroup) { + final parentPath = path.split('/')..removeLast(); + body['parent'] = '${_documentsRoot(app)}/${parentPath.join('/')}'; + } else { + body['parent'] = _documentsRoot(app); + } + + final decoded = await _request( + app, + method: 'POST', + suffix: ':runQuery', + body: body, + ); + + final documents = []; + if (decoded is List) { + for (final entry in decoded) { + if (entry is! Map) continue; + final document = entry['document']; + if (document is! Map) continue; + final name = document['name']?.toString() ?? ''; + final docPath = _pathFromDocumentName(name); + documents.add(_toPigeonDocumentSnapshot(docPath, document)); + } + } + + return InternalQuerySnapshot( + documents: documents, + documentChanges: documents + .whereType() + .map( + (document) => InternalDocumentChange( + type: DocumentChangeType.added, + document: document, + oldIndex: -1, + newIndex: documents.indexOf(document), + ), + ) + .toList(), + metadata: InternalSnapshotMetadata( + hasPendingWrites: false, + isFromCache: options.source == Source.cache, + ), + ); + } + + Future _request( + FirestorePigeonFirebaseApp app, { + required String method, + String? path, + String suffix = '', + List updateMaskPaths = const [], + Map? body, + }) async { + final projectId = _projectIdForApp(app.appName); + final idToken = _idTokenForApp(app.appName); + final encodedPath = + path == null ? '' : path.split('/').map(Uri.encodeComponent).join('/'); + final queryParts = [ + for (final fieldPath in updateMaskPaths) + 'updateMask.fieldPaths=${Uri.encodeQueryComponent(fieldPath)}', + ]; + final uri = Uri.parse( + 'https://firestore.googleapis.com/v1/projects/$projectId/databases/(default)/documents' + '${encodedPath.isEmpty ? '' : '/$encodedPath'}$suffix' + '${queryParts.isEmpty ? '' : '?${queryParts.join('&')}'}', + ); + + final client = HttpClient(); + try { + late final HttpClientRequest request; + switch (method) { + case 'GET': + request = await client.getUrl(uri); + case 'DELETE': + request = await client.deleteUrl(uri); + case 'PATCH': + request = await client.patchUrl(uri); + case 'POST': + request = await client.postUrl(uri); + default: + throw UnsupportedError('Unsupported HTTP method: $method'); + } + request.headers.set(HttpHeaders.contentTypeHeader, 'application/json'); + request.headers.set(HttpHeaders.authorizationHeader, 'Bearer $idToken'); + if (body != null) { + request.write(jsonEncode(body)); + } + + final response = await request.close(); + final responseBody = await response.transform(utf8.decoder).join(); + + if (response.statusCode == 404 && method == 'GET') { + return null; + } + if (response.statusCode < 200 || response.statusCode >= 300) { + throw PlatformException( + code: 'firestore', + message: 'HTTP ${response.statusCode} $method $uri: $responseBody', + ); + } + if (responseBody.isEmpty) { + return null; + } + return jsonDecode(responseBody); + } finally { + client.close(); + } + } + + String _documentsRoot(FirestorePigeonFirebaseApp app) { + final projectId = _projectIdForApp(app.appName); + return 'projects/$projectId/databases/(default)/documents'; + } +} + +class _EncodedDocument { + _EncodedDocument({ + required this.fields, + required this.transforms, + required this.fieldPaths, + }); + + final Map fields; + final List> transforms; + final List fieldPaths; +} + +_EncodedDocument _encodeDocumentFields(Map data) { + final fields = {}; + final fieldPaths = []; + + data.forEach((key, value) { + final fieldPath = key?.toString(); + if (fieldPath == null || fieldPath.isEmpty) return; + + fields[fieldPath] = _encodeValue(value); + fieldPaths.add(fieldPath); + }); + + return _EncodedDocument( + fields: fields, + transforms: const [], + fieldPaths: fieldPaths, + ); +} + +Map _encodeValue(Object? value) { + if (value == null) return {'nullValue': null}; + if (value is bool) return {'booleanValue': value}; + if (value is int) return {'integerValue': value.toString()}; + if (value is double) return {'doubleValue': value}; + if (value is String) return {'stringValue': value}; + if (value is Timestamp) { + return {'timestampValue': value.toDate().toUtc().toIso8601String()}; + } + if (value is GeoPoint) { + return { + 'geoPointValue': { + 'latitude': value.latitude, + 'longitude': value.longitude, + }, + }; + } + if (value is Map) { + final nested = {}; + value.forEach((key, nestedValue) { + final nestedKey = key?.toString(); + if (nestedKey == null || nestedKey.isEmpty) return; + nested[nestedKey] = _encodeValue(nestedValue); + }); + return { + 'mapValue': {'fields': nested} + }; + } + if (value is Iterable) { + return { + 'arrayValue': { + 'values': value.map(_encodeValue).toList(), + }, + }; + } + return {'stringValue': value.toString()}; +} + +InternalDocumentSnapshot _toPigeonDocumentSnapshot( + String path, + Map document, +) { + final fields = document['fields']; + return InternalDocumentSnapshot( + path: path, + data: fields is Map + ? _decodeFields(fields.cast()) + : {}, + metadata: InternalSnapshotMetadata( + hasPendingWrites: false, + isFromCache: false, + ), + ); +} + +Map _decodeFields(Map fields) { + final decoded = {}; + fields.forEach((key, value) { + decoded[key?.toString()] = _decodeValue(value); + }); + return decoded; +} + +Object? _decodeValue(Object? value) { + if (value is! Map) return value; + final map = value.cast(); + if (map.containsKey('stringValue')) return map['stringValue']; + if (map.containsKey('booleanValue')) return map['booleanValue']; + if (map.containsKey('integerValue')) { + return int.tryParse(map['integerValue'].toString()) ?? map['integerValue']; + } + if (map.containsKey('doubleValue')) { + final raw = map['doubleValue']; + return raw is num ? raw.toDouble() : double.tryParse(raw.toString()); + } + if (map.containsKey('nullValue')) return null; + if (map.containsKey('timestampValue')) { + return Timestamp.fromDate(DateTime.parse(map['timestampValue'].toString())); + } + if (map.containsKey('geoPointValue')) { + final geo = map['geoPointValue'] as Map; + return GeoPoint( + (geo['latitude'] as num).toDouble(), + (geo['longitude'] as num).toDouble(), + ); + } + if (map.containsKey('mapValue')) { + final nested = (map['mapValue'] as Map?)?['fields']; + if (nested is Map) { + return _decodeFields(nested.cast()); + } + return {}; + } + if (map.containsKey('arrayValue')) { + final values = (map['arrayValue'] as Map?)?['values']; + if (values is List) { + return values.map(_decodeValue).toList(); + } + return []; + } + return map; +} + +String _pathFromDocumentName(String name) { + const marker = '/documents/'; + final index = name.indexOf(marker); + if (index < 0) return name; + return name.substring(index + marker.length); +} + +String _projectIdForApp(String appName) { + final fromInit = firebaseProjectIdsByApp[appName]; + if (fromInit != null && fromInit.isNotEmpty) { + return fromInit; + } + try { + final app = appName.isEmpty || appName == defaultFirebaseAppName + ? Firebase.app() + : Firebase.app(appName); + return app.options.projectId; + } catch (_) { + throw StateError( + 'Could not resolve Firebase projectId for Firestore call (app: $appName).', + ); + } +} + +String _idTokenForApp(String appName) { + final session = liveAuthSessionForApp(appName) ?? + liveAuthSessionForApp(defaultFirebaseAppName); + final token = session?.idToken; + if (token == null || token.isEmpty) { + throw PlatformException( + code: 'firestore', + message: + 'No signed-in Firebase user for Firestore REST call (app: $appName).', + ); + } + return token; +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart new file mode 100644 index 000000000..849395363 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_functions_test_setup.dart @@ -0,0 +1,146 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:cloud_functions_platform_interface/src/pigeon/messages.pigeon.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +bool _cloudFunctionsBridgeInstalled = false; + +/// Bridges Firebase Cloud Functions pigeon calls to real HTTPS callable endpoints +/// during [flutter test]. Native macOS/iOS/Android plugins are unavailable in the +/// VM test binding, so without this handler [httpsCallable] fails immediately. +void ensureLiveCloudFunctionsForTest() { + if (_cloudFunctionsBridgeInstalled) return; + TestWidgetsFlutterBinding.ensureInitialized(); + + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + + messenger.setMockDecodedMessageHandler( + BasicMessageChannel( + 'dev.flutter.pigeon.cloud_functions_platform_interface.CloudFunctionsHostApi.call', + CloudFunctionsHostApi.pigeonChannelCodec, + ), + (Object? message) async { + final args = (message! as List)[0]! as Map; + final arguments = args.cast(); + try { + final result = await _invokeCallableOverHttp(arguments); + return [result]; + } on PlatformException catch (error) { + return [error.code, error.message, error.details]; + } catch (error) { + return ['error', error.toString(), null]; + } + }, + ); + + messenger.setMockDecodedMessageHandler( + BasicMessageChannel( + 'dev.flutter.pigeon.cloud_functions_platform_interface.CloudFunctionsHostApi.registerEventChannel', + CloudFunctionsHostApi.pigeonChannelCodec, + ), + (_) async => [], + ); + + _cloudFunctionsBridgeInstalled = true; +} + +Future _invokeCallableOverHttp(Map arguments) async { + final uri = _resolveCallableUri(arguments); + final parameters = arguments['parameters']; + final timeoutMs = arguments['timeout'] as int? ?? 60000; + + final client = HttpClient(); + client.connectionTimeout = Duration(milliseconds: timeoutMs); + + try { + final request = await client.postUrl(uri); + request.headers.set(HttpHeaders.contentTypeHeader, 'application/json'); + request.write(jsonEncode({'data': parameters})); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + + if (response.statusCode < 200 || response.statusCode >= 300) { + throw PlatformException( + code: 'firebase_functions', + message: 'HTTP ${response.statusCode} calling $uri: $body', + ); + } + + final decoded = jsonDecode(body); + if (decoded is! Map) { + throw PlatformException( + code: 'firebase_functions', + message: 'Unexpected callable response from $uri: $body', + ); + } + + final map = Map.from(decoded); + if (map.containsKey('error')) { + throw PlatformException( + code: 'firebase_functions', + message: map['error'].toString(), + ); + } + + final result = map['result']; + if (result is Map) { + return Map.from(result); + } + return result; + } finally { + client.close(); + } +} + +Uri _resolveCallableUri(Map arguments) { + final functionUri = arguments['functionUri'] as String?; + if (functionUri != null && functionUri.isNotEmpty) { + return Uri.parse(functionUri); + } + + final origin = arguments['origin'] as String?; + if (origin != null && origin.isNotEmpty) { + return Uri.parse(origin); + } + + final functionName = arguments['functionName'] as String?; + if (functionName == null || functionName.isEmpty) { + throw ArgumentError('Cloud Functions call missing functionName'); + } + + final region = (arguments['region'] as String?) ?? 'us-central1'; + final projectId = _projectIdForApp(arguments['appName'] as String?); + if (projectId == null || projectId.isEmpty) { + throw StateError( + 'Could not resolve Firebase projectId for Cloud Functions call. ' + 'Ensure firebase_config is loaded before invoking callable APIs.', + ); + } + + return Uri.parse( + 'https://$region-$projectId.cloudfunctions.net/$functionName'); +} + +String? _projectIdForApp(String? appName) { + if (appName != null && appName.isNotEmpty) { + final fromInit = firebaseProjectIdsByApp[appName]; + if (fromInit != null && fromInit.isNotEmpty) { + return fromInit; + } + } + try { + final app = (appName == null || + appName.isEmpty || + appName == defaultFirebaseAppName) + ? Firebase.app() + : Firebase.app(appName); + return app.options.projectId; + } catch (_) { + return null; + } +} diff --git a/tools/ensemble_test_runner/lib/mocks/firebase_test_setup.dart b/tools/ensemble_test_runner/lib/mocks/firebase_test_setup.dart new file mode 100644 index 000000000..d866bb89a --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/firebase_test_setup.dart @@ -0,0 +1,69 @@ +import 'package:ensemble_test_runner/mocks/firebase_auth_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_firestore_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_functions_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/live_sign_in_with_custom_token.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:firebase_core_platform_interface/test.dart'; + +bool _firebaseCoreMocksInstalled = false; + +final Map firebaseProjectIdsByApp = {}; + +/// Preserves [CoreFirebaseOptions] from [Firebase.initializeApp] instead of +/// the stock mock that always returns projectId `123`. +class _PreservingFirebaseCoreHostApi implements TestFirebaseCoreHostApi { + @override + Future initializeApp( + String appName, + CoreFirebaseOptions initializeAppRequest, + ) async { + firebaseProjectIdsByApp[appName] = initializeAppRequest.projectId; + return CoreInitializeResponse( + name: appName, + options: initializeAppRequest, + pluginConstants: {}, + ); + } + + @override + Future> initializeCore() async { + return [ + CoreInitializeResponse( + name: defaultFirebaseAppName, + options: CoreFirebaseOptions( + apiKey: 'test-api-key', + projectId: 'test-project', + appId: 'test-app-id', + messagingSenderId: 'test-sender', + ), + pluginConstants: {}, + ), + ]; + } + + @override + Future optionsFromResource() async { + return CoreFirebaseOptions( + apiKey: 'test-api-key', + projectId: 'test-project', + appId: 'test-app-id', + messagingSenderId: 'test-sender', + ); + } +} + +/// Installs Firebase Core platform mocks so [Firebase.initializeApp] works under +/// [flutter test] (no native Firebase host). Required before app bootstrap. +void ensureFirebaseCoreMocksForTest() { + if (_firebaseCoreMocksInstalled) return; + TestFirebaseCoreHostApi.setUp(_PreservingFirebaseCoreHostApi()); + ensureLiveCloudFunctionsForTest(); + ensureLiveFirebaseAuthForTest(); + ensureLiveFirestoreForTest(); + _firebaseCoreMocksInstalled = true; +} + +/// Call after app module bootstrap so [signInWithCustomToken] uses live Firebase. +void ensureLiveAuthActionsForTest() { + ensureLiveSignInWithCustomTokenForTest(); +} diff --git a/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart b/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart new file mode 100644 index 000000000..75105497b --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/live_firebase_auth_http.dart @@ -0,0 +1,95 @@ +import 'dart:convert'; +import 'dart:io'; + +/// Signs in with a Firebase custom token via the Identity Toolkit REST API. +Future> postIdentityToolkitSignInWithCustomToken({ + required String customToken, + required String apiKey, + int maxAttempts = 3, +}) async { + Object? lastError; + for (var attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await _postIdentityToolkitSignInOnce( + customToken: customToken, + apiKey: apiKey, + ); + } catch (error) { + lastError = error; + final retryable = _isRetryableAuthError(error); + if (!retryable || attempt >= maxAttempts) { + rethrow; + } + await Future.delayed(Duration(milliseconds: 250 * attempt)); + } + } + throw lastError ?? StateError('Identity Toolkit sign-in failed.'); +} + +bool _isRetryableAuthError(Object error) { + final message = error.toString().toLowerCase(); + return message.contains('connection reset') || + message.contains('connection closed') || + message.contains('timed out') || + message.contains('socketexception'); +} + +Future> _postIdentityToolkitSignInOnce({ + required String customToken, + required String apiKey, +}) async { + final uri = Uri.parse( + 'https://identitytoolkit.googleapis.com/v1/accounts:signInWithCustomToken?key=$apiKey', + ); + final client = HttpClient(); + client.connectionTimeout = const Duration(seconds: 60); + try { + final request = await client.postUrl(uri); + request.headers.set(HttpHeaders.contentTypeHeader, 'application/json'); + request.write(jsonEncode({ + 'token': customToken, + 'returnSecureToken': true, + })); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + if (response.statusCode < 200 || response.statusCode >= 300) { + throw HttpException( + 'HTTP ${response.statusCode} calling $uri: $body', + uri: uri, + ); + } + final decoded = jsonDecode(body); + if (decoded is! Map) { + throw HttpException('Unexpected auth response from $uri', uri: uri); + } + return Map.from(decoded); + } finally { + client.close(); + } +} + +/// Firebase ID tokens use `user_id`; custom tokens may expose `localId`. +String? uidFromFirebaseAuthResponse(Map authBody) { + final localId = authBody['localId']?.toString(); + if (localId != null && localId.isNotEmpty) { + return localId; + } + return uidFromFirebaseIdToken(authBody['idToken']?.toString() ?? ''); +} + +String? uidFromFirebaseIdToken(String idToken) { + final parts = idToken.split('.'); + if (parts.length < 2) { + return null; + } + try { + final normalized = base64Url.normalize(parts[1]); + final payload = jsonDecode(utf8.decode(base64Url.decode(normalized))); + if (payload is Map) { + return payload['user_id']?.toString() ?? payload['sub']?.toString(); + } + } catch (_) { + return null; + } + return null; +} diff --git a/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart b/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart new file mode 100644 index 000000000..c94bfa491 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/live_sign_in_with_custom_token.dart @@ -0,0 +1,180 @@ +import 'package:ensemble/framework/action.dart'; +import 'package:ensemble/framework/apiproviders/firebase_functions/firebase_functions_api_provider.dart'; +import 'package:ensemble/framework/error_handling.dart'; +import 'package:ensemble/framework/event.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble/framework/stub/auth_context_manager.dart'; +import 'package:ensemble/screen_controller.dart'; +import 'package:ensemble/widget/stub_widgets.dart'; +import 'package:ensemble_test_runner/mocks/firebase_auth_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/live_firebase_auth_http.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter/material.dart'; +import 'package:get_it/get_it.dart'; + +/// Signs in with a Firebase custom token via the real Identity Toolkit REST API. +/// +/// HTTP runs through [LiveAsyncCallSupport] (same [WidgetTester.runAsync] queue +/// as live [invokeAPI] calls) because direct sockets from action callbacks fail +/// under [flutter test]. +class LiveSignInWithCustomToken implements SignInWithCustomToken { + @override + Future signInWithCustomToken( + BuildContext context, { + required SignInWithCustomTokenAction action, + }) async { + try { + final token = _evaluatedToken(context, action.jwtToken); + if (token.isEmpty) { + throw LanguageError( + "signInWithCustomToken requires jwtToken as 'token' parameter.", + recovery: + "Fix: pass valid jwtToken as 'token' under signInWithCustomToken", + ); + } + + final appName = _resolveAppName(); + final apiKey = _resolveApiKey(); + final authBody = await _signInOverHttp(token: token, apiKey: apiKey); + + final idToken = authBody['idToken']?.toString(); + final refreshToken = authBody['refreshToken']?.toString(); + final localId = uidFromFirebaseAuthResponse(authBody); + if (idToken == null || localId == null || refreshToken == null) { + throw StateError( + 'Firebase signInWithCustomToken response missing tokens ' + '(keys: ${authBody.keys.toList()}).', + ); + } + + final expiresInSec = + int.tryParse(authBody['expiresIn']?.toString() ?? '') ?? 3600; + recordLiveAuthSession( + appName: appName, + idToken: idToken, + refreshToken: refreshToken, + uid: localId, + expiresAtMs: DateTime.now() + .add(Duration(seconds: expiresInSec)) + .millisecondsSinceEpoch, + email: authBody['email']?.toString(), + isEmailVerified: authBody['emailVerified'] == true, + ); + + await StorageManager().writeToSystemStorage('user.id', localId); + await StorageManager().writeToSystemStorage('user.isAnonymous', false); + + if (action.onAuthenticated != null) { + final user = AuthenticatedUser( + provider: SignInProvider.firebase, + id: localId, + ); + ScreenController().executeAction( + context, + action.onAuthenticated!, + event: EnsembleEvent( + null, + data: {'user': user, 'idToken': idToken}, + ), + ); + } + } catch (error) { + if (action.onError != null) { + ScreenController().executeAction( + context, + action.onError!, + event: EnsembleEvent( + null, + error: {'error': _redactSecrets(error.toString())}, + ), + ); + } + } + } + + /// Avoid dumping JWTs / refresh tokens into test logs via onError. + static String _redactSecrets(String message) { + return message + .replaceAllMapped( + RegExp(r'eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+'), + (_) => '[redacted-jwt]', + ) + .replaceAllMapped( + RegExp(r'(refreshToken|idToken|accessToken|token)["\s:=]+[^\s,"}]+', + caseSensitive: false), + (match) => '${match.group(1)}=[redacted]', + ); + } + + Future> _signInOverHttp({ + required String token, + required String apiKey, + }) async { + final result = await LiveAsyncCallSupport.run( + () => postIdentityToolkitSignInWithCustomToken( + customToken: token, + apiKey: apiKey, + ), + ); + if (result == null) { + throw StateError( + 'Firebase signInWithCustomToken did not complete under runAsync.', + ); + } + return result; + } + + String _evaluatedToken(BuildContext context, String? jwtToken) { + var token = jwtToken ?? ''; + final scopeManager = ScreenController().getScopeManager(context); + if (scopeManager != null) { + token = scopeManager.dataContext.eval(token)?.toString() ?? token; + } + return token; + } + + String _resolveApiKey() { + final apiKey = _resolveFirebaseApp().options.apiKey; + if (apiKey.isEmpty) { + throw ConfigError('Firebase apiKey is missing for live auth sign-in.'); + } + return apiKey; + } + + String _resolveAppName() { + try { + return _resolveFirebaseApp().name; + } catch (_) { + return defaultFirebaseAppName; + } + } + + FirebaseApp _resolveFirebaseApp() { + final functionsApp = FirebaseFunctionsAPIProvider.getFirebaseAppContext(); + if (functionsApp != null) { + return functionsApp; + } + if (Firebase.apps.isNotEmpty) { + return Firebase.apps.first; + } + throw ConfigError( + 'No Firebase app is initialized for live auth sign-in.', + ); + } +} + +/// Registers [LiveSignInWithCustomToken] so YAML `signInWithCustomToken` +/// actions use the real Identity Toolkit API during tests. +void ensureLiveSignInWithCustomTokenForTest() { + if (!GetIt.I.isRegistered()) { + GetIt.I.registerFactory( + () => LiveSignInWithCustomToken(), + ); + return; + } + GetIt.I.unregister(); + GetIt.I.registerFactory( + () => LiveSignInWithCustomToken(), + ); +} diff --git a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart b/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart deleted file mode 100644 index 5e5407851..000000000 --- a/tools/ensemble_test_runner/lib/mocks/mock_api_provider.dart +++ /dev/null @@ -1,186 +0,0 @@ -import 'dart:convert'; - -import 'package:ensemble/framework/apiproviders/api_provider.dart'; -import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; -import 'package:ensemble/framework/data_context.dart'; -import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -import 'package:flutter/widgets.dart'; -import 'package:yaml/yaml.dart'; - -class APICallRecord { - final String name; - final YamlMap apiDefinition; - final DateTime timestamp; - final dynamic body; - final Map? query; - final Map? headers; - - APICallRecord({ - required this.name, - required this.apiDefinition, - required this.timestamp, - this.body, - this.query, - this.headers, - }); -} - -/// Records API calls and returns YAML-configured mock responses by API name. -class MockAPIProvider extends HTTPAPIProvider { - MockAPIProvider({ - required Map mocks, - HTTPAPIProvider? delegate, - }) : _mocks = mocks, - _delegate = delegate ?? HTTPAPIProvider(); - - final Map _mocks; - final HTTPAPIProvider _delegate; - final List calls = []; - - int callCount(String apiName) => - calls.where((c) => c.name == apiName).length; - - List callsFor(String apiName) => - calls.where((c) => c.name == apiName).toList(); - - void setMock(String apiName, MockAPIResponse response) { - _mocks[apiName] = response; - } - - void resetCalls() => calls.clear(); - - void clearMocks() => _mocks.clear(); - - final Map _forcedExceptions = {}; - - void setApiException(String apiName, Exception error) { - _forcedExceptions[apiName] = error; - } - - void clearApiExceptions() => _forcedExceptions.clear(); - - @override - Future init(String appId, Map config) => - _delegate.init(appId, config); - - /// When true, [invokeApi] returns an offline error without calling the delegate. - bool simulateNetworkOffline = false; - - @override - Future invokeApi( - BuildContext context, - YamlMap api, - DataContext eContext, - String apiName, - ) async { - if (simulateNetworkOffline) { - calls.add(APICallRecord( - name: apiName, - apiDefinition: api, - timestamp: DateTime.now(), - )); - return HttpResponse.fromBody( - {'message': 'Network offline (test)'}, - null, - 503, - null, - APIState.error, - ); - } - - final forced = _forcedExceptions[apiName]; - if (forced != null) { - calls.add(APICallRecord( - name: apiName, - apiDefinition: api, - timestamp: DateTime.now(), - )); - throw forced; - } - - final captured = _captureRequest(api, eContext); - calls.add(APICallRecord( - name: apiName, - apiDefinition: api, - timestamp: DateTime.now(), - body: captured.body, - query: captured.query, - headers: captured.headers, - )); - - final mock = _mocks[apiName]; - if (mock == null) { - return _delegate.invokeApi(context, api, eContext, apiName); - } - - if (mock.delayMs != null && mock.delayMs! > 0) { - await Future.delayed(Duration(milliseconds: mock.delayMs!)); - } - - return HttpResponse.fromBody( - mock.body, - mock.headers?.map((k, v) => MapEntry(k, v.toString())), - mock.statusCode, - null, - APIState.success, - ); - } - - @override - Future invokeMockAPI(DataContext eContext, dynamic mock) => - _delegate.invokeMockAPI(eContext, mock); - - /// Same instance as config — keeps call recording aligned with [EnsembleTestContext]. - @override - MockAPIProvider clone() => this; - - @override - void dispose() => _delegate.dispose(); -} - -class _CapturedRequest { - final dynamic body; - final Map? query; - final Map? headers; - - const _CapturedRequest({this.body, this.query, this.headers}); -} - -_CapturedRequest _captureRequest(YamlMap api, DataContext eContext) { - Map? headers; - if (api['headers'] is YamlMap) { - headers = {}; - (api['headers'] as YamlMap).forEach((key, value) { - if (value != null) { - headers![key.toString().toLowerCase()] = - eContext.eval(value)?.toString() ?? ''; - } - }); - } - - dynamic body; - if (api['body'] != null) { - final evaluated = eContext.eval(api['body']); - if (evaluated is Map || evaluated is List) { - body = evaluated; - } else if (evaluated is String) { - try { - body = json.decode(evaluated); - } catch (_) { - body = evaluated; - } - } else { - body = evaluated; - } - } - - Map? query; - if (api['parameters'] is YamlMap) { - query = {}; - (api['parameters'] as YamlMap).forEach((key, value) { - query![key.toString()] = eContext.eval(value)?.toString() ?? ''; - }); - } - - return _CapturedRequest(body: body, query: query, headers: headers); -} diff --git a/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart b/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart new file mode 100644 index 000000000..fefdf90c4 --- /dev/null +++ b/tools/ensemble_test_runner/lib/mocks/test_api_provider_overlay.dart @@ -0,0 +1,253 @@ +import 'dart:async'; + +import 'package:ensemble/framework/apiproviders/api_provider.dart'; +import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; +import 'package:ensemble/framework/data_context.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:flutter/widgets.dart'; +import 'package:yaml/yaml.dart'; + +class APICallRecord { + final String name; + final YamlMap apiDefinition; + final DateTime timestamp; + + APICallRecord({ + required this.name, + required this.apiDefinition, + required this.timestamp, + }); +} + +class ApiCallRecorder { + final List calls = []; + + int callCount(String apiName) => calls.where((c) => c.name == apiName).length; + + List callsFor(String apiName) => + calls.where((c) => c.name == apiName).toList(); + + void record(APICallRecord call) => calls.add(call); + + void reset() => calls.clear(); +} + +/// Test provider overlay: observes API calls, applies test overrides, otherwise delegates. +class TestApiProviderOverlay extends HTTPAPIProvider { + TestApiProviderOverlay({ + required Map mocks, + HTTPAPIProvider? delegate, + ApiCallRecorder? recorder, + }) : _mocks = mocks, + _delegate = delegate ?? HTTPAPIProvider(), + recorder = recorder ?? ApiCallRecorder(); + + final Map _mocks; + HTTPAPIProvider _delegate; + HTTPAPIProvider get delegate => _delegate; + final ApiCallRecorder recorder; + Future Function(Future Function())? liveAsyncRunner; + + List get calls => recorder.calls; + + int callCount(String apiName) => recorder.callCount(apiName); + + List callsFor(String apiName) => recorder.callsFor(apiName); + + void bindHttpDelegate(HTTPAPIProvider delegate) { + _delegate = delegate; + } + + void setMock(String apiName, MockAPIResponse response) { + _mocks[apiName] = response; + } + + void resetCalls() => recorder.reset(); + + void clearMocks() => _mocks.clear(); + + final Map _forcedExceptions = {}; + + void setApiException(String apiName, Exception error) { + _forcedExceptions[apiName] = error; + } + + void clearApiExceptions() => _forcedExceptions.clear(); + + bool get hasPendingLiveCalls => LiveAsyncCallSupport.hasPendingLiveCalls; + + Future waitForLiveCalls() => LiveAsyncCallSupport.waitForLiveCalls(); + + @override + Future init(String appId, Map config) => + _delegate.init(appId, config); + + @override + Future invokeApi( + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + final response = await invokeApiWithDelegate( + _delegate, + context, + api, + eContext, + apiName, + ); + if (response is HttpResponse) { + return response; + } + return HttpResponse.fromBody( + response.body, + response.headers?.map((k, v) => MapEntry(k, v.toString())), + response.statusCode, + response.reasonPhrase, + response.apiState, + ); + } + + Future invokeApiWithDelegate( + APIProvider delegate, + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + final forced = _forcedExceptions[apiName]; + if (forced != null) { + recorder.record(APICallRecord( + name: apiName, + apiDefinition: api, + timestamp: DateTime.now(), + )); + throw forced; + } + + recorder.record(APICallRecord( + name: apiName, + apiDefinition: api, + timestamp: DateTime.now(), + )); + + final mock = _mocks[apiName]; + if (mock == null) { + return _invokeLiveDelegate( + delegate, + context, + api, + eContext, + apiName, + ); + } + + if (mock.delayMs != null && mock.delayMs! > 0) { + await Future.delayed(Duration(milliseconds: mock.delayMs!)); + } + + return delegate.invokeMockAPI(eContext, _toRuntimeMockResponse(mock)); + } + + Future _invokeLiveDelegate( + APIProvider delegate, + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + final runner = liveAsyncRunner; + Future call() => + delegate.invokeApi(context, api, eContext, apiName); + + if (runner != null) { + try { + final response = await _runLiveApiCall(call); + if (response != null) { + return response; + } + return HttpResponse.fromBody( + 'Live API call did not return a response.', + {'Content-Type': 'text/plain'}, + 500, + 'Internal Server Error', + APIState.error, + ); + } catch (error) { + return HttpResponse.fromBody( + 'Unexpected error: $error.', + {'Content-Type': 'text/plain'}, + 500, + 'Internal Server Error', + APIState.error, + ); + } + } + + return call(); + } + + Future _runLiveApiCall(Future Function() call) { + LiveAsyncCallSupport.runner ??= liveAsyncRunner; + return LiveAsyncCallSupport.run(call); + } + + @override + Future invokeMockAPI(DataContext eContext, dynamic mock) => + _delegate.invokeMockAPI(eContext, mock); + + /// Same instance as config — keeps call recording aligned with [EnsembleTestContext]. + @override + TestApiProviderOverlay clone() => this; + + @override + void dispose() => _delegate.dispose(); +} + +/// Delegates to a real provider unless [host] has a test override for the API name. +class TestApiOverlay implements APIProvider { + TestApiOverlay(this._host, this._delegate); + + final TestApiProviderOverlay _host; + final APIProvider _delegate; + APIProvider get delegate => _delegate; + + @override + Future init(String appId, Map config) => + _delegate.init(appId, config); + + @override + Future invokeApi( + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) { + return _host.invokeApiWithDelegate( + _delegate, + context, + api, + eContext, + apiName, + ); + } + + @override + Future invokeMockAPI(DataContext eContext, dynamic mock) => + _delegate.invokeMockAPI(eContext, mock); + + @override + TestApiOverlay clone() => TestApiOverlay(_host, _delegate.clone()); + + @override + void dispose() => _delegate.dispose(); +} + +Map _toRuntimeMockResponse(MockAPIResponse mock) { + return { + 'body': mock.body, + if (mock.headers != null) 'headers': mock.headers, + 'statusCode': mock.statusCode, + }; +} diff --git a/tools/ensemble_test_runner/lib/mocks/test_logger.dart b/tools/ensemble_test_runner/lib/mocks/test_logger.dart index ee3ee0b1c..d47736b9a 100644 --- a/tools/ensemble_test_runner/lib/mocks/test_logger.dart +++ b/tools/ensemble_test_runner/lib/mocks/test_logger.dart @@ -1,3 +1,5 @@ +import 'dart:io'; + /// Simple in-memory logger for test runs. class TestLogger { final List logs = []; @@ -6,5 +8,32 @@ class TestLogger { logs.add(message); } + Future writeLogFile({ + required String testId, + required String name, + required String content, + String extension = 'log', + }) async { + final directory = Directory('build/ensemble_test_runner/logs'); + await directory.create(recursive: true); + final safeTestId = _safeFileName(testId); + final safeName = _safeFileName(name); + final fileName = safeTestId.isEmpty + ? '$safeName.${_safeExtension(extension)}' + : '${safeTestId}_$safeName.${_safeExtension(extension)}'; + final file = File('${directory.path}/$fileName'); + await file.writeAsString(content); + return file.path; + } + void clear() => logs.clear(); + + static String _safeFileName(String value) { + return value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); + } + + static String _safeExtension(String value) { + final extension = value.replaceAll(RegExp(r'[^A-Za-z0-9]+'), ''); + return extension.isEmpty ? 'log' : extension; + } } diff --git a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart index 99863d7fd..d08f47ba4 100644 --- a/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart +++ b/tools/ensemble_test_runner/lib/models/ensemble_test_models.dart @@ -7,6 +7,7 @@ class EnsembleTestRunRequest { final String? appHome; final String? i18nPath; final List tests; + final EnsembleTestConfig config; final EnsembleTestEnvironment environment; const EnsembleTestRunRequest({ @@ -14,6 +15,7 @@ class EnsembleTestRunRequest { this.appHome, this.i18nPath, required this.tests, + this.config = const EnsembleTestConfig(), this.environment = const EnsembleTestEnvironment(), }); } @@ -41,7 +43,11 @@ class EnsembleTestCase { /// Test [id] that must run before this one (same app session). final String? prerequisite; + final List mockFiles; + final List scenarios; final Map initialState; + + /// Runtime API mocks resolved from [mockFiles]. final TestMocks mocks; final List steps; @@ -56,6 +62,8 @@ class EnsembleTestCase { this.priority, this.startScreen, this.prerequisite, + this.mockFiles = const [], + this.scenarios = const [], this.initialState = const {}, this.mocks = const TestMocks(), required this.steps, @@ -74,6 +82,100 @@ class EnsembleTestCase { }; } +/// One scenario from a scenario-based test suite. +class TestScenario { + final String id; + final String? description; + final Map vars; + + const TestScenario({ + required this.id, + this.description, + this.vars = const {}, + }); +} + +class EnsembleTestConfig { + final ScreenshotConfig screenshots; + final PerformanceConfig performance; + final DumpTreeConfig dumpTree; + final LogApiCallsConfig logApiCalls; + final LogStorageConfig logStorage; + + const EnsembleTestConfig({ + this.screenshots = const ScreenshotConfig(), + this.performance = const PerformanceConfig(), + this.dumpTree = const DumpTreeConfig(), + this.logApiCalls = const LogApiCallsConfig(), + this.logStorage = const LogStorageConfig(), + }); +} + +class PerformanceConfig { + final bool enabled; + + const PerformanceConfig({ + this.enabled = false, + }); +} + +class DumpTreeConfig { + final bool enabled; + + const DumpTreeConfig({ + this.enabled = false, + }); +} + +class LogApiCallsConfig { + final bool enabled; + + const LogApiCallsConfig({ + this.enabled = false, + }); +} + +class LogStorageConfig { + final bool enabled; + final String? key; + + const LogStorageConfig({ + this.enabled = false, + this.key, + }); +} + +class ScreenshotConfig { + final bool enabled; + final String platform; + final String model; + final List includeSteps; + final List excludeSteps; + + const ScreenshotConfig({ + this.enabled = false, + this.platform = 'ios', + this.model = 'iPhone 15 Pro', + this.includeSteps = const [], + this.excludeSteps = const [], + }); + + bool shouldCaptureStep(String stepType) { + if (!enabled) return false; + if (excludeSteps.contains(stepType)) return false; + if (includeSteps.isNotEmpty && !includeSteps.contains(stepType)) { + return false; + } + return true; + } + + Map toScreenshotArgs([Map? overrides]) => { + 'platform': platform, + 'model': model, + ...?overrides, + }; +} + /// Mock configuration attached to a test case. class TestMocks { final Map apis; @@ -130,8 +232,12 @@ class TestStep { /// Aggregate result for a YAML test run. class EnsembleTestRunResult { final List results; + final List suiteLogs; - const EnsembleTestRunResult({required this.results}); + const EnsembleTestRunResult({ + required this.results, + this.suiteLogs = const [], + }); int get passedCount => results.where((r) => r.status == TestStatus.passed).length; @@ -147,6 +253,7 @@ class EnsembleTestRunResult { 'passed': passedCount, 'failed': failedCount, 'results': results.map((r) => r.toJson()).toList(), + if (suiteLogs.isNotEmpty) 'suiteLogs': suiteLogs, }; } @@ -253,7 +360,6 @@ class EnsembleSingleTestResult { String _failureKind(String text) { final lower = text.toLowerCase(); - if (lower.contains('fixture')) return 'parseError'; if (lower.contains('timeout') || lower.contains('timed out')) { return 'timeout'; } @@ -288,7 +394,7 @@ class EnsembleSingleTestResult { case 'apiMismatch': return [ 'Check API name spelling against --inspect-app output.', - 'Use root mocks.apis for onLoad APIs and mockApi for later user-triggered APIs.', + 'Use root mocks with .mock.json files for mocked API responses.', ]; case 'navigationMismatch': return [ @@ -302,7 +408,7 @@ class EnsembleSingleTestResult { ]; case 'parseError': return [ - 'Run --validate-only to find schema, fixture, and prerequisite issues.', + 'Run --validate-only to find schema and prerequisite issues.', ]; default: return [ diff --git a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart index 7f9ed1100..5df554a1a 100644 --- a/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart +++ b/tools/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -1,42 +1,36 @@ import 'dart:io'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; -import 'package:flutter/services.dart'; -import 'package:flutter/widgets.dart'; import 'package:yaml/yaml.dart'; class EnsembleTestParser { - /// Loads a test file from [rootBundle] (widget/integration tests) or [File] (CLI). - static Future parseFile(String path) async { - Object? bundleError; - try { - final content = await rootBundle.loadString(path); - return parseString(content, sourcePath: path); - } catch (error) { - bundleError = error; - } - - // dart:io can hang under widget test bindings' fake async zone. - if (_isWidgetTestBinding) { - throw EnsembleTestFailure( - 'Test file not found in asset bundle: $path ($bundleError)', - ); - } - + /// Loads a test file from disk. + static Future parseFile( + String path, { + Map inputs = const {}, + Map scenario = const {}, + String? scenarioId, + }) async { final file = File(path); if (!await file.exists()) { throw EnsembleTestFailure('Test file not found: $path'); } - return parseString(await file.readAsString(), sourcePath: path); - } - - static bool get _isWidgetTestBinding { - final type = WidgetsBinding.instance.runtimeType.toString(); - return type.contains('TestWidgetsFlutterBinding') || - type.contains('LiveTestWidgetsFlutterBinding'); + return parseString( + await file.readAsString(), + sourcePath: path, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } - static EnsembleTestCase parseString(String content, {String? sourcePath}) { + static EnsembleTestCase parseString( + String content, { + String? sourcePath, + Map inputs = const {}, + Map scenario = const {}, + String? scenarioId, + }) { final dynamic doc = loadYaml(content); if (doc is! YamlMap) { throw EnsembleTestFailure( @@ -50,10 +44,34 @@ class EnsembleTestParser { ); } - return _parseTestCase(doc, sourcePath: sourcePath); + return _parseTestCase( + doc, + sourcePath: sourcePath, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } - static EnsembleTestCase _parseTestCase(YamlMap map, {String? sourcePath}) { + static EnsembleTestCase _parseTestCase( + YamlMap map, { + String? sourcePath, + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + if (map.containsKey('options')) { + throw EnsembleTestFailure( + 'Root-level "options" is no longer supported in *.test.yaml files. ' + 'Move shared screenshots/performance settings to tests/config.yaml.', + ); + } + if (map.containsKey('mocks') && map['mocks'] is! YamlList) { + throw EnsembleTestFailure( + 'Root-level "mocks" must be a list of .mock.json files.', + ); + } + final id = map['id']?.toString(); if (id == null || id.isEmpty) { throw EnsembleTestFailure('Each test must have an "id"'); @@ -92,69 +110,204 @@ class EnsembleTestParser { priority: map['priority']?.toString(), startScreen: hasStartScreen ? startScreen : null, prerequisite: hasPrerequisite ? prerequisite : null, - initialState: _toStringDynamicMap(map['initialState']), - mocks: _parseMocks(map['mocks']), - steps: _parseSteps(stepsNode, testId: id), + mockFiles: _toStringList( + map['mocks'], + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + scenarios: _parseScenarios(map['scenarios'], inputs: inputs), + initialState: _toStringDynamicMap( + map['initialState'], + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + steps: _parseSteps( + stepsNode, + testId: id, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), ); } - static TestMocks _parseMocks(dynamic node) { - if (node == null) return const TestMocks(); - if (node is! YamlMap) { - throw EnsembleTestFailure('"mocks" must be a map'); + static EnsembleTestConfig parseConfigString( + String content, { + String? sourcePath, + }) { + if (content.trim().isEmpty) return const EnsembleTestConfig(); + final dynamic doc = loadYaml(content); + if (doc == null) return const EnsembleTestConfig(); + if (doc is! YamlMap) { + throw EnsembleTestFailure( + 'Invalid test config${sourcePath != null ? ' ($sourcePath)' : ''}: root must be a map', + ); } + return _parseConfig(doc); + } - final apisNode = node['apis']; - if (apisNode == null) return const TestMocks(); - if (apisNode is! YamlMap) { - throw EnsembleTestFailure('"mocks.apis" must be a map'); + static EnsembleTestConfig _parseConfig(YamlMap node) { + const allowedKeys = { + 'screenshots', + 'performance', + 'dumpTree', + 'logApiCalls', + 'logStorage', + }; + for (final key in node.keys) { + if (!allowedKeys.contains(key)) { + throw EnsembleTestFailure( + 'Unsupported test config key "$key". Supported keys: ' + '${allowedKeys.join(', ')}', + ); + } } - final apis = {}; - apisNode.forEach((key, value) { - if (value is! YamlMap) { - throw EnsembleTestFailure('Mock for API "$key" must be a map'); - } - apis[key.toString()] = _parseMockApiResponse(value); - }); + final screenshotsNode = node['screenshots']; + final performanceNode = node['performance']; + final dumpTreeNode = node['dumpTree']; + final logApiCallsNode = node['logApiCalls']; + final logStorageNode = node['logStorage']; + if (screenshotsNode != null && screenshotsNode is! YamlMap) { + throw EnsembleTestFailure('"screenshots" must be a map'); + } + if (performanceNode != null && performanceNode is! YamlMap) { + throw EnsembleTestFailure('"performance" must be a map'); + } + if (dumpTreeNode != null && dumpTreeNode is! YamlMap) { + throw EnsembleTestFailure('"dumpTree" must be a map'); + } + if (logApiCallsNode != null && logApiCallsNode is! YamlMap) { + throw EnsembleTestFailure('"logApiCalls" must be a map'); + } + if (logStorageNode != null && logStorageNode is! YamlMap) { + throw EnsembleTestFailure('"logStorage" must be a map'); + } - return TestMocks(apis: apis); + return EnsembleTestConfig( + screenshots: screenshotsNode == null + ? const ScreenshotConfig() + : ScreenshotConfig( + enabled: screenshotsNode['enabled'] == true, + platform: screenshotsNode['platform']?.toString() ?? 'ios', + model: screenshotsNode['model']?.toString() ?? 'iPhone 15 Pro', + includeSteps: _toStringList(screenshotsNode['includeSteps']), + excludeSteps: _toStringList(screenshotsNode['excludeSteps']), + ), + performance: performanceNode == null + ? const PerformanceConfig() + : PerformanceConfig( + enabled: performanceNode['enabled'] == true, + ), + dumpTree: dumpTreeNode == null + ? const DumpTreeConfig() + : DumpTreeConfig( + enabled: dumpTreeNode['enabled'] == true, + ), + logApiCalls: logApiCallsNode == null + ? const LogApiCallsConfig() + : LogApiCallsConfig( + enabled: logApiCallsNode['enabled'] == true, + ), + logStorage: logStorageNode == null + ? const LogStorageConfig() + : LogStorageConfig( + enabled: logStorageNode['enabled'] == true, + key: logStorageNode['key']?.toString(), + ), + ); } - static MockAPIResponse _parseMockApiResponse(YamlMap map) { - final response = map['response']; - if (response is! YamlMap) { - throw EnsembleTestFailure('API mock must include a "response" map'); + static List _parseScenarios( + dynamic node, { + required Map inputs, + }) { + if (node == null) return const []; + if (node is! YamlList) { + throw EnsembleTestFailure('"scenarios" must be a list'); } - - return MockAPIResponse( - statusCode: response['statusCode'] as int? ?? 200, - body: _unwrapYaml(response['body']), - headers: response['headers'] is YamlMap - ? _toStringDynamicMap(response['headers']) - : null, - delayMs: map['delayMs'] as int?, - ); + final scenarios = []; + final ids = {}; + for (final item in node) { + if (item is! YamlMap) { + throw EnsembleTestFailure('Each scenario must be a map'); + } + final id = item['id']?.toString(); + if (id == null || id.isEmpty) { + throw EnsembleTestFailure('Each scenario must have an "id"'); + } + if (!ids.add(id)) { + throw EnsembleTestFailure('Duplicate scenario id "$id"'); + } + final vars = _toStringDynamicMap( + item['vars'], + inputs: inputs, + scenario: const {}, + scenarioId: id, + ); + scenarios.add( + TestScenario( + id: id, + description: item['description']?.toString(), + vars: vars, + ), + ); + } + return scenarios; } - static List _parseSteps(YamlList steps, {required String testId}) => - _parseStepsList(steps, testId: testId); + static List _parseSteps( + YamlList steps, { + required String testId, + required Map inputs, + required Map scenario, + String? scenarioId, + }) => + _parseStepsList( + steps, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); - static List _parseStepsList(dynamic steps, - {required String testId}) { + static List _parseStepsList( + dynamic steps, { + required String testId, + required Map inputs, + required Map scenario, + String? scenarioId, + }) { if (steps is! List || steps.isEmpty) { throw EnsembleTestFailure( 'Test "$testId" requires a non-empty "steps" list'); } final result = []; for (var i = 0; i < steps.length; i++) { - result.add(_parseStep(steps[i], testId: testId, index: i)); + result.add( + _parseStep( + steps[i], + testId: testId, + index: i, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ); } return result; } - static TestStep _parseStep(dynamic step, - {required String testId, int? index}) { + static TestStep _parseStep( + dynamic step, { + required String testId, + int? index, + required Map inputs, + Map scenario = const {}, + String? scenarioId, + }) { final String type; final dynamic argsNode; @@ -171,72 +324,238 @@ class EnsembleTestParser { ); } - final args = _argsFromNode(argsNode); + final args = _argsFromNode( + argsNode, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); List nested = const []; if (type == 'group' || type == 'repeat') { final stepsNode = args['steps']; if (stepsNode is List && stepsNode.isNotEmpty) { - nested = _parseStepsList(stepsNode, testId: testId); + nested = _parseStepsList( + stepsNode, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } else { throw EnsembleTestFailure('"$type" requires a non-empty "steps" list'); } } else if (type == 'optional' || type == 'ifVisible') { final single = args['step']; if (single is YamlMap && single.length == 1) { - nested = [_parseStep(single, testId: testId)]; + nested = [ + _parseStep( + single, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ]; } else if (single is Map && single.length == 1) { - nested = [_parseStep(single, testId: testId)]; + nested = [ + _parseStep( + single, + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ]; } else if (args['steps'] is List && (args['steps'] as List).isNotEmpty) { - nested = _parseStepsList(args['steps'], testId: testId); + nested = _parseStepsList( + args['steps'], + testId: testId, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } } return TestStep(type: type, args: args, nestedSteps: nested); } - static Map _argsFromNode(dynamic node) { + static Map _argsFromNode( + dynamic node, { + required Map inputs, + required Map scenario, + String? scenarioId, + }) { if (node is YamlMap) { - return _toStringDynamicMap(node); + return _toStringDynamicMap( + node, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } if (node is Map) { return node.map( - (key, value) => MapEntry(key.toString(), _unwrapYaml(value)), + (key, value) => MapEntry( + key.toString(), + _unwrapYaml( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ), ); } if (node != null) { - return {'value': _unwrapYaml(node)}; + return { + 'value': _unwrapYaml( + node, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + }; } return {}; } - static Map _toStringDynamicMap(dynamic node) { + static Map _toStringDynamicMap( + dynamic node, { + required Map inputs, + Map scenario = const {}, + String? scenarioId, + }) { if (node == null) return {}; if (node is! YamlMap) { throw EnsembleTestFailure('Expected a map'); } final out = {}; node.forEach((key, value) { - out[key.toString()] = _unwrapYaml(value); + out[key.toString()] = _unwrapYaml( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); }); return out; } - static List _toStringList(dynamic node) { + static List _toStringList( + dynamic node, { + Map inputs = const {}, + Map scenario = const {}, + String? scenarioId, + }) { if (node == null) return const []; if (node is! Iterable) { throw EnsembleTestFailure('Expected a list'); } - return node.map((value) => value.toString()).toList(); + return node + .map( + (value) => _unwrapYaml( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ).toString(), + ) + .toList(); } - static dynamic _unwrapYaml(dynamic value) { + static dynamic _unwrapYaml( + dynamic value, { + required Map inputs, + Map scenario = const {}, + String? scenarioId, + }) { if (value is YamlMap) { - return _toStringDynamicMap(value); + return _toStringDynamicMap( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } if (value is YamlList) { - return value.map(_unwrapYaml).toList(); + return value + .map( + (item) => _unwrapYaml( + item, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ), + ) + .toList(); + } + if (value is String) { + return _resolvePlaceholders( + value, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); } return value; } + + static dynamic _resolvePlaceholders( + String value, { + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + final exact = RegExp(r'^\$\{(inputs|scenario)\.([A-Za-z0-9_.-]+)\}$') + .firstMatch(value); + if (exact != null) { + return _placeholderValue( + exact.group(1)!, + exact.group(2)!, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ); + } + + return value.replaceAllMapped( + RegExp(r'\$\{(inputs|scenario)\.([A-Za-z0-9_.-]+)\}'), + (match) => _placeholderValue( + match.group(1)!, + match.group(2)!, + inputs: inputs, + scenario: scenario, + scenarioId: scenarioId, + ).toString(), + ); + } + + static dynamic _placeholderValue( + String namespace, + String key, { + required Map inputs, + required Map scenario, + String? scenarioId, + }) { + if (namespace == 'inputs') { + if (!inputs.containsKey(key)) { + throw EnsembleTestFailure( + 'Missing CLI input "$key". Pass it with --input $key=value.', + ); + } + return inputs[key]; + } + if (scenarioId == null && scenario.isEmpty) { + return '\${scenario.$key}'; + } + if (!scenario.containsKey(key)) { + throw EnsembleTestFailure( + 'Missing scenario value "$key"' + '${scenarioId != null ? ' in scenario "$scenarioId"' : ''}.', + ); + } + return scenario[key]; + } } diff --git a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart index 89a5c0cc3..75979a8ff 100644 --- a/tools/ensemble_test_runner/lib/reporters/test_reporter.dart +++ b/tools/ensemble_test_runner/lib/reporters/test_reporter.dart @@ -99,6 +99,14 @@ class TestReporter { _writeTestCase(buffer, r); } + if (result.suiteLogs.isNotEmpty) { + buffer.writeln('│'); + buffer.writeln('│ suite artifacts:'); + for (final log in result.suiteLogs) { + buffer.writeln('│ $log'); + } + } + buffer.writeln('│'); buffer.writeln( '└─ ${result.summary} · ${totalMs}ms total', @@ -106,6 +114,55 @@ class TestReporter { return buffer.toString(); } + /// Formats the short failure passed to Flutter's test framework. + /// + /// The full boxed report is printed separately. Keeping this compact avoids + /// Flutter echoing the entire report again with its framework stack trace. + String formatFailureSummary( + EnsembleTestRunResult result, { + Iterable failedPaths = const [], + Iterable pendingFrameworkExceptions = const [], + }) { + final failed = result.results + .where((r) => r.status == TestStatus.failed) + .toList(growable: false); + final paths = failedPaths.toList(growable: false); + final buffer = StringBuffer() + ..writeln( + 'Failed YAML tests (${result.failedCount}/${result.results.length}):', + ); + + for (var i = 0; i < failed.length; i++) { + final r = failed[i]; + final path = i < paths.length ? paths[i] : null; + final label = path == null || r.testId.contains(path) + ? r.testId + : '${r.testId} ($path)'; + final failedStep = r.failedStep != null + ? ' (failed: ${formatStepBrief(r.failedStep!)})' + : r.failedStepIndex != null + ? ' (failed: step ${r.failedStepIndex! + 1})' + : ''; + buffer.writeln('- $label: ${r.message ?? 'failed'}$failedStep'); + } + + final pending = pendingFrameworkExceptions + .map((e) => e?.toString().trim() ?? '') + .where((e) => e.isNotEmpty) + .toList(growable: false); + if (pending.isNotEmpty) { + buffer.writeln(); + buffer.writeln('Pending Flutter framework exceptions:'); + for (final exception in pending) { + buffer.writeln('- ${_firstLine(exception)}'); + } + } + + buffer.writeln(); + buffer.write('See the Ensemble YAML tests report above.'); + return buffer.toString(); + } + void _writeTestCase(StringBuffer buffer, EnsembleSingleTestResult r) { final icon = r.status == TestStatus.passed ? '✓' : '✗'; buffer.writeln('│ $icon ${r.testId} (${r.durationMs}ms)'); @@ -143,5 +200,17 @@ class TestReporter { buffer.writeln('│ at step: ${r.failedStepIndex! + 1}'); } } + + if (r.logs.isNotEmpty) { + buffer.writeln('│ artifacts:'); + for (final log in r.logs) { + buffer.writeln('│ $log'); + } + } } } + +String _firstLine(String value) { + final index = value.indexOf('\n'); + return index == -1 ? value : value.substring(0, index); +} diff --git a/tools/ensemble_test_runner/lib/runner/app_performance_log.dart b/tools/ensemble_test_runner/lib/runner/app_performance_log.dart new file mode 100644 index 000000000..794a7f462 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/app_performance_log.dart @@ -0,0 +1,265 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; + +Future writeAppPerformanceLog(EnsembleTestContext context) { + return writePerformanceLog( + logger: context.logger, + filePrefix: context.testCase.id, + name: 'app_performance', + frames: context.runtime.appFrameTimings, + markers: context.runtime.performanceMarkers, + apiCalls: context.apiOverlay.calls, + ); +} + +Future writePerformanceLog({ + required TestLogger logger, + required String filePrefix, + required String name, + required List frames, + List markers = const [], + List apiCalls = const [], +}) { + final jankyFrames = frames.where((frame) => frame.isJanky).length; + final attributedFrames = frames + .map((frame) => _AttributedFrame(frame, _markerForFrame(frame, markers))) + .toList(growable: false); + final slowestFrames = attributedFrames.toList() + ..sort((a, b) => b.frame.totalSpanMs.compareTo(a.frame.totalSpanMs)); + final worstSteps = _worstSteps(attributedFrames); + final worstScreens = _worstScreens(attributedFrames); + final jankClusters = _jankClusters(attributedFrames); + final apiCorrelation = _apiCorrelation(attributedFrames, apiCalls); + final content = const JsonEncoder.withIndent(' ').convert({ + 'testId': filePrefix.isEmpty ? 'suite' : filePrefix, + 'frameBudgetMs': AppFrameTimingEntry.frameBudgetMs, + 'summary': { + 'totalFrames': frames.length, + 'jankyFrames': jankyFrames, + 'jankyFrameRate': _ratio(jankyFrames, frames.length), + 'averageBuildMs': _average(frames.map((frame) => frame.buildMs)), + 'averageRasterMs': _average(frames.map((frame) => frame.rasterMs)), + 'averageTotalSpanMs': _average(frames.map((frame) => frame.totalSpanMs)), + 'maxBuildMs': _max(frames.map((frame) => frame.buildMs)), + 'maxRasterMs': _max(frames.map((frame) => frame.rasterMs)), + 'maxTotalSpanMs': _max(frames.map((frame) => frame.totalSpanMs)), + if (worstSteps.isNotEmpty) 'worstStep': worstSteps.first['step'], + if (worstScreens.isNotEmpty) 'worstScreen': worstScreens.first['screen'], + }, + 'totalFrames': frames.length, + 'jankyFrames': jankyFrames, + 'jankyFrameRate': _ratio(jankyFrames, frames.length), + 'averageBuildMs': _average(frames.map((frame) => frame.buildMs)), + 'averageRasterMs': _average(frames.map((frame) => frame.rasterMs)), + 'averageTotalSpanMs': _average(frames.map((frame) => frame.totalSpanMs)), + 'maxBuildMs': _max(frames.map((frame) => frame.buildMs)), + 'maxRasterMs': _max(frames.map((frame) => frame.rasterMs)), + 'maxTotalSpanMs': _max(frames.map((frame) => frame.totalSpanMs)), + 'worstSteps': worstSteps, + 'worstScreens': worstScreens, + 'jankClusters': jankClusters, + 'apiCorrelation': apiCorrelation, + 'slowestFrames': + slowestFrames.take(10).map((frame) => frame.toJson()).toList(), + 'frames': attributedFrames.map((frame) => frame.toJson()).toList(), + }); + + return logger.writeLogFile( + testId: filePrefix, + name: name, + content: content, + extension: 'json', + ); +} + +PerformanceMarker? _markerForFrame( + AppFrameTimingEntry frame, + List markers, +) { + for (final marker in markers.reversed) { + if (marker.containsFrame(frame.frameNumber)) return marker; + } + return null; +} + +List> _worstSteps(List<_AttributedFrame> frames) { + final byStep = >{}; + for (final frame in frames) { + final marker = frame.marker; + if (marker == null) continue; + byStep.putIfAbsent(marker.label, () => []).add(frame); + } + final rows = byStep.entries.map((entry) { + final stepFrames = entry.value; + final janky = stepFrames.where((frame) => frame.frame.isJanky).toList(); + return { + 'step': entry.key, + 'testId': stepFrames.first.marker?.testId, + if (stepFrames.first.marker?.stepIndex != null) + 'stepIndex': stepFrames.first.marker?.stepIndex, + 'screen': stepFrames.first.marker?.screen, + 'phase': stepFrames.first.marker?.phase, + 'totalFrames': stepFrames.length, + 'jankyFrames': janky.length, + 'jankyFrameRate': _ratio(janky.length, stepFrames.length), + 'maxTotalSpanMs': + _max(stepFrames.map((frame) => frame.frame.totalSpanMs)), + 'totalJankOverBudgetMs': _jankOverBudget(janky), + }; + }).toList() + ..sort(_rankJankRows); + return rows.take(10).toList(); +} + +List> _worstScreens(List<_AttributedFrame> frames) { + final byScreen = >{}; + for (final frame in frames) { + final screen = frame.marker?.screen; + if (screen == null || screen.isEmpty) continue; + byScreen.putIfAbsent(screen, () => []).add(frame); + } + final rows = byScreen.entries.map((entry) { + final screenFrames = entry.value; + final janky = screenFrames.where((frame) => frame.frame.isJanky).toList(); + return { + 'screen': entry.key, + 'totalFrames': screenFrames.length, + 'jankyFrames': janky.length, + 'jankyFrameRate': _ratio(janky.length, screenFrames.length), + 'maxTotalSpanMs': + _max(screenFrames.map((frame) => frame.frame.totalSpanMs)), + 'totalJankOverBudgetMs': _jankOverBudget(janky), + }; + }).toList() + ..sort(_rankJankRows); + return rows.take(10).toList(); +} + +int _rankJankRows(Map a, Map b) { + final byJank = (b['jankyFrames'] as int).compareTo(a['jankyFrames'] as int); + if (byJank != 0) return byJank; + return (b['maxTotalSpanMs'] as double) + .compareTo(a['maxTotalSpanMs'] as double); +} + +List> _jankClusters(List<_AttributedFrame> frames) { + final clusters = >[]; + var current = <_AttributedFrame>[]; + for (final frame in frames) { + if (!frame.frame.isJanky) { + if (current.isNotEmpty) { + clusters.add(current); + current = <_AttributedFrame>[]; + } + continue; + } + if (current.isEmpty || + frame.frame.frameNumber - current.last.frame.frameNumber <= 2) { + current.add(frame); + } else { + clusters.add(current); + current = [frame]; + } + } + if (current.isNotEmpty) clusters.add(current); + + final rows = clusters.map((cluster) { + final slowest = cluster.toList() + ..sort((a, b) => b.frame.totalSpanMs.compareTo(a.frame.totalSpanMs)); + final marker = slowest.first.marker; + return { + 'startFrame': cluster.first.frame.frameNumber, + 'endFrame': cluster.last.frame.frameNumber, + 'jankyFrames': cluster.length, + 'maxTotalSpanMs': slowest.first.frame.totalSpanMs, + 'totalJankOverBudgetMs': _jankOverBudget(cluster), + if (marker != null) ..._markerJson(marker), + }; + }).toList() + ..sort(_rankJankRows); + return rows.take(10).toList(); +} + +List> _apiCorrelation( + List<_AttributedFrame> frames, + List apiCalls, +) { + final rows = >[]; + final jankyMarkers = frames + .where((frame) => frame.frame.isJanky && frame.marker != null) + .map((frame) => frame.marker!) + .toSet(); + for (final marker in jankyMarkers) { + final calls = apiCalls + .where((call) => marker.overlaps(call.timestamp, + tolerance: const Duration(milliseconds: 250))) + .toList(); + if (calls.isEmpty) continue; + rows.add({ + ..._markerJson(marker), + 'apiCalls': [ + for (final call in calls) + { + 'name': call.name, + 'timestamp': call.timestamp.toIso8601String(), + }, + ], + }); + } + return rows.take(25).toList(); +} + +double _jankOverBudget(Iterable<_AttributedFrame> frames) { + return _round( + frames.fold( + 0, + (sum, frame) => + sum + + (frame.frame.totalSpanMs - AppFrameTimingEntry.frameBudgetMs) + .clamp(0, double.infinity), + ), + ); +} + +Map _markerJson(PerformanceMarker marker) => { + 'testId': marker.testId, + if (marker.stepIndex != null) 'stepIndex': marker.stepIndex, + 'step': marker.label, + if (marker.screen != null) 'screen': marker.screen, + 'phase': marker.phase, + }; + +class _AttributedFrame { + final AppFrameTimingEntry frame; + final PerformanceMarker? marker; + + const _AttributedFrame(this.frame, this.marker); + + Map toJson() => { + ...frame.toJson(), + if (marker != null) ..._markerJson(marker!), + }; +} + +double _average(Iterable values) { + final list = values.toList(growable: false); + if (list.isEmpty) return 0; + return _round(list.reduce((a, b) => a + b) / list.length); +} + +double _max(Iterable values) { + final list = values.toList(growable: false); + if (list.isEmpty) return 0; + return _round(list.reduce((a, b) => a > b ? a : b)); +} + +double _ratio(int numerator, int denominator) { + if (denominator == 0) return 0; + return _round(numerator / denominator); +} + +double _round(double value) => double.parse(value.toStringAsFixed(3)); diff --git a/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart b/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart new file mode 100644 index 000000000..bb268cb57 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart @@ -0,0 +1,127 @@ +import 'dart:convert'; + +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:flutter/material.dart'; + +Future writeDumpTreeLog(EnsembleTestContext context) { + return writeDumpTreeLogFile( + logger: context.logger, + filePrefix: context.testCase.id, + ); +} + +Future writeDumpTreeLogFile({ + required TestLogger logger, + required String filePrefix, +}) { + return logger.writeLogFile( + testId: filePrefix, + name: 'dump_tree', + content: captureDebugDumpApp(), + extension: 'txt', + ); +} + +Future writeApiCallsLog(EnsembleTestContext context) { + return writeApiCallsLogFile( + logger: context.logger, + filePrefix: context.testCase.id, + calls: context.apiOverlay.calls, + ); +} + +Future writeApiCallsLogFile({ + required TestLogger logger, + required String filePrefix, + required List calls, +}) { + final callsByName = >{}; + for (final call in calls) { + callsByName.putIfAbsent(call.name, () => []).add(call.timestamp); + } + final content = const JsonEncoder.withIndent(' ').convert({ + 'total': calls.length, + 'calls': callsByName.entries + .map( + (entry) => { + 'name': entry.key, + 'count': entry.value.length, + 'timestamps': [ + for (final timestamp in entry.value) timestamp.toIso8601String(), + ], + }, + ) + .toList(), + }); + + return logger.writeLogFile( + testId: filePrefix, + name: 'api_calls', + content: content, + extension: 'json', + ); +} + +Future writeStorageLog( + EnsembleTestContext context, { + String? key, +}) { + return writeStorageLogFile( + logger: context.logger, + filePrefix: context.testCase.id, + key: key, + ); +} + +Future writeStorageLogFile({ + required TestLogger logger, + required String filePrefix, + String? key, +}) { + if (key == null || key.isEmpty) { + final storage = StorageManager(); + final entries = storage.getKeys().map( + (key) => MapEntry(key, storage.read(key)), + ); + return logger.writeLogFile( + testId: filePrefix, + name: 'storage', + content: _prettyJson(Map.fromEntries(entries)), + extension: 'json', + ); + } + + return logger.writeLogFile( + testId: filePrefix, + name: 'storage_$key', + content: _prettyJson(StorageManager().read(key)), + extension: 'json', + ); +} + +String captureDebugDumpApp() { + final previousDebugPrint = debugPrint; + final lines = []; + debugPrint = (String? message, {int? wrapWidth}) { + if (message != null) { + lines.add(message); + } + }; + try { + debugDumpApp(); + } finally { + debugPrint = previousDebugPrint; + } + if (lines.isEmpty) { + return ''; + } + return lines.join('\n'); +} + +String _prettyJson(Object? value) { + return JsonEncoder.withIndent(' ', (value) => value.toString()) + .convert(value); +} diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart index 046bdcbbb..5f69bacce 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart @@ -1,6 +1,6 @@ import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/storage_manager.dart'; -import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/mocks/test_logger.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; @@ -8,7 +8,8 @@ import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; class EnsembleTestContext { final EnsembleTestCase testCase; - final MockAPIProvider mockApiProvider; + final EnsembleTestConfig config; + final TestApiProviderOverlay apiOverlay; final TestLogger logger; final EnsembleTestSetup setup; @@ -19,34 +20,40 @@ class EnsembleTestContext { EnsembleTestContext({ required this.testCase, - required this.mockApiProvider, + this.config = const EnsembleTestConfig(), + required this.apiOverlay, required this.logger, required this.setup, }); - factory EnsembleTestContext.fromTestCase(EnsembleTestCase testCase) { + factory EnsembleTestContext.fromTestCase( + EnsembleTestCase testCase, { + EnsembleTestConfig config = const EnsembleTestConfig(), + }) { final logger = TestLogger(); - final mockApi = MockAPIProvider( + final apiOverlay = TestApiProviderOverlay( mocks: Map.from(testCase.mocks.apis), ); final storage = testCase.initialState['storage']; + final keychain = testCase.initialState['keychain']; final env = testCase.initialState['env']; - final envMap = env is Map - ? Map.from(env) - : {}; + final envMap = + env is Map ? Map.from(env) : {}; final setup = EnsembleTestSetup( envOverrides: envMap.isEmpty ? null : envMap, - initialPublicStorage: storage is Map - ? Map.from(storage) - : null, + initialPublicStorage: + storage is Map ? Map.from(storage) : null, + initialKeychain: + keychain is Map ? Map.from(keychain) : null, ); final ctx = EnsembleTestContext( testCase: testCase, - mockApiProvider: mockApi, + config: config, + apiOverlay: apiOverlay, logger: logger, setup: setup, ); @@ -75,19 +82,4 @@ class EnsembleTestContext { Future clearStorage() async { await StorageManager().clearPublicStorage(); } - - MockAPIResponse mockFromStepArgs(Map args) { - final response = args['response']; - if (response is! Map) { - throw EnsembleTestFailure('mockApi requires a "response" map'); - } - return MockAPIResponse( - statusCode: response['statusCode'] as int? ?? 200, - body: response['body'], - headers: response['headers'] is Map - ? Map.from(response['headers'] as Map) - : null, - delayMs: args['delayMs'] as int?, - ); - } } diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index 05d1cde70..52d125bf8 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -1,48 +1,72 @@ +import 'dart:convert'; import 'dart:io'; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/ensemble_app.dart'; +import 'package:ensemble/framework/apiproviders/api_provider.dart'; import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; import 'package:ensemble/framework/definition_providers/local_provider.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; import 'package:ensemble/page_model.dart'; -import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble_device_preview/ensemble_device_preview.dart'; +import 'package:ensemble_test_runner/actions/screenshot_device.dart'; +import 'package:ensemble_test_runner/mocks/adobe_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:yaml/yaml.dart'; /// Per-test bootstrap data applied before the widget tree mounts. class EnsembleTestSetup { final Map? envOverrides; final Map? initialPublicStorage; + final Map? initialKeychain; const EnsembleTestSetup({ this.envOverrides, this.initialPublicStorage, + this.initialKeychain, }); } /// Applies YAML test environment and storage bootstrap data to [config]. -void applyYamlTestBootstrap(EnsembleConfig config, EnsembleTestSetup setup) { +Future applyYamlTestBootstrap( + EnsembleConfig config, EnsembleTestSetup setup) async { if (setup.envOverrides != null && setup.envOverrides!.isNotEmpty) { config.updateEnvOverrides(setup.envOverrides!); } - setup.initialPublicStorage?.forEach((key, value) { - StorageManager().write(key, value); - }); + await applyYamlTestStorageBootstrap(setup); +} + +Future applyYamlTestStorageBootstrap(EnsembleTestSetup setup) async { + for (final entry in setup.initialPublicStorage?.entries ?? + const Iterable>.empty()) { + await StorageManager().write(entry.key, entry.value); + } + for (final entry in setup.initialKeychain?.entries ?? + const Iterable>.empty()) { + await StorageManager().writeSecurely(key: entry.key, value: entry.value); + } } /// Boots the real Ensemble runtime for widget tests. class EnsembleTestHarness { static final String _testStoragePath = Directory.systemTemp.createTempSync('ensemble_test_runner_storage_').path; + static bool _appFontsLoaded = false; static void ensureTestPlugins() { TestWidgetsFlutterBinding.ensureInitialized(); + ensureFirebaseCoreMocksForTest(); + ensureAdobeAnalyticsMocksForTest(); + HttpOverrides.global = _RealNetworkHttpOverrides(); const pathProviderChannel = MethodChannel('plugins.flutter.io/path_provider'); TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger @@ -62,12 +86,7 @@ class EnsembleTestHarness { TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger .setMockMethodCallHandler(packageInfoChannel, (call) async { if (call.method == 'getAll') { - return { - 'appName': 'EnsembleTest', - 'packageName': 'com.ensemble.test', - 'version': '1.0.0', - 'buildNumber': '1', - }; + return _resolvePackageInfo(Directory.current); } return null; }); @@ -180,11 +199,13 @@ class EnsembleTestHarness { final String appPath; final String appHome; final String? i18nPath; + final Map? externalMethods; EnsembleTestHarness({ required this.appPath, required this.appHome, this.i18nPath, + this.externalMethods, }); static String normalizeAppPath(String path) { @@ -208,56 +229,280 @@ class EnsembleTestHarness { return updated; } - static void installHttpMockProvider( + static Future initializeRealApiProviders(EnsembleConfig config) async { + await Ensemble.initializeAPIProviders(config); + config.apiProviders ??= {}; + config.apiProviders!.putIfAbsent('http', () => HTTPAPIProvider()); + + final firebase = config.apiProviders!['firebase']; + if (firebase != null) { + config.apiProviders!['firebaseFunction'] = firebase; + } + } + + /// Wraps real providers with [mock] for call recording and optional overrides. + static void installTestApiOverlay( EnsembleConfig config, - MockAPIProvider mock, + TestApiProviderOverlay mock, ) { - config.apiProviders = { - ...?config.apiProviders, - 'http': mock, - }; + final realProviders = Map.from( + config.apiProviders ?? const {}, + ); + + final installed = {}; + for (final entry in realProviders.entries) { + var delegate = entry.value; + if (delegate is TestApiProviderOverlay) { + delegate = delegate.delegate; + } else if (delegate is TestApiOverlay) { + delegate = delegate.delegate; + } + + if (entry.key == 'http') { + mock.bindHttpDelegate(delegate as HTTPAPIProvider); + installed['http'] = mock; + continue; + } + installed[entry.key] = TestApiOverlay(mock, delegate); + } + + if (!installed.containsKey('http')) { + mock.bindHttpDelegate(HTTPAPIProvider()); + installed['http'] = mock; + } + + final firebase = realProviders['firebase']; + if (firebase != null && !installed.containsKey('firebaseFunction')) { + installed['firebaseFunction'] = + installed['firebase'] ?? TestApiOverlay(mock, firebase); + } + + config.apiProviders = installed; } Future bootstrapRuntime( EnsembleConfig config, EnsembleTestSetup setup, { - MockAPIProvider? httpMock, + TestApiProviderOverlay? apiOverlay, }) async { ensureTestPlugins(); + await ensureAppFontsLoaded(); - applyYamlTestBootstrap(config, setup); + final env = Map.from(config.envOverrides ?? {}); + env['firebase_app_check'] = 'false'; + if (setup.envOverrides != null && setup.envOverrides!.isNotEmpty) { + env.addAll(setup.envOverrides!); + } + config.updateEnvOverrides(env); Ensemble().setEnsembleConfig(config); - - if (httpMock != null) { - installHttpMockProvider(config, httpMock); + if (externalMethods != null && externalMethods!.isNotEmpty) { + Ensemble().setExternalMethods(externalMethods!); } await Ensemble().initManagers(); + await initializeRealApiProviders(config); - config.apiProviders ??= {'http': HTTPAPIProvider()}; - config.apiProviders!.putIfAbsent('http', () => HTTPAPIProvider()); + if (apiOverlay != null) { + installTestApiOverlay(config, apiOverlay); + } + await applyYamlTestStorageBootstrap(setup); YamlTestSession.markRuntimeBootstrapped(); return config; } + static Future ensureAppFontsLoaded() async { + if (_appFontsLoaded) return; + _appFontsLoaded = true; + + List manifest; + try { + final rawManifest = await rootBundle.loadString('FontManifest.json'); + manifest = jsonDecode(rawManifest) as List; + } catch (_) { + return; + } + + for (final familyEntry in manifest.whereType()) { + final family = familyEntry['family']?.toString(); + final fonts = familyEntry['fonts']; + if (family == null || fonts is! List) continue; + + for (final alias in _fontFamilyAliases(family)) { + await _loadFontFamily(alias, fonts); + } + } + } + + static List fontFamilyAliasesForTest(String family) => + _fontFamilyAliases(family); + + static List fontAssetCandidatesForTest(String asset) => + _fontAssetCandidates(asset); + + static Map packageInfoForTest(String appDir) => + _resolvePackageInfo(Directory(appDir)); + + static Map _resolvePackageInfo(Directory appDir) { + final pubspec = _readPubspec(appDir); + final properties = _readProperties( + File('${appDir.path}/ensemble/ensemble.properties'), + ); + + final packageName = + properties['appId'] ?? pubspec['name'] ?? 'com.ensemble.test'; + final appName = properties['appName'] ?? + _titleFromPackageName(pubspec['name']) ?? + 'EnsembleTest'; + final versionParts = _splitVersion(pubspec['version']); + + return { + 'appName': appName, + 'packageName': packageName, + 'version': versionParts.version, + 'buildNumber': versionParts.buildNumber, + }; + } + + static Map _readPubspec(Directory appDir) { + final file = File('${appDir.path}/pubspec.yaml'); + if (!file.existsSync()) return const {}; + + try { + final yaml = loadYaml(file.readAsStringSync()); + if (yaml is! YamlMap) return const {}; + return { + for (final entry in yaml.entries) + if (entry.value != null) entry.key.toString(): entry.value.toString(), + }; + } catch (_) { + return const {}; + } + } + + static Map _readProperties(File file) { + if (!file.existsSync()) return const {}; + + final values = {}; + for (final rawLine in file.readAsLinesSync()) { + final line = rawLine.trim(); + if (line.isEmpty || line.startsWith('#')) continue; + final separator = line.indexOf('='); + if (separator <= 0) continue; + values[line.substring(0, separator).trim()] = + line.substring(separator + 1).trim(); + } + return values; + } + + static ({String version, String buildNumber}) _splitVersion(String? raw) { + if (raw == null || raw.trim().isEmpty) { + return (version: '1.0.0', buildNumber: '1'); + } + final parts = raw.trim().split('+'); + final version = parts.first.trim(); + final buildNumber = parts.length > 1 ? parts.sublist(1).join('+') : '1'; + return ( + version: version.isEmpty ? '1.0.0' : version, + buildNumber: buildNumber.trim().isEmpty ? '1' : buildNumber.trim(), + ); + } + + static String? _titleFromPackageName(String? name) { + if (name == null || name.trim().isEmpty) return null; + return name + .trim() + .split(RegExp(r'[_\s]+')) + .where((part) => part.isNotEmpty) + .map((part) => '${part[0].toUpperCase()}${part.substring(1)}') + .join(' '); + } + + static List _fontFamilyAliases(String family) { + final aliases = {family, family.toLowerCase()}; + + const packagePrefix = 'packages/'; + if (family.startsWith(packagePrefix)) { + final slashIndex = family.lastIndexOf('/'); + if (slashIndex != -1 && slashIndex < family.length - 1) { + final unqualifiedFamily = family.substring(slashIndex + 1); + aliases.add(unqualifiedFamily); + aliases.add(unqualifiedFamily.toLowerCase()); + } + } + + return aliases.toList(growable: false); + } + + static List _fontAssetCandidates(String asset) { + final candidates = {asset}; + if (asset.contains('%')) { + try { + candidates.add(Uri.decodeFull(asset)); + } catch (_) { + // Keep the manifest-provided asset key if decoding is not valid URI + // escaping. + } + } + return candidates.toList(growable: false); + } + + static Future _loadFontFamily( + String family, + List fontEntries, + ) async { + final loader = FontLoader(family); + var hasFonts = false; + + for (final fontEntry in fontEntries.whereType()) { + final asset = fontEntry['asset']?.toString(); + if (asset == null || asset.isEmpty) continue; + + for (final assetKey in _fontAssetCandidates(asset)) { + try { + final fontData = await rootBundle.load(assetKey); + loader.addFont(Future.value(fontData)); + hasFonts = true; + break; + } catch (_) { + // Ignore missing assets so a bad font entry does not fail unrelated + // behavioral tests. + } + } + } + + if (hasFonts) { + await loader.load(); + } + } + Future loadScreen({ required WidgetTester tester, required EnsembleTestCase testCase, EnsembleConfig? existingConfig, EnsembleTestContext? context, + EnsembleTestConfig suiteConfig = const EnsembleTestConfig(), }) async { resetTestRuntime(); ScreenTracker().clearAll(); YamlTestSession.navigationFlow.clear(); - final ctx = context ?? EnsembleTestContext.fromTestCase(testCase); + final ctx = context ?? + EnsembleTestContext.fromTestCase( + testCase, + config: suiteConfig, + ); + final screenshotDevice = screenshotDeviceForTestCase(testCase, ctx.config); + if (screenshotDevice != null) { + await _setViewportForDevice(tester, ctx, screenshotDevice); + } + await _ensureDefaultViewport(tester, ctx); var config = existingConfig ?? await buildConfig(); final bootstrapped = await tester.runAsync(() async { return bootstrapRuntime( config, ctx.setup, - httpMock: ctx.mockApiProvider, + apiOverlay: ctx.apiOverlay, ); }); config = bootstrapped!; @@ -269,8 +514,8 @@ class EnsembleTestHarness { ); } - // Let EnsembleApp load the bundle through its existing config path, which - // preserves the test-installed API providers and avoids real provider init. + // Skip re-initializing providers in EnsembleApp.initApp; bootstrapRuntime + // already installed real providers and mock overlays. config.appBundle = null; await tester.pumpWidget( EnsembleApp( @@ -283,6 +528,42 @@ class EnsembleTestHarness { return config; } + static Future _ensureDefaultViewport( + WidgetTester tester, + EnsembleTestContext context, + ) async { + if (context.runtime.deviceSize != null) return; + const size = Size(800, 844); + await tester.binding.setSurfaceSize(size); + tester.view.physicalSize = size; + tester.view.devicePixelRatio = 1.0; + _setViewPadding(tester, EdgeInsets.zero); + context.runtime.deviceSize = size; + } + + static Future _setViewportForDevice( + WidgetTester tester, + EnsembleTestContext context, + DeviceInfo device, + ) async { + await tester.binding.setSurfaceSize(device.screenSize); + tester.view.devicePixelRatio = 1.0; + tester.view.physicalSize = device.screenSize; + _setViewPadding(tester, device.safeAreas); + context.runtime.deviceSize = device.screenSize; + } + + static void _setViewPadding(WidgetTester tester, EdgeInsets padding) { + final viewPadding = FakeViewPadding( + left: padding.left, + top: padding.top, + right: padding.right, + bottom: padding.bottom, + ); + tester.view.padding = viewPadding; + tester.view.viewPadding = viewPadding; + } + static Future waitForInitialWidgets( WidgetTester tester, { EnsembleTestCase? testCase, @@ -319,22 +600,28 @@ class EnsembleTestHarness { } static Future _yieldToRealAsyncWork(WidgetTester tester) async { - await tester.runAsync(() async { + Future yieldOnce() async { await Future.delayed(Duration.zero); - }); + } + + if (LiveAsyncCallSupport.runner != null) { + await LiveAsyncCallSupport.run(yieldOnce); + } else { + await tester.runAsync(yieldOnce); + } } - static void applyInPlaceSetup(EnsembleTestContext ctx) { + static Future applyInPlaceSetup(EnsembleTestContext ctx) async { final config = Ensemble().getConfig(); if (config != null) { - applyYamlTestBootstrap(config, ctx.setup); + await applyYamlTestBootstrap(config, ctx.setup); } ctx.applyRuntimeEnv(); for (final entry in ctx.testCase.mocks.apis.entries) { - ctx.mockApiProvider.setMock(entry.key, entry.value); + ctx.apiOverlay.setMock(entry.key, entry.value); } if (config != null) { - installHttpMockProvider(config, ctx.mockApiProvider); + installTestApiOverlay(config, ctx.apiOverlay); } } @@ -342,3 +629,10 @@ class EnsembleTestHarness { YamlTestSession.reset(); } } + +class _RealNetworkHttpOverrides extends HttpOverrides { + @override + HttpClient createHttpClient(SecurityContext? context) { + return super.createHttpClient(context); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index 1f6612828..be6cec03e 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -1,20 +1,45 @@ +import 'dart:ui' as ui; + import 'package:ensemble/ensemble.dart'; +import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; import 'package:ensemble_test_runner/actions/test_step_executor.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; import 'package:ensemble_test_runner/reporters/test_reporter.dart'; +import 'package:ensemble_test_runner/runner/app_performance_log.dart'; +import 'package:ensemble_test_runner/runner/debug_artifact_logs.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; +import 'package:ensemble_test_runner/runner/screenshot_contact_sheet.dart'; import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter/cupertino.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter/scheduler.dart'; import 'package:flutter_test/flutter_test.dart'; typedef EnsembleTestRunOutput = ({ EnsembleSingleTestResult result, EnsembleConfig config, + EnsembleTestContext context, }); +class EnsembleTestPlanRunResult { + final Map resultsById; + final List suiteLogs; + + const EnsembleTestPlanRunResult({ + required this.resultsById, + this.suiteLogs = const [], + }); +} + /// Executes parsed Ensemble YAML test plans against a widget tester. class EnsembleTestRunner { /// Harness used to boot and reset the real Ensemble runtime. @@ -24,11 +49,16 @@ class EnsembleTestRunner { EnsembleTestRunner({required this.harness}); /// Runs every test in [plan] and returns results keyed by test id. - Future> runPlan( + Future runPlan( EnsembleTestExecutionPlan plan, WidgetTester tester, ) async { final resultsById = {}; + final suiteLogger = TestLogger(); + final suiteFrames = []; + final suiteMarkers = []; + final suiteApiCalls = []; + EnsembleTestContext? lastContext; var config = await harness.buildConfig(); for (final def in plan.ordered) { @@ -56,28 +86,65 @@ class EnsembleTestRunner { final out = await runOne( test, tester, + suiteConfig: plan.config, existingConfig: config, continuation: test.hasPrerequisite, ); resultsById[test.id] = out.result; config = out.config; + lastContext = out.context; + final frameOffset = suiteFrames.length; + _appendSuiteFrames(suiteFrames, out.context.runtime.appFrameTimings); + suiteMarkers.addAll( + out.context.runtime.performanceMarkers.map( + (marker) => marker.shiftedFrames(frameOffset), + ), + ); + suiteApiCalls.addAll(out.context.apiOverlay.calls); } - return resultsById; + final suiteLogs = await _writeSuiteLogs( + tester: tester, + config: plan.config, + logger: suiteLogger, + frames: suiteFrames, + markers: suiteMarkers, + apiCalls: suiteApiCalls, + lastContext: lastContext, + ); + + return EnsembleTestPlanRunResult( + resultsById: resultsById, + suiteLogs: suiteLogs, + ); } /// Runs a single [test], optionally continuing an existing app session. Future runOne( EnsembleTestCase test, WidgetTester tester, { + EnsembleTestConfig suiteConfig = const EnsembleTestConfig(), EnsembleConfig? existingConfig, bool continuation = false, }) async { final stopwatch = Stopwatch()..start(); + void Function(List)? timingsCallback; + final ctx = EnsembleTestContext.fromTestCase( + test, + config: suiteConfig, + ); try { - final ctx = EnsembleTestContext.fromTestCase(test); + timingsCallback = (List timings) { + ctx.runtime.addFrameTimings(timings); + }; + + SchedulerBinding.instance.addTimingsCallback(timingsCallback); + ctx.apiOverlay.liveAsyncRunner = tester.runAsync; + LiveAsyncCallSupport.runner = tester.runAsync; TestErrorTracker.install(ctx.runtime); + final startupStartFrame = ctx.runtime.appFrameTimings.length + 1; + final startupStartTime = DateTime.now(); late final EnsembleConfig config; if (continuation) { @@ -87,7 +154,7 @@ class EnsembleTestRunner { 'runtime is not bootstrapped — ensure the prerequisite test runs first', ); } - EnsembleTestHarness.applyInPlaceSetup(ctx); + await EnsembleTestHarness.applyInPlaceSetup(ctx); config = existingConfig ?? Ensemble().getConfig()!; await EnsembleTestHarness.waitForInitialWidgets(tester, testCase: test); } else { @@ -96,8 +163,22 @@ class EnsembleTestRunner { testCase: test, existingConfig: existingConfig, context: ctx, + suiteConfig: suiteConfig, ); } + await YamlTestSession.navigationFlow.flushPending(); + YamlTestSession.navigationFlow.beginTest( + ScreenTracker().getCurrentScreenIdentifier(), + ); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: null, + label: '${test.id} startup', + phase: 'startup', + startFrame: startupStartFrame, + startTime: startupStartTime, + ); final result = await _executeSteps( test: test, @@ -106,7 +187,7 @@ class EnsembleTestRunner { config: config, stopwatch: stopwatch, ); - return (result: result, config: config); + return (result: result, config: config, context: ctx); } catch (error, stackTrace) { final config = existingConfig ?? Ensemble().getConfig(); return ( @@ -119,9 +200,14 @@ class EnsembleTestRunner { report: buildTestReportDetails(test), ), config: config ?? await harness.buildConfig(), + context: ctx, ); } finally { TestErrorTracker.reset(); + final callback = timingsCallback; + if (callback != null) { + SchedulerBinding.instance.removeTimingsCallback(callback); + } } } @@ -140,12 +226,54 @@ class EnsembleTestRunner { harness: harness, config: config, ); - for (var i = 0; i < test.steps.length; i++) { final step = test.steps[i]; + final startFrame = ctx.runtime.appFrameTimings.length + 1; + final startTime = DateTime.now(); try { + if (i == 0 && ctx.config.screenshots.enabled) { + await executor.settle(); + } + await _captureAutomaticScreenshotForStep( + executor: executor, + step: step, + stepIndex: i, + ); await executor.execute(step); + await YamlTestSession.navigationFlow.flushPending(); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: i + 1, + label: '${test.id} step ${i + 1} ${formatStepBrief(step)}', + phase: _phaseForStep(step), + startFrame: startFrame, + startTime: startTime, + ); } catch (error, stackTrace) { + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: i + 1, + label: '${test.id} step ${i + 1} ${formatStepBrief(step)}', + phase: _phaseForStep(step), + startFrame: startFrame, + startTime: startTime, + ); + final idleStartFrame = ctx.runtime.appFrameTimings.length + 1; + final idleStartTime = DateTime.now(); + await _settleLiveApiWork(tester, ctx); + await _flushPendingScreenshots(ctx); + await YamlTestSession.navigationFlow.flushPending(); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: null, + label: '${test.id} failure cleanup', + phase: 'idle', + startFrame: idleStartFrame, + startTime: idleStartTime, + ); return EnsembleSingleTestResult.failed( testId: test.id, metadata: test.metadataJson, @@ -160,6 +288,21 @@ class EnsembleTestRunner { } } + await YamlTestSession.navigationFlow.flushPending(); + final idleStartFrame = ctx.runtime.appFrameTimings.length + 1; + final idleStartTime = DateTime.now(); + await _settleLiveApiWork(tester, ctx); + await _flushPendingScreenshots(ctx); + _recordPerformanceMarker( + ctx: ctx, + testId: test.id, + stepIndex: null, + label: '${test.id} idle', + phase: 'idle', + startFrame: idleStartFrame, + startTime: idleStartTime, + ); + return EnsembleSingleTestResult.passed( testId: test.id, metadata: test.metadataJson, @@ -168,4 +311,342 @@ class EnsembleTestRunner { report: buildTestReportDetails(test), ); } + + Future _captureAutomaticScreenshotForStep({ + required TestStepExecutor executor, + required TestStep step, + required int stepIndex, + }) async { + final options = executor.context.config.screenshots; + if (!options.shouldCaptureStep(step.type)) return; + + await _ensureHighlightTargetVisible(executor, step); + + var image = ExtendedStepHandlers.captureScreenshotImage(executor.tester); + final highlightRect = _highlightRectForStep(executor, step); + if (highlightRect != null) { + try { + final highlighted = await _highlightScreenshotImage( + tester: executor.tester, + image: image, + rect: highlightRect, + ); + image.dispose(); + image = highlighted; + } catch (_) { + // Screenshot annotations must never change test behavior. + } + } + + executor.context.runtime.addScreenshotSheetFrame( + ScreenshotSheetFrame( + label: '${stepIndex + 1}. ${formatStepBrief(step)}', + image: image, + ), + ); + } + + ui.Rect? _highlightRectForStep(TestStepExecutor executor, TestStep step) { + if (!_shouldHighlightStep(step)) return null; + + final id = step.args['id']?.toString(); + if (id == null || id.isEmpty) return null; + + final elements = executor.assertions.finderForId(id).evaluate(); + if (elements.isEmpty) return null; + + final renderObject = elements.first.renderObject; + if (renderObject == null) return null; + return _rectForRenderObject(renderObject); + } + + Future _ensureHighlightTargetVisible( + TestStepExecutor executor, + TestStep step, + ) async { + if (!_shouldHighlightStep(step)) return; + + final id = step.args['id']?.toString(); + if (id == null || id.isEmpty) return; + + final finder = executor.assertions.finderForId(id); + if (finder.evaluate().isEmpty) return; + + var visibilityTarget = finder; + if (step.type == 'toggle' || + step.type == 'check' || + step.type == 'uncheck') { + final control = find.descendant( + of: finder, + matching: find.byWidgetPredicate( + (widget) => + widget is Switch || + widget is CupertinoSwitch || + widget is Checkbox, + ), + ); + if (control.evaluate().isNotEmpty) { + visibilityTarget = control.first; + } + } + + if (visibilityTarget.hitTestable().evaluate().isNotEmpty) return; + + await executor.tester.ensureVisible(visibilityTarget); + await executor.tester.pump(); + } + + ui.Rect? _rectForRenderObject(RenderObject renderObject) { + if (renderObject is RenderBox && + renderObject.hasSize && + renderObject.size.longestSide > 0) { + final topLeft = renderObject.localToGlobal(ui.Offset.zero); + final rect = topLeft & renderObject.size; + if (rect.isFinite && !rect.isEmpty) { + return rect; + } + } + + final rects = []; + + void collect(RenderObject object) { + if (object is RenderBox && + object.hasSize && + object.size.longestSide > 0) { + final topLeft = object.localToGlobal(ui.Offset.zero); + final rect = topLeft & object.size; + if (rect.isFinite && !rect.isEmpty) { + rects.add(rect); + } + } + object.visitChildren(collect); + } + + collect(renderObject); + if (rects.isEmpty) return null; + + return rects.reduce((a, b) => a.expandToInclude(b)); + } + + bool _shouldHighlightStep(TestStep step) { + switch (step.type) { + case 'tap': + case 'doubleTap': + case 'longPress': + case 'toggle': + case 'check': + case 'uncheck': + case 'enterText': + case 'clearText': + case 'replaceText': + case 'submitText': + case 'focus': + case 'select': + case 'selectIndex': + case 'setSlider': + return true; + default: + return false; + } + } + + Future _highlightScreenshotImage({ + required WidgetTester tester, + required ui.Image image, + required ui.Rect rect, + }) async { + final renderView = tester.binding.renderViews.first; + final paintBounds = renderView.paintBounds; + final scaleX = image.width / paintBounds.width; + final scaleY = image.height / paintBounds.height; + final scaledRect = ui.Rect.fromLTRB( + (rect.left - paintBounds.left) * scaleX, + (rect.top - paintBounds.top) * scaleY, + (rect.right - paintBounds.left) * scaleX, + (rect.bottom - paintBounds.top) * scaleY, + ).inflate(6 * scaleX); + + final recorder = ui.PictureRecorder(); + final canvas = ui.Canvas( + recorder, + ui.Rect.fromLTWH(0, 0, image.width.toDouble(), image.height.toDouble()), + ); + canvas.drawImage(image, ui.Offset.zero, ui.Paint()); + + final radius = ui.Radius.circular(10 * scaleX); + final rrect = ui.RRect.fromRectAndRadius(scaledRect, radius); + canvas.drawRRect( + rrect, + ui.Paint() + ..color = const ui.Color(0x33FFD400) + ..style = ui.PaintingStyle.fill, + ); + canvas.drawRRect( + rrect, + ui.Paint() + ..color = const ui.Color(0xFF006BFF) + ..style = ui.PaintingStyle.stroke + ..strokeWidth = 5 * scaleX, + ); + canvas.drawCircle( + scaledRect.center, + 9 * scaleX, + ui.Paint() + ..color = const ui.Color(0xFF006BFF) + ..style = ui.PaintingStyle.fill, + ); + + final picture = recorder.endRecording(); + final highlighted = await picture.toImage(image.width, image.height); + picture.dispose(); + return highlighted; + } + + void _recordPerformanceMarker({ + required EnsembleTestContext ctx, + required String testId, + required int? stepIndex, + required String label, + required String phase, + required int startFrame, + required DateTime startTime, + }) { + final endFrame = ctx.runtime.appFrameTimings.length; + if (endFrame < startFrame) return; + ctx.runtime.recordPerformanceMarker( + PerformanceMarker( + testId: testId, + stepIndex: stepIndex, + label: label, + screen: ScreenTracker().getCurrentScreenIdentifier(), + phase: phase, + startFrame: startFrame, + endFrame: endFrame, + startTime: startTime, + endTime: DateTime.now(), + ), + ); + } + + String _phaseForStep(TestStep step) { + switch (step.type) { + case 'waitForNavigation': + case 'openScreen': + case 'goBack': + case 'restartApp': + case 'reloadScreen': + case 'launchApp': + return 'navigation'; + case 'settle': + case 'wait': + case 'waitFor': + case 'waitForApi': + return 'settle'; + default: + return 'step'; + } + } + + void _appendSuiteFrames( + List suiteFrames, + List testFrames, + ) { + for (final frame in testFrames) { + suiteFrames.add( + AppFrameTimingEntry( + frameNumber: suiteFrames.length + 1, + buildStartMicros: frame.buildStartMicros, + buildMs: frame.buildMs, + rasterMs: frame.rasterMs, + vsyncOverheadMs: frame.vsyncOverheadMs, + totalSpanMs: frame.totalSpanMs, + ), + ); + } + } + + Future> _writeSuiteLogs({ + required WidgetTester tester, + required EnsembleTestConfig config, + required TestLogger logger, + required List frames, + required List markers, + required List apiCalls, + required EnsembleTestContext? lastContext, + }) async { + final logs = []; + + if (config.performance.enabled) { + final path = await writePerformanceLog( + logger: logger, + filePrefix: '', + name: 'app_performance', + frames: frames, + markers: markers, + apiCalls: apiCalls, + ); + logs.add('appPerformance: $path'); + } + + if (config.dumpTree.enabled && lastContext != null) { + final path = await writeDumpTreeLogFile(logger: logger, filePrefix: ''); + logs.add('dumpTree: $path'); + } + + if (config.logApiCalls.enabled) { + final path = await writeApiCallsLogFile( + logger: logger, + filePrefix: '', + calls: apiCalls, + ); + logs.add('apiCalls: $path'); + } + + if (config.logStorage.enabled && lastContext != null) { + final key = config.logStorage.key; + final path = await writeStorageLogFile( + logger: logger, + filePrefix: '', + key: key, + ); + logs.add( + key == null || key.isEmpty ? 'storage: $path' : 'storage[$key]: $path', + ); + } + + return logs; + } + + Future _settleLiveApiWork( + WidgetTester tester, + EnsembleTestContext ctx, + ) async { + for (var i = 0; i < 20; i++) { + await ctx.apiOverlay.waitForLiveCalls(); + await tester.pump(); + await YamlTestSession.navigationFlow.flushPending(); + if (!ctx.apiOverlay.hasPendingLiveCalls) { + return; + } + } + } + + Future _flushPendingScreenshots(EnsembleTestContext ctx) async { + final sheetFrames = List.from( + ctx.runtime.screenshotSheetFrames, + ); + ctx.runtime.screenshotSheetFrames.clear(); + if (sheetFrames.isEmpty) return; + + final path = await LiveAsyncCallSupport.run( + () => writeScreenshotContactSheet( + testId: ctx.testCase.id, + config: ctx.config.screenshots, + frames: sheetFrames, + ), + ); + if (path != null) { + ctx.logger.log('screenshots: $path'); + } + } } diff --git a/tools/ensemble_test_runner/lib/runner/live_async_call.dart b/tools/ensemble_test_runner/lib/runner/live_async_call.dart new file mode 100644 index 000000000..4e2d0fccf --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/live_async_call.dart @@ -0,0 +1,60 @@ +import 'dart:async'; + +/// Serializes real async work (HTTP, etc.) through [WidgetTester.runAsync]. +/// +/// Flutter rejects reentrant [WidgetTester.runAsync] calls, so live network +/// requests from the app must be queued when multiple actions run in parallel. +class LiveAsyncCallSupport { + static Future Function(Future Function())? runner; + + static int _pendingLiveCalls = 0; + static final Set> _inFlightLiveCalls = {}; + static Future _liveCallQueue = Future.value(); + + static bool get hasPendingLiveCalls => _pendingLiveCalls > 0; + + static Future waitForLiveCalls() async { + if (_inFlightLiveCalls.isEmpty) { + return; + } + await Future.wait(_inFlightLiveCalls.toList()); + } + + /// Backwards-compatible alias for callers that need real async work. + static Future runWithPlatformHttp(Future Function() call) => + run(call); + + static Future run(Future Function() call) { + final liveRunner = runner; + if (liveRunner == null) { + return call(); + } + + _pendingLiveCalls++; + final completer = Completer(); + final tracked = completer.future; + _inFlightLiveCalls.add(tracked); + + _liveCallQueue = _liveCallQueue.then((_) async { + try { + completer.complete(await liveRunner(call)); + } catch (error, stackTrace) { + if (!completer.isCompleted) { + completer.completeError(error, stackTrace); + } + } finally { + _pendingLiveCalls--; + _inFlightLiveCalls.remove(tracked); + } + }); + + return tracked; + } + + static void reset() { + runner = null; + _pendingLiveCalls = 0; + _inFlightLiveCalls.clear(); + _liveCallQueue = Future.value(); + } +} diff --git a/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart new file mode 100644 index 000000000..775d64833 --- /dev/null +++ b/tools/ensemble_test_runner/lib/runner/screenshot_contact_sheet.dart @@ -0,0 +1,208 @@ +import 'dart:io'; +import 'dart:math' as math; + +import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; +import 'package:ensemble_test_runner/actions/screenshot_device.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:image/image.dart' as img; + +Future writeScreenshotContactSheet({ + required String testId, + required ScreenshotConfig config, + required List frames, +}) async { + if (frames.isEmpty) return null; + + final tiles = []; + final device = resolveScreenshotDevice(config.toScreenshotArgs()); + try { + for (final frame in frames) { + final pngBytes = await ExtendedStepHandlers.encodeScreenshotImage( + frame.image, + device, + ); + final source = img.decodePng(pngBytes); + if (source == null) continue; + tiles.add(_buildTile(source, frame.label)); + } + } finally { + for (final frame in frames) { + frame.image.dispose(); + } + } + + if (tiles.isEmpty) return null; + + final sheet = _composeSheet(tiles); + + final directory = Directory('build/ensemble_test_runner/screenshots'); + directory.createSync(recursive: true); + final file = File('${directory.path}/${_safeFileName(testId)}_sheet.png'); + file.writeAsBytesSync(img.encodePng(sheet, level: 1)); + return file.path; +} + +img.Image _composeSheet(List tiles) { + const columns = 5; + const gap = 16; + final rows = (tiles.length / columns).ceil(); + final tileWidth = tiles.map((tile) => tile.width).reduce(math.max); + final tileHeight = tiles.map((tile) => tile.height).reduce(math.max); + final sheet = img.Image( + width: columns * tileWidth + (columns + 1) * gap, + height: rows * tileHeight + (rows + 1) * gap, + )..clear(img.ColorRgb8(240, 240, 240)); + + for (var i = 0; i < tiles.length; i++) { + final tile = tiles[i]; + final row = i ~/ columns; + final column = i % columns; + final remainingTiles = tiles.length - row * columns; + final rowTiles = math.min(columns, remainingTiles); + final rowWidth = rowTiles * tileWidth + (rowTiles - 1) * gap; + final rowStartX = ((sheet.width - rowWidth) / 2).round(); + final x = rowStartX + column * (tileWidth + gap); + final y = gap + row * (tileHeight + gap); + img.compositeImage(sheet, tile, dstX: x, dstY: y); + } + + return sheet; +} + +img.Image _buildTile(img.Image source, String label) { + const width = 420; + const labelHeight = 72; + final thumbnail = img.copyResize( + source, + width: width, + interpolation: img.Interpolation.linear, + ); + final tile = img.Image(width: width, height: thumbnail.height + labelHeight) + ..clear(img.ColorRgb8(255, 255, 255)); + img.fillRect( + tile, + x1: 0, + y1: 0, + x2: width - 1, + y2: labelHeight - 1, + color: img.ColorRgb8(255, 255, 255), + ); + img.drawLine( + tile, + x1: 0, + y1: labelHeight - 1, + x2: width - 1, + y2: labelHeight - 1, + color: img.ColorRgb8(224, 229, 236), + ); + img.compositeImage(tile, thumbnail, dstX: 0, dstY: labelHeight); + _drawLabel(tile, label, width: width, height: labelHeight); + return tile; +} + +void _drawLabel( + img.Image image, + String label, { + required int width, + required int height, +}) { + const horizontalPadding = 12; + final availableWidth = width - horizontalPadding * 2; + final lines = _fitLabelLines(label, img.arial24, availableWidth); + final lineHeight = img.arial24.lineHeight; + final contentHeight = lines.length * lineHeight; + final startY = ((height - contentHeight) / 2).round(); + + for (var i = 0; i < lines.length; i++) { + final x = ((width - _textWidth(lines[i], img.arial24)) / 2).round(); + img.drawString( + image, + lines[i], + font: img.arial24, + x: math.max(horizontalPadding, x), + y: startY + i * lineHeight, + color: img.ColorRgb8(20, 26, 36), + ); + } +} + +List _fitLabelLines(String label, img.BitmapFont font, int maxWidth) { + if (_textWidth(label, font) <= maxWidth) { + return [label]; + } + + final breakIndex = _bestBreakIndex(label, font, maxWidth); + if (breakIndex != null) { + final first = label.substring(0, breakIndex).trimRight(); + final second = label.substring(breakIndex).trimLeft(); + if (_textWidth(second, font) <= maxWidth) { + return [first, second]; + } + return [first, _ellipsis(second, font, maxWidth)]; + } + + return [_ellipsis(label, font, maxWidth)]; +} + +int? _bestBreakIndex(String label, img.BitmapFont font, int maxWidth) { + final candidates = []; + for (var i = 1; i < label.length; i++) { + final previous = label[i - 1]; + final current = label[i]; + if (previous == ' ' || + previous == '_' || + previous == '(' || + current == ')' || + current == '_' || + current == '(') { + candidates.add(i); + } + } + + int? best; + var bestScore = double.infinity; + for (final index in candidates) { + final first = label.substring(0, index).trimRight(); + final second = label.substring(index).trimLeft(); + if (first.isEmpty || second.isEmpty) continue; + if (_textWidth(first, font) > maxWidth) continue; + + final overflow = math.max(0, _textWidth(second, font) - maxWidth); + final balance = (first.length - second.length).abs(); + final score = overflow * 100 + balance; + if (score < bestScore) { + bestScore = score.toDouble(); + best = index; + } + } + return best; +} + +String _ellipsis(String text, img.BitmapFont font, int maxWidth) { + const suffix = '...'; + if (_textWidth(text, font) <= maxWidth) return text; + if (_textWidth(suffix, font) > maxWidth) return ''; + + var end = text.length; + while (end > 0) { + final candidate = '${text.substring(0, end).trimRight()}$suffix'; + if (_textWidth(candidate, font) <= maxWidth) { + return candidate; + } + end--; + } + return suffix; +} + +int _textWidth(String text, img.BitmapFont font) { + var width = 0; + for (final codeUnit in text.codeUnits) { + final character = font.characters[codeUnit]; + width += character?.xAdvance ?? font.base ~/ 2; + } + return width; +} + +String _safeFileName(String value) => + value.replaceAll(RegExp(r'[^A-Za-z0-9._-]+'), '_'); diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index a87b17712..9b90dcb31 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -1,3 +1,5 @@ +import 'dart:ui' as ui; + import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; @@ -6,24 +8,166 @@ class TestRuntimeState { bool networkOffline = false; final List consoleLogs = []; final List flutterErrors = []; + final List appFrameTimings = []; + final List performanceMarkers = []; + final List screenshotSheetFrames = []; Map? authUser; final Map permissions = {}; Size? deviceSize; Locale? locale; String? themeMode; - final Map fixtures = {}; void clear() { networkOffline = false; consoleLogs.clear(); flutterErrors.clear(); + appFrameTimings.clear(); + performanceMarkers.clear(); + screenshotSheetFrames.clear(); authUser = null; permissions.clear(); deviceSize = null; locale = null; themeMode = null; - fixtures.clear(); } + + void addFrameTimings(List timings) { + for (final timing in timings) { + appFrameTimings.add( + AppFrameTimingEntry.fromFrameTiming( + frameNumber: appFrameTimings.length + 1, + timing: timing, + ), + ); + } + } + + void recordPerformanceMarker(PerformanceMarker marker) { + performanceMarkers.add(marker); + } + + void addScreenshotSheetFrame(ScreenshotSheetFrame frame) { + screenshotSheetFrames.add(frame); + } +} + +class ScreenshotSheetFrame { + final String label; + final ui.Image image; + + const ScreenshotSheetFrame({ + required this.label, + required this.image, + }); +} + +class PerformanceMarker { + final String testId; + final int? stepIndex; + final String label; + final String? screen; + final String phase; + final int startFrame; + final int endFrame; + final DateTime startTime; + final DateTime endTime; + + const PerformanceMarker({ + required this.testId, + required this.stepIndex, + required this.label, + required this.screen, + required this.phase, + required this.startFrame, + required this.endFrame, + required this.startTime, + required this.endTime, + }); + + PerformanceMarker shiftedFrames(int offset) => PerformanceMarker( + testId: testId, + stepIndex: stepIndex, + label: label, + screen: screen, + phase: phase, + startFrame: startFrame + offset, + endFrame: endFrame + offset, + startTime: startTime, + endTime: endTime, + ); + + bool containsFrame(int frameNumber) => + frameNumber >= startFrame && frameNumber <= endFrame; + + bool overlaps(DateTime timestamp, {Duration tolerance = Duration.zero}) { + final lower = startTime.subtract(tolerance); + final upper = endTime.add(tolerance); + return !timestamp.isBefore(lower) && !timestamp.isAfter(upper); + } + + Map toJson() => { + 'testId': testId, + if (stepIndex != null) 'stepIndex': stepIndex, + 'label': label, + if (screen != null) 'screen': screen, + 'phase': phase, + 'startFrame': startFrame, + 'endFrame': endFrame, + 'startTime': startTime.toIso8601String(), + 'endTime': endTime.toIso8601String(), + }; +} + +class AppFrameTimingEntry { + static const double frameBudgetMs = 16.67; + + final int frameNumber; + final int buildStartMicros; + final double buildMs; + final double rasterMs; + final double vsyncOverheadMs; + final double totalSpanMs; + + const AppFrameTimingEntry({ + required this.frameNumber, + required this.buildStartMicros, + required this.buildMs, + required this.rasterMs, + required this.vsyncOverheadMs, + required this.totalSpanMs, + }); + + factory AppFrameTimingEntry.fromFrameTiming({ + required int frameNumber, + required ui.FrameTiming timing, + }) { + return AppFrameTimingEntry( + frameNumber: frameNumber, + buildStartMicros: timing.timestampInMicroseconds( + ui.FramePhase.buildStart, + ), + buildMs: _durationMs(timing.buildDuration), + rasterMs: _durationMs(timing.rasterDuration), + vsyncOverheadMs: _durationMs(timing.vsyncOverhead), + totalSpanMs: _durationMs(timing.totalSpan), + ); + } + + bool get isJanky => + totalSpanMs > frameBudgetMs || buildMs + rasterMs > frameBudgetMs; + + Map toJson() => { + 'frameNumber': frameNumber, + 'buildStartMicros': buildStartMicros, + 'buildMs': buildMs, + 'rasterMs': rasterMs, + 'vsyncOverheadMs': vsyncOverheadMs, + 'totalSpanMs': totalSpanMs, + 'janky': isJanky, + }; + + static double _durationMs(Duration duration) => + double.parse((duration.inMicroseconds / 1000).toStringAsFixed(3)); } /// Captures Flutter framework errors for quality assertion steps. @@ -34,7 +178,6 @@ class TestErrorTracker { _previousHandler = FlutterError.onError; FlutterError.onError = (FlutterErrorDetails details) { runtime.flutterErrors.add(details.exceptionAsString()); - _previousHandler?.call(details); }; } diff --git a/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart b/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart index bf90c5d19..15bcde7dd 100644 --- a/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart +++ b/tools/ensemble_test_runner/lib/runner/yaml_test_session.dart @@ -2,11 +2,14 @@ import 'dart:async'; import 'package:ensemble/ensemble.dart'; import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble_test_runner/runner/live_async_call.dart'; /// Records screen visits for YAML tests (including back-navigation revisits). class NavigationFlowRecorder { final List _flow = []; StreamSubscription? _subscription; + VisibleScreen? _pendingScreen; + bool _flushScheduled = false; List get flow => List.unmodifiable(_flow); @@ -15,7 +18,18 @@ class NavigationFlowRecorder { _subscription = ScreenTracker().onScreenChange.listen(_onScreenChange); } - void clear() => _flow.clear(); + void clear() { + _flow.clear(); + _pendingScreen = null; + _flushScheduled = false; + } + + void beginTest(String? currentScreen) { + clear(); + if (currentScreen != null && currentScreen.isNotEmpty) { + _flow.add(currentScreen); + } + } /// For unit tests only. void seed(Iterable names) { @@ -34,6 +48,24 @@ class NavigationFlowRecorder { void recordScreenChange(VisibleScreen? screen) => _onScreenChange(screen); void _onScreenChange(VisibleScreen? screen) { + if (screen == null) return; + _pendingScreen = screen; + if (_flushScheduled) return; + _flushScheduled = true; + scheduleMicrotask(_flushPendingScreen); + } + + Future flushPending() async { + await Future.microtask(() {}); + if (_flushScheduled) { + _flushPendingScreen(); + } + } + + void _flushPendingScreen() { + _flushScheduled = false; + final screen = _pendingScreen; + _pendingScreen = null; if (screen == null) return; final name = screen.screenName ?? screen.screenId; if (name == null) return; @@ -59,6 +91,7 @@ class YamlTestSession { static void reset() { runtimeBootstrapped = false; navigationFlow.clear(); + LiveAsyncCallSupport.reset(); Ensemble.resetInitManagersForTest(); } diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index 4bd70ebf4..328d176b2 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -6,48 +6,30 @@ import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; class EnsembleTestSchemaBuilder { static const schemaId = 'https://cdn.ensembleui.com/schemas/ensemble_tests_schema.json'; + static const configSchemaId = + 'https://cdn.ensembleui.com/schemas/ensemble_test_config_schema.json'; static const schemaVersion = 'https://json-schema.org/draft/2020-12/schema'; static Map build() { final defs = { - 'mockResponse': { - 'type': 'object', - 'additionalProperties': false, - 'properties': { - 'statusCode': {'type': 'integer'}, - 'body': true, - 'headers': { - 'type': 'object', - 'additionalProperties': true, - }, - }, - }, - 'mockApiEntry': { - 'type': 'object', - 'additionalProperties': false, - 'properties': { - 'response': {'\$ref': '#/\$defs/mockResponse'}, - 'delayMs': {'type': 'integer'}, - }, - 'required': ['response'], - }, 'initialState': { 'type': 'object', 'additionalProperties': false, 'properties': { 'storage': {'type': 'object', 'additionalProperties': true}, + 'keychain': {'type': 'object', 'additionalProperties': true}, 'env': {'type': 'object', 'additionalProperties': true}, }, }, - 'mocks': { + 'scenario': { 'type': 'object', 'additionalProperties': false, 'properties': { - 'apis': { - 'type': 'object', - 'additionalProperties': {'\$ref': '#/\$defs/mockApiEntry'}, - }, + 'id': {'type': 'string', 'minLength': 1}, + 'description': {'type': 'string'}, + 'vars': {'type': 'object', 'additionalProperties': true}, }, + 'required': ['id'], }, 'testCase': { 'type': 'object', @@ -96,7 +78,15 @@ class EnsembleTestSchemaBuilder { 'ID of another test that must run before this one in the same app session', }, 'initialState': {'\$ref': '#/\$defs/initialState'}, - 'mocks': {'\$ref': '#/\$defs/mocks'}, + 'mocks': { + 'type': 'array', + 'items': {'type': 'string', 'minLength': 1}, + }, + 'scenarios': { + 'type': 'array', + 'items': {'\$ref': '#/\$defs/scenario'}, + 'minItems': 1, + }, 'steps': { 'type': 'array', 'minItems': 1, @@ -183,4 +173,69 @@ class EnsembleTestSchemaBuilder { pretty ? const JsonEncoder.withIndent(' ') : const JsonEncoder(); return encoder.convert(build()); } + + static Map buildConfig() { + return { + '\$schema': schemaVersion, + '\$id': configSchemaId, + 'title': 'Ensemble declarative test config', + 'description': 'Suite-wide config for app-local tests/config.yaml', + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'screenshots': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + 'platform': {'type': 'string'}, + 'model': {'type': 'string'}, + 'includeSteps': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'excludeSteps': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + }, + }, + 'performance': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + }, + }, + 'dumpTree': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + }, + }, + 'logApiCalls': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + }, + }, + 'logStorage': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'enabled': {'type': 'boolean'}, + 'key': {'type': 'string'}, + }, + }, + }, + }; + } + + static String buildConfigJson({bool pretty = true}) { + final encoder = + pretty ? const JsonEncoder.withIndent(' ') : const JsonEncoder(); + return encoder.convert(buildConfig()); + } } diff --git a/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart b/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart index 3f49b7692..3e5998c4e 100644 --- a/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart +++ b/tools/ensemble_test_runner/lib/validation/ensemble_test_validator.dart @@ -2,6 +2,7 @@ import 'dart:convert'; import 'dart:io'; import 'package:ensemble_test_runner/inspect/ensemble_app_inspector.dart'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; import 'package:path/path.dart' as p; import 'package:yaml/yaml.dart'; @@ -122,6 +123,23 @@ class EnsembleTestValidator { return EnsembleTestValidationResult(issues); } + final configFile = File(p.join(testsDir.path, 'config.yaml')); + if (configFile.existsSync()) { + try { + EnsembleTestParser.parseConfigString( + configFile.readAsStringSync(), + sourcePath: p.relative(configFile.path, from: appDir), + ); + } catch (error) { + add( + ValidationSeverity.error, + 'config', + error.toString(), + path: p.relative(configFile.path, from: appDir), + ); + } + } + final screensByName = { for (final screen in inspection.screens) screen.name: screen }; @@ -141,6 +159,16 @@ class EnsembleTestValidator { continue; } + if (doc.containsKey('options')) { + add( + ValidationSeverity.error, + 'options', + 'Root-level "options" is no longer supported. Move shared settings to tests/config.yaml.', + path: relativePath, + ); + continue; + } + final id = doc['id']?.toString(); if (id == null || id.isEmpty) { add(ValidationSeverity.error, 'missingId', 'Each test must have an id', @@ -191,6 +219,7 @@ class EnsembleTestValidator { final stepInfo = _collectStepInfo(steps); for (final widgetId in stepInfo.widgetIds) { + if (_isPlaceholder(widgetId)) continue; if (!knownWidgetIds.contains(widgetId)) { add( ValidationSeverity.warning, @@ -202,6 +231,7 @@ class EnsembleTestValidator { } } for (final api in stepInfo.apiNames) { + if (_isPlaceholder(api)) continue; if (!knownApis.contains(api)) { add( ValidationSeverity.warning, @@ -212,39 +242,6 @@ class EnsembleTestValidator { ); } } - for (final fixture in stepInfo.fixtures) { - final fixtureFile = File( - fixture.startsWith('fixtures/') - ? p.join(testsDir.path, fixture) - : p.join(testsDir.path, 'fixtures', fixture), - ); - if (!fixtureFile.existsSync()) { - add( - ValidationSeverity.error, - 'missingFixture', - 'Fixture not found: tests/fixtures/$fixture', - path: relativePath, - testId: id, - ); - } - } - - final rootMocks = _rootMockApis(doc); - final onLoadApis = - startScreen != null && screensByName[startScreen] != null - ? screensByName[startScreen]!.apis - : const []; - for (final api in onLoadApis) { - if (stepInfo.mockApis.contains(api) && !rootMocks.contains(api)) { - add( - ValidationSeverity.warning, - 'mockPlacement', - 'API "$api" may run during onLoad. Prefer root mocks.apis for startup APIs.', - path: relativePath, - testId: id, - ); - } - } } for (final entry in prerequisites.entries) { @@ -266,15 +263,11 @@ class EnsembleTestValidator { typedef _StepInfo = ({ Set widgetIds, Set apiNames, - Set mockApis, - Set fixtures, }); _StepInfo _collectStepInfo(dynamic steps) { final widgetIds = {}; final apiNames = {}; - final mockApis = {}; - final fixtures = {}; void visit(dynamic node) { if (node is! Iterable) return; @@ -289,11 +282,6 @@ _StepInfo _collectStepInfo(dynamic steps) { if (name != null && type.toLowerCase().contains('api')) { apiNames.add(name.toString()); } - if (type == 'mockApi' || type == 'mockApiFromFixture') { - if (name != null) mockApis.add(name.toString()); - } - final fixture = args['fixture']; - if (fixture != null) fixtures.add(fixture.toString()); visit(args['steps']); final single = args['step']; if (single != null) visit([single]); @@ -305,15 +293,8 @@ _StepInfo _collectStepInfo(dynamic steps) { return ( widgetIds: widgetIds, apiNames: apiNames, - mockApis: mockApis, - fixtures: fixtures, ); } -Set _rootMockApis(YamlMap doc) { - final mocks = doc['mocks']; - if (mocks is! YamlMap) return const {}; - final apis = mocks['apis']; - if (apis is! YamlMap) return const {}; - return apis.keys.map((key) => key.toString()).toSet(); -} +bool _isPlaceholder(String value) => + RegExp(r'\$\{(inputs|scenario)\.[A-Za-z0-9_.-]+\}').hasMatch(value); diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart index d9250cfa9..fcec134c5 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -18,7 +18,6 @@ enum TestStepArgKind { waitFor, waitForGone, waitForNavigation, - waitUntil, textRequired, expectEquals, expectChecked, @@ -27,24 +26,12 @@ enum TestStepArgKind { expectListCount, screenRequired, expectVisited, - mockApi, - mockApiError, - mockApiFromFixture, - mockApiException, - mockTimeout, apiName, - apiRequest, - expectApiHeader, - setState, - expectState, storageKey, group, repeat, optional, ifVisible, - screenshot, - expectStatePath, - resetStatePath, setAuth, setPermission, setDevice, @@ -53,7 +40,6 @@ enum TestStepArgKind { runScript, expectConsoleLog, expectErrorContains, - fixturePath, expectApiCallOrder, expectListContains, expectListItem, @@ -121,7 +107,10 @@ extension TestStepArgKindSchema on TestStepArgKind { ); case TestStepArgKind.setSlider: return _object( - properties: {'id': _string, 'value': {'type': 'number'}}, + properties: { + 'id': _string, + 'value': {'type': 'number'} + }, required: ['id'], ); case TestStepArgKind.chooseValue: @@ -168,17 +157,6 @@ extension TestStepArgKindSchema on TestStepArgKind { properties: {'screen': _string, 'timeoutMs': _integer}, required: ['screen'], ); - case TestStepArgKind.waitUntil: - return _object( - properties: { - 'path': _string, - 'equals': _any, - 'state': _object( - properties: {'path': _string, 'equals': _any}, - ), - 'timeoutMs': _integer, - }, - ); case TestStepArgKind.textRequired: return _object(properties: {'text': _string}, required: ['text']); case TestStepArgKind.expectEquals: @@ -216,80 +194,11 @@ extension TestStepArgKindSchema on TestStepArgKind { properties: {'screen': _string}, required: ['screen'], ); - case TestStepArgKind.mockApi: - return _object( - properties: { - 'name': _string, - 'response': _ref('mockResponse'), - 'delayMs': _integer, - }, - required: ['name', 'response'], - ); - case TestStepArgKind.mockApiError: - return _object( - properties: { - 'name': _string, - 'statusCode': _integer, - 'body': _any, - 'delayMs': _integer, - }, - required: ['name'], - ); - case TestStepArgKind.mockApiFromFixture: - return _object( - properties: { - 'name': _string, - 'fixture': _string, - 'statusCode': _integer, - }, - required: ['name', 'fixture'], - ); - case TestStepArgKind.mockApiException: - return _object( - properties: {'name': _string, 'message': _string}, - required: ['name'], - ); - case TestStepArgKind.mockTimeout: - return _object( - properties: {'name': _string, 'delayMs': _integer}, - required: ['name'], - ); case TestStepArgKind.apiName: return _object( properties: {'name': _string, 'times': _integer}, required: ['name'], ); - case TestStepArgKind.apiRequest: - return _object( - properties: { - 'name': _string, - 'body': _any, - 'query': _any, - 'headers': _any, - 'times': _integer, - }, - required: ['name'], - ); - case TestStepArgKind.expectApiHeader: - return _object( - properties: { - 'name': _string, - 'header': _string, - 'equals': _any, - 'times': _integer, - }, - required: ['name', 'header', 'equals'], - ); - case TestStepArgKind.setState: - return _object( - properties: {'path': _string, 'value': _any}, - required: ['path'], - ); - case TestStepArgKind.expectState: - return _object( - properties: {'path': _string, 'equals': _any, 'contains': _any}, - required: ['path'], - ); case TestStepArgKind.storageKey: return _object( properties: {'key': _string, 'equals': _any, 'value': _any}, @@ -327,12 +236,6 @@ extension TestStepArgKindSchema on TestStepArgKind { }, required: ['id'], ); - case TestStepArgKind.screenshot: - return _object(properties: {'name': _string}); - case TestStepArgKind.expectStatePath: - return _object(properties: {'path': _string}, required: ['path']); - case TestStepArgKind.resetStatePath: - return _object(properties: {'path': _string}); case TestStepArgKind.setAuth: return _object( properties: { @@ -367,15 +270,6 @@ extension TestStepArgKindSchema on TestStepArgKind { ); case TestStepArgKind.expectErrorContains: return _object(properties: {'contains': _string}); - case TestStepArgKind.fixturePath: - return _object( - properties: { - 'key': _string, - 'path': _string, - 'fixture': _string, - 'statePath': _string, - }, - ); case TestStepArgKind.expectApiCallOrder: return _object( properties: { diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart index d57562d26..1156648b5 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -234,9 +234,8 @@ abstract final class TestStepRegistry { category: TestStepCategory.wait, tier: TestStepTier.extended, argKind: TestStepArgKind.pump, - description: 'Alias for pump — advance frame clock by durationMs', + description: 'Real-time delay using runAsync, followed by a frame pump', example: const {'durationMs': 100}, - executorCanonical: 'pump', ), 'pump': TestStepRegistryEntry( category: TestStepCategory.wait, @@ -288,13 +287,6 @@ abstract final class TestStepRegistry { description: 'Poll until the given screen is visible', example: const {'screen': 'Home', 'timeoutMs': 5000}, ), - 'waitUntil': TestStepRegistryEntry( - category: TestStepCategory.wait, - tier: TestStepTier.core, - argKind: TestStepArgKind.waitUntil, - description: 'Poll until app state at path equals expected value', - example: const {'path': 'user.name', 'equals': 'Jane'}, - ), 'expectVisible': TestStepRegistryEntry( category: TestStepCategory.uiAssertion, tier: TestStepTier.core, @@ -488,65 +480,6 @@ abstract final class TestStepRegistry { description: 'Navigate back (Ensemble navigateBack or Navigator.pop)', example: const {}, ), - 'mockApi': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApi, - description: 'Register a mock HTTP API response by API name', - example: const { - 'name': 'login', - 'response': const { - 'statusCode': 200, - 'body': const {'token': 'test-token'} - } - }, - ), - 'mockApiError': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApiError, - description: 'Mock an API to return an error status/body', - example: const { - 'name': 'login', - 'statusCode': 401, - 'body': const {'error': 'Unauthorized'} - }, - ), - 'mockApiFromFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApiFromFixture, - description: 'Load mock response body from a JSON fixture asset', - example: const {'name': 'users', 'fixture': 'users.json'}, - ), - 'mockApiException': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockApiException, - description: 'Force an API call to throw an exception', - example: const {'name': 'login', 'message': 'Network error'}, - ), - 'mockTimeout': TestStepRegistryEntry( - category: TestStepCategory.network, - tier: TestStepTier.core, - argKind: TestStepArgKind.mockTimeout, - description: 'Mock an API with a long delay (simulate timeout)', - example: const {'name': 'slow_api', 'delayMs': 60000}, - ), - 'mockNetworkOffline': TestStepRegistryEntry( - category: TestStepCategory.network, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Simulate offline network for API calls', - example: const {}, - ), - 'mockNetworkOnline': TestStepRegistryEntry( - category: TestStepCategory.network, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Restore online network for API calls', - example: const {}, - ), 'resetApiCalls': TestStepRegistryEntry( category: TestStepCategory.apiMock, tier: TestStepTier.core, @@ -554,13 +487,6 @@ abstract final class TestStepRegistry { description: 'Clear recorded API call history', example: const {}, ), - 'clearApiMocks': TestStepRegistryEntry( - category: TestStepCategory.apiMock, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Remove all registered API mocks', - example: const {}, - ), 'expectApiCalled': TestStepRegistryEntry( category: TestStepCategory.apiAssertion, tier: TestStepTier.core, @@ -575,37 +501,6 @@ abstract final class TestStepRegistry { description: 'Assert an API was never called', example: const {'name': 'login', 'times': 1}, ), - 'expectApiRequest': TestStepRegistryEntry( - category: TestStepCategory.apiAssertion, - tier: TestStepTier.core, - argKind: TestStepArgKind.apiRequest, - description: 'Assert last API request body/query/headers match', - example: const { - 'name': 'login', - 'body': const {'email': 'user@test.com', 'password': 'secret'} - }, - ), - 'expectApiRequestContains': TestStepRegistryEntry( - category: TestStepCategory.apiAssertion, - tier: TestStepTier.core, - argKind: TestStepArgKind.apiRequest, - description: 'Assert API request contains partial body/query', - example: const { - 'name': 'login', - 'body': const {'email': 'user@test.com', 'password': 'secret'} - }, - ), - 'expectApiHeader': TestStepRegistryEntry( - category: TestStepCategory.apiAssertion, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectApiHeader, - description: 'Assert an API request header equals expected', - example: const { - 'name': 'login', - 'header': 'Authorization', - 'equals': 'Bearer test-token' - }, - ), 'expectApiCallOrder': TestStepRegistryEntry( category: TestStepCategory.apiAssertion, tier: TestStepTier.core, @@ -622,48 +517,6 @@ abstract final class TestStepRegistry { description: 'Assert the most recent API call name', example: const {'name': 'login', 'times': 1}, ), - 'setState': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.setState, - description: 'Set app data-context state at path to value', - example: const {'path': 'user.name', 'value': 'Jane'}, - ), - 'expectState': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectState, - description: 'Assert app state at path equals expected', - example: const {'path': 'user.name', 'equals': 'Jane'}, - ), - 'expectStateContains': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectState, - description: 'Assert app state at path contains subset', - example: const {'path': 'user.name', 'equals': 'Jane'}, - ), - 'expectStateExists': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectStatePath, - description: 'Assert state path resolves without error', - example: const {'path': 'user.id'}, - ), - 'expectStateNotExists': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectStatePath, - description: 'Assert state path is null or absent', - example: const {'path': 'user.id'}, - ), - 'resetState': TestStepRegistryEntry( - category: TestStepCategory.state, - tier: TestStepTier.core, - argKind: TestStepArgKind.resetStatePath, - description: 'Clear state at path (set to null)', - example: const {'path': 'cart'}, - ), 'setStorage': TestStepRegistryEntry( category: TestStepCategory.storage, tier: TestStepTier.core, @@ -826,34 +679,6 @@ abstract final class TestStepRegistry { description: 'Log all recorded API calls to the test log', example: const {}, ), - 'screenshot': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.screenshot, - description: 'Capture golden or dump widget tree for debugging', - example: const {'name': 'home_screen'}, - ), - 'dumpTree': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.empty, - description: 'Print the widget tree to the debug console', - example: const {}, - ), - 'logState': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.expectStatePath, - description: 'Log resolved state at path', - example: const {'path': 'user.id'}, - ), - 'logStorage': TestStepRegistryEntry( - category: TestStepCategory.debug, - tier: TestStepTier.core, - argKind: TestStepArgKind.storageKey, - description: 'Log public storage value for key', - example: const {'key': 'onboarding_done', 'value': true}, - ), 'expectNoConsoleErrors': TestStepRegistryEntry( category: TestStepCategory.quality, tier: TestStepTier.core, @@ -903,26 +728,5 @@ abstract final class TestStepRegistry { description: 'Assert widget renders without overflow issues', example: const {'id': 'my_widget'}, ), - 'loadFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.fixturePath, - description: 'Load a JSON fixture into the test fixture map', - example: const {'fixture': 'user.json'}, - ), - 'setStateFromFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.fixturePath, - description: 'Apply all keys from a JSON fixture to state', - example: const {'fixture': 'user.json'}, - ), - 'expectMatchesFixture': TestStepRegistryEntry( - category: TestStepCategory.fixture, - tier: TestStepTier.core, - argKind: TestStepArgKind.fixturePath, - description: 'Assert state or path matches a JSON fixture', - example: const {'fixture': 'user.json'}, - ), }; } diff --git a/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart b/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart index cffcc153f..a23be9a53 100644 --- a/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart +++ b/tools/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart @@ -21,12 +21,10 @@ enum TestStepCategory { navigation, apiMock, apiAssertion, - state, storage, runtime, script, network, - fixture, control, debug, quality, diff --git a/tools/ensemble_test_runner/pubspec.yaml b/tools/ensemble_test_runner/pubspec.yaml index a67a02483..084c995ef 100644 --- a/tools/ensemble_test_runner/pubspec.yaml +++ b/tools/ensemble_test_runner/pubspec.yaml @@ -24,15 +24,26 @@ dependencies: url: https://github.com/EnsembleUI/ensemble.git ref: ensemble-v1.2.49 path: modules/ensemble + device_frame: ^1.3.0 + ensemble_device_preview: ^1.1.3 flutter: sdk: flutter flutter_test: sdk: flutter yaml: ^3.1.2 path: ^1.9.0 + image: ^4.8.0 + firebase_core_platform_interface: ^7.0.1 + cloud_functions_platform_interface: ^6.0.1 + cloud_firestore_platform_interface: ^8.0.1 + firebase_auth_platform_interface: ^9.0.3 + http: ^1.2.2 + get_it: ^8.0.3 dev_dependencies: flutter_lints: ^2.0.3 + cloud_functions: ^6.0.3 + firebase_auth: ^6.5.4 flutter: uses-material-design: true diff --git a/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart b/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart index 388592e5c..e0270105a 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_cli_output_test.dart @@ -24,6 +24,23 @@ name is ={first: John} ); }); + test('extractSuiteReport can omit streamed screen tracker lines', () { + const noisy = ''' +SCREEN TRACKER: Login +┌─ Ensemble YAML tests ───────────────────────────── +│ ✓ login_test +└─ 1 passed, 0 failed (1 total) · 653ms total +'''; + + expect( + extractSuiteReport(noisy, includeScreenTracker: false), + '''┌─ Ensemble YAML tests ───────────────────────────── +│ ✓ login_test +└─ 1 passed, 0 failed (1 total) · 653ms total +''', + ); + }); + test('flutterTestArguments strips CLI-only flags', () { expect( flutterTestArguments([ @@ -41,6 +58,9 @@ name is ={first: John} '--tag=smoke', '--path=auth/', '--timeout=30s', + '--input', + 'adminPassword=s4C>M7U6t~', + '--input=expectedDeviceCount=2', '--name', 'x', ]), @@ -83,4 +103,54 @@ The test description was: 'No declarative tests found. Add *.test.yaml files under ensemble/apps/inhome/tests/', ); }); + + test('extractKnownFailure keeps actionable EnsembleTestFailure message', () { + const noisy = ''' +══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════ +The following EnsembleTestFailure was thrown running a test: +Missing CLI input "password". Pass it with --input password=value. + +When the exception was thrown, this was the stack: +#0 EnsembleTestParser._placeholderValue +════════════════════════════════════════════════════════════════════════════════ +'''; + + expect( + extractKnownFailure(noisy), + 'Missing CLI input "password". Pass it with --input password=value.', + ); + }); + + test('live output filter emits app logs before final report only', () { + final filter = LiveFlutterTestOutputFilter(); + + expect(filter.shouldEmit('SCREEN TRACKER: Login'), isTrue); + expect( + filter.shouldEmit('DEBUG [InitApp] token stored successfully'), isTrue); + expect( + filter.shouldEmit('ENSEMBLE_TEST_JSON_REPORT:{"status":"passed"}'), + isFalse, + ); + expect( + filter.shouldEmit( + '(The following exception is now available via WidgetTester.takeException:)', + ), + isFalse, + ); + expect(filter.shouldEmit('┌─ Ensemble YAML tests ─────'), isFalse); + expect(filter.shouldEmit('SCREEN TRACKER: Home'), isFalse); + }); + + test('live output filter suppresses framework exception noise', () { + final filter = LiveFlutterTestOutputFilter(); + + expect(filter.shouldEmit('SCREEN TRACKER: Login'), isTrue); + expect( + filter.shouldEmit( + '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════', + ), + isFalse, + ); + expect(filter.shouldEmit('The following TestFailure was thrown'), isFalse); + }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart index 9cd4f2a42..4e898a319 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart @@ -98,6 +98,154 @@ steps: expect(order, ['login', 'profile']); }); + + test('expands scenarios with bracketed ids and prerequisite chain', + () async { + const yaml = ''' +id: home_scenarios +prerequisite: signin_to_gateway +scenarios: + - id: v12_online + vars: + expectedDeviceCount: 2 + - id: v14_empty + vars: + expectedDeviceCount: 0 +steps: + - expectText: + text: \${scenario.expectedDeviceCount} +'''; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'ensemble/apps/inhome/tests/home.test.yaml', + yaml, + ); + + expect( + definitions.map((definition) => definition.testCase.id), + [ + 'home_scenarios[v12_online]', + 'home_scenarios[v14_empty]', + ], + ); + expect( + definitions[0].testCase.prerequisite, + 'signin_to_gateway', + ); + expect( + definitions[1].testCase.prerequisite, + 'home_scenarios[v12_online]', + ); + expect(definitions[0].testCase.steps.single.args['text'], 2); + expect(definitions[1].testCase.steps.single.args['text'], 0); + }); + + test('skips empty discovered test files', () async { + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'ensemble/apps/inhome/tests/commented.test.yaml', + ''' +# id: disabled_test +# startScreen: Home +# steps: +# - expectVisible: +# id: home_body +''', + ); + + expect(definitions, isEmpty); + }); + + test('loads JSON mock files and applies override order', () async { + const yaml = ''' +id: home_scenarios +startScreen: Home +mocks: + - mocks/common.mock.json + - mocks/\${scenario.behavior}.mock.json +scenarios: + - id: online + vars: + behavior: online +steps: + - expectVisible: + id: home +'''; + final assets = { + 'suite/tests/mocks/common.mock.json': ''' +{ + "getDevices": { + "statusCode": 200, + "body": {"count": 1} + }, + "rootApi": { + "body": {"from": "layer"} + } +} +''', + 'suite/tests/mocks/online.mock.json': ''' +{ + "getDevices": { + "statusCode": 200, + "delayMs": 125, + "body": {"count": 2} + }, + "scenarioApi": { + "body": {"from": "scenario-layer"} + } +} +''', + }; + + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + assetLoader: (path) async => assets[path]!, + ); + + final mocks = definitions.single.testCase.mocks.apis; + expect((mocks['getDevices']!.body as Map)['count'], 2); + expect(mocks['getDevices']!.delayMs, 125); + expect((mocks['rootApi']!.body as Map)['from'], 'layer'); + expect((mocks['scenarioApi']!.body as Map)['from'], 'scenario-layer'); + }); + + test('selection by base scenario suite id includes expanded scenarios', + () async { + const yaml = ''' +id: home_scenarios +startScreen: Home +scenarios: + - id: first + vars: {} + - id: second + vars: {} +steps: + - expectVisible: + id: home +'''; + final definitions = + await EnsembleTestExecutionPlanner.parseDefinitionsForTest( + 'suite/tests/home.test.yaml', + yaml, + ); + final byId = { + for (final definition in definitions) + definition.testCase.id: definition, + }; + + final order = EnsembleTestExecutionPlanner.selectAndOrderIdsForTest( + byId, + const EnsembleTestSelection(ids: {'home_scenarios'}), + ); + + expect(order, [ + 'home_scenarios[first]', + 'home_scenarios[second]', + ]); + }); }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart new file mode 100644 index 000000000..9a3b49780 --- /dev/null +++ b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart @@ -0,0 +1,106 @@ +import 'dart:io'; + +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('app font bootstrap aliases package icon font families', () { + final aliases = EnsembleTestHarness.fontFamilyAliasesForTest( + 'packages/font_awesome_flutter/FontAwesomeSolid', + ); + + expect( + aliases, + containsAll([ + 'packages/font_awesome_flutter/FontAwesomeSolid', + 'FontAwesomeSolid', + 'fontawesomesolid', + ]), + ); + }); + + test('app font bootstrap can try decoded asset keys', () { + final candidates = EnsembleTestHarness.fontAssetCandidatesForTest( + 'packages/font_awesome_flutter/lib/fonts/Font%20Awesome%207%20Free-Solid-900.otf', + ); + + expect( + candidates, + contains( + 'packages/font_awesome_flutter/lib/fonts/Font Awesome 7 Free-Solid-900.otf', + ), + ); + }); + + test('package info bootstrap reads app metadata from app files', () { + final dir = Directory.systemTemp.createTempSync('ensemble_package_info_'); + try { + File('${dir.path}/pubspec.yaml').writeAsStringSync(''' +name: inhome +version: 0.6.0+60 +'''); + Directory('${dir.path}/ensemble').createSync(); + File('${dir.path}/ensemble/ensemble.properties').writeAsStringSync(''' +appId=com.kpn.inhome.dev +appName=KPN InHome Dev +'''); + + expect( + EnsembleTestHarness.packageInfoForTest(dir.path), + { + 'appName': 'KPN InHome Dev', + 'packageName': 'com.kpn.inhome.dev', + 'version': '0.6.0', + 'buildNumber': '60', + }, + ); + } finally { + dir.deleteSync(recursive: true); + } + }); + + test('package info bootstrap falls back without app metadata files', () { + final dir = Directory.systemTemp.createTempSync('ensemble_package_info_'); + try { + expect( + EnsembleTestHarness.packageInfoForTest(dir.path), + { + 'appName': 'EnsembleTest', + 'packageName': 'com.ensemble.test', + 'version': '1.0.0', + 'buildNumber': '1', + }, + ); + } finally { + dir.deleteSync(recursive: true); + } + }); + + testWidgets('app font bootstrap is safe when font manifest is unavailable', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + + await EnsembleTestHarness.ensureAppFontsLoaded(); + }); + + testWidgets('initial keychain state is written to secure storage', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + + await applyYamlTestStorageBootstrap( + const EnsembleTestSetup( + initialKeychain: { + 'kpnPsi': 'test-psi', + 'authPayload': {'token': 'abc'}, + }, + ), + ); + + expect(await StorageManager().readSecurely('kpnPsi'), 'test-psi'); + expect( + await StorageManager().readSecurely('authPayload'), + {'token': 'abc'}, + ); + }); +} diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 241018977..7415a7939 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -20,7 +20,7 @@ steps: expect(test.steps.first.args['id'], 'emailInput'); }); - test('parses API mocks', () { + test('rejects inline root-level mocks', () { const yaml = ''' id: login_success startScreen: Login @@ -38,15 +38,251 @@ steps: times: 1 '''; + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Root-level "mocks" must be a list of .mock.json files'), + ), + ), + ); + }); + + test('parses initial keychain state', () { + const yaml = ''' +id: resumes_session +startScreen: Login +initialState: + storage: + languageSet: true + keychain: + kpnPsi: test-psi + authPayload: + token: abc +steps: + - expectVisible: + id: login_button +'''; + final test = EnsembleTestParser.parseString(yaml); - expect(test.mocks.apis['loginApi']?.statusCode, 200); - expect(test.mocks.apis['loginApi']?.delayMs, 100); + expect(test.initialState['keychain'], isA()); + expect((test.initialState['keychain'] as Map)['kpnPsi'], 'test-psi'); + expect( + ((test.initialState['keychain'] as Map)['authPayload'] as Map)['token'], + 'abc', + ); + }); + + test('resolves CLI inputs in initial state and steps', () { + const yaml = ''' +id: input_test +startScreen: Login +initialState: + keychain: + adminPassword: \${inputs.adminPassword} +steps: + - expectText: + text: \${inputs.expectedDeviceCount} + - expectText: + text: "Devices: \${inputs.expectedDeviceCount}" +'''; + + final test = EnsembleTestParser.parseString( + yaml, + inputs: { + 'adminPassword': 's4C>M7U6t~', + 'expectedDeviceCount': 2, + }, + ); + + expect( + (test.initialState['keychain'] as Map)['adminPassword'], + 's4C>M7U6t~', + ); + expect(test.steps[0].args['text'], 2); + expect(test.steps[1].args['text'], 'Devices: 2'); + }); + + test('fails clearly when CLI input is missing', () { + const yaml = ''' +id: input_test +startScreen: Login +steps: + - expectText: + text: \${inputs.expectedDeviceCount} +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Missing CLI input "expectedDeviceCount"'), + ), + ), + ); + }); + + test('parses scenarios and resolves scenario placeholders', () { + const yaml = ''' +id: home_scenarios +startScreen: Home +mocks: + - mocks/\${scenario.device}.mock.json +scenarios: + - id: v14_online + vars: + device: v14 + expectedDeviceCount: 2 +steps: + - expectText: + text: \${scenario.expectedDeviceCount} +'''; + + final base = EnsembleTestParser.parseString(yaml); + expect(base.scenarios.single.id, 'v14_online'); + expect(base.scenarios.single.vars['device'], 'v14'); + expect(base.mockFiles.single, 'mocks/\${scenario.device}.mock.json'); + + final expanded = EnsembleTestParser.parseString( + yaml, + scenario: base.scenarios.single.vars, + scenarioId: base.scenarios.single.id, + ); + expect(expanded.mockFiles.single, 'mocks/v14.mock.json'); + expect(expanded.steps.single.args['text'], 2); + }); + + test('fails clearly when scenario value is missing', () { + const yaml = ''' +id: home_scenarios +startScreen: Home +scenarios: + - id: missing_value + vars: {} +steps: + - expectText: + text: \${scenario.expectedDeviceCount} +'''; + + final base = EnsembleTestParser.parseString(yaml); + expect( + () => EnsembleTestParser.parseString( + yaml, + scenario: base.scenarios.single.vars, + scenarioId: base.scenarios.single.id, + ), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains( + 'Missing scenario value "expectedDeviceCount" in scenario "missing_value"', + ), + ), + ), + ); + }); + + test('rejects root-level options in test files', () { + const yaml = ''' +id: visual_debug +startScreen: Home +options: + screenshots: + enabled: true + platform: android + model: Samsung Galaxy S20 + includeSteps: [tap, waitForNavigation] + excludeSteps: [wait] +steps: + - tap: + id: start_button +'''; + expect( - (test.mocks.apis['loginApi']?.body as Map)['token'], - 'test-token', + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Move shared screenshots/performance settings'), + ), + ), ); }); + test('parses suite config screenshots', () { + const yaml = ''' +screenshots: + enabled: true + platform: android + model: Samsung Galaxy S20 + includeSteps: [tap, waitForNavigation] + excludeSteps: [wait] +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + final screenshots = config.screenshots; + expect(screenshots.enabled, isTrue); + expect(screenshots.platform, 'android'); + expect(screenshots.model, 'Samsung Galaxy S20'); + expect(screenshots.includeSteps, ['tap', 'waitForNavigation']); + expect(screenshots.excludeSteps, ['wait']); + expect(screenshots.shouldCaptureStep('tap'), isTrue); + expect(screenshots.shouldCaptureStep('wait'), isFalse); + expect(screenshots.shouldCaptureStep('settle'), isFalse); + }); + + test('parses suite config performance', () { + const yaml = ''' +performance: + enabled: true +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + expect(config.performance.enabled, isTrue); + }); + + test('rejects removed record config', () { + const yaml = ''' +record: + enabled: true +'''; + + expect( + () => EnsembleTestParser.parseConfigString(yaml), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('Unsupported test config key "record"'), + ), + ), + ); + }); + + test('parses suite config debug artifacts', () { + const yaml = ''' +dumpTree: + enabled: true +logApiCalls: + enabled: true +logStorage: + enabled: true + key: auth +'''; + + final config = EnsembleTestParser.parseConfigString(yaml); + expect(config.dumpTree.enabled, isTrue); + expect(config.logApiCalls.enabled, isTrue); + expect(config.logStorage.enabled, isTrue); + expect(config.logStorage.key, 'auth'); + }); + test('parses AI-friendly metadata fields', () { const yaml = ''' id: login_valid diff --git a/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart b/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart index 8c25ee42b..ffad03202 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_scaffold_test.dart @@ -26,8 +26,7 @@ void main() { expect(content, contains('tags: [smoke]')); expect(content, contains('startScreen: Login')); expect( - Directory('${dir.path}/ensemble/apps/inhome/tests/fixtures') - .existsSync(), + Directory('${dir.path}/ensemble/apps/inhome/tests/mocks').existsSync(), isTrue); }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart index 6b5098f12..e7cfa008c 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart @@ -11,12 +11,10 @@ void main() { final stepDef = schema['\$defs']['step'] as Map; final oneOf = stepDef['oneOf'] as List; - final yamlKeys = oneOf - .map((e) { - final props = (e as Map)['properties'] as Map; - return props.keys.first as String; - }) - .toSet(); + final yamlKeys = oneOf.map((e) { + final props = (e as Map)['properties'] as Map; + return props.keys.first as String; + }).toSet(); for (final name in TestStepRegistry.entries.keys) { expect(yamlKeys, contains(name), reason: 'missing step $name in schema'); @@ -38,7 +36,9 @@ void main() { } }); - test('generated JSON is valid and requires id, steps with XOR start/prerequisite', () { + test( + 'generated JSON is valid and requires id, steps with XOR start/prerequisite', + () { final json = EnsembleTestSchemaBuilder.buildJson(); final decoded = jsonDecode(json) as Map; expect(decoded['\$schema'], EnsembleTestSchemaBuilder.schemaVersion); @@ -46,8 +46,55 @@ void main() { expect((decoded['required'] as List), isNot(contains('startScreen'))); expect(decoded['properties'], contains('id')); expect(decoded['properties'], isNot(contains('tests'))); + expect(decoded['properties'], isNot(contains('options'))); + expect(decoded['properties'], contains('mocks')); expect(decoded['properties'], contains('startScreen')); expect(decoded['properties'], contains('prerequisite')); + expect(decoded['properties'], isNot(contains('mockLayers'))); + expect(decoded['properties'], contains('scenarios')); expect(decoded['oneOf'], isA()); }); + + test('schema keeps mocks at the root only', () { + final schema = EnsembleTestSchemaBuilder.build(); + final defs = schema['\$defs'] as Map; + final scenario = defs['scenario'] as Map; + final scenarioProperties = scenario['properties'] as Map; + + expect(scenarioProperties, contains('vars')); + expect(scenarioProperties, isNot(contains('mocks'))); + }); + + test('config schema includes suite screenshots and performance settings', () { + final json = EnsembleTestSchemaBuilder.buildConfigJson(); + final decoded = jsonDecode(json) as Map; + final properties = decoded['properties'] as Map; + + expect(decoded['\$schema'], EnsembleTestSchemaBuilder.schemaVersion); + expect(properties, contains('screenshots')); + expect(properties, isNot(contains('record'))); + expect(properties, contains('performance')); + expect(properties, contains('dumpTree')); + expect(properties, contains('logApiCalls')); + expect(properties, contains('logStorage')); + expect( + (properties['screenshots'] as Map)['properties'], + containsPair('model', {'type': 'string'}), + ); + }); + + test('initialState schema accepts storage, keychain, and env maps', () { + final schema = EnsembleTestSchemaBuilder.build(); + final initialState = + schema['\$defs']['initialState'] as Map; + final properties = initialState['properties'] as Map; + + expect(properties, contains('storage')); + expect(properties, contains('keychain')); + expect(properties, contains('env')); + expect(properties['keychain'], { + 'type': 'object', + 'additionalProperties': true, + }); + }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart index 04ed2ad9a..fa41e8b31 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart @@ -5,14 +5,22 @@ import 'package:flutter_test/flutter_test.dart'; void main() { test('committed schema matches generator output', () { - final path = 'assets/schema/ensemble_tests_schema.json'; - final committed = File(path).readAsStringSync().trim(); - final generated = EnsembleTestSchemaBuilder.buildJson().trim(); - expect( - committed, - generated, - reason: - 'Run: cd tools/ensemble_test_runner && dart run tool/generate_schema.dart', - ); + final schemas = { + 'assets/schema/ensemble_tests_schema.json': + EnsembleTestSchemaBuilder.buildJson(), + 'assets/schema/ensemble_test_config_schema.json': + EnsembleTestSchemaBuilder.buildConfigJson(), + }; + + for (final entry in schemas.entries) { + final committed = File(entry.key).readAsStringSync().trim(); + final generated = entry.value.trim(); + expect( + committed, + generated, + reason: + 'Run: cd tools/ensemble_test_runner && dart run tool/generate_schema.dart', + ); + } }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart b/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart index 95bde0f52..01092a772 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_validator_test.dart @@ -4,22 +4,14 @@ import 'package:ensemble_test_runner/validation/ensemble_test_validator.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { - test('validates generated tests with structured issues', () { + test('validates generated tests with structured warning issues', () { final dir = Directory.systemTemp.createTempSync('ensemble_validate_'); addTearDown(() => dir.deleteSync(recursive: true)); _writeApp(dir); _writeTest(dir, 'login.test.yaml', ''' id: login_happy startScreen: Login -mocks: - apis: - login: - response: - body: {ok: true} steps: - - mockApiFromFixture: - name: profile - fixture: missing.json - tap: id: missing_button - expectApiCalled: @@ -29,35 +21,23 @@ steps: final result = EnsembleTestValidator(dir.path).validate(); final codes = result.issues.map((issue) => issue.code).toList(); - expect(result.hasErrors, isTrue); - expect(codes, contains('missingFixture')); + expect(result.hasErrors, isFalse); expect(codes, contains('unknownWidgetId')); expect(codes, contains('unknownApi')); - expect(codes, contains('mockPlacement')); }); test('passes with warnings only as non-blocking', () { final dir = Directory.systemTemp.createTempSync('ensemble_validate_ok_'); addTearDown(() => dir.deleteSync(recursive: true)); _writeApp(dir); - Directory('${dir.path}/ensemble/apps/inhome/tests/fixtures') - .createSync(recursive: true); - File('${dir.path}/ensemble/apps/inhome/tests/fixtures/profile.json') - .writeAsStringSync('{"ok": true}'); _writeTest(dir, 'login.test.yaml', ''' id: login_happy startScreen: Login -mocks: - apis: - profile: - response: - body: {ok: true} steps: - tap: id: login_button - - mockApiFromFixture: + - expectApiCalled: name: login - fixture: profile.json '''); final result = EnsembleTestValidator(dir.path).validate(); diff --git a/tools/ensemble_test_runner/test/firebase_auth_test_setup_test.dart b/tools/ensemble_test_runner/test/firebase_auth_test_setup_test.dart new file mode 100644 index 000000000..e16f0c824 --- /dev/null +++ b/tools/ensemble_test_runner/test/firebase_auth_test_setup_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/mocks/live_firebase_auth_http.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('extracts Firebase uid from idToken payload', () { + const uid = 'test-user-uid'; + final payload = base64Url.encode(utf8.encode(jsonEncode({ + 'user_id': uid, + 'sub': uid, + }))); + final idToken = 'header.$payload.signature'; + expect(uidFromFirebaseIdToken(idToken), uid); + }); +} diff --git a/tools/ensemble_test_runner/test/firebase_functions_test_setup_test.dart b/tools/ensemble_test_runner/test/firebase_functions_test_setup_test.dart new file mode 100644 index 000000000..6706c98b2 --- /dev/null +++ b/tools/ensemble_test_runner/test/firebase_functions_test_setup_test.dart @@ -0,0 +1,29 @@ +import 'package:ensemble_test_runner/mocks/firebase_test_setup.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:firebase_core/firebase_core.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + testWidgets( + 'firebase core mock preserves projectId from initializeApp', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + + await tester.runAsync(() async { + const appName = 'firebaseFunctionsTest'; + await Firebase.initializeApp( + name: appName, + options: const FirebaseOptions( + apiKey: 'test-api-key', + appId: 'test-app-id', + messagingSenderId: 'test-sender', + projectId: 'test-project-id', + ), + ); + + expect(firebaseProjectIdsByApp[appName], 'test-project-id'); + expect(Firebase.app(appName).options.projectId, 'test-project-id'); + }); + }, + ); +} diff --git a/tools/ensemble_test_runner/test/navigation_flow_recorder_test.dart b/tools/ensemble_test_runner/test/navigation_flow_recorder_test.dart index f337763a0..92619bf9c 100644 --- a/tools/ensemble_test_runner/test/navigation_flow_recorder_test.dart +++ b/tools/ensemble_test_runner/test/navigation_flow_recorder_test.dart @@ -3,25 +3,48 @@ import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { - test('dedupes consecutive screen names', () { + test('dedupes consecutive screen names', () async { final recorder = NavigationFlowRecorder(); recorder.recordScreenChange( VisibleScreen(screenName: 'Hello Home', visibleSince: DateTime.now()), ); + await recorder.flushPending(); recorder.recordScreenChange( VisibleScreen(screenName: 'Goodbye', visibleSince: DateTime.now()), ); + await recorder.flushPending(); recorder.recordScreenChange( VisibleScreen(screenName: 'Hello Home', visibleSince: DateTime.now()), ); + await recorder.flushPending(); recorder.recordScreenChange( VisibleScreen(screenName: 'Hello Home', visibleSince: DateTime.now()), ); + await recorder.flushPending(); expect( recorder.flow, ['Hello Home', 'Goodbye', 'Hello Home'], ); }); + + test('coalesces transient route removal screen changes', () async { + final recorder = NavigationFlowRecorder(); + + recorder.recordScreenChange( + VisibleScreen(screenName: 'InitApp', visibleSince: DateTime.now()), + ); + await recorder.flushPending(); + + recorder.recordScreenChange( + VisibleScreen(screenName: 'Login', visibleSince: DateTime.now()), + ); + recorder.recordScreenChange( + VisibleScreen(screenName: 'AutoSignIn', visibleSince: DateTime.now()), + ); + await recorder.flushPending(); + + expect(recorder.flow, ['InitApp', 'AutoSignIn']); + }); } diff --git a/tools/ensemble_test_runner/test/screenshot_device_test.dart b/tools/ensemble_test_runner/test/screenshot_device_test.dart new file mode 100644 index 000000000..7c8939136 --- /dev/null +++ b/tools/ensemble_test_runner/test/screenshot_device_test.dart @@ -0,0 +1,41 @@ +import 'package:ensemble_test_runner/actions/screenshot_device.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('resolves screenshot model by platform and model name', () { + final device = resolveScreenshotDevice({ + 'platform': 'android', + 'model': 'Samsung Galaxy S20', + }); + + expect(device.name, 'Samsung Galaxy S20'); + }); + + test('defaults screenshot device to iPhone 15 Pro', () { + final device = resolveScreenshotDevice({}); + + expect(device.name, 'iPhone 15 Pro'); + }); + + test('uses suite screenshot config device when enabled', () { + final device = screenshotDeviceForTestCase( + const EnsembleTestCase( + id: 'screenshots', + startScreen: 'Home', + steps: [ + TestStep(type: 'tap', args: {'id': 'button'}), + ], + ), + const EnsembleTestConfig( + screenshots: ScreenshotConfig( + enabled: true, + platform: 'android', + model: 'Samsung Galaxy S20', + ), + ), + ); + + expect(device?.name, 'Samsung Galaxy S20'); + }); +} diff --git a/tools/ensemble_test_runner/test/test_api_provider_overlay_test.dart b/tools/ensemble_test_runner/test/test_api_provider_overlay_test.dart new file mode 100644 index 000000000..0e3e19d2d --- /dev/null +++ b/tools/ensemble_test_runner/test/test_api_provider_overlay_test.dart @@ -0,0 +1,184 @@ +import 'package:ensemble/framework/apiproviders/api_provider.dart'; +import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; +import 'package:ensemble/framework/data_context.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + testWidgets('unmocked firebase functions delegate to the real provider', + (tester) async { + await tester.pumpWidget(const MaterialApp(home: SizedBox.shrink())); + + final delegate = _RecordingAPIProvider(); + final provider = TestApiProviderOverlay(mocks: {}); + final context = tester.element(find.byType(SizedBox)); + final api = + YamlMap.wrap({'type': 'firebaseFunction', 'name': 'exampleCallable'}); + + await provider.invokeApiWithDelegate( + delegate, + context, + api, + DataContext(buildContext: context), + 'exampleCallable', + ); + + expect(delegate.invoked, isTrue); + expect(provider.callCount('exampleCallable'), 1); + }); + + testWidgets('mocked APIs return configured responses without delegating', + (tester) async { + await tester.pumpWidget(const MaterialApp(home: SizedBox.shrink())); + + final delegate = _RecordingAPIProvider(); + final provider = TestApiProviderOverlay( + mocks: { + 'login': MockAPIResponse(statusCode: 200, body: {'token': 'abc'}), + }, + delegate: delegate, + ); + final context = tester.element(find.byType(SizedBox)); + final response = await provider.invokeApi( + context, + YamlMap.wrap({'type': 'http', 'url': 'https://example.test/login'}), + DataContext(buildContext: context), + 'login', + ); + + expect(delegate.invoked, isFalse); + expect(response.body, {'token': 'abc'}); + }); + + test('mocked APIs honor delayMs before returning', () async { + final delegate = _RecordingAPIProvider(); + final provider = TestApiProviderOverlay( + mocks: { + 'slowProfile': MockAPIResponse( + statusCode: 200, + body: {'name': 'Jane'}, + delayMs: 50, + ), + }, + delegate: delegate, + ); + final context = _FakeBuildContext(); + var completed = false; + + final responseFuture = provider + .invokeApiWithDelegate( + delegate, + context, + YamlMap.wrap({'type': 'http', 'url': 'https://example.test/profile'}), + DataContext(buildContext: context), + 'slowProfile', + ) + .then((response) { + completed = true; + return response; + }); + + await Future.delayed(const Duration(milliseconds: 10)); + expect(completed, isFalse); + + final response = await responseFuture; + expect(completed, isTrue); + expect(delegate.invoked, isFalse); + expect(response.body, {'name': 'Jane'}); + }); + + test('serializes live runner calls to avoid reentrant runAsync', () async { + var inFlight = 0; + var maxInFlight = 0; + final delegate = _ConcurrentRecordingAPIProvider( + onStart: () { + inFlight++; + if (inFlight > maxInFlight) { + maxInFlight = inFlight; + } + }, + onEnd: () { + inFlight--; + }, + ); + final provider = TestApiProviderOverlay(mocks: {}, delegate: delegate); + provider.liveAsyncRunner = (Future Function() fn) => fn(); + + final buildContext = _FakeBuildContext(); + final first = provider.invokeApiWithDelegate( + delegate, + buildContext, + YamlMap.wrap({'type': 'http', 'url': 'https://example.test/a'}), + DataContext(buildContext: buildContext), + 'first', + ); + final second = provider.invokeApiWithDelegate( + delegate, + buildContext, + YamlMap.wrap({'type': 'http', 'url': 'https://example.test/b'}), + DataContext(buildContext: buildContext), + 'second', + ); + + await Future.wait([first, second]); + expect(maxInFlight, 1); + expect(delegate.invoked, 2); + }); +} + +class _ConcurrentRecordingAPIProvider extends HTTPAPIProvider { + _ConcurrentRecordingAPIProvider({ + required this.onStart, + required this.onEnd, + }); + + final void Function() onStart; + final void Function() onEnd; + int invoked = 0; + + @override + Future invokeApi( + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + onStart(); + invoked++; + await Future.delayed(const Duration(milliseconds: 50)); + onEnd(); + return HttpResponse.fromBody( + {'ok': true}, + {'Content-Type': 'application/json'}, + 200, + null, + APIState.success, + ); + } +} + +class _RecordingAPIProvider extends HTTPAPIProvider { + bool invoked = false; + + @override + Future invokeApi( + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + invoked = true; + return HttpResponse.fromBody( + {'ok': true}, + {'Content-Type': 'application/json'}, + 200, + null, + APIState.success, + ); + } +} + +class _FakeBuildContext extends Fake implements BuildContext {} diff --git a/tools/ensemble_test_runner/test/test_reporter_test.dart b/tools/ensemble_test_runner/test/test_reporter_test.dart index b7cb93205..b1f78d01b 100644 --- a/tools/ensemble_test_runner/test/test_reporter_test.dart +++ b/tools/ensemble_test_runner/test/test_reporter_test.dart @@ -1,3 +1,4 @@ +import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; import 'package:ensemble_test_runner/reporters/test_reporter.dart'; @@ -30,6 +31,41 @@ void main() { ['Hello Home', 'Goodbye', 'Hello Home'], ); }); + + test('beginTest starts a fresh report flow for continuation tests', + () async { + YamlTestSession.navigationFlow.seed([ + 'Login', + 'InitApp', + 'AutoSignIn', + 'AutoSignIn_Gateway', + 'Home', + ]); + + YamlTestSession.navigationFlow.beginTest('Home'); + YamlTestSession.navigationFlow.recordScreenChange( + VisibleScreen(screenName: 'Devices', visibleSince: DateTime.now()), + ); + await YamlTestSession.navigationFlow.flushPending(); + YamlTestSession.navigationFlow.recordScreenChange( + VisibleScreen(screenName: 'Home', visibleSince: DateTime.now()), + ); + await YamlTestSession.navigationFlow.flushPending(); + YamlTestSession.navigationFlow.recordScreenChange( + VisibleScreen( + screenName: 'Settings_Wifi', visibleSince: DateTime.now()), + ); + await YamlTestSession.navigationFlow.flushPending(); + YamlTestSession.navigationFlow.recordScreenChange( + VisibleScreen(screenName: 'Home', visibleSince: DateTime.now()), + ); + await YamlTestSession.navigationFlow.flushPending(); + + expect( + collectScreensVisited('Home'), + ['Home', 'Devices', 'Home', 'Settings_Wifi', 'Home'], + ); + }); }); group('TestReporter', () { @@ -87,5 +123,93 @@ void main() { expect(output, contains('>> 2. tap(submit)')); expect(output, contains('error: not found')); }); + + test('prints step logs', () { + final output = TestReporter().formatSummary( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.passed( + testId: 'api_log_test', + durationMs: 42, + logs: const [ + 'apiCalls: build/ensemble_test_runner/logs/api_log_test_api_calls.json', + 'dumpTree: build/ensemble_test_runner/logs/api_log_test_dump_tree.txt', + ], + ), + ], + ), + ); + + expect(output, contains('artifacts:')); + expect( + output, + contains( + 'apiCalls: build/ensemble_test_runner/logs/api_log_test_api_calls.json')); + expect(output, contains('dumpTree:')); + expect(output, isNot(contains('MaterialApp'))); + }); + + test('prints suite logs separately from test logs', () { + final output = TestReporter().formatSummary( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.passed( + testId: 'login_test', + durationMs: 42, + ), + ], + suiteLogs: const [ + 'storage: build/ensemble_test_runner/logs/storage.json', + ], + ), + ); + + expect(output, contains('suite artifacts:')); + expect( + output, + contains('storage: build/ensemble_test_runner/logs/storage.json'), + ); + }); + + test('formats compact failure summary without repeating boxed report', () { + final output = TestReporter().formatFailureSummary( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.failed( + testId: + 'login_test (ensemble/apps/inhome/tests/signin.test.yaml)', + durationMs: 100, + failedStep: const TestStep( + type: 'waitForNavigation', + args: {'screen': 'Home'}, + ), + error: 'Timed out waiting for navigation', + ), + EnsembleSingleTestResult.failed( + testId: + 'smoke_navigations (ensemble/apps/inhome/tests/smoke.test.yaml)', + durationMs: 0, + error: 'Prerequisite "login_test" failed', + ), + ], + ), + failedPaths: const [ + 'ensemble/apps/inhome/tests/signin.test.yaml', + 'ensemble/apps/inhome/tests/smoke.test.yaml', + ], + pendingFrameworkExceptions: const [ + 'Reentrant call to runAsync() denied.\nextra framework text', + ], + ); + + expect(output, contains('Failed YAML tests (2/2):')); + expect(output, contains('Timed out waiting for navigation')); + expect(output, contains('failed: waitForNavigation(Home)')); + expect(output, contains('Pending Flutter framework exceptions:')); + expect(output, contains('Reentrant call to runAsync() denied.')); + expect(output, contains('See the Ensemble YAML tests report above.')); + expect(output, isNot(contains('┌─ Ensemble YAML tests'))); + expect(output, isNot(contains('extra framework text'))); + }); }); } diff --git a/tools/ensemble_test_runner/test/test_step_executor_test.dart b/tools/ensemble_test_runner/test/test_step_executor_test.dart index 8e128a273..5b7ac9d5d 100644 --- a/tools/ensemble_test_runner/test/test_step_executor_test.dart +++ b/tools/ensemble_test_runner/test/test_step_executor_test.dart @@ -1,11 +1,69 @@ +import 'dart:convert'; +import 'dart:io'; + import 'package:ensemble_test_runner/actions/test_step_executor.dart'; -import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/mocks/test_api_provider_overlay.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; +import 'package:ensemble_test_runner/runner/app_performance_log.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:flutter/cupertino.dart'; +import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:yaml/yaml.dart'; void main() { + testWidgets('toggle taps the switch inside a keyed input wrapper', + (tester) async { + var value = false; + await tester.pumpWidget( + MaterialApp( + home: StatefulBuilder( + builder: (context, setState) => Scaffold( + body: KeyedSubtree( + key: const ValueKey('notifications'), + child: SizedBox( + width: 400, + child: Row( + children: [ + const Expanded(child: Text('Notifications')), + CupertinoSwitch( + value: value, + onChanged: (next) => setState(() => value = next), + ), + ], + ), + ), + ), + ), + ), + ), + ); + + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 't', + startScreen: 'Home', + steps: [], + ), + ); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'), + ); + + await executor.execute( + const TestStep(type: 'toggle', args: {'id': 'notifications'}), + ); + + expect(value, isTrue); + }); + testWidgets('waitFor requires id or text', (tester) async { final context = EnsembleTestContext.fromTestCase( const EnsembleTestCase( @@ -14,7 +72,8 @@ void main() { steps: [], ), ); - final harness = EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); final executor = TestStepExecutor( tester: tester, context: context, @@ -33,4 +92,146 @@ void main() { ), ); }); + + testWidgets('logApiCalls writes structured json', (tester) async { + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 'api_log_test', + startScreen: 'Home', + steps: [], + ), + ); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: harness, + ); + + await executor.execute(const TestStep(type: 'logApiCalls', args: {})); + + expect(context.logger.logs.single, startsWith('apiCalls: ')); + final path = context.logger.logs.single.substring('apiCalls: '.length); + expect(path, endsWith('.json')); + final content = File(path).readAsStringSync(); + expect(content, isNot(contains('API '))); + expect(jsonDecode(content), { + 'total': 0, + 'calls': [], + }); + }); + + test('performance log attributes frames and ranks jank context', () async { + final logger = TestLogger(); + final start = DateTime.now(); + final apiTimestamp = start.add(const Duration(milliseconds: 20)); + final path = await writePerformanceLog( + logger: logger, + filePrefix: 'suite', + name: 'app_performance', + frames: const [ + AppFrameTimingEntry( + frameNumber: 1, + buildStartMicros: 1000, + buildMs: 30, + rasterMs: 0, + vsyncOverheadMs: 1, + totalSpanMs: 45, + ), + AppFrameTimingEntry( + frameNumber: 2, + buildStartMicros: 2000, + buildMs: 5, + rasterMs: 0, + vsyncOverheadMs: 1, + totalSpanMs: 10, + ), + AppFrameTimingEntry( + frameNumber: 3, + buildStartMicros: 3000, + buildMs: 40, + rasterMs: 0, + vsyncOverheadMs: 1, + totalSpanMs: 60, + ), + ], + markers: [ + PerformanceMarker( + testId: 'login_test', + stepIndex: 1, + label: 'login_test step 1 tap(login_button)', + screen: 'Login', + phase: 'step', + startFrame: 1, + endFrame: 3, + startTime: start, + endTime: start.add(const Duration(milliseconds: 100)), + ), + ], + apiCalls: [ + APICallRecord( + name: 'loginApi', + apiDefinition: loadYaml('{}') as YamlMap, + timestamp: apiTimestamp, + ), + ], + ); + + final json = jsonDecode(File(path).readAsStringSync()) as Map; + expect(json['summary'], containsPair('worstScreen', 'Login')); + expect( + json['summary'], + containsPair('worstStep', 'login_test step 1 tap(login_button)'), + ); + expect((json['frames'] as List).first, containsPair('screen', 'Login')); + expect((json['frames'] as List).first, containsPair('phase', 'step')); + expect( + ((json['worstSteps'] as List).first as Map)['step'], + 'login_test step 1 tap(login_button)', + ); + expect( + ((json['worstScreens'] as List).first as Map)['screen'], + 'Login', + ); + expect((json['jankClusters'] as List), isNotEmpty); + expect( + (((json['apiCorrelation'] as List).first as Map)['apiCalls'] as List) + .single, + containsPair('name', 'loginApi'), + ); + expect( + (json['slowestFrames'] as List).first, containsPair('screen', 'Login')); + }); + + testWidgets('setDevice updates the render surface size', (tester) async { + await tester.pumpWidget( + const MaterialApp(home: SizedBox.expand()), + ); + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 'set_device_test', + startScreen: 'Home', + steps: [], + ), + ); + final harness = + EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: harness, + ); + + await executor.execute( + const TestStep( + type: 'setDevice', + args: {'width': 393, 'height': 852}, + ), + ); + + expect(tester.binding.renderViews.first.size, const Size(393, 852)); + }); } diff --git a/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart b/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart index ce77c823e..bb01aab9e 100644 --- a/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart +++ b/tools/ensemble_test_runner/test/yaml_test_app_patcher_test.dart @@ -45,7 +45,7 @@ steps: [] expect( File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') .readAsStringSync(), - contains('runEnsembleYamlTests'), + contains('EnsembleModules().init()'), ); patcher.restore(); @@ -58,7 +58,7 @@ steps: [] ); }); - test('restore deletes leftover generated test entry from a prior run', () { + test('upgrades legacy stub and keeps it after restore', () { final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); addTearDown(() => dir.deleteSync(recursive: true)); @@ -82,16 +82,68 @@ steps: [] '''); Directory('${dir.path}/test').createSync(recursive: true); File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') - .writeAsStringSync(YamlTestAppPatcher.testEntryContents); + .writeAsStringSync(YamlTestAppPatcher.legacyTestEntryContents); final patcher = YamlTestAppPatcher(dir.path); patcher.enable(); patcher.restore(); + final restored = + File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') + .readAsStringSync(); + expect(restored, contains('EnsembleModules().init()')); + }); + + test('does not overwrite a customized test entry', () { + final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); + addTearDown(() => dir.deleteSync(recursive: true)); + + File('${dir.path}/pubspec.yaml').writeAsStringSync(''' +name: sample_app +dev_dependencies: + flutter_test: + sdk: flutter +flutter: + assets: + - ensemble/ +'''); + _writeConfig(dir); + Directory('${dir.path}/ensemble/apps/helloApp/tests') + .createSync(recursive: true); + File('${dir.path}/ensemble/apps/helloApp/tests/sample.test.yaml') + .writeAsStringSync(''' +id: sample +startScreen: Home +steps: [] +'''); + const customEntry = ''' +import 'package:ensemble_test_runner/entry/ensemble_test_entry.dart'; + +Future main() async { + await runEnsembleYamlTests( + bootstrap: () async {}, + ); +} +'''; + Directory('${dir.path}/test').createSync(recursive: true); + File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') + .writeAsStringSync(customEntry); + + final patcher = YamlTestAppPatcher(dir.path); + patcher.enable(); + expect( File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') - .existsSync(), - isFalse, + .readAsStringSync(), + customEntry, + ); + + patcher.restore(); + + expect( + File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') + .readAsStringSync(), + customEntry, ); }); @@ -130,6 +182,44 @@ steps: [] patcher.restore(); }); + test('enable adds nested test asset directories for mock files', () { + final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); + addTearDown(() => dir.deleteSync(recursive: true)); + + const pubspec = ''' +name: sample_app +flutter: + assets: + - ensemble/ + - ensemble/apps/helloApp/tests/ +'''; + File('${dir.path}/pubspec.yaml').writeAsStringSync(pubspec); + _writeConfig(dir); + Directory('${dir.path}/ensemble/apps/helloApp/tests/mocks/common') + .createSync(recursive: true); + File('${dir.path}/ensemble/apps/helloApp/tests/sample.test.yaml') + .writeAsStringSync(''' +id: sample +startScreen: Home +steps: + - expectVisible: + id: home +'''); + File('${dir.path}/ensemble/apps/helloApp/tests/mocks/common/base.mock.json') + .writeAsStringSync('{}'); + + final patcher = YamlTestAppPatcher(dir.path); + patcher.enable(); + + final enabled = File('${dir.path}/pubspec.yaml').readAsStringSync(); + expect( + enabled, + contains(' - ensemble/apps/helloApp/tests/mocks/common/'), + ); + + patcher.restore(); + }); + test('exposes configured tests directory and test presence', () { final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); addTearDown(() => dir.deleteSync(recursive: true)); diff --git a/tools/ensemble_test_runner/tool/generate_schema.dart b/tools/ensemble_test_runner/tool/generate_schema.dart index 669e77528..63de542bc 100644 --- a/tools/ensemble_test_runner/tool/generate_schema.dart +++ b/tools/ensemble_test_runner/tool/generate_schema.dart @@ -2,7 +2,7 @@ import 'dart:io'; import 'package:ensemble_test_runner/schema/ensemble_test_schema_builder.dart'; -/// Regenerates [assets/schema/ensemble_tests_schema.json] from the step registry. +/// Regenerates JSON Schemas from the step registry. /// /// Run from this package: /// dart run tool/generate_schema.dart @@ -17,4 +17,11 @@ void main() { outFile.parent.createSync(recursive: true); outFile.writeAsStringSync('${EnsembleTestSchemaBuilder.buildJson()}\n'); stdout.writeln('Wrote ${outFile.path}'); + + final configOutFile = File('assets/schema/ensemble_test_config_schema.json'); + configOutFile.parent.createSync(recursive: true); + configOutFile.writeAsStringSync( + '${EnsembleTestSchemaBuilder.buildConfigJson()}\n', + ); + stdout.writeln('Wrote ${configOutFile.path}'); } diff --git a/tools/ensemble_test_runner/tool/generate_step_registry.dart b/tools/ensemble_test_runner/tool/generate_step_registry.dart index e0ba1fe9f..e1704115b 100644 --- a/tools/ensemble_test_runner/tool/generate_step_registry.dart +++ b/tools/ensemble_test_runner/tool/generate_step_registry.dart @@ -193,7 +193,7 @@ void main() { 'wait', 'extended', 'pump', - 'Alias for pump — advance frame clock by durationMs', + 'Real-time delay using runAsync, followed by a frame pump', ), 'pump': step( 'wait', @@ -237,12 +237,6 @@ void main() { 'waitForNavigation', 'Poll until the given screen is visible', ), - 'waitUntil': step( - 'wait', - 'core', - 'waitUntil', - 'Poll until app state at path equals expected value', - ), 'expectVisible': step( 'uiAssertion', 'core', @@ -405,60 +399,12 @@ void main() { 'empty', 'Navigate back (Ensemble navigateBack or Navigator.pop)', ), - 'mockApi': step( - 'apiMock', - 'core', - 'mockApi', - 'Register a mock HTTP API response by API name', - ), - 'mockApiError': step( - 'apiMock', - 'core', - 'mockApiError', - 'Mock an API to return an error status/body', - ), - 'mockApiFromFixture': step( - 'fixture', - 'core', - 'mockApiFromFixture', - 'Load mock response body from a JSON fixture asset', - ), - 'mockApiException': step( - 'apiMock', - 'core', - 'mockApiException', - 'Force an API call to throw an exception', - ), - 'mockTimeout': step( - 'network', - 'core', - 'mockTimeout', - 'Mock an API with a long delay (simulate timeout)', - ), - 'mockNetworkOffline': step( - 'network', - 'core', - 'empty', - 'Simulate offline network for API calls', - ), - 'mockNetworkOnline': step( - 'network', - 'core', - 'empty', - 'Restore online network for API calls', - ), 'resetApiCalls': step( 'apiMock', 'core', 'empty', 'Clear recorded API call history', ), - 'clearApiMocks': step( - 'apiMock', - 'core', - 'empty', - 'Remove all registered API mocks', - ), 'expectApiCalled': step( 'apiAssertion', 'core', @@ -471,24 +417,6 @@ void main() { 'apiName', 'Assert an API was never called', ), - 'expectApiRequest': step( - 'apiAssertion', - 'core', - 'apiRequest', - 'Assert last API request body/query/headers match', - ), - 'expectApiRequestContains': step( - 'apiAssertion', - 'core', - 'apiRequest', - 'Assert API request contains partial body/query', - ), - 'expectApiHeader': step( - 'apiAssertion', - 'core', - 'expectApiHeader', - 'Assert an API request header equals expected', - ), 'expectApiCallOrder': step( 'apiAssertion', 'core', @@ -501,42 +429,6 @@ void main() { 'apiName', 'Assert the most recent API call name', ), - 'setState': step( - 'state', - 'core', - 'setState', - 'Set app data-context state at path to value', - ), - 'expectState': step( - 'state', - 'core', - 'expectState', - 'Assert app state at path equals expected', - ), - 'expectStateContains': step( - 'state', - 'core', - 'expectState', - 'Assert app state at path contains subset', - ), - 'expectStateExists': step( - 'state', - 'core', - 'expectStatePath', - 'Assert state path resolves without error', - ), - 'expectStateNotExists': step( - 'state', - 'core', - 'expectStatePath', - 'Assert state path is null or absent', - ), - 'resetState': step( - 'state', - 'core', - 'resetStatePath', - 'Clear state at path (set to null)', - ), 'setStorage': step( 'storage', 'core', @@ -651,30 +543,6 @@ void main() { 'empty', 'Log all recorded API calls to the test log', ), - 'screenshot': step( - 'debug', - 'core', - 'screenshot', - 'Capture golden or dump widget tree for debugging', - ), - 'dumpTree': step( - 'debug', - 'core', - 'empty', - 'Print the widget tree to the debug console', - ), - 'logState': step( - 'debug', - 'core', - 'expectStatePath', - 'Log resolved state at path', - ), - 'logStorage': step( - 'debug', - 'core', - 'storageKey', - 'Log public storage value for key', - ), 'expectNoConsoleErrors': step( 'quality', 'core', @@ -717,30 +585,11 @@ void main() { 'idRequired', 'Assert widget renders without overflow issues', ), - 'loadFixture': step( - 'fixture', - 'core', - 'fixturePath', - 'Load a JSON fixture into the test fixture map', - ), - 'setStateFromFixture': step( - 'fixture', - 'core', - 'fixturePath', - 'Apply all keys from a JSON fixture to state', - ), - 'expectMatchesFixture': step( - 'fixture', - 'core', - 'fixturePath', - 'Assert state or path matches a JSON fixture', - ), }; const executorAliases = { 'waitForText': 'waitFor', 'expectScreen': 'expectNavigateTo', - 'wait': 'pump', 'launchApp': 'restartApp', 'expectScript': 'expectScriptResult', }; @@ -853,8 +702,6 @@ Map defaultExampleForArg(String arg) { return {'id': 'loading_spinner', 'timeoutMs': 5000}; case 'waitForNavigation': return {'screen': 'Home', 'timeoutMs': 5000}; - case 'waitUntil': - return {'path': 'user.name', 'equals': 'Jane'}; case 'textRequired': return {'text': 'Welcome'}; case 'expectEquals': @@ -871,43 +718,8 @@ Map defaultExampleForArg(String arg) { return {'screen': 'Home'}; case 'expectVisited': return {'screen': 'Login'}; - case 'mockApi': - return { - 'name': 'login', - 'response': { - 'statusCode': 200, - 'body': {'token': 'test-token'}, - }, - }; - case 'mockApiError': - return { - 'name': 'login', - 'statusCode': 401, - 'body': {'error': 'Unauthorized'} - }; - case 'mockApiFromFixture': - return {'name': 'users', 'fixture': 'fixtures/users.json'}; - case 'mockApiException': - return {'name': 'login', 'message': 'Network error'}; - case 'mockTimeout': - return {'name': 'slow_api', 'delayMs': 60000}; case 'apiName': return {'name': 'login', 'times': 1}; - case 'apiRequest': - return { - 'name': 'login', - 'body': {'email': 'user@test.com', 'password': 'secret'}, - }; - case 'expectApiHeader': - return { - 'name': 'login', - 'header': 'Authorization', - 'equals': 'Bearer test-token', - }; - case 'setState': - return {'path': 'user.name', 'value': 'Jane'}; - case 'expectState': - return {'path': 'user.name', 'equals': 'Jane'}; case 'storageKey': return {'key': 'onboarding_done', 'value': true}; case 'group': @@ -945,12 +757,6 @@ Map defaultExampleForArg(String arg) { }, ], }; - case 'screenshot': - return {'name': 'home_screen'}; - case 'expectStatePath': - return {'path': 'user.id'}; - case 'resetStatePath': - return {'path': 'cart'}; case 'setAuth': return { 'user': {'id': '1', 'email': 'user@test.com'}, @@ -969,8 +775,6 @@ Map defaultExampleForArg(String arg) { return {'contains': 'Screen loaded'}; case 'expectErrorContains': return {'contains': 'overflow'}; - case 'fixturePath': - return {'fixture': 'fixtures/user.json'}; case 'expectApiCallOrder': return { 'names': ['auth', 'profile']