From 2dff4d3d698ea5ce5e15f80bed45736c329fb4d3 Mon Sep 17 00:00:00 2001 From: Sean Poulter Date: Mon, 9 Feb 2026 15:41:40 -0500 Subject: [PATCH 1/3] Configure prettier --- .prettierignore | 2 ++ .prettierrc.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .prettierignore diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..a9baa04 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +.vale/ +pnpm-lock.yaml diff --git a/.prettierrc.yaml b/.prettierrc.yaml index eaae201..06808f9 100644 --- a/.prettierrc.yaml +++ b/.prettierrc.yaml @@ -1,2 +1,2 @@ printWidth: 100 -proseWrap: always +proseWrap: preserve From 28507988b21d35fffb42d3d3752fb29c8183b587 Mon Sep 17 00:00:00 2001 From: Sean Poulter Date: Mon, 9 Feb 2026 15:42:24 -0500 Subject: [PATCH 2/3] Run `npm run fmt` --- docs/cloud/deploys/integrations/index.mdx | 3 +- .../devbox-global-pull/index.mdx | 3 +- .../devbox-global-push/index.mdx | 3 +- .../devbox-services-restart/index.mdx | 3 +- .../languages/python/index.mdx | 29 ++--- .../devbox-examples/stacks/lapp/index.mdx | 3 +- .../devbox-examples/stacks/laravel/index.mdx | 11 +- .../devbox-examples/stacks/lepp/index.mdx | 4 +- docs/devbox/guides/plugins/index.mdx | 3 + docs/devbox/guides/services/index.mdx | 6 +- .../devbox/ide-configuration/direnv/index.mdx | 12 +- .../devbox/ide-configuration/vscode/index.mdx | 17 ++- docs/devbox/installing-devbox.mdx | 20 ++-- docs/devbox/quickstart/index.mdx | 10 +- docs/testpilot/advanced/bash-tool.mdx | 7 +- docs/testpilot/advanced/index.mdx | 5 +- docs/testpilot/ai-test-generation.mdx | 1 + docs/testpilot/best-practices/index.mdx | 11 +- .../analyze-json-report-evaluations.mdx | 110 ++++++++++-------- .../testpilot-json-report-reference.mdx | 81 +++++++------ docs/testpilot/getting-started/index.mdx | 6 +- .../getting-started/testing-android-apps.mdx | 10 +- .../getting-started/testing-ios-apps.mdx | 8 +- .../getting-started/testing-web-apps.mdx | 2 +- ...rameterize-tests-environment-variables.mdx | 2 +- docs/testpilot/guides/test-authentication.mdx | 7 +- .../guides/test-responsive-design.mdx | 6 +- docs/testpilot/integrations/gitlab-cicd.mdx | 14 +-- docs/testpilot/integrations/index.mdx | 14 ++- 29 files changed, 225 insertions(+), 186 deletions(-) diff --git a/docs/cloud/deploys/integrations/index.mdx b/docs/cloud/deploys/integrations/index.mdx index ed56189..3f821f8 100644 --- a/docs/cloud/deploys/integrations/index.mdx +++ b/docs/cloud/deploys/integrations/index.mdx @@ -1,7 +1,6 @@ --- title: "Integration Guides" -description: - "This section contains guides on how to integrate common services with your Jetify deployments." +description: "This section contains guides on how to integrate common services with your Jetify deployments." --- - [PostgresQL with Supabase](/docs/cloud/deploys/integrations/supabase/) diff --git a/docs/devbox/cli-reference/devbox-global-pull/index.mdx b/docs/devbox/cli-reference/devbox-global-pull/index.mdx index 3da718c..c77ffef 100644 --- a/docs/devbox/cli-reference/devbox-global-pull/index.mdx +++ b/docs/devbox/cli-reference/devbox-global-pull/index.mdx @@ -1,7 +1,6 @@ --- title: "devbox global pull" -description: - "Pulls a global config from a file or URL. URLs must be prefixed with 'http://' or 'https://'." +description: "Pulls a global config from a file or URL. URLs must be prefixed with 'http://' or 'https://'." --- ```bash diff --git a/docs/devbox/cli-reference/devbox-global-push/index.mdx b/docs/devbox/cli-reference/devbox-global-push/index.mdx index 081fa71..2629559 100644 --- a/docs/devbox/cli-reference/devbox-global-push/index.mdx +++ b/docs/devbox/cli-reference/devbox-global-push/index.mdx @@ -1,7 +1,6 @@ --- title: "devbox global push" -description: - "Push a [global] config. Leave empty to use Jetify sync. Can be a git repo for self storage." +description: "Push a [global] config. Leave empty to use Jetify sync. Can be a git repo for self storage." --- ```bash diff --git a/docs/devbox/cli-reference/devbox-services-restart/index.mdx b/docs/devbox/cli-reference/devbox-services-restart/index.mdx index 068bdcc..ec6a733 100644 --- a/docs/devbox/cli-reference/devbox-services-restart/index.mdx +++ b/docs/devbox/cli-reference/devbox-services-restart/index.mdx @@ -1,7 +1,6 @@ --- title: "devbox services restart" -description: - "Restarts service. If no service is specified, restarts all services and process-compose." +description: "Restarts service. If no service is specified, restarts all services and process-compose." --- ```bash diff --git a/docs/devbox/devbox-examples/languages/python/index.mdx b/docs/devbox/devbox-examples/languages/python/index.mdx index b6b0fb6..063b8eb 100644 --- a/docs/devbox/devbox-examples/languages/python/index.mdx +++ b/docs/devbox/devbox-examples/languages/python/index.mdx @@ -55,16 +55,14 @@ this path by setting the `VENV_DIR` environment variable in your devbox.json: ```json { - "packages": [ - "[email protected]" - ], - "env": { - // Install your virtual environment in `.venv` - "VENV_DIR": ".venv" - }, - "shell": { - "init_hook": ". $VENV_DIR/bin/activate" - } + "packages": ["[email protected]"], + "env": { + // Install your virtual environment in `.venv` + "VENV_DIR": ".venv" + }, + "shell": { + "init_hook": ". $VENV_DIR/bin/activate" + } } ``` @@ -74,13 +72,10 @@ install: ```json { - "packages": [ - "[email protected]", - "python310Packages.pip" - ], - "shell": { - "init_hook": ". $VENV_DIR/bin/activate" - } + "packages": ["[email protected]", "python310Packages.pip"], + "shell": { + "init_hook": ". $VENV_DIR/bin/activate" + } } ``` diff --git a/docs/devbox/devbox-examples/stacks/lapp/index.mdx b/docs/devbox/devbox-examples/stacks/lapp/index.mdx index 34eb939..d4f42cd 100644 --- a/docs/devbox/devbox-examples/stacks/lapp/index.mdx +++ b/docs/devbox/devbox-examples/stacks/lapp/index.mdx @@ -23,7 +23,7 @@ description: 2. Add the packages using the command below. Installing the packages with `devbox add` will ensure that the plugins are activated: -```bash +````bash devbox add postgresql@14 php php83Extensions.pgsql@latest [email protected] ```bash @@ -32,3 +32,4 @@ devbox add postgresql@14 php php83Extensions.pgsql@latest [email protected] 2. Follow the instructions above in the How to Run section to initialize your project [Edit this page](https://github.com/jetify-com/docs/tree/main/docs/devbox/devbox-examples/stacks/lapp/index.mdx) +```` diff --git a/docs/devbox/devbox-examples/stacks/laravel/index.mdx b/docs/devbox/devbox-examples/stacks/laravel/index.mdx index e6d2dba..be7f220 100644 --- a/docs/devbox/devbox-examples/stacks/laravel/index.mdx +++ b/docs/devbox/devbox-examples/stacks/laravel/index.mdx @@ -46,6 +46,8 @@ Devbox Plugins for all 3 Nix packages to simplify configuration ```bash devbox add mariadb@latest, [email protected], nodejs@18, redis@latest, php81Packages.composer@latest + ``` + ```bash 3. Run `devbox shell` to start your shell. This will also initialize your database by running @@ -53,8 +55,10 @@ Devbox Plugins for all 3 Nix packages to simplify configuration 4. Create your laravel project by running: - ``` - composer create-project laravel/laravel tmpmv tmp/* tmp/.* . +``` + +composer create-project laravel/laravel tmpmv tmp/_ tmp/._ . + ```bash ### Setting up MariaDB[​](#setting-up-mariadb "Direct link to Setting up MariaDB") @@ -63,7 +67,10 @@ To use MariaDB, you need to create the default Laravel database. You can do this following commands in your `devbox shell`: ``` + # Start the MariaDB servicedevbox services up mariadb -b# Create the databasemysql -u root -e "CREATE DATABASE laravel;"# Once you're done, stop the MariaDB servicedevbox services stop mariadb + ```bash [Edit this page](https://github.com/jetify-com/docs/tree/main/docs/devbox/devbox-examples/stacks/laravel/index.mdx) +``` diff --git a/docs/devbox/devbox-examples/stacks/lepp/index.mdx b/docs/devbox/devbox-examples/stacks/lepp/index.mdx index 79d70ee..f57eea3 100644 --- a/docs/devbox/devbox-examples/stacks/lepp/index.mdx +++ b/docs/devbox/devbox-examples/stacks/lepp/index.mdx @@ -35,7 +35,7 @@ You can query Nginx on port 80, which will route to the PHP example. 2. Add the packages using the command below. Installing the packages with `devbox add` will ensure that the plugins are activated: -```bash +````bash devbox add postgresql@14 [email protected] php81Extensions.pgsql@latest [email protected] ```bash @@ -50,6 +50,6 @@ path by setting the `PGHOST` env variable in your `devbox.json` as follows: "env": { "PGHOST": "/" } -``` +```` [Edit this page](https://github.com/jetify-com/docs/tree/main/docs/devbox/devbox-examples/stacks/lepp/index.mdx) diff --git a/docs/devbox/guides/plugins/index.mdx b/docs/devbox/guides/plugins/index.mdx index 4f7017f..987dcd7 100644 --- a/docs/devbox/guides/plugins/index.mdx +++ b/docs/devbox/guides/plugins/index.mdx @@ -56,16 +56,19 @@ provide a Github reference to a plugin hosted on Github. To install a github hos following to the include section of your devbox.json Default - `master` branch: + ``` "include": [ "github:/?dir=" ] ``` Branch override: + ``` "include": [ "github://?dir=" ] ``` Git Tag Pinning: + ``` "include": [ "github://tags/?dir=" ] ``` diff --git a/docs/devbox/guides/services/index.mdx b/docs/devbox/guides/services/index.mdx index 532ce90..cd9ae39 100644 --- a/docs/devbox/guides/services/index.mdx +++ b/docs/devbox/guides/services/index.mdx @@ -49,9 +49,9 @@ Django server, you could add the following yaml: version: "0.5" processes: django: - command: python todo_project/manage.py runserver - availability: - restart: "always" + command: python todo_project/manage.py runserver + availability: + restart: "always" ``` This will now start your django service whenever you run `devbox services up`. diff --git a/docs/devbox/ide-configuration/direnv/index.mdx b/docs/devbox/ide-configuration/direnv/index.mdx index d678c73..37a64fe 100644 --- a/docs/devbox/ide-configuration/direnv/index.mdx +++ b/docs/devbox/ide-configuration/direnv/index.mdx @@ -22,7 +22,7 @@ direnv to seamlessly work with a devbox project. If you have direnv installed, Devbox will generate an .envrc file when you run `devbox generate direnv` and enables it by running `direnv allow` in the background: -```bash +````bash ➜ devbox generate direnvSuccess: generated .envrc fileSuccess: ran `direnv allow`direnv: loading ~/src/docs/devbox/.envrcdirenv: using devbox ```bash @@ -38,7 +38,7 @@ For an existing project, you can add a `.envrc` file by running `devbox generate ```bash ➜ devbox generate direnvSuccess: generated .envrc fileSuccess: ran `direnv allow`direnv: loading ~/src/docs/devbox/.envrcdirenv: using devbox -``` +```` The generated `.envrc` file doesn't need any further configuration. Just having the generated file along with installed direnv and Devbox, is enough to make direnv integration with Devbox work. @@ -55,7 +55,7 @@ key-value pairs, where the key is the name of the environment variable and the v the environment variable. For example, if you wanted to add a `MY_CUSTOM_ENV_VAR` environment variable with a value of `my-custom-value`, you would run the following command: -```bash +````bash devbox generate direnv --env MY_CUSTOM_ENV_VAR=my-value ```bash @@ -63,7 +63,7 @@ The resulting .envrc will have the following: ```bash # Automatically sets up your devbox environment whenever you cd into this# directory via our direnv integration:eval "$(devbox generate direnv --print-envrc --env MY_CUSTOM_ENV_VAR=my-value)"# check out https://www.jetify.com/docs/devbox/ide-configuration/direnv/# for more details -``` +```` You can also tell direnv to read environment variables from a custom `.env` file by passing the `--env-file` flag to `devbox generate direnv`. This flag takes a path to a file containing @@ -71,7 +71,7 @@ environment variables to set in the devbox environment. If the file does not exi parameter is ignored. For example, if you wanted to add a `.env.devbox` file located in your project root, you would run the following command: -```bash +````bash devbox generate direnv --env-file .env.devbox ```bash @@ -79,7 +79,7 @@ The resulting .envrc will have the following: ```bash # Automatically sets up your devbox environment whenever you cd into this# directory via our direnv integration:eval "$(devbox generate direnv --print-envrc --env-file .env.devbox)"# check out https://www.jetify.com/docs/devbox/ide-configuration/direnv/# for more details -``` +```` Note that if Devbox cannot find the env file provided to the flag, it will ignore the flag and load your Devbox shell environment as normal diff --git a/docs/devbox/ide-configuration/vscode/index.mdx b/docs/devbox/ide-configuration/vscode/index.mdx index 6c2b18a..71fe9c7 100644 --- a/docs/devbox/ide-configuration/vscode/index.mdx +++ b/docs/devbox/ide-configuration/vscode/index.mdx @@ -102,7 +102,7 @@ follow the installation guide first. Then follow the steps below: ```json /nix/store/qaf9fysymdoj19qtyg7209s83lajz65b-zulu17.34.19-ca-jdk-17.0.3/bin/java - ``` +``` 4. Open VS Code and create a new Java project if you don't have already. If VS Code prompts for installing Java support choose yes. @@ -124,6 +124,8 @@ follow the installation guide first. Then follow the steps below: "projectName": "", "javaExec": "" } + ``` + ```json Update the values in between \< and > to match your project and environment. @@ -151,23 +153,28 @@ macOS terminal. 5. Use `javac` command to compile your Java project. As an example, if you have a simple hello world project and the directory structure such as: - ``` - my_java_project/-- src/-- -- main/-- -- -- hello.java -```json +``` + +my_java_project/-- src/-- -- main/-- -- -- hello.java + +````json You can use the following command to compile: to compile: ```bash javac my_java_project/src/main/hello.java - ``` +```` 6. Use `java` command to run the compiled project. For example, to run the sample project from above: ```bash cd src/java main/hello + ``` + ```json If this guide is missing something, feel free to contribute by opening a [pull request](https://github.com/jetify-com/devbox/pulls) in Github. [Edit this page](https://github.com/jetify-com/docs/tree/main/docs/devbox/ide-configuration/vscode/index.mdx) +``` diff --git a/docs/devbox/installing-devbox.mdx b/docs/devbox/installing-devbox.mdx index 69bd2b6..dca1e11 100644 --- a/docs/devbox/installing-devbox.mdx +++ b/docs/devbox/installing-devbox.mdx @@ -4,11 +4,11 @@ title: "Overview" ### Prerequisites -- Nix Package Manager +- Nix Package Manager -If Nix is not detected when running a command, Devbox will automatically install it for you. +If Nix is not detected when running a command, Devbox will automatically install it for you. Don't worry: You can use Devbox without needing to learn the Nix Language. -If you would like to install Nix yourself, we recommend the +If you would like to install Nix yourself, we recommend the [Determinate Nix Installer](https://determinate.systems/nix-installer/). ### Installation Script @@ -26,12 +26,13 @@ curl -fsSL https://get.jetify.com/devbox | bash Run the install script as a **non-root user**. Devbox will install Nix in single-user mode for Linux if not already present. - - - Devbox will install Nix in multi-user mode for macOS if not already present. + + Devbox will install Nix in multi-user mode for macOS if not already present. + + You can use Devbox on Windows using [**Windows Subsystem for Linux 2**](https://learn.microsoft.com/en-us/windows/wsl/install). @@ -51,8 +52,8 @@ curl -fsSL https://get.jetify.com/devbox | bash ``` If you are running an older version of Windows, you may need to follow the - [manual installation steps](https://learn.microsoft.com/en-us/windows/wsl/install-manual) - to enable virtualization and WSL2 on your system. + [manual installation steps](https://learn.microsoft.com/en-us/windows/wsl/install-manual) + to enable virtualization and WSL2 on your system. See the [official docs](https://learn.microsoft.com/en-us/windows/wsl/install) for more details. @@ -60,6 +61,7 @@ curl -fsSL https://get.jetify.com/devbox | bash Run the install script in your WSL2 terminal as a **non-root user**. Devbox will install Nix in single-user mode for WSL2 if not already present. + @@ -89,6 +91,7 @@ curl -fsSL https://get.jetify.com/devbox | bash installation. If you want to use the latest version of Devbox, you can install it using the [Nix Flake](/docs/devbox/installing-devbox/?install-method=flake). + @@ -107,6 +110,7 @@ curl -fsSL https://get.jetify.com/devbox | bash ```bash nix profile install github:jetify-com/devbox/0.13.2 ``` + diff --git a/docs/devbox/quickstart/index.mdx b/docs/devbox/quickstart/index.mdx index 8562f0e..ba308de 100644 --- a/docs/devbox/quickstart/index.mdx +++ b/docs/devbox/quickstart/index.mdx @@ -125,10 +125,7 @@ README.md to add the badge ```html - Built with Devbox + Built with Devbox ``` @@ -143,10 +140,7 @@ README.md to add the badge ```html - Built with Devbox + Built with Devbox ``` diff --git a/docs/testpilot/advanced/bash-tool.mdx b/docs/testpilot/advanced/bash-tool.mdx index 122a290..662f5f2 100644 --- a/docs/testpilot/advanced/bash-tool.mdx +++ b/docs/testpilot/advanced/bash-tool.mdx @@ -24,7 +24,10 @@ export BASH_TOOL=true Once the Bash tool is enabled, you can use it in your test cases by adding a step that describes a script or command you want to execute. You should specify the relative path to the script, as well as any arguments you want Testpilot to pass to the script. -Testpilot will use your current working directory and shell environment as the context for executing scripts. + + Testpilot will use your current working directory and shell environment as the context for + executing scripts. + Here is an example of how to define a test case that uses the Bash tool: @@ -66,4 +69,4 @@ echo "Generated $COUNT test items" exit 0 ``` -When you execute this test case, Testpilot will run the `generate-test-data.sh` script with the specified arguments, capture its output, and use that output to fill out the todo items on the TodoMVC application. \ No newline at end of file +When you execute this test case, Testpilot will run the `generate-test-data.sh` script with the specified arguments, capture its output, and use that output to fill out the todo items on the TodoMVC application. diff --git a/docs/testpilot/advanced/index.mdx b/docs/testpilot/advanced/index.mdx index 951afa6..e930435 100644 --- a/docs/testpilot/advanced/index.mdx +++ b/docs/testpilot/advanced/index.mdx @@ -7,7 +7,9 @@ description: "Advanced features for power users. Explore AI test generation, MCP This section covers advanced Testpilot features designed for specific use cases and power users. These features are optional but powerful tools that can enhance your testing workflow. -Before exploring advanced features, you should be comfortable with the basics covered in [Getting Started](/docs/testpilot/getting-started). These features are designed for users who have already run tests successfully and want to extend Testpilot's capabilities. + Before exploring advanced features, you should be comfortable with the basics covered in [Getting + Started](/docs/testpilot/getting-started). These features are designed for users who have already + run tests successfully and want to extend Testpilot's capabilities. ## What You'll Find Here @@ -38,4 +40,3 @@ These features complement the core testing workflow but are not required for eff - [Getting Started](/docs/testpilot/getting-started) - Learn the basics first - [Guides](/docs/testpilot/guides) - Practical solutions for common testing scenarios - [CLI Reference](/docs/testpilot/reference/cli) - Command-line options for advanced features - diff --git a/docs/testpilot/ai-test-generation.mdx b/docs/testpilot/ai-test-generation.mdx index a09b5c4..453e694 100644 --- a/docs/testpilot/ai-test-generation.mdx +++ b/docs/testpilot/ai-test-generation.mdx @@ -165,6 +165,7 @@ Test the login functionality at https://example.com ```markdown Test the login functionality at https://example.com including: + - Successful login with valid credentials - Failed login with incorrect password - Failed login with non-existent user diff --git a/docs/testpilot/best-practices/index.mdx b/docs/testpilot/best-practices/index.mdx index 9b5d00a..f1fa4ac 100644 --- a/docs/testpilot/best-practices/index.mdx +++ b/docs/testpilot/best-practices/index.mdx @@ -21,9 +21,13 @@ This approach creates tests that are more resilient to UI changes and easier to This section provides guidance on writing effective tests, optimizing test performance, and maintaining your test suite over time. - - Tips and patterns for creating reliable, maintainable tests. - + + Tips and patterns for creating reliable, maintainable tests. + ## Key Principles @@ -38,4 +42,3 @@ This section provides guidance on writing effective tests, optimizing test perfo - [Getting Started](/docs/testpilot/getting-started) - Learn the basics of writing tests - [Guides](/docs/testpilot/guides) - Practical examples of common testing scenarios - [Pilotfile Reference](/docs/testpilot/pilotfile-reference) - Complete reference for test file format - diff --git a/docs/testpilot/evaluation/analyze-json-report-evaluations.mdx b/docs/testpilot/evaluation/analyze-json-report-evaluations.mdx index e413a5b..49940c8 100644 --- a/docs/testpilot/evaluation/analyze-json-report-evaluations.mdx +++ b/docs/testpilot/evaluation/analyze-json-report-evaluations.mdx @@ -40,10 +40,11 @@ Test-level latency provides the most important metric for overall user experienc - **Resource planning**: Understanding typical execution times for capacity planning Key fields for analyzing Per test latency include: -* **Test.id**: Stable Identifier for analyzing your test across runs -* **Test.duration**: Wall time for how long the test took to run -* **Test.startTime**: Useful for organizing your tests chronologically -* **Test.profilingMetrics**: Splits test time into LLM Duration, Tool Duration, and Attempts, which can help you understand what is driving the test's overall duration. + +- **Test.id**: Stable Identifier for analyzing your test across runs +- **Test.duration**: Wall time for how long the test took to run +- **Test.startTime**: Useful for organizing your tests chronologically +- **Test.profilingMetrics**: Splits test time into LLM Duration, Tool Duration, and Attempts, which can help you understand what is driving the test's overall duration. ### Per-Step Latency Analysis @@ -56,20 +57,20 @@ Step-level latency analysis helps identify bottlenecks in test execution by exam Extract the duration from each step and track it alongside the step title and status to build comprehensive performance profiles. Key fields for analyzing Per-Step latency include: -* Step.title: The actual wording of the step provided to the LLM. You can use this, or the step's index for analysis across tests. -* Step.duration: Wall time for how long the step took -* Step.profilingMetrics: Shows the breakdown of duration by attempts, LLM Duration, and Tool Duration -* Step.executionMode: Shows how the step was executed (cache, agent, script, etc.) - see [Enum Reference](#enum-reference) -* Step.actions: The actual actions Taken by Testpilot, which can give you a granular understanding of what occurred during the step. -* Step.cacheStatus: Shows whether the step successfully used the Action cache, or had to fall back. See [Enum Reference](#enum-reference) -* Step.explanation: The LLM's interpretation of how the step ended. -Some things to evaluate when analyzing steps include: +- Step.title: The actual wording of the step provided to the LLM. You can use this, or the step's index for analysis across tests. +- Step.duration: Wall time for how long the step took +- Step.profilingMetrics: Shows the breakdown of duration by attempts, LLM Duration, and Tool Duration +- Step.executionMode: Shows how the step was executed (cache, agent, script, etc.) - see [Enum Reference](#enum-reference) +- Step.actions: The actual actions Taken by Testpilot, which can give you a granular understanding of what occurred during the step. +- Step.cacheStatus: Shows whether the step successfully used the Action cache, or had to fall back. See [Enum Reference](#enum-reference) +- Step.explanation: The LLM's interpretation of how the step ended. -* Do the profiling metrics indicate frequent retries of the step, or issues with any of the tool calls you are making? -* Does cache status consistently suggest an issue with caching capability? What reasons are preventing the cache from hitting on the action. -* Do you see a large number of waits, or repetitive actions in the step? +Some things to evaluate when analyzing steps include: +- Do the profiling metrics indicate frequent retries of the step, or issues with any of the tool calls you are making? +- Does cache status consistently suggest an issue with caching capability? What reasons are preventing the cache from hitting on the action. +- Do you see a large number of waits, or repetitive actions in the step? ## 2. Pass/Fail Metrics @@ -88,6 +89,7 @@ Step-level analysis helps identify which specific operations are most prone to f Pay attention to the `stepType` field as verification steps failing might indicate different issues than action steps failing. See the [StepType enum table](#steptype) for complete values. **Key aggregation strategies:** + - **Binary classification**: Treat STATUS_SUCCEEDED as "pass" and all others as "fail" for most analyses - **Step type segmentation**: Analyze action steps vs verification steps separately - **Failure categorization**: Use the `failureReasonCategory` field to group similar failure types @@ -103,12 +105,14 @@ The `explanation` field often contains valuable context about why a test failed, Tracking pass/fail rates over time is essential for identifying flaky tests and overall test suite stability: **Critical metrics to track:** + - **Overall pass rate**: Percentage of tests passing in recent runs - **Test stability**: Individual tests with inconsistent results across runs - **Flaky test identification**: Tests with pass rates between 20-80% indicating intermittent issues - **Failure clustering**: Whether failures concentrate around specific time periods or code changes **Analysis techniques:** + - **Stability scoring**: Calculate pass rates over rolling windows to identify consistently failing vs intermittent tests - **Trend detection**: Monitor whether test suite health is improving or degrading over time - **Outlier identification**: Flag tests that deviate significantly from expected pass rates @@ -120,6 +124,7 @@ Tracking pass/fail rates over time is essential for identifying flaky tests and TestPilot's caching system speeds up test execution by reusing previous step results when conditions are similar. Understanding cache performance helps optimize test execution time and identify optimization opportunities. **Key fields for cache analysis:** + - **Step.cacheStatus**: Indicates what happened with cache for this step (hit, miss, or unused with reason) - **Step.executionMode**: Shows how the step was actually executed (cache, agent, script, etc.) - **Test.cacheSourceId**: When present, indicates this test used another test as a cache source @@ -133,6 +138,7 @@ The most effective cache rate calculation focuses on **non-assertion steps** sin - **Calculate hit rate**: Cached steps divided by total non-assertion steps **Assertion step identification:** + - Step titles beginning with "verify," "assert," or "expect" (case-insensitive). - Steps with `stepType` of `STEP_TYPE_VERIFICATION` (3). @@ -157,6 +163,7 @@ See the [CacheStatus enum table](#cachestatus) for complete cache status values Track cache performance over time to understand optimization effectiveness: **Key trend indicators:** + - **Declining hit rates**: May indicate tests becoming more dynamic or environmental instability - **Consistent unused reasons**: Suggest systematic issues addressable through test design changes - **Cache source stability**: Tests frequently serving as cache sources should be prioritized for stability @@ -167,18 +174,21 @@ Track cache performance over time to understand optimization effectiveness: ### Essential Fields by Analysis Type **Latency Analysis:** + - `Test.duration` and `Step.duration`: Core timing metrics - `Test.id`: For grouping across runs - `ProfilingMetrics.totalDuration`, `llmDuration`, `toolDuration`: For detailed breakdowns - `Report.startTime`: For chronological ordering **Pass/Fail Analysis:** + - `Test.status` and `Step.status`: Core success metrics (focus on value 2 = SUCCESS) - `Test.explanation` and `Step.explanation`: Context for failures - `Step.stepType`: To differentiate verification vs action failures - `Step.failureReasonCategory`: For categorizing failure types **Cache Analysis:** + - `Step.executionMode`: How step was executed (primary metric) - `Step.cacheStatus`: Detailed cache behavior - `Test.cacheSourceId`: Cache dependency relationships @@ -192,56 +202,56 @@ TestPilot uses numeric enum values throughout the JSON reports. This section pro The `status` field appears on both `Test` and `Step` objects to indicate execution outcomes: -| Number | Name | Meaning | -|---|---|---| -| 0 | STATUS_UNSPECIFIED | Status is unknown or not set | -| 1 | STATUS_PENDING | Test or step is still in progress | -| 2 | STATUS_SUCCEEDED | Completed successfully (passed) | -| 3 | STATUS_FAILED | Completed with failure | -| 4 | STATUS_INCOMPLETE | Did not finish all steps (aborted or skipped) | +| Number | Name | Meaning | +| ------ | ------------------ | --------------------------------------------- | +| 0 | STATUS_UNSPECIFIED | Status is unknown or not set | +| 1 | STATUS_PENDING | Test or step is still in progress | +| 2 | STATUS_SUCCEEDED | Completed successfully (passed) | +| 3 | STATUS_FAILED | Completed with failure | +| 4 | STATUS_INCOMPLETE | Did not finish all steps (aborted or skipped) | ### ExecutionMode The `executionMode` field indicates how each step was executed: -| Number | Name | Meaning | -|---|---|---| -| 0 | EXECUTION_MODE_UNSPECIFIED | Mode not specified | -| 1 | EXECUTION_MODE_CUA | Executed by Computer Use Agent (LLM-driven) | -| 2 | EXECUTION_MODE_CACHE | Step executed from cache (no live LLM run) | -| 3 | EXECUTION_MODE_CACHE_FALLBACK_TO_CUA | Cache attempted but fell back to agent execution | -| 4 | EXECUTION_MODE_FALLBACK_TO_CUA | Non-CUA execution failed, fell back to agent | -| 5 | EXECUTION_MODE_NON_CUA | Non-CUA mode (deterministic/scripted) | -| 6 | EXECUTION_MODE_SCRIPT | Script mode execution | -| 7 | EXECUTION_MODE_FORM_FILLER | Form filler mode execution | +| Number | Name | Meaning | +| ------ | ------------------------------------ | ------------------------------------------------ | +| 0 | EXECUTION_MODE_UNSPECIFIED | Mode not specified | +| 1 | EXECUTION_MODE_CUA | Executed by Computer Use Agent (LLM-driven) | +| 2 | EXECUTION_MODE_CACHE | Step executed from cache (no live LLM run) | +| 3 | EXECUTION_MODE_CACHE_FALLBACK_TO_CUA | Cache attempted but fell back to agent execution | +| 4 | EXECUTION_MODE_FALLBACK_TO_CUA | Non-CUA execution failed, fell back to agent | +| 5 | EXECUTION_MODE_NON_CUA | Non-CUA mode (deterministic/scripted) | +| 6 | EXECUTION_MODE_SCRIPT | Script mode execution | +| 7 | EXECUTION_MODE_FORM_FILLER | Form filler mode execution | ### CacheStatus The `cacheStatus` field indicates what happened with cache for each step: -| Number | Name | Meaning | -|---|---|---| -| 0 | CACHE_STATUS_UNSPECIFIED | Cache status not specified | -| 1 | CACHE_STATUS_HIT | Cache entry found and used - optimal performance | -| 2 | CACHE_STATUS_MISS | No suitable cache entry found - expected for first runs | -| 3 | CACHE_STATUS_UNUSED_IS_RETRY | Cache ignored because step needed retry - may indicate flaky steps | -| 4 | CACHE_STATUS_UNUSED_IS_ASSERTION | Cache ignored for assertion-only step - expected behavior | -| 5 | CACHE_STATUS_UNUSED_NON_CACHEABLE_EXECUTION_MODE | Cache ignored due to non-cacheable execution mode | -| 6 | CACHE_STATUS_UNUSED_CONTAINS_NON_CACHEABLE_ACTIONS | Cache ignored due to non-cacheable actions | -| 7 | CACHE_STATUS_UNUSED_ELEMENT_IS_MISSING | Cache ignored because required element missing - UI changes may have broken cache assumptions | -| 8 | CACHE_STATUS_UNUSED_HAS_TOOL_CALLS | Cache ignored because step involved tool calls - dynamic behavior prevented caching | -| 9 | CACHE_STATUS_UNUSED_IS_SCRIPT | Cache ignored for script-based step | +| Number | Name | Meaning | +| ------ | -------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| 0 | CACHE_STATUS_UNSPECIFIED | Cache status not specified | +| 1 | CACHE_STATUS_HIT | Cache entry found and used - optimal performance | +| 2 | CACHE_STATUS_MISS | No suitable cache entry found - expected for first runs | +| 3 | CACHE_STATUS_UNUSED_IS_RETRY | Cache ignored because step needed retry - may indicate flaky steps | +| 4 | CACHE_STATUS_UNUSED_IS_ASSERTION | Cache ignored for assertion-only step - expected behavior | +| 5 | CACHE_STATUS_UNUSED_NON_CACHEABLE_EXECUTION_MODE | Cache ignored due to non-cacheable execution mode | +| 6 | CACHE_STATUS_UNUSED_CONTAINS_NON_CACHEABLE_ACTIONS | Cache ignored due to non-cacheable actions | +| 7 | CACHE_STATUS_UNUSED_ELEMENT_IS_MISSING | Cache ignored because required element missing - UI changes may have broken cache assumptions | +| 8 | CACHE_STATUS_UNUSED_HAS_TOOL_CALLS | Cache ignored because step involved tool calls - dynamic behavior prevented caching | +| 9 | CACHE_STATUS_UNUSED_IS_SCRIPT | Cache ignored for script-based step | ### StepType The `stepType` field classifies the type of step being executed: -| Number | Name | Meaning | -|---|---|---| -| 0 | STEP_TYPE_UNSPECIFIED | Step type not specified | -| 1 | STEP_TYPE_REGULAR_ACTION | Regular user actions (click, type, navigate) | -| 2 | STEP_TYPE_FORM_FILLING | Data entry or form completion | -| 3 | STEP_TYPE_VERIFICATION | Assertion or verification of expected state | +| Number | Name | Meaning | +| ------ | ------------------------ | -------------------------------------------- | +| 0 | STEP_TYPE_UNSPECIFIED | Step type not specified | +| 1 | STEP_TYPE_REGULAR_ACTION | Regular user actions (click, type, navigate) | +| 2 | STEP_TYPE_FORM_FILLING | Data entry or form completion | +| 3 | STEP_TYPE_VERIFICATION | Assertion or verification of expected state | ## Key Takeaways diff --git a/docs/testpilot/evaluation/testpilot-json-report-reference.mdx b/docs/testpilot/evaluation/testpilot-json-report-reference.mdx index 328c0aa..f4c592f 100644 --- a/docs/testpilot/evaluation/testpilot-json-report-reference.mdx +++ b/docs/testpilot/evaluation/testpilot-json-report-reference.mdx @@ -7,6 +7,7 @@ sidebarTitle: "report.json Reference" ## Overview TestPilot generates JSON reports in two formats: + - **Aggregated Report**: Top-level report containing all tests from a run - **Per-test Results**: Individual `results.json` files for each test @@ -315,23 +316,27 @@ Both formats share the same core object structures and use `camelCase` field nam ### Other Object Fields #### LLMMetrics + - **`promptTokens`** (integer): Number of input tokens sent to LLM - **`completionTokens`** (integer): Number of output tokens received from LLM - **`anthropicCachedInputTokens`** (integer): Number of cached input tokens (Anthropic-specific) - **`maxScreenshots`** (integer): Maximum number of screenshots allowed #### ProfilingMetrics + - **`totalDuration`** (string): Total time spent - **`llmDuration`** (string): Time spent waiting for LLM responses - **`toolDuration`** (string): Time spent executing tools/actions - **`attempts`** (integer): Number of execution attempts #### PlatformConfig + - **`url`** (string): Target URL for web testing - **`androidPkg`** (string): Android package name for mobile testing - **`iosPkg`** (string): iOS package name for mobile testing #### Viewport + - **`width`** (integer): Viewport width in pixels - **`height`** (integer): Viewport height in pixels @@ -339,55 +344,55 @@ Both formats share the same core object structures and use `camelCase` field nam ### Status -| Number | Name | Meaning | -|---|---|---| -| 0 | STATUS_UNSPECIFIED | Status is unknown or not set | -| 1 | STATUS_PENDING | Test or step is still in progress | -| 2 | STATUS_SUCCEEDED | Completed successfully (passed) | -| 3 | STATUS_FAILED | Completed with failure | -| 4 | STATUS_INCOMPLETE | Did not finish all steps (aborted or skipped) | +| Number | Name | Meaning | +| ------ | ------------------ | --------------------------------------------- | +| 0 | STATUS_UNSPECIFIED | Status is unknown or not set | +| 1 | STATUS_PENDING | Test or step is still in progress | +| 2 | STATUS_SUCCEEDED | Completed successfully (passed) | +| 3 | STATUS_FAILED | Completed with failure | +| 4 | STATUS_INCOMPLETE | Did not finish all steps (aborted or skipped) | ### ExecutionMode -| Number | Name | Meaning | -|---|---|---| -| 0 | EXECUTION_MODE_UNSPECIFIED | Mode not specified | -| 1 | EXECUTION_MODE_CUA | Executed by Computer Use Agent (LLM-driven) | -| 2 | EXECUTION_MODE_CACHE | Step executed from cache (no live LLM run) | -| 3 | EXECUTION_MODE_CACHE_FALLBACK_TO_CUA | Cache attempted but fell back to agent execution | -| 4 | EXECUTION_MODE_FALLBACK_TO_CUA | Non-CUA execution failed, fell back to agent | -| 5 | EXECUTION_MODE_NON_CUA | Non-CUA mode (deterministic/scripted) | -| 6 | EXECUTION_MODE_SCRIPT | Script mode execution | -| 7 | EXECUTION_MODE_FORM_FILLER | Form filler mode execution | +| Number | Name | Meaning | +| ------ | ------------------------------------ | ------------------------------------------------ | +| 0 | EXECUTION_MODE_UNSPECIFIED | Mode not specified | +| 1 | EXECUTION_MODE_CUA | Executed by Computer Use Agent (LLM-driven) | +| 2 | EXECUTION_MODE_CACHE | Step executed from cache (no live LLM run) | +| 3 | EXECUTION_MODE_CACHE_FALLBACK_TO_CUA | Cache attempted but fell back to agent execution | +| 4 | EXECUTION_MODE_FALLBACK_TO_CUA | Non-CUA execution failed, fell back to agent | +| 5 | EXECUTION_MODE_NON_CUA | Non-CUA mode (deterministic/scripted) | +| 6 | EXECUTION_MODE_SCRIPT | Script mode execution | +| 7 | EXECUTION_MODE_FORM_FILLER | Form filler mode execution | ### CacheStatus -| Number | Name | Meaning | -|---|---|---| -| 0 | CACHE_STATUS_UNSPECIFIED | Cache status not specified | -| 1 | CACHE_STATUS_HIT | Cache entry found and used | -| 2 | CACHE_STATUS_MISS | No suitable cache entry found | -| 3 | CACHE_STATUS_UNUSED_IS_RETRY | Cache ignored because step needed retry | -| 4 | CACHE_STATUS_UNUSED_IS_ASSERTION | Cache ignored for assertion-only step | -| 5 | CACHE_STATUS_UNUSED_NON_CACHEABLE_EXECUTION_MODE | Cache ignored due to non-cacheable execution mode | -| 6 | CACHE_STATUS_UNUSED_CONTAINS_NON_CACHEABLE_ACTIONS | Cache ignored due to non-cacheable actions | -| 7 | CACHE_STATUS_UNUSED_ELEMENT_IS_MISSING | Cache ignored because required element missing | -| 8 | CACHE_STATUS_UNUSED_HAS_TOOL_CALLS | Cache ignored because step involved tool calls | -| 9 | CACHE_STATUS_UNUSED_IS_SCRIPT | Cache ignored for script-based step | +| Number | Name | Meaning | +| ------ | -------------------------------------------------- | ------------------------------------------------- | +| 0 | CACHE_STATUS_UNSPECIFIED | Cache status not specified | +| 1 | CACHE_STATUS_HIT | Cache entry found and used | +| 2 | CACHE_STATUS_MISS | No suitable cache entry found | +| 3 | CACHE_STATUS_UNUSED_IS_RETRY | Cache ignored because step needed retry | +| 4 | CACHE_STATUS_UNUSED_IS_ASSERTION | Cache ignored for assertion-only step | +| 5 | CACHE_STATUS_UNUSED_NON_CACHEABLE_EXECUTION_MODE | Cache ignored due to non-cacheable execution mode | +| 6 | CACHE_STATUS_UNUSED_CONTAINS_NON_CACHEABLE_ACTIONS | Cache ignored due to non-cacheable actions | +| 7 | CACHE_STATUS_UNUSED_ELEMENT_IS_MISSING | Cache ignored because required element missing | +| 8 | CACHE_STATUS_UNUSED_HAS_TOOL_CALLS | Cache ignored because step involved tool calls | +| 9 | CACHE_STATUS_UNUSED_IS_SCRIPT | Cache ignored for script-based step | ### StepType -| Number | Name | Meaning | -|---|---|---| -| 0 | STEP_TYPE_UNSPECIFIED | Step type not specified | -| 1 | STEP_TYPE_REGULAR_ACTION | Regular user actions (click, type, navigate) | -| 2 | STEP_TYPE_FORM_FILLING | Data entry or form completion | -| 3 | STEP_TYPE_VERIFICATION | Assertion or verification of expected state | - +| Number | Name | Meaning | +| ------ | ------------------------ | -------------------------------------------- | +| 0 | STEP_TYPE_UNSPECIFIED | Step type not specified | +| 1 | STEP_TYPE_REGULAR_ACTION | Regular user actions (click, type, navigate) | +| 2 | STEP_TYPE_FORM_FILLING | Data entry or form completion | +| 3 | STEP_TYPE_VERIFICATION | Assertion or verification of expected state | ## Example Usage ### Basic Report Structure + ```json { "id": "tpreport_01abc123...", @@ -395,8 +400,8 @@ Both formats share the same core object structures and use `camelCase` field nam "duration": "45.123 s", "platforms": ["Linux - Chrome"], "statusSummary": [ - {"count": 3, "status": 2}, - {"count": 1, "status": 3} + { "count": 3, "status": 2 }, + { "count": 1, "status": 3 } ], "tests": [ { diff --git a/docs/testpilot/getting-started/index.mdx b/docs/testpilot/getting-started/index.mdx index fb6ac11..dc8073c 100644 --- a/docs/testpilot/getting-started/index.mdx +++ b/docs/testpilot/getting-started/index.mdx @@ -12,7 +12,11 @@ quickly. Install Testpilot and configure your environment in minutes. - + Write and run your first browser-based test with Testpilot. diff --git a/docs/testpilot/getting-started/testing-android-apps.mdx b/docs/testpilot/getting-started/testing-android-apps.mdx index 916060a..df8d52b 100644 --- a/docs/testpilot/getting-started/testing-android-apps.mdx +++ b/docs/testpilot/getting-started/testing-android-apps.mdx @@ -13,10 +13,10 @@ Before you can run tests on Android devices, you'll need to set up Appium: ```bash # Install Appium globally npm install -g appium - + # Install Appium Doctor for diagnostics npm install -g appium-doctor - + # Install the Android driver appium driver install uiautomator2 ``` @@ -90,7 +90,7 @@ cases: name: "Basic Android Navigation" description: "Test basic navigation in the Android app" platform_config: - android_pkg: "com.example.myapp" # Replace with your app's package name + android_pkg: "com.example.myapp" # Replace with your app's package name steps: - "Verify the app launches successfully" - "Tap on the 'Login' button" @@ -120,8 +120,8 @@ You can specify an app and URL for both browser based and native application tes ```yaml platform_config: - url: "https://example.com" # For browser-based tests - android_pkg: "com.example.myapp" # For native app tests + url: "https://example.com" # For browser-based tests + android_pkg: "com.example.myapp" # For native app tests ``` - Use `url` for browser-based tests on Android Chrome diff --git a/docs/testpilot/getting-started/testing-ios-apps.mdx b/docs/testpilot/getting-started/testing-ios-apps.mdx index cdfca22..7ccc259 100644 --- a/docs/testpilot/getting-started/testing-ios-apps.mdx +++ b/docs/testpilot/getting-started/testing-ios-apps.mdx @@ -14,10 +14,10 @@ dependencies: ```bash # Install Appium globally npm install -g appium - + # Install Appium Doctor for diagnostics npm install -g appium-doctor - + # Install the XCUITest driver for iOS testing appium driver install xcuitest ``` @@ -70,7 +70,7 @@ cases: name: "Basic iOS Navigation" description: "Test basic navigation in the iOS app" platform_config: - ios_bundle: "com.example.MyApp" # Replace with your app's bundle identifier + ios_bundle: "com.example.MyApp" # Replace with your app's bundle identifier steps: - "Verify the app launches successfully" - "Tap on the 'Login' button" @@ -116,7 +116,7 @@ The `platform_config` section of your test file is crucial for iOS testing: ```yaml platform_config: - url: "https://example.com" # For browser-based tests + url: "https://example.com" # For browser-based tests ios_bundle: "com.example.MyApp" # For native app tests ``` diff --git a/docs/testpilot/getting-started/testing-web-apps.mdx b/docs/testpilot/getting-started/testing-web-apps.mdx index 90fc55f..fb11187 100644 --- a/docs/testpilot/getting-started/testing-web-apps.mdx +++ b/docs/testpilot/getting-started/testing-web-apps.mdx @@ -22,7 +22,7 @@ cases: name: "Search for Devbox on Hacker News" description: "Navigate to news.ycombinator.com, search for Devbox, and click the first search result" url: "https://news.ycombinator.com" - + # List of Test Steps steps: - "Use the search bar on the front page to search for Devbox" diff --git a/docs/testpilot/guides/parameterize-tests-environment-variables.mdx b/docs/testpilot/guides/parameterize-tests-environment-variables.mdx index 8f87ac5..ad221a5 100644 --- a/docs/testpilot/guides/parameterize-tests-environment-variables.mdx +++ b/docs/testpilot/guides/parameterize-tests-environment-variables.mdx @@ -106,7 +106,7 @@ test_staging: test_production: stage: test - when: manual # Only run when triggered manually + when: manual # Only run when triggered manually script: - export TEST_HOST=https://example.com - testpilot test tests/*.pilot.yaml diff --git a/docs/testpilot/guides/test-authentication.mdx b/docs/testpilot/guides/test-authentication.mdx index a7fae6a..1c09727 100644 --- a/docs/testpilot/guides/test-authentication.mdx +++ b/docs/testpilot/guides/test-authentication.mdx @@ -21,6 +21,7 @@ manager of your choice to set these credentials before running Testpilot ## Configuring Custom Environment Variables For Login[](#configuring-custom-environment-variables-for-login "Direct link to Configuring Custom Environment Variables for Login") Alternatively, you can also use custom env-vars of your own choosing by specifying them in the pilot file: + ```yaml name: "Configuring Environment Variables for Login" login: @@ -127,6 +128,6 @@ cases: android_pkg: com.mymobileapp.android ``` -If you are using a custom env-var for the password (i.e. other than `TESTPILOT_PASSWORD`) then you are -encouraged to specify the top-level `login.password` field as well. This helps Testpilot handle the -password string literal more securely. \ No newline at end of file +If you are using a custom env-var for the password (i.e. other than `TESTPILOT_PASSWORD`) then you are +encouraged to specify the top-level `login.password` field as well. This helps Testpilot handle the +password string literal more securely. diff --git a/docs/testpilot/guides/test-responsive-design.mdx b/docs/testpilot/guides/test-responsive-design.mdx index cabcd4c..d160872 100644 --- a/docs/testpilot/guides/test-responsive-design.mdx +++ b/docs/testpilot/guides/test-responsive-design.mdx @@ -80,11 +80,11 @@ viewports: ```yaml viewports: - name: "Mobile Large" - size: [428, 926] # iPhone 13 Pro Max + size: [428, 926] # iPhone 13 Pro Max - name: "Mobile Medium" - size: [390, 844] # iPhone 13 Pro + size: [390, 844] # iPhone 13 Pro - name: "Mobile Small" - size: [375, 667] # iPhone SE + size: [375, 667] # iPhone SE ``` ## Tips for Viewport Testing diff --git a/docs/testpilot/integrations/gitlab-cicd.mdx b/docs/testpilot/integrations/gitlab-cicd.mdx index 7d2fe0f..6b66528 100644 --- a/docs/testpilot/integrations/gitlab-cicd.mdx +++ b/docs/testpilot/integrations/gitlab-cicd.mdx @@ -37,9 +37,9 @@ stages: # Your test job run-my-tests: stage: test - needs: ["testpilot-install-linux-amd64"] # Wait for Testpilot installation + needs: ["testpilot-install-linux-amd64"] # Wait for Testpilot installation script: - - ./bin/testpilot test your_test.pilot.yaml # Replace with your test file + - ./bin/testpilot test your_test.pilot.yaml # Replace with your test file # preserve artifacts and reports artifacts: reports: @@ -120,7 +120,7 @@ setup. Here's how to configure your pipeline to use GitLab's SaaS macOS runners: # macOS runner configuration .macos_saas_runners: tags: - - saas-macos-medium-m1 # Use GitLab's SaaS macOS runners + - saas-macos-medium-m1 # Use GitLab's SaaS macOS runners ``` **Step 2:** Include the component with macOS-specific settings: @@ -130,9 +130,9 @@ setup. Here's how to configure your pipeline to use GitLab's SaaS macOS runners: include: - component: gitlab.com/jetify-com/gitlab-components/testpilot-install@main inputs: - stage: install # Install Testpilot in the 'install' stage - platform: macos # Target the macOS platform - runner-extends: .macos_saas_runners # Use the macOS SaaS runner config + stage: install # Install Testpilot in the 'install' stage + platform: macos # Target the macOS platform + runner-extends: .macos_saas_runners # Use the macOS SaaS runner config # Define pipeline stages stages: @@ -145,7 +145,7 @@ run-tests-macos: extends: .macos_saas_runners needs: ["testpilot-install-macos"] script: - - ./bin/testpilot test your_test.pilot.yaml # Replace with your test file + - ./bin/testpilot test your_test.pilot.yaml # Replace with your test file ``` ### Setting up Custom Runners[​](#setting-up-custom-runners "Direct link to Setting up Custom Runners") diff --git a/docs/testpilot/integrations/index.mdx b/docs/testpilot/integrations/index.mdx index 184bc55..f0d76ff 100644 --- a/docs/testpilot/integrations/index.mdx +++ b/docs/testpilot/integrations/index.mdx @@ -11,13 +11,18 @@ Testpilot integrates seamlessly with your existing CI/CD workflows, allowing you This section covers integrations with popular CI/CD platforms and development tools. You'll learn how to set up Testpilot in your pipeline, configure test execution, and manage test results. - - Set up Testpilot in your GitLab pipelines with the official component. - + + Set up Testpilot in your GitLab pipelines with the official component. + -Additional integrations for GitHub Actions, Jenkins, and other platforms are coming soon. For now, you can run Testpilot in any CI/CD environment that supports command-line tools. + Additional integrations for GitHub Actions, Jenkins, and other platforms are coming soon. For now, + you can run Testpilot in any CI/CD environment that supports command-line tools. ## Setting Up CI/CD Testing @@ -35,4 +40,3 @@ Each integration guide provides specific instructions for your platform. - [CLI Reference](/docs/testpilot/reference/cli) - Command-line options for running tests - [Best Practices](/docs/testpilot/best-practices) - Tips for writing effective CI/CD tests - From f291fc801c3baf7b912efb7816a1dd437031a9fd Mon Sep 17 00:00:00 2001 From: Sean Poulter Date: Mon, 9 Feb 2026 15:20:13 -0500 Subject: [PATCH 3/3] Fix formatting --- .../devbox-examples/stacks/lapp/index.mdx | 11 +++++----- .../devbox-examples/stacks/laravel/index.mdx | 22 +++++++++---------- .../devbox-examples/stacks/lepp/index.mdx | 6 ++--- .../devbox/ide-configuration/direnv/index.mdx | 20 ++++++++--------- .../devbox/ide-configuration/vscode/index.mdx | 19 +++++----------- 5 files changed, 35 insertions(+), 43 deletions(-) diff --git a/docs/devbox/devbox-examples/stacks/lapp/index.mdx b/docs/devbox/devbox-examples/stacks/lapp/index.mdx index d4f42cd..863569c 100644 --- a/docs/devbox/devbox-examples/stacks/lapp/index.mdx +++ b/docs/devbox/devbox-examples/stacks/lapp/index.mdx @@ -23,13 +23,12 @@ description: 2. Add the packages using the command below. Installing the packages with `devbox add` will ensure that the plugins are activated: -````bash -devbox add postgresql@14 php php83Extensions.pgsql@latest [email protected] -```bash + ```bash + devbox add postgresql@14 php php83Extensions.pgsql@latest [email protected] + ``` -1. Update `devbox.d/apache/httpd.conf` to point to the directory with your PHP files. You'll need to +3. Update `devbox.d/apache/httpd.conf` to point to the directory with your PHP files. You'll need to update the `DocumentRoot` and `Directory` directives. -2. Follow the instructions above in the How to Run section to initialize your project +4. Follow the instructions above in the How to Run section to initialize your project [Edit this page](https://github.com/jetify-com/docs/tree/main/docs/devbox/devbox-examples/stacks/lapp/index.mdx) -```` diff --git a/docs/devbox/devbox-examples/stacks/laravel/index.mdx b/docs/devbox/devbox-examples/stacks/laravel/index.mdx index be7f220..7fe8fdd 100644 --- a/docs/devbox/devbox-examples/stacks/laravel/index.mdx +++ b/docs/devbox/devbox-examples/stacks/laravel/index.mdx @@ -48,29 +48,29 @@ Devbox Plugins for all 3 Nix packages to simplify configuration devbox add mariadb@latest, [email protected], nodejs@18, redis@latest, php81Packages.composer@latest ``` -```bash - 3. Run `devbox shell` to start your shell. This will also initialize your database by running `initdb` in the init hook. 4. Create your laravel project by running: -``` - -composer create-project laravel/laravel tmpmv tmp/_ tmp/._ . - -```bash + ```bash + composer create-project laravel/laravel tmpmv tmp/* tmp/.* . + ``` ### Setting up MariaDB[​](#setting-up-mariadb "Direct link to Setting up MariaDB") To use MariaDB, you need to create the default Laravel database. You can do this by running the following commands in your `devbox shell`: -``` +```bash +# Start the MariaDB service +devbox services up mariadb -b -# Start the MariaDB servicedevbox services up mariadb -b# Create the databasemysql -u root -e "CREATE DATABASE laravel;"# Once you're done, stop the MariaDB servicedevbox services stop mariadb +# Create the database +mysql -u root -e "CREATE DATABASE laravel;" -```bash +# Once you're done, stop the MariaDB service +devbox services stop mariadb +``` [Edit this page](https://github.com/jetify-com/docs/tree/main/docs/devbox/devbox-examples/stacks/laravel/index.mdx) -``` diff --git a/docs/devbox/devbox-examples/stacks/lepp/index.mdx b/docs/devbox/devbox-examples/stacks/lepp/index.mdx index f57eea3..de2ee05 100644 --- a/docs/devbox/devbox-examples/stacks/lepp/index.mdx +++ b/docs/devbox/devbox-examples/stacks/lepp/index.mdx @@ -35,9 +35,9 @@ You can query Nginx on port 80, which will route to the PHP example. 2. Add the packages using the command below. Installing the packages with `devbox add` will ensure that the plugins are activated: -````bash -devbox add postgresql@14 [email protected] php81Extensions.pgsql@latest [email protected] ```bash +devbox add postgresql@14 [email protected] php81Extensions.pgsql@latest [email protected] +``` 1. Update `devbox.d/nginx/httpd.conf` to point to the directory with your PHP files. You'll need to update the `root` directive to point to your project folder @@ -50,6 +50,6 @@ path by setting the `PGHOST` env variable in your `devbox.json` as follows: "env": { "PGHOST": "/" } -```` +``` [Edit this page](https://github.com/jetify-com/docs/tree/main/docs/devbox/devbox-examples/stacks/lepp/index.mdx) diff --git a/docs/devbox/ide-configuration/direnv/index.mdx b/docs/devbox/ide-configuration/direnv/index.mdx index 37a64fe..84b6e69 100644 --- a/docs/devbox/ide-configuration/direnv/index.mdx +++ b/docs/devbox/ide-configuration/direnv/index.mdx @@ -22,9 +22,9 @@ direnv to seamlessly work with a devbox project. If you have direnv installed, Devbox will generate an .envrc file when you run `devbox generate direnv` and enables it by running `direnv allow` in the background: -````bash +``` ➜ devbox generate direnvSuccess: generated .envrc fileSuccess: ran `direnv allow`direnv: loading ~/src/docs/devbox/.envrcdirenv: using devbox -```bash +``` This will generate a `.envrc` file in your project directory that contains `devbox.json`. Run `direnv allow` to activate your shell upon directory navigation. Run `direnv revoke` to stop. @@ -36,9 +36,9 @@ installed direnv and Devbox is enough to make direnv integrate with Devbox. For an existing project, you can add a `.envrc` file by running `devbox generate direnv`: -```bash +``` ➜ devbox generate direnvSuccess: generated .envrc fileSuccess: ran `direnv allow`direnv: loading ~/src/docs/devbox/.envrcdirenv: using devbox -```` +``` The generated `.envrc` file doesn't need any further configuration. Just having the generated file along with installed direnv and Devbox, is enough to make direnv integration with Devbox work. @@ -55,15 +55,15 @@ key-value pairs, where the key is the name of the environment variable and the v the environment variable. For example, if you wanted to add a `MY_CUSTOM_ENV_VAR` environment variable with a value of `my-custom-value`, you would run the following command: -````bash -devbox generate direnv --env MY_CUSTOM_ENV_VAR=my-value ```bash +devbox generate direnv --env MY_CUSTOM_ENV_VAR=my-value +``` The resulting .envrc will have the following: ```bash # Automatically sets up your devbox environment whenever you cd into this# directory via our direnv integration:eval "$(devbox generate direnv --print-envrc --env MY_CUSTOM_ENV_VAR=my-value)"# check out https://www.jetify.com/docs/devbox/ide-configuration/direnv/# for more details -```` +``` You can also tell direnv to read environment variables from a custom `.env` file by passing the `--env-file` flag to `devbox generate direnv`. This flag takes a path to a file containing @@ -71,15 +71,15 @@ environment variables to set in the devbox environment. If the file does not exi parameter is ignored. For example, if you wanted to add a `.env.devbox` file located in your project root, you would run the following command: -````bash -devbox generate direnv --env-file .env.devbox ```bash +devbox generate direnv --env-file .env.devbox +``` The resulting .envrc will have the following: ```bash # Automatically sets up your devbox environment whenever you cd into this# directory via our direnv integration:eval "$(devbox generate direnv --print-envrc --env-file .env.devbox)"# check out https://www.jetify.com/docs/devbox/ide-configuration/direnv/# for more details -```` +``` Note that if Devbox cannot find the env file provided to the flag, it will ignore the flag and load your Devbox shell environment as normal diff --git a/docs/devbox/ide-configuration/vscode/index.mdx b/docs/devbox/ide-configuration/vscode/index.mdx index 71fe9c7..3838de4 100644 --- a/docs/devbox/ide-configuration/vscode/index.mdx +++ b/docs/devbox/ide-configuration/vscode/index.mdx @@ -100,9 +100,9 @@ follow the installation guide first. Then follow the steps below: executable java binary inside the devbox shell. Copy and save that path. It should look something like this: -```json + ```json /nix/store/qaf9fysymdoj19qtyg7209s83lajz65b-zulu17.34.19-ca-jdk-17.0.3/bin/java -``` + ``` 4. Open VS Code and create a new Java project if you don't have already. If VS Code prompts for installing Java support choose yes. @@ -126,8 +126,6 @@ follow the installation guide first. Then follow the steps below: } ``` -```json - Update the values in between \< and > to match your project and environment. 8. Click on **Run and Debug** or the green triangle at the top of the left sidebar to run and debug @@ -153,17 +151,15 @@ macOS terminal. 5. Use `javac` command to compile your Java project. As an example, if you have a simple hello world project and the directory structure such as: -``` - -my_java_project/-- src/-- -- main/-- -- -- hello.java - -````json + ``` + my_java_project/-- src/-- -- main/-- -- -- hello.java + ``` You can use the following command to compile: to compile: ```bash javac my_java_project/src/main/hello.java -```` + ``` 6. Use `java` command to run the compiled project. For example, to run the sample project from above: @@ -171,10 +167,7 @@ my_java_project/-- src/-- -- main/-- -- -- hello.java cd src/java main/hello ``` -```json - If this guide is missing something, feel free to contribute by opening a [pull request](https://github.com/jetify-com/devbox/pulls) in Github. [Edit this page](https://github.com/jetify-com/docs/tree/main/docs/devbox/ide-configuration/vscode/index.mdx) -```