From 0bea36fcdf6f2724af3779593264a8eb763b2959 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Mon, 24 Nov 2025 20:52:15 -0300 Subject: [PATCH] ci(tests): Add support for multiple duts --- .github/scripts/sketch_utils.sh | 13 +- .github/scripts/tests_build.sh | 168 +++++++++++- .github/scripts/tests_run.sh | 255 ++++++++++++++++- .github/scripts/tests_utils.sh | 36 +++ docs/en/contributing.rst | 184 +++++++++++-- libraries/BLE/examples/Server/Server.ino | 2 +- .../Server_secure_authorization.ino | 2 +- .../Server_secure_static_passkey.ino | 2 +- tests/conftest.py | 21 +- tests/validation/ble/ci.yml | 17 ++ tests/validation/ble/client/client.ino | 257 ++++++++++++++++++ tests/validation/ble/server/server.ino | 185 +++++++++++++ tests/validation/ble/test_ble.py | 151 ++++++++++ tests/validation/wifi_ap/ap/ap.ino | 75 +++++ tests/validation/wifi_ap/ci.yml | 28 ++ tests/validation/wifi_ap/client/client.ino | 107 ++++++++ tests/validation/wifi_ap/test_wifi_ap.py | 79 ++++++ 17 files changed, 1531 insertions(+), 51 deletions(-) create mode 100644 .github/scripts/tests_utils.sh create mode 100644 tests/validation/ble/ci.yml create mode 100644 tests/validation/ble/client/client.ino create mode 100644 tests/validation/ble/server/server.ino create mode 100644 tests/validation/ble/test_ble.py create mode 100644 tests/validation/wifi_ap/ap/ap.ino create mode 100644 tests/validation/wifi_ap/ci.yml create mode 100644 tests/validation/wifi_ap/client/client.ino create mode 100644 tests/validation/wifi_ap/test_wifi_ap.py diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh index 668d2ab7bd8..8093b350bfb 100755 --- a/.github/scripts/sketch_utils.sh +++ b/.github/scripts/sketch_utils.sh @@ -396,14 +396,25 @@ function count_sketches { # count_sketches [target] [ignore-requirements] local sketchdirname local sketchname local has_requirements + local parent_dir sketchdir=$(dirname "$sketch") sketchdirname=$(basename "$sketchdir") sketchname=$(basename "$sketch") + parent_dir=$(dirname "$sketchdir") if [[ "$sketchdirname.ino" != "$sketchname" ]]; then continue - elif [[ -n $target ]] && [[ -f $sketchdir/ci.yml ]]; then + # Skip sketches that are part of multi-device tests (they are built separately) + elif [[ -f "$parent_dir/ci.yml" ]]; then + local has_multi_device + has_multi_device=$(yq eval '.multi_device' "$parent_dir/ci.yml" 2>/dev/null) + if [[ "$has_multi_device" != "null" && "$has_multi_device" != "" ]]; then + continue + fi + fi + + if [[ -n $target ]] && [[ -f $sketchdir/ci.yml ]]; then # If the target is listed as false, skip the sketch. Otherwise, include it. is_target=$(yq eval ".targets.${target}" "$sketchdir"/ci.yml 2>/dev/null) if [[ "$is_target" == "false" ]]; then diff --git a/.github/scripts/tests_build.sh b/.github/scripts/tests_build.sh index 93342c83299..d27cbc33c15 100755 --- a/.github/scripts/tests_build.sh +++ b/.github/scripts/tests_build.sh @@ -18,6 +18,111 @@ function clean { find tests/ -name 'result_*.json' -exec rm -rf "{}" \+ } +# Check if a test is a multi-device test +function is_multi_device_test { + local test_dir=$1 + local has_multi_device + if [ -f "$test_dir/ci.yml" ]; then + has_multi_device=$(yq eval '.multi_device' "$test_dir/ci.yml" 2>/dev/null) + if [[ "$has_multi_device" != "null" && "$has_multi_device" != "" ]]; then + echo "1" + return + fi + fi + echo "0" +} + +# Extract target from arguments and return target and remaining arguments +# Usage: extract_target_and_args "$@" +# Returns: Sets global variables 'target' and 'remaining_args' array +function extract_target_and_args { + target="" + remaining_args=() + while [ -n "$1" ]; do + case $1 in + -t ) + shift + target=$1 + remaining_args+=("-t" "$target") + ;; + * ) + remaining_args+=("$1") + ;; + esac + shift + done +} + +# Build a single sketch for multi-device tests +function build_multi_device_sketch { + local test_name=$1 + local sketch_path=$2 + local target=$3 + shift 3 + local build_args=("$@") + local sketch_dir + local sketch_name + local test_dir + local build_dir + + sketch_dir=$(dirname "$sketch_path") + sketch_name=$(basename "$sketch_dir") + test_dir=$(dirname "$sketch_dir") + + # Override build directory to use _ pattern + build_dir="$HOME/.arduino/tests/$target/${test_name}_${sketch_name}/build.tmp" + + echo "Building sketch $sketch_name for multi-device test $test_name" + + # Call sketch_utils.sh build function with custom build directory + ARDUINO_BUILD_DIR="$build_dir" ${SCRIPTS_DIR}/sketch_utils.sh build "${build_args[@]}" -s "$sketch_dir" + return $? +} + +# Build all sketches for a multi-device test +function build_multi_device_test { + local test_dir=$1 + local target=$2 + shift 2 + local build_args=("$@") + local test_name + local devices + + test_name=$(basename "$test_dir") + + echo "Building multi-device test $test_name" + + # Get the list of devices from ci.yml + devices=$(yq eval '.multi_device | keys | .[]' "$test_dir/ci.yml" 2>/dev/null) + + if [[ -z "$devices" ]]; then + echo "ERROR: No devices found in multi_device configuration for $test_name" + return 1 + fi + + local result=0 + local sketch_name + local sketch_path + for device in $devices; do + sketch_name=$(yq eval ".multi_device.$device" "$test_dir/ci.yml" 2>/dev/null) + sketch_path="$test_dir/$sketch_name/$sketch_name.ino" + + if [ ! -f "$sketch_path" ]; then + echo "ERROR: Sketch not found: $sketch_path" + return 1 + fi + + build_multi_device_sketch "$test_name" "$sketch_path" "$target" "${build_args[@]}" + result=$? + if [ $result -ne 0 ]; then + echo "ERROR: Failed to build sketch $sketch_name for test $test_name" + return $result + fi + done + + return 0 +} + SCRIPTS_DIR="./.github/scripts" BUILD_CMD="" @@ -53,15 +158,13 @@ done source "${SCRIPTS_DIR}/install-arduino-cli.sh" source "${SCRIPTS_DIR}/install-arduino-core-esp32.sh" +source "${SCRIPTS_DIR}/tests_utils.sh" args=("-ai" "$ARDUINO_IDE_PATH" "-au" "$ARDUINO_USR_PATH") if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then if [ -n "$sketch" ]; then - tmp_sketch_path=$(find tests -name "$sketch".ino) - test_type=$(basename "$(dirname "$(dirname "$tmp_sketch_path")")") - echo "Sketch $sketch test type: $test_type" - test_folder="$PWD/tests/$test_type" + detect_test_type_and_folder "$sketch" else test_folder="$PWD/tests" fi @@ -70,11 +173,58 @@ else fi if [ $chunk_build -eq 1 ]; then + # For chunk builds, we need to handle multi-device tests separately + # First, build all multi-device tests, then build regular tests + + # Extract target from remaining arguments + extract_target_and_args "$@" + + if [ -z "$target" ]; then + echo "ERROR: Target (-t) is required for chunk builds" + exit 1 + fi + + # Find and build all multi-device tests in the test folder + multi_device_error=0 + if [ -d "$test_folder" ]; then + for test_dir in "$test_folder"/*; do + if [ -d "$test_dir" ] && [ "$(is_multi_device_test "$test_dir")" -eq 1 ]; then + build_multi_device_test "$test_dir" "$target" "${args[@]}" "${remaining_args[@]}" + result=$? + if [ $result -ne 0 ]; then + multi_device_error=$result + fi + fi + done + fi + + # Now build regular (non-multi-device) tests using chunk_build BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build" - args+=("-p" "$test_folder" "-i" "0" "-m" "1") + args+=("-p" "$test_folder") + ${BUILD_CMD} "${args[@]}" "${remaining_args[@]}" + regular_error=$? + + # Return error if either multi-device or regular builds failed + if [ $multi_device_error -ne 0 ]; then + exit $multi_device_error + fi + exit $regular_error else - BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build" - args+=("-s" "$test_folder/$sketch") -fi + # Check if this is a multi-device test + test_dir="$test_folder/$sketch" + if [ -d "$test_dir" ] && [ "$(is_multi_device_test "$test_dir")" -eq 1 ]; then + # Extract target from remaining arguments + extract_target_and_args "$@" -${BUILD_CMD} "${args[@]}" "$@" + if [ -z "$target" ]; then + echo "ERROR: Target (-t) is required for multi-device tests" + exit 1 + fi + + build_multi_device_test "$test_dir" "$target" "${args[@]}" "${remaining_args[@]}" + else + BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build" + args+=("-s" "$test_folder/$sketch") + ${BUILD_CMD} "${args[@]}" "$@" + fi +fi diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index e56518e9745..f1a04cf3c87 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -1,5 +1,184 @@ #!/bin/bash +# Check if a test is a multi-device test +function is_multi_device_test { + local test_dir=$1 + local has_multi_device + if [ -f "$test_dir/ci.yml" ]; then + has_multi_device=$(yq eval '.multi_device' "$test_dir/ci.yml" 2>/dev/null) + if [[ "$has_multi_device" != "null" && "$has_multi_device" != "" ]]; then + echo "1" + return + fi + fi + echo "0" +} + +function run_multi_device_test { + local target=$1 + local test_dir=$2 + local options=$3 + local erase_flash=$4 + local test_name + local test_type + local result=0 + local error=0 + local devices + + test_name=$(basename "$test_dir") + test_type=$(basename "$(dirname "$test_dir")") + + printf "\033[95mRunning multi-device test: %s\033[0m\n" "$test_name" + + # Get the list of devices from ci.yml + devices=$(yq eval '.multi_device | keys | .[]' "$test_dir/ci.yml" 2>/dev/null | sort) + + if [[ -z "$devices" ]]; then + echo "ERROR: No devices found in multi_device configuration for $test_name" + return 1 + fi + + # Build the build-dir argument for pytest-embedded + local build_dirs="" + local device_count=0 + local sketch_name + local build_dir + local sdkconfig_path + local compiled_target + for device in $devices; do + sketch_name=$(yq eval ".multi_device.$device" "$test_dir/ci.yml" 2>/dev/null) + build_dir="$HOME/.arduino/tests/$target/${test_name}_${sketch_name}/build.tmp" + + if [ ! -d "$build_dir" ]; then + printf "\033[93mSkipping multi-device test %s: build not found for device %s in %s\033[0m\n" "$test_name" "$device" "$build_dir" + return 0 + fi + + # Check if the build is for the correct target + sdkconfig_path="$build_dir/sdkconfig" + if [ -f "$sdkconfig_path" ]; then + compiled_target=$(grep -E "CONFIG_IDF_TARGET=" "$sdkconfig_path" | cut -d'"' -f2) + if [ "$compiled_target" != "$target" ]; then + printf "\033[91mError: Device %s compiled for %s, expected %s\033[0m\n" "$device" "$compiled_target" "$target" + return 1 + fi + fi + + if [ $device_count -gt 0 ]; then + build_dirs="$build_dirs|$build_dir" + else + build_dirs="$build_dir" + fi + device_count=$((device_count + 1)) + done + + if [ $device_count -lt 2 ]; then + echo "ERROR: Multi-device test $test_name requires at least 2 devices, found $device_count" + return 1 + fi + + # Check platform support + if [ -f "$test_dir/ci.yml" ]; then + is_target=$(yq eval ".targets.${target}" "$test_dir/ci.yml" 2>/dev/null) + selected_platform=$(yq eval ".platforms.${platform}" "$test_dir/ci.yml" 2>/dev/null) + + if [[ $is_target == "false" ]] || [[ $selected_platform == "false" ]]; then + printf "\033[93mSkipping %s test for %s, platform: %s\033[0m\n" "$test_name" "$target" "$platform" + printf "\n\n\n" + return 0 + fi + fi + + # Build embedded-services argument + if [ $platform == "wokwi" ]; then + echo "ERROR: Wokwi platform not supported for multi-device tests" + return 1 + elif [ $platform == "qemu" ]; then + echo "ERROR: QEMU platform not supported for multi-device tests" + return 1 + else + # For hardware platform, use esp,arduino for each device + local services="esp,arduino" + for ((i=1; i/dev/null || true - local wifi_args="" + # Build pytest command as an array to properly handle arguments + local pytest_cmd=( + pytest -s "$sketchdir/test_$sketchname.py" + --build-dir "$build_dir" + --junit-xml="$report_file" + -o "junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i}" + "${extra_args[@]}" + ) + if [ -n "$wifi_ssid" ]; then - wifi_args="--wifi-ssid \"$wifi_ssid\"" + pytest_cmd+=(--wifi-ssid "$wifi_ssid") fi if [ -n "$wifi_password" ]; then - wifi_args="$wifi_args --wifi-password \"$wifi_password\"" + pytest_cmd+=(--wifi-password "$wifi_password") fi result=0 - printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}" "$wifi_args" - bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q} $wifi_args; exit \$?" || result=$? + printf "\033[95m%s\033[0m\n" "${pytest_cmd[*]}" + set +e + "${pytest_cmd[@]}" + result=$? + set -e printf "\n" if [ $result -ne 0 ]; then result=0 printf "\033[95mRetrying test: %s -- Config: %s\033[0m\n" "$sketchname" "$i" - printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}" "$wifi_args" - bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q} $wifi_args; exit \$?" || result=$? + printf "\033[95m%s\033[0m\n" "${pytest_cmd[*]}" + set +e + "${pytest_cmd[@]}" + result=$? + set -e printf "\n" if [ $result -ne 0 ]; then printf "\033[91mFailed test: %s -- Config: %s\033[0m\n\n" "$sketchname" "$i" @@ -147,6 +340,21 @@ erase=0 wifi_ssid="" wifi_password="" +# Check for supplied port. Single dut tests will use ESPPORT. +if [ -n "${ESPPORT}" ]; then + echo "Supplied port [ESPPORT]: ${ESPPORT}" +elif [ -n "${ESPPORT1}" ]; then + echo "Supplied port [ESPPORT1]: ${ESPPORT1}" + if [ -n "${ESPPORT2}" ]; then + echo "Supplied port [ESPPORT2]: ${ESPPORT2}" + fi +else + echo "No port supplied. Using ESPPORT=/dev/ttyUSB0, ESPPORT1=/dev/ttyUSB0, ESPPORT2=/dev/ttyUSB1" + ESPPORT="/dev/ttyUSB0" + ESPPORT1="/dev/ttyUSB0" + ESPPORT2="/dev/ttyUSB1" +fi + while [ -n "$1" ]; do case $1 in -c ) @@ -215,13 +423,12 @@ if [ ! $platform == "qemu" ]; then source "${SCRIPTS_DIR}/install-arduino-ide.sh" fi +source "${SCRIPTS_DIR}/tests_utils.sh" + # If sketch is provided and test type is not, test type is inferred from the sketch path if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then if [ -n "$sketch" ]; then - tmp_sketch_path=$(find tests -name "$sketch".ino) - test_type=$(basename "$(dirname "$(dirname "$tmp_sketch_path")")") - echo "Sketch $sketch test type: $test_type" - test_folder="$PWD/tests/$test_type" + detect_test_type_and_folder "$sketch" else test_folder="$PWD/tests" fi @@ -234,8 +441,16 @@ if [ $chunk_run -eq 0 ]; then echo "ERROR: Sketch name is required for single test run" exit 1 fi - run_test "$target" "$test_folder"/"$sketch"/"$sketch".ino $options $erase - exit $? + + # Check if this is a multi-device test + test_dir="$test_folder/$sketch" + if [ -d "$test_dir" ] && [ "$(is_multi_device_test "$test_dir")" -eq 1 ]; then + run_multi_device_test "$target" "$test_dir" $options $erase + exit $? + else + run_test "$target" "$test_folder"/"$sketch"/"$sketch".ino $options $erase + exit $? + fi else if [ "$chunk_max" -le 0 ]; then echo "ERROR: Chunks count must be positive number" @@ -281,6 +496,20 @@ else sketchnum=0 error=0 + # First, run all multi-device tests in the test folder + if [ -d "$test_folder" ]; then + for test_dir in "$test_folder"/*; do + if [ -d "$test_dir" ] && [ "$(is_multi_device_test "$test_dir")" -eq 1 ]; then + exit_code=0 + run_multi_device_test "$target" "$test_dir" $options $erase || exit_code=$? + if [ $exit_code -ne 0 ]; then + error=$exit_code + fi + fi + done + fi + + # Then run regular tests for sketch in $sketches; do sketchnum=$((sketchnum + 1)) diff --git a/.github/scripts/tests_utils.sh b/.github/scripts/tests_utils.sh new file mode 100644 index 00000000000..44451ccca00 --- /dev/null +++ b/.github/scripts/tests_utils.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Shared utility functions for test scripts + +# Detect test type and folder from sketch name +# This function handles both multi-device tests (which have ci.yml at test level) +# and regular tests (which have .ino files in sketch directories) +# +# Usage: detect_test_type_and_folder "sketch_name" +# Returns: Sets global variables 'test_type' and 'test_folder' +# Exits with error if sketch is not found +function detect_test_type_and_folder { + local sketch=$1 + + # shellcheck disable=SC2034 # test_type and test_folder are used by caller + # For multi-device tests, we need to find the test directory, not the device sketch directory + # First, try to find a test directory with this name + if [ -d "tests/validation/$sketch" ] && [ -f "tests/validation/$sketch/ci.yml" ]; then + test_type="validation" + test_folder="$PWD/tests/$test_type" + elif [ -d "tests/performance/$sketch" ] && [ -f "tests/performance/$sketch/ci.yml" ]; then + test_type="performance" + test_folder="$PWD/tests/$test_type" + else + # Fall back to finding by .ino file (for regular tests) + tmp_sketch_path=$(find tests -name "$sketch".ino | head -1) + if [ -z "$tmp_sketch_path" ]; then + echo "ERROR: Sketch $sketch not found" + exit 1 + fi + test_type=$(basename "$(dirname "$(dirname "$tmp_sketch_path")")") + test_folder="$PWD/tests/$test_type" + fi + echo "Sketch $sketch test type: $test_type" +} + diff --git a/docs/en/contributing.rst b/docs/en/contributing.rst index c9f83530d36..f410d7c15b7 100644 --- a/docs/en/contributing.rst +++ b/docs/en/contributing.rst @@ -214,6 +214,17 @@ If you are contributing to the documentation, please follow the instructions des Testing and CI -------------- +In our repository, we have a Continuous Integration (CI) system that runs tests and code formatting fixes on every Pull Request. +This system will run the tests on all supported targets and check if everything is working as expected. + +We have many types of tests and checks, including: + +* Compilation tests; +* Runtime tests; +* Documentation checks; +* Code style checks; +* And more. + After you have made your changes, you should test them. You can do this in different ways depending on the type of change you have made. Examples @@ -239,24 +250,8 @@ Core Changes If you have made changes to the core, it is important to ensure that the changes do not break the existing functionality. You can do this by running the tests on all supported targets. You can refer to the `Runtime Tests`_ section for more information. -CI -** - -In our repository, we have a Continuous Integration (CI) system that runs tests and fixes on every Pull Request. This system will run the tests -on all supported targets and check if everything is working as expected. - -We have many types of tests and checks, including: - -* Compilation tests; -* Runtime tests; -* Documentation checks; -* Code style checks; -* And more. - -Let's go deeper into each type of test in the CI system: - Compilation Tests -^^^^^^^^^^^^^^^^^ +***************** The compilation tests are the first type of tests in the CI system. They check if the code compiles on all supported targets. If the code does not compile, the CI system will fail the test and the Pull Request will not be merged. @@ -270,7 +265,7 @@ a sketch that uses the changes you made (you can also add the sketch as an examp Make sure to set ``Compiler Warnings`` to ``All`` in the Arduino IDE to catch any potential issues. Runtime Tests -^^^^^^^^^^^^^ +************* Another type of test is the runtime tests. They check if the code runs and behaves as expected on all supported targets. If the code does not run as expected, the CI system will fail the test and the Pull Request will not be merged. This is important to ensure that the code @@ -381,7 +376,7 @@ And to run the ``uart`` test using QEMU, you would run: Wokwi support depends on the `currently implemented peripherals `_. Adding a New Test Suite -####################### +^^^^^^^^^^^^^^^^^^^^^^^ If you want to add a new test suite, you can create a new folder inside ``tests/validation`` or ``tests/performance`` with the name of the test suite. You can use the ``hello_world`` test suite as a starting point and the other test suites as a reference. @@ -400,10 +395,12 @@ For more information about the Unity testing framework, you can check the `Unity After creating the test suite, make sure to test it locally and run it in the CI system to ensure that it works as expected. CI YAML File -############ +"""""""""""" -The ``ci.yml`` file is used to specify how the test suite and sketches will handled by the CI system. It can contain the following fields: +The ``ci.yml`` file is used to specify how the test suite and sketches will be handled by the CI system. It can contain the following fields: +* ``multi_device``: A dictionary that matches the device names to their sketch directories. Each device must have its own sketch folder. + There are two devices in a multi-device test suite: ``device0`` and ``device1``. This field is only valid for multi-device test suites. * ``requires``: A list of configurations in ``sdkconfig`` that are required to run the test suite. The test suite will only run on the targets that have **ALL** the required configurations. By default, no configurations are required. * ``requires_any``: A list of configurations in ``sdkconfig`` that are required to run the test suite. The test suite will only run on the targets @@ -430,14 +427,155 @@ The ``wifi`` test suite is a good example of how to use the ``ci.yml`` file: .. literalinclude:: ../../tests/validation/wifi/ci.yml :language: yaml +Adding a Multi-Device Test Suite +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Multi-device tests allow you to test interactions between multiple ESP32 devices, such as WiFi AP/client communication or BLE connections. +Each device runs its own sketch, and the test coordinates communication between them. + +Creating a Multi-Device Test +"""""""""""""""""""""""""""" + +To create a multi-device test, follow these steps. The examples below use the ``wifi_ap`` test as a reference, but you should create your own test with an appropriate name: + +1. **Create the test directory structure**: Create a folder for each device sketch within your test directory. + For example, using the ``wifi_ap`` test structure: + + .. code-block:: bash + + mkdir -p tests/validation/wifi_ap/ap + mkdir -p tests/validation/wifi_ap/client + +2. **Create the ``ci.yml`` file**: Create ``ci.yml`` in your test directory with a ``multi_device`` configuration. + Example from the ``wifi_ap`` test: + + .. code-block:: yaml + + multi_device: + device0: ap + device1: client + + platforms: + qemu: false + wokwi: false + + requires: + - CONFIG_SOC_WIFI_SUPPORTED=y + + The ``multi_device`` field maps device names to their sketch directories. Each device must have its own sketch folder. + +3. **Create device sketches**: Create an ``.ino`` file in each sketch directory with the same name as the directory. + Example from the ``wifi_ap`` test, ``tests/validation/wifi_ap/ap/ap.ino``: + + .. code-block:: arduino + + void setup() { + Serial.begin(115200); + Serial.println("[AP] Device ready for WiFi credentials"); + } + + void loop() { + // Your AP logic + } + + And ``tests/validation/wifi_ap/client/client.ino`` (example): + + .. code-block:: arduino + + void setup() { + Serial.begin(115200); + Serial.println("[CLIENT] Device ready for WiFi credentials"); + } + + void loop() { + // Your client logic + } + +4. **Create the test file**: Create ``test_.py`` in your test directory. + Example from the ``wifi_ap`` test, ``tests/validation/wifi_ap/test_wifi_ap.py``: + + .. code-block:: python + + def test_wifi_ap(dut): + ap = dut[0] + client = dut[1] + + # Wait for devices to be ready + ap.expect_exact("[AP] Device ready for WiFi credentials") + client.expect_exact("[CLIENT] Device ready for WiFi credentials") + + # Your test logic here + # Example: Send commands and verify responses + ap.write("command") + client.expect("expected_response") + + The ``dut`` parameter is a list where each element corresponds to a device in the order specified in ``ci.yml``. + +.. note:: + + The examples above use the ``wifi_ap`` test as a reference. You can find the complete working example in ``tests/validation/wifi_ap/``. + When creating your own test, replace ``wifi_ap`` with your test name and adapt the device names (``ap``, ``client``) to match your use case. + +Building Multi-Device Tests +""""""""""""""""""""""""""" + +To build a specific multi-device test (using ``wifi_ap`` as an example): + +.. code-block:: bash + + ./.github/scripts/tests_build.sh -s wifi_ap -t esp32 + +To build all tests (chunk build): + +.. code-block:: bash + + ./.github/scripts/tests_build.sh -c -type validation -t esp32 -i 0 -m 1 + +The build output will be stored in: + +.. code-block:: bash + + $HOME/.arduino/tests//_/build.tmp + +For example, for a ``wifi_ap`` test on ``esp32``: +* Device0: ``$HOME/.arduino/tests/esp32/wifi_ap_ap/build.tmp`` +* Device1: ``$HOME/.arduino/tests/esp32/wifi_ap_client/build.tmp`` + +Running Multi-Device Tests +"""""""""""""""""""""""""" + +Before running multi-device tests, you need to set up the environment variables for the serial ports: + +.. code-block:: bash + + export ESPPORT1=/dev/ttyUSB0 # Port for device0 + export ESPPORT2=/dev/ttyUSB1 # Port for device1 + +Then, to run a specific test (using ``wifi_ap`` as an example): + +.. code-block:: bash + + ./.github/scripts/tests_run.sh -s wifi_ap -t esp32 + +To run all tests (chunk run): + +.. code-block:: bash + + ./.github/scripts/tests_run.sh -c -type validation -t esp32 -i 0 -m 1 + +.. note:: + + Both ``ESPPORT1`` and ``ESPPORT2`` environment variables must be set before running multi-device tests. + The sketch directory name must match the value in the ``multi_device`` configuration in ``ci.yml``, and the ``.ino`` file must have the same name as the directory. + Documentation Checks -^^^^^^^^^^^^^^^^^^^^ +******************** The CI also checks the documentation for any compilation errors. This is important to ensure that the documentation layout is not broken. To build the documentation locally, please refer to the `documentation guidelines `_. Code Style Checks -^^^^^^^^^^^^^^^^^ +***************** For checking the code style and other code quality checks, we use pre-commit hooks. These hooks will be automatically run by the CI when a Pull Request is marked as ``Status: Pending Merge``. diff --git a/libraries/BLE/examples/Server/Server.ino b/libraries/BLE/examples/Server/Server.ino index e86ed723267..4d70d4e897d 100644 --- a/libraries/BLE/examples/Server/Server.ino +++ b/libraries/BLE/examples/Server/Server.ino @@ -31,7 +31,7 @@ void setup() { pAdvertising->addServiceUUID(SERVICE_UUID); pAdvertising->setScanResponse(true); pAdvertising->setMinPreferred(0x06); // functions that help with iPhone connections issue - pAdvertising->setMinPreferred(0x12); + pAdvertising->setMaxPreferred(0x12); BLEDevice::startAdvertising(); Serial.println("Characteristic defined! Now you can read it in your phone!"); } diff --git a/libraries/BLE/examples/Server_secure_authorization/Server_secure_authorization.ino b/libraries/BLE/examples/Server_secure_authorization/Server_secure_authorization.ino index b1ab9cd5931..2056e793393 100644 --- a/libraries/BLE/examples/Server_secure_authorization/Server_secure_authorization.ino +++ b/libraries/BLE/examples/Server_secure_authorization/Server_secure_authorization.ino @@ -140,7 +140,7 @@ void setup() { pAdvertising->addServiceUUID(SERVICE_UUID); pAdvertising->setScanResponse(true); pAdvertising->setMinPreferred(0x06); // helps with iPhone connections - pAdvertising->setMinPreferred(0x12); + pAdvertising->setMaxPreferred(0x12); BLEDevice::startAdvertising(); diff --git a/libraries/BLE/examples/Server_secure_static_passkey/Server_secure_static_passkey.ino b/libraries/BLE/examples/Server_secure_static_passkey/Server_secure_static_passkey.ino index fef8c0bd15f..3ccba21bf71 100644 --- a/libraries/BLE/examples/Server_secure_static_passkey/Server_secure_static_passkey.ino +++ b/libraries/BLE/examples/Server_secure_static_passkey/Server_secure_static_passkey.ino @@ -185,7 +185,7 @@ void setup() { pAdvertising->addServiceUUID(SERVICE_UUID); pAdvertising->setScanResponse(true); pAdvertising->setMinPreferred(0x06); // functions that help with iPhone connections issue - pAdvertising->setMinPreferred(0x12); + pAdvertising->setMaxPreferred(0x12); BLEDevice::startAdvertising(); Serial.println("Characteristic defined! Now you can read it in your phone!"); } diff --git a/tests/conftest.py b/tests/conftest.py index 449dabc4758..fa34dee995f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,17 +1,21 @@ import pytest import os +import ipaddress +import random +import string +REGEX_IPV4 = r"(\b(?:(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)\b)" +# Pytest arguments def pytest_addoption(parser): parser.addoption("--wifi-password", action="store", default=None, help="Wi-Fi password.") parser.addoption("--wifi-ssid", action="store", default=None, help="Wi-Fi SSID.") - +# Fixtures @pytest.fixture(scope="session") def wifi_ssid(request): return request.config.getoption("--wifi-ssid") - @pytest.fixture(scope="session") def wifi_pass(request): return request.config.getoption("--wifi-password") @@ -20,3 +24,16 @@ def wifi_pass(request): @pytest.fixture(scope="session") def ci_job_id(request): return os.environ.get("CI_JOB_ID") + +# Helper functions +def is_valid_ipv4(ip): + # Check if the IP address is a valid IPv4 address + try: + ipaddress.IPv4Address(ip) + return True + except ipaddress.AddressValueError: + return False + +def rand_str4(): + # Generate a random string of 4 characters + return "".join(random.choices(string.ascii_letters + string.digits, k=4)) diff --git a/tests/validation/ble/ci.yml b/tests/validation/ble/ci.yml new file mode 100644 index 00000000000..71a333edd1e --- /dev/null +++ b/tests/validation/ble/ci.yml @@ -0,0 +1,17 @@ +tags: + - generic_multi_device + +multi_device: + device0: server + device1: client + +platforms: + # Wokwi and QEMU do not support multi-device tests + wokwi: false + qemu: false + # Hardware runners not yet assigned + hardware: false + +requires: + - CONFIG_SOC_BLE_SUPPORTED=y + diff --git a/tests/validation/ble/client/client.ino b/tests/validation/ble/client/client.ino new file mode 100644 index 00000000000..fd1f4206bd3 --- /dev/null +++ b/tests/validation/ble/client/client.ino @@ -0,0 +1,257 @@ +#include "BLEDevice.h" +#include "BLESecurity.h" +#include "nvs_flash.h" + +static BLEUUID serviceUUID("4fafc201-1fb5-459e-8fcc-c5c9c331914b"); +static BLEUUID insecureCharUUID("beb5483e-36e1-4688-b7f5-ea07361b26a8"); +static BLEUUID secureCharUUID("ff1d2614-e2d6-4c87-9154-6625d39ca7f8"); + +String targetServerName = ""; +static boolean doConnect = false; +static boolean connected = false; +static boolean doScan = false; +static boolean testCompleted = false; +static BLEClient *pClient = nullptr; +static BLERemoteCharacteristic *pRemoteInsecureCharacteristic = nullptr; +static BLERemoteCharacteristic *pRemoteSecureCharacteristic = nullptr; +static BLEAdvertisedDevice *myDevice = nullptr; + +class MyClientCallback : public BLEClientCallbacks { + void onConnect(BLEClient *pclient) { + Serial.println("[CLIENT] Connected to server"); + } + + void onDisconnect(BLEClient *pclient) { + connected = false; + Serial.println("[CLIENT] Disconnected from server"); + } +}; + +class MySecurityCallbacks : public BLESecurityCallbacks { + // Numeric Comparison callback - both devices display the same PIN + bool onConfirmPIN(uint32_t pin) override { + Serial.printf("[CLIENT] Numeric comparison PIN: %lu\n", (unsigned long)pin); + Serial.println("[CLIENT] Confirming PIN match"); + // Automatically confirm for testing + return true; + } + +#if defined(CONFIG_BLUEDROID_ENABLED) + void onAuthenticationComplete(esp_ble_auth_cmpl_t desc) override { + BLEAddress peerAddr(desc.bd_addr); + Serial.println("[CLIENT] Authentication complete"); + + uint8_t irk[16]; + if (BLEDevice::getPeerIRK(peerAddr, irk)) { + Serial.print("[CLIENT] Successfully retrieved peer IRK: "); + for (int i = 0; i < 16; i++) { + if (irk[i] < 0x10) Serial.print("0"); + Serial.print(irk[i], HEX); + if (i < 15) Serial.print(":"); + } + Serial.println(); + } + } +#endif + +#if defined(CONFIG_NIMBLE_ENABLED) + void onAuthenticationComplete(ble_gap_conn_desc *desc) override { + BLEAddress peerAddr(desc->peer_id_addr.val, desc->peer_id_addr.type); + Serial.println("[CLIENT] Authentication complete"); + + uint8_t irk[16]; + if (BLEDevice::getPeerIRK(peerAddr, irk)) { + Serial.print("[CLIENT] Successfully retrieved peer IRK: "); + for (int i = 0; i < 16; i++) { + if (irk[i] < 0x10) Serial.print("0"); + Serial.print(irk[i], HEX); + if (i < 15) Serial.print(":"); + } + Serial.println(); + } + } +#endif +}; + +bool connectToServer() { + Serial.printf("[CLIENT] Connecting to %s\n", myDevice->getAddress().toString().c_str()); + + pClient = BLEDevice::createClient(); + pClient->setClientCallbacks(new MyClientCallback()); + + pClient->connect(myDevice); + Serial.println("[CLIENT] Physical connection established"); + + pClient->setMTU(517); + + BLERemoteService *pRemoteService = pClient->getService(serviceUUID); + if (pRemoteService == nullptr) { + Serial.println("[CLIENT] ERROR: Failed to find service"); + pClient->disconnect(); + return false; + } + Serial.println("[CLIENT] Found service"); + + pRemoteInsecureCharacteristic = pRemoteService->getCharacteristic(insecureCharUUID); + if (pRemoteInsecureCharacteristic == nullptr) { + Serial.println("[CLIENT] ERROR: Failed to find insecure characteristic"); + pClient->disconnect(); + return false; + } + Serial.println("[CLIENT] Found insecure characteristic"); + + pRemoteSecureCharacteristic = pRemoteService->getCharacteristic(secureCharUUID); + if (pRemoteSecureCharacteristic == nullptr) { + Serial.println("[CLIENT] ERROR: Failed to find secure characteristic"); + pClient->disconnect(); + return false; + } + Serial.println("[CLIENT] Found secure characteristic"); + + // Read insecure characteristic + if (pRemoteInsecureCharacteristic->canRead()) { + String value = pRemoteInsecureCharacteristic->readValue(); + Serial.print("[CLIENT] Insecure characteristic value: "); + Serial.println(value.c_str()); + } + + // Set auth requirement for secure characteristic (Bluedroid) + pRemoteSecureCharacteristic->setAuth(ESP_GATT_AUTH_REQ_MITM); + + // Read secure characteristic (triggers authentication in NimBLE) + if (pRemoteSecureCharacteristic->canRead()) { + Serial.println("[CLIENT] Reading secure characteristic..."); + String value = pRemoteSecureCharacteristic->readValue(); + Serial.print("[CLIENT] Secure characteristic value: "); + Serial.println(value.c_str()); + } + + connected = true; + Serial.println("[CLIENT] Connection and authentication successful"); + return true; +} + +class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks { + void onResult(BLEAdvertisedDevice advertisedDevice) { + Serial.print("[CLIENT] Found device: "); + Serial.println(advertisedDevice.toString().c_str()); + + // Check if device has the correct service UUID and name + if (advertisedDevice.haveServiceUUID() && advertisedDevice.isAdvertisingService(serviceUUID)) { + String deviceName = advertisedDevice.getName().c_str(); + Serial.print("[CLIENT] Device has matching service UUID, name: "); + Serial.println(deviceName); + + if (deviceName == targetServerName) { + Serial.println("[CLIENT] Found target server!"); + BLEDevice::getScan()->stop(); + myDevice = new BLEAdvertisedDevice(advertisedDevice); + doConnect = true; + doScan = true; + } + } + } +}; + +void readServerName() { + Serial.println("[CLIENT] Waiting for server name..."); + Serial.println("[CLIENT] Send server name:"); + + // Wait for server name + while (targetServerName.length() == 0) { + if (Serial.available()) { + targetServerName = Serial.readStringUntil('\n'); + targetServerName.trim(); + } + delay(100); + } + + Serial.print("[CLIENT] Target server name: "); + Serial.println(targetServerName); +} + +void setup() { + Serial.begin(115200); + while (!Serial) { + delay(100); + } + + Serial.println("[CLIENT] Device ready for server name"); + + // Read server name from serial + readServerName(); + + Serial.println("[CLIENT] Starting BLE Secure Client"); + + // Clear NVS to ensure fresh pairing for testing + nvs_flash_erase(); + nvs_flash_init(); + + BLEDevice::init("BLE_Test_Client"); + + // Configure security for Numeric Comparison + BLESecurity *pSecurity = new BLESecurity(); + // Use DisplayYesNo capability for Numeric Comparison pairing + pSecurity->setCapability(ESP_IO_CAP_IO); + // Enable bonding, MITM (required for Numeric Comparison), and secure connection + pSecurity->setAuthenticationMode(true, true, true); + BLEDevice::setSecurityCallbacks(new MySecurityCallbacks()); + + Serial.print("[CLIENT] Scanning for server: "); + Serial.println(targetServerName); + + // Start scanning + BLEScan *pBLEScan = BLEDevice::getScan(); + pBLEScan->setAdvertisedDeviceCallbacks(new MyAdvertisedDeviceCallbacks()); + pBLEScan->setInterval(1349); + pBLEScan->setWindow(449); + pBLEScan->setActiveScan(true); + pBLEScan->start(30, false); +} + +void loop() { + if (doConnect == true) { + if (connectToServer()) { + Serial.println("[CLIENT] Successfully connected and authenticated"); + } else { + Serial.println("[CLIENT] Connection failed"); + } + doConnect = false; + } + + if (connected && !testCompleted) { + // Test write and read operations once + testCompleted = true; + + // Write to insecure characteristic + String insecureWriteValue = "Test Insecure Write"; + if (pRemoteInsecureCharacteristic && pRemoteInsecureCharacteristic->canWrite()) { + pRemoteInsecureCharacteristic->writeValue(insecureWriteValue.c_str(), insecureWriteValue.length()); + Serial.printf("[CLIENT] Wrote to insecure characteristic: %s\n", insecureWriteValue.c_str()); + } + + // Read back insecure characteristic + if (pRemoteInsecureCharacteristic && pRemoteInsecureCharacteristic->canRead()) { + String insecureReadValue = pRemoteInsecureCharacteristic->readValue(); + Serial.printf("[CLIENT] Read from insecure characteristic: %s\n", insecureReadValue.c_str()); + } + + // Write to secure characteristic + String secureWriteValue = "Test Secure Write"; + if (pRemoteSecureCharacteristic && pRemoteSecureCharacteristic->canWrite()) { + pRemoteSecureCharacteristic->writeValue(secureWriteValue.c_str(), secureWriteValue.length()); + Serial.printf("[CLIENT] Wrote to secure characteristic: %s\n", secureWriteValue.c_str()); + } + + // Read back secure characteristic + if (pRemoteSecureCharacteristic && pRemoteSecureCharacteristic->canRead()) { + String secureReadValue = pRemoteSecureCharacteristic->readValue(); + Serial.printf("[CLIENT] Read from secure characteristic: %s\n", secureReadValue.c_str()); + } + + Serial.println("[CLIENT] Test operations completed"); + } + + delay(1000); +} + diff --git a/tests/validation/ble/server/server.ino b/tests/validation/ble/server/server.ino new file mode 100644 index 00000000000..368de83ef59 --- /dev/null +++ b/tests/validation/ble/server/server.ino @@ -0,0 +1,185 @@ +#include +#include +#include +#include +#include + +#define SERVICE_UUID "4fafc201-1fb5-459e-8fcc-c5c9c331914b" +#define INSECURE_CHARACTERISTIC_UUID "beb5483e-36e1-4688-b7f5-ea07361b26a8" +#define SECURE_CHARACTERISTIC_UUID "ff1d2614-e2d6-4c87-9154-6625d39ca7f8" + +String serverName = ""; +static bool deviceConnected = false; + +class MyServerCallbacks : public BLEServerCallbacks { + void onConnect(BLEServer *pServer) { + deviceConnected = true; + Serial.println("[SERVER] Client connected"); + } + + void onDisconnect(BLEServer *pServer) { + deviceConnected = false; + Serial.println("[SERVER] Client disconnected"); + // Restart advertising after disconnect + BLEDevice::startAdvertising(); + Serial.println("[SERVER] Advertising restarted"); + } +}; + +class MySecurityCallbacks : public BLESecurityCallbacks { + // Numeric Comparison callback - both devices display the same PIN + bool onConfirmPIN(uint32_t pin) override { + Serial.printf("[SERVER] Numeric comparison PIN: %lu\n", (unsigned long)pin); + Serial.println("[SERVER] Confirming PIN match"); + // Automatically confirm for testing + return true; + } + +#if defined(CONFIG_BLUEDROID_ENABLED) + void onAuthenticationComplete(esp_ble_auth_cmpl_t desc) override { + BLEAddress peerAddr(desc.bd_addr); + Serial.println("[SERVER] Authentication complete"); + + uint8_t irk[16]; + if (BLEDevice::getPeerIRK(peerAddr, irk)) { + Serial.print("[SERVER] Successfully retrieved peer IRK: "); + for (int i = 0; i < 16; i++) { + if (irk[i] < 0x10) Serial.print("0"); + Serial.print(irk[i], HEX); + if (i < 15) Serial.print(":"); + } + Serial.println(); + } + } +#endif + +#if defined(CONFIG_NIMBLE_ENABLED) + void onAuthenticationComplete(ble_gap_conn_desc *desc) override { + BLEAddress peerAddr(desc->peer_id_addr.val, desc->peer_id_addr.type); + Serial.println("[SERVER] Authentication complete"); + + uint8_t irk[16]; + if (BLEDevice::getPeerIRK(peerAddr, irk)) { + Serial.print("[SERVER] Successfully retrieved peer IRK: "); + for (int i = 0; i < 16; i++) { + if (irk[i] < 0x10) Serial.print("0"); + Serial.print(irk[i], HEX); + if (i < 15) Serial.print(":"); + } + Serial.println(); + } + } +#endif +}; + +class MyCharacteristicCallbacks : public BLECharacteristicCallbacks { + void onWrite(BLECharacteristic *pCharacteristic) { + String value = pCharacteristic->getValue(); + if (value.length() > 0) { + Serial.printf("[SERVER] Received write: %s\n", value.c_str()); + } + } + + void onRead(BLECharacteristic *pCharacteristic) { + Serial.printf("[SERVER] Characteristic read: %s\n", pCharacteristic->getUUID().toString().c_str()); + } +}; + +void readServerName() { + Serial.println("[SERVER] Waiting for server name..."); + Serial.println("[SERVER] Send server name:"); + + // Wait for server name + while (serverName.length() == 0) { + if (Serial.available()) { + serverName = Serial.readStringUntil('\n'); + serverName.trim(); + } + delay(100); + } + + Serial.printf("[SERVER] Server name: %s\n", serverName.c_str()); +} + +void setup() { + Serial.begin(115200); + while (!Serial) { + delay(100); + } + + Serial.println("[SERVER] Device ready for server name"); + + // Read server name from serial + readServerName(); + + Serial.println("[SERVER] Starting BLE Secure Server"); + + // Clear NVS to ensure fresh pairing for testing + nvs_flash_erase(); + nvs_flash_init(); + + Serial.printf("[SERVER] BLE stack: %s\n", BLEDevice::getBLEStackString().c_str()); + + BLEDevice::init(serverName.c_str()); + + // Configure security for Numeric Comparison + BLESecurity *pSecurity = new BLESecurity(); + // Use DisplayYesNo capability for Numeric Comparison pairing + pSecurity->setCapability(ESP_IO_CAP_IO); + // Enable bonding, MITM (required for Numeric Comparison), and secure connection + pSecurity->setAuthenticationMode(true, true, true); + BLEDevice::setSecurityCallbacks(new MySecurityCallbacks()); + + // Create server + BLEServer *pServer = BLEDevice::createServer(); + pServer->setCallbacks(new MyServerCallbacks()); + pServer->advertiseOnDisconnect(true); + + // Create service + BLEService *pService = pServer->createService(SERVICE_UUID); + + // Create characteristics + uint32_t insecure_properties = BLECharacteristic::PROPERTY_READ | BLECharacteristic::PROPERTY_WRITE; + uint32_t secure_properties = insecure_properties | BLECharacteristic::PROPERTY_READ_AUTHEN | BLECharacteristic::PROPERTY_WRITE_AUTHEN; + + BLECharacteristic *pSecureChar = pService->createCharacteristic(SECURE_CHARACTERISTIC_UUID, secure_properties); + BLECharacteristic *pInsecureChar = pService->createCharacteristic(INSECURE_CHARACTERISTIC_UUID, insecure_properties); + + // Set permissions for Bluedroid + pSecureChar->setAccessPermissions(ESP_GATT_PERM_READ_ENC_MITM | ESP_GATT_PERM_WRITE_ENC_MITM); + pInsecureChar->setAccessPermissions(ESP_GATT_PERM_READ | ESP_GATT_PERM_WRITE); + + // Set callbacks + pSecureChar->setCallbacks(new MyCharacteristicCallbacks()); + pInsecureChar->setCallbacks(new MyCharacteristicCallbacks()); + + // Set initial values + pSecureChar->setValue("Secure Hello World!"); + pInsecureChar->setValue("Insecure Hello World!"); + + Serial.println("[SERVER] Characteristics configured"); + + // Start service + pService->start(); + + // Start advertising + BLEAdvertising *pAdvertising = BLEDevice::getAdvertising(); + pAdvertising->addServiceUUID(SERVICE_UUID); + pAdvertising->setScanResponse(true); + pAdvertising->setMinPreferred(0x06); + pAdvertising->setMaxPreferred(0x12); + BLEDevice::startAdvertising(); + + Serial.printf("[SERVER] Advertising started with name: %s\n", serverName.c_str()); + Serial.printf("[SERVER] Service UUID: %s\n", SERVICE_UUID); +} + +void loop() { + static unsigned long lastStatus = 0; + if (millis() - lastStatus > 3000) { + lastStatus = millis(); + Serial.printf("[SERVER] Status: %s\n", deviceConnected ? "Connected" : "Waiting for connection"); + } + delay(100); +} + diff --git a/tests/validation/ble/test_ble.py b/tests/validation/ble/test_ble.py new file mode 100644 index 00000000000..333dec216a7 --- /dev/null +++ b/tests/validation/ble/test_ble.py @@ -0,0 +1,151 @@ +import re +import logging +from conftest import rand_str4 + + +def test_ble(dut, ci_job_id): + LOGGER = logging.getLogger(__name__) + + server = dut[0] + client = dut[1] + + # Generate unique server name for this test run + server_name = "BLE_SRV_" + ci_job_id if ci_job_id else "BLE_SRV_" + rand_str4() + + LOGGER.info(f"Server Name: {server_name}") + + # Wait for devices to be ready and send server name + # Add longer timeout as the test starts running in one device while it is uploaded to the other + LOGGER.info("Waiting for devices to be ready...") + server.expect_exact("[SERVER] Device ready for server name", timeout=120) + client.expect_exact("[CLIENT] Device ready for server name", timeout=120) + + server.expect_exact("[SERVER] Send server name:") + client.expect_exact("[CLIENT] Send server name:") + LOGGER.info(f"Sending server name: {server_name}") + server.write(f"{server_name}") + client.write(f"{server_name}") + + # Verify server name was received + server.expect_exact(f"[SERVER] Server name: {server_name}") + client.expect_exact(f"[CLIENT] Target server name: {server_name}") + LOGGER.info("Server name received by both devices") + + # Wait for devices to initialize + LOGGER.info("Waiting for devices to initialize...") + server.expect_exact("[SERVER] Starting BLE Secure Server", timeout=30) + client.expect_exact("[CLIENT] Starting BLE Secure Client", timeout=30) + + # Verify server is using correct BLE stack + LOGGER.info("Checking BLE stack...") + server.expect(r"\[SERVER\] BLE stack: (Bluedroid|NimBLE)", timeout=10) + + # Wait for server to be ready + LOGGER.info("Waiting for server to start advertising...") + server.expect_exact("[SERVER] Characteristics configured", timeout=10) + server.expect_exact(f"[SERVER] Advertising started with name: {server_name}", timeout=10) + LOGGER.info(f"Server advertising with name: {server_name}") + + m = server.expect_exact("[SERVER] Service UUID: 4fafc201-1fb5-459e-8fcc-c5c9c331914b", timeout=10) + LOGGER.info(f"Server Service UUID: 4fafc201-1fb5-459e-8fcc-c5c9c331914b") + + # Wait for client to start scanning + LOGGER.info("Waiting for client to start scanning...") + client.expect_exact(f"[CLIENT] Scanning for server: {server_name}", timeout=10) + + # Client finds server + LOGGER.info("Waiting for client to discover server...") + client.expect_exact("[CLIENT] Found target server!", timeout=30) + + # Client connects to server + LOGGER.info("Client connecting to server...") + client.expect_exact("[CLIENT] Physical connection established", timeout=20) + server.expect_exact("[SERVER] Client connected", timeout=10) + + # Verify service discovery starts + LOGGER.info("Verifying service discovery...") + client.expect_exact("[CLIENT] Found service", timeout=10) + + # Wait for numeric comparison PIN display on both devices + # Note: Bluedroid (ESP32) initiates security on-connect, NimBLE initiates on-demand + # So numeric comparison may happen during service discovery (Bluedroid) or later (NimBLE) + LOGGER.info("Waiting for numeric comparison...") + m_server = server.expect(r"\[SERVER\] Numeric comparison PIN: (\d+)", timeout=30) + server_pin = m_server.group(1).decode() + LOGGER.info(f"Server PIN: {server_pin}") + server.expect_exact("[SERVER] Confirming PIN match", timeout=5) + + m_client = client.expect(r"\[CLIENT\] Numeric comparison PIN: (\d+)", timeout=30) + client_pin = m_client.group(1).decode() + LOGGER.info(f"Client PIN: {client_pin}") + client.expect_exact("[CLIENT] Confirming PIN match", timeout=5) + + # Verify both devices show the same PIN + assert server_pin == client_pin, f"PIN mismatch! Server: {server_pin}, Client: {client_pin}" + LOGGER.info(f"PIN match confirmed: {server_pin}") + + # Continue with characteristics discovery + client.expect_exact("[CLIENT] Found insecure characteristic", timeout=10) + client.expect_exact("[CLIENT] Found secure characteristic", timeout=10) + + # Verify insecure characteristic read + LOGGER.info("Verifying insecure characteristic access...") + + # Wait for authentication to complete on both devices + LOGGER.info("Waiting for authentication to complete...") + client.expect_exact("[CLIENT] Authentication complete", timeout=30) + + # Verify IRK retrieval on client side + LOGGER.info("Verifying IRK retrieval on client...") + client.expect(r"\[CLIENT\] Successfully retrieved peer IRK: ([0-9a-fA-F:]+)", timeout=10) + + # Server-side authentication may complete after characteristic read + server.expect_exact("[SERVER] Characteristic read: beb5483e-36e1-4688-b7f5-ea07361b26a8", timeout=10) + server.expect_exact("[SERVER] Authentication complete", timeout=30) + + # Verify IRK retrieval on server side + LOGGER.info("Verifying IRK retrieval on server...") + server.expect(r"\[SERVER\] Successfully retrieved peer IRK: ([0-9a-fA-F:]+)", timeout=10) + + client.expect_exact("[CLIENT] Insecure characteristic value: Insecure Hello World!", timeout=10) + + # Verify secure characteristic read + LOGGER.info("Verifying secure characteristic access...") + client.expect_exact("[CLIENT] Reading secure characteristic...", timeout=10) + + # Verify secure characteristic was read successfully + server.expect_exact("[SERVER] Characteristic read: ff1d2614-e2d6-4c87-9154-6625d39ca7f8", timeout=10) + client.expect_exact("[CLIENT] Secure characteristic value: Secure Hello World!", timeout=10) + + # Verify connection is established + client.expect_exact("[CLIENT] Connection and authentication successful", timeout=10) + client.expect_exact("[CLIENT] Successfully connected and authenticated", timeout=10) + + # Verify write and read operations on insecure characteristic + LOGGER.info("Testing insecure characteristic write/read...") + client.expect_exact("[CLIENT] Wrote to insecure characteristic: Test Insecure Write", timeout=10) + server.expect_exact("[SERVER] Received write: Test Insecure Write", timeout=10) + + client.expect_exact("[CLIENT] Read from insecure characteristic: Test Insecure Write", timeout=10) + server.expect_exact("[SERVER] Characteristic read: beb5483e-36e1-4688-b7f5-ea07361b26a8", timeout=10) + LOGGER.info("Insecure characteristic write/read verified") + + # Verify write and read operations on secure characteristic + LOGGER.info("Testing secure characteristic write/read...") + client.expect_exact("[CLIENT] Wrote to secure characteristic: Test Secure Write", timeout=10) + server.expect_exact("[SERVER] Received write: Test Secure Write", timeout=10) + + client.expect_exact("[CLIENT] Read from secure characteristic: Test Secure Write", timeout=10) + server.expect_exact("[SERVER] Characteristic read: ff1d2614-e2d6-4c87-9154-6625d39ca7f8", timeout=10) + LOGGER.info("Secure characteristic write/read verified") + + # Verify test completion + client.expect_exact("[CLIENT] Test operations completed", timeout=10) + LOGGER.info("All characteristic operations completed successfully") + + # Verify connection status + LOGGER.info("Verifying connection status...") + server.expect_exact("[SERVER] Status: Connected", timeout=5) + + LOGGER.info("BLE test passed!") + diff --git a/tests/validation/wifi_ap/ap/ap.ino b/tests/validation/wifi_ap/ap/ap.ino new file mode 100644 index 00000000000..7379ecc3533 --- /dev/null +++ b/tests/validation/wifi_ap/ap/ap.ino @@ -0,0 +1,75 @@ +#include + +String ssid = ""; +String password = ""; + +void readWiFiCredentials() { + Serial.println("[AP] Waiting for WiFi credentials..."); + Serial.println("[AP] Send SSID:"); + + // Wait for SSID + while (ssid.length() == 0) { + if (Serial.available()) { + ssid = Serial.readStringUntil('\n'); + ssid.trim(); + } + delay(100); + } + + Serial.println("[AP] Send Password:"); + + // Wait for password (allow empty password) + bool password_received = false; + while (!password_received) { + if (Serial.available()) { + password = Serial.readStringUntil('\n'); + password.trim(); + password_received = true; // Accept even empty password + } + delay(100); + } + + Serial.print("[AP] SSID: "); + Serial.println(ssid); + Serial.print("[AP] Password: "); + Serial.println(password); +} + +void setup() { + Serial.begin(115200); + while (!Serial) { + delay(100); + } + + // delete old config + WiFi.disconnect(true); + delay(1000); + + // Wait for test to be ready + Serial.println("[AP] Device ready for WiFi credentials"); + + // Read WiFi credentials from serial + readWiFiCredentials(); + + WiFi.mode(WIFI_AP); + bool ok = WiFi.softAP(ssid, password); + + if (!ok) { + Serial.println("[AP] Failed to start AP"); + return; + } + + IPAddress ip = WiFi.softAPIP(); + Serial.printf("[AP] Started SSID=%s Password=%s IP=%s\n", + ssid.c_str(), password.c_str(), ip.toString().c_str()); +} + +void loop() { + // periodically report stations + static unsigned long last = 0; + if (millis() - last > 3000) { + last = millis(); + int count = WiFi.softAPgetStationNum(); + Serial.printf("[AP] Stations connected: %d\n", count); + } +} diff --git a/tests/validation/wifi_ap/ci.yml b/tests/validation/wifi_ap/ci.yml new file mode 100644 index 00000000000..4edf00869a9 --- /dev/null +++ b/tests/validation/wifi_ap/ci.yml @@ -0,0 +1,28 @@ +tags: + - wifi_router + +multi_device: + device0: ap + device1: client + +fqbn: + esp32: + - espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio + - espressif:esp32:esp32:PSRAM=disabled,PartitionScheme=huge_app,FlashMode=dio + esp32s2: + - espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio + - espressif:esp32:esp32s2:PSRAM=disabled,PartitionScheme=huge_app,FlashMode=dio + esp32s3: + - espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,PartitionScheme=huge_app,FlashMode=qio + - espressif:esp32:esp32s3:PSRAM=disabled,USBMode=default,PartitionScheme=huge_app,FlashMode=qio + - espressif:esp32:esp32s3:PSRAM=enabled,USBMode=default,PartitionScheme=huge_app,FlashMode=qio + +platforms: + # Wokwi and QEMU do not support multi-device tests + wokwi: false + qemu: false + # Hardware runners not yet assigned + hardware: false + +requires: + - CONFIG_SOC_WIFI_SUPPORTED=y diff --git a/tests/validation/wifi_ap/client/client.ino b/tests/validation/wifi_ap/client/client.ino new file mode 100644 index 00000000000..7c2d9bc102a --- /dev/null +++ b/tests/validation/wifi_ap/client/client.ino @@ -0,0 +1,107 @@ +#include + +String ssid = ""; +String password = ""; + +void readWiFiCredentials() { + Serial.println("[CLIENT] Waiting for WiFi credentials..."); + Serial.println("[CLIENT] Send SSID:"); + + // Wait for SSID + while (ssid.length() == 0) { + if (Serial.available()) { + ssid = Serial.readStringUntil('\n'); + ssid.trim(); + } + delay(100); + } + + Serial.println("[CLIENT] Send Password:"); + + // Wait for password (allow empty password) + bool password_received = false; + while (!password_received) { + if (Serial.available()) { + password = Serial.readStringUntil('\n'); + password.trim(); + password_received = true; // Accept even empty password + } + delay(100); + } + + Serial.print("[CLIENT] SSID: "); + Serial.println(ssid); + Serial.print("[CLIENT] Password: "); + Serial.println(password); +} + +void setup() { + Serial.begin(115200); + while (!Serial) { + delay(100); + } + + // delete old config + WiFi.disconnect(true); + delay(1000); + + // Wait for test to be ready + Serial.println("[CLIENT] Device ready for WiFi credentials"); + + // Read WiFi credentials from serial + readWiFiCredentials(); + + WiFi.mode(WIFI_STA); + + bool found = false; + do { + Serial.println("[CLIENT] Scan start"); + // WiFi.scanNetworks will return the number of networks found. + int n = WiFi.scanNetworks(); + Serial.println("[CLIENT] Scan done"); + if (n == 0) { + Serial.println("[CLIENT] no networks found"); + } else { + Serial.print("[CLIENT] "); + Serial.print(n); + Serial.println(" networks found:"); + for (int i = 0; i < n; ++i) { + // Print SSID for each network found + Serial.printf("%s\n", WiFi.SSID(i).c_str()); + Serial.println(); + delay(10); + if (WiFi.SSID(i) == ssid) { + found = true; + break; + } + } + } + Serial.println(""); + delay(5000); + } while (!found); + + // Delete the scan result to free memory for code below. + WiFi.scanDelete(); + + WiFi.begin(ssid, password); + + Serial.printf("[CLIENT] Connecting to SSID=%s Password=%s ...\n", ssid.c_str(), password.c_str()); + + int retries = 50; + while (WiFi.status() != WL_CONNECTED && retries--) { + delay(200); + } + + if (WiFi.status() == WL_CONNECTED) { + Serial.printf("[CLIENT] Connected! IP=%s\n", + WiFi.localIP().toString().c_str()); + } else { + Serial.println("[CLIENT] Failed to connect"); + } +} + +void loop() { + delay(2000); + wl_status_t st = WiFi.status(); + Serial.printf("[CLIENT] Status=%d\n", st); +} diff --git a/tests/validation/wifi_ap/test_wifi_ap.py b/tests/validation/wifi_ap/test_wifi_ap.py new file mode 100644 index 00000000000..3d9ab0dee14 --- /dev/null +++ b/tests/validation/wifi_ap/test_wifi_ap.py @@ -0,0 +1,79 @@ +import re +import logging +from conftest import REGEX_IPV4, is_valid_ipv4, rand_str4 + + +def test_wifi_ap(dut, ci_job_id): + LOGGER = logging.getLogger(__name__) + + ap = dut[0] + client = dut[1] + + ap_ssid = "AP_SSID_" + ci_job_id if ci_job_id else "AP_SSID_" + rand_str4() + ap_password = "AP_PW_" + ci_job_id if ci_job_id else "AP_PW_" + rand_str4() + + LOGGER.info(f"AP SSID: {ap_ssid}") + LOGGER.info(f"AP Password: {ap_password}") + + # Wait for devices to be ready and send WiFi credentials + # Add longer timeout as the test start running in one device while it is uploaded to the other + LOGGER.info("Waiting for devices to be ready...") + ap.expect_exact("[AP] Device ready for WiFi credentials", timeout=120) + client.expect_exact("[CLIENT] Device ready for WiFi credentials", timeout=120) + + ap.expect_exact("[AP] Send SSID:") + client.expect_exact("[CLIENT] Send SSID:") + LOGGER.info(f"Sending WiFi credentials: SSID={ap_ssid}") + ap.write(f"{ap_ssid}") + client.write(f"{ap_ssid}") + + ap.expect_exact("[AP] Send Password:") + client.expect_exact("[CLIENT] Send Password:") + LOGGER.info(f"Sending WiFi password: Password={ap_password}") + ap.write(f"{ap_password}") + client.write(f"{ap_password}") + + # Verify credentials were received + ap.expect_exact(f"[AP] SSID: {ap_ssid}") + ap.expect_exact(f"[AP] Password: {ap_password}") + client.expect_exact(f"[CLIENT] SSID: {ap_ssid}") + client.expect_exact(f"[CLIENT] Password: {ap_password}") + LOGGER.info(f"Credentials received") + + # Verify AP started + LOGGER.info(f"Starting AP") + m = ap.expect(rf"\[AP\] Started SSID={re.escape(ap_ssid)} Password={re.escape(ap_password)} IP={REGEX_IPV4}", timeout=20) + ap_ip = m.group(1).decode() + LOGGER.info(f"AP started successfully. IP: {ap_ip}") + assert is_valid_ipv4(ap_ip) + + # Wait for AP to begin reporting station count + LOGGER.info(f"Waiting for AP to begin reporting station count") + ap.expect(r"\[AP\] Stations connected: \d+", timeout=20) + + # Client scans for AP + LOGGER.info(f"Scanning for AP: {ap_ssid}") + client.expect_exact("[CLIENT] Scan start") + client.expect_exact("[CLIENT] Scan done") + client.expect_exact(f"{ap_ssid}") + LOGGER.info(f"AP found") + + # Client connects to AP + LOGGER.info(f"Connecting to AP") + client.expect_exact(f"[CLIENT] Connecting to SSID={ap_ssid} Password={ap_password}", timeout=20) + + # Client connection success + m = client.expect(rf"\[CLIENT\] Connected! IP={REGEX_IPV4}", timeout=30) + client_ip = m.group(1).decode() + LOGGER.info(f"Client connection successful. IP: {client_ip}") + assert is_valid_ipv4(client_ip) + + # Verify AP reports 1 station connected + LOGGER.info(f"Waiting for AP to report 1 station connected") + ap.expect_exact("[AP] Stations connected: 1", timeout=30) + + # Verify client stays connected + LOGGER.info(f"Waiting for client to report connected status") + client.expect_exact("[CLIENT] Status=3", timeout=10) # WL_CONNECTED + + LOGGER.info(f"WiFi AP test passed")