diff --git a/.github/workflows/port-status-contract.yml b/.github/workflows/port-status-contract.yml index 7cf2634b717..2f2fd4dbb64 100644 --- a/.github/workflows/port-status-contract.yml +++ b/.github/workflows/port-status-contract.yml @@ -75,6 +75,18 @@ jobs: python-version: '3.13' - name: Validate every test and golden mapping run: python3 scripts/hellocodenameone/conformance/port_status.py validate + # The reports under docs/website/data/port_status_reports are CI output. + # Nothing about registering a test requires editing one -- each port picks + # the test up on its next master run -- so a change to a report's results + # that keeps the stamp naming the run is a forged result, not a fix. + - name: Refuse hand-edited port status reports + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + run: | + if [ -n "${BASE_SHA}" ] && [ "${BASE_SHA}" != "0000000000000000000000000000000000000000" ]; then + git fetch --no-tags --depth=1 origin "${BASE_SHA}" || true + fi + ./scripts/hellocodenameone/conformance/check_port_status_provenance.sh "${BASE_SHA:-HEAD^}" - name: Run normalizer tests working-directory: scripts/hellocodenameone/conformance run: python3 -m unittest -v test_port_status.py diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidAsyncView.java b/Ports/Android/src/com/codename1/impl/android/AndroidAsyncView.java index be8eca6ff01..69ab44f5c1c 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidAsyncView.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidAsyncView.java @@ -218,15 +218,31 @@ protected void dispatchDraw(Canvas c) { //final HashMap slowest = new HashMap<>(); //final HashMap counts = new HashMap<>(); - int count = renderingOperations.size(); + // Snapshot under the lock this method already takes to clear the queue. + // Reading size() and then copying without it let flushGraphics swap the + // list in between, and ArrayList.addAll copies through toArray(): a + // concurrent mutation there returns an array sized for the new contents + // and padded with NULLS. Those nulls arrived here as AsyncOps and threw + // out of executeWithClip below, which is a hard crash on the UI thread. + // flushGraphics has carried an "if (o != null)" guard against the same + // corruption since a user reported it; this is the other end of it, and + // the reason the nulls exist at all. + // + // Only the copy is synchronized. The ops are executed below without the + // lock because that is the frame's actual drawing, and holding it there + // would park the EDT in flushGraphics for the whole paint. + int count; + synchronized (RENDERING_OPERATIONS_LOCK) { + count = renderingOperations.size(); - // this works around the case of a blank screen when an invalidate occurs out of nowhere - // and no operations are in the queue - if(count > 0) { - currentlyRendering.clear(); - currentlyRendering.addAll(renderingOperations); - } else { - count = currentlyRendering.size(); + // this works around the case of a blank screen when an invalidate occurs out of nowhere + // and no operations are in the queue + if(count > 0) { + currentlyRendering.clear(); + currentlyRendering.addAll(renderingOperations); + } else { + count = currentlyRendering.size(); + } } int offset = 0; for(; offset < count ; offset++) { @@ -364,13 +380,23 @@ public void run() { // When this reaches 10, the rendering operations are flushed. private int timeoutCounter=0; + // Not the crash, but the same field: dispatchDraw clears renderingOperations + // holding the lock, so reading it without one has no happens-before against + // that clear and can spin on a stale size. Leaving one access of a + // lock-guarded field unguarded is how the next one gets written that way. + private boolean renderingOperationsPending() { + synchronized (RENDERING_OPERATIONS_LOCK) { + return !renderingOperations.isEmpty(); + } + } + @Override public void flushGraphics(Rect rect) { //Log.d(Display.getInstance().getProperty("AppName", "CodenameOne"), "Flush graphics invoked with pending: " + pendingRenderingOperations.size() + " and current " + renderingOperations.size()); // we might have pending entries in the rendering queue int counter = 0; - while (!renderingOperations.isEmpty()) { + while (renderingOperationsPending()) { try { synchronized (RENDERING_OPERATIONS_LOCK) { RENDERING_OPERATIONS_LOCK.wait(5); @@ -404,9 +430,14 @@ public void run() { } } timeoutCounter = 0; - ArrayList tmp = renderingOperations; - renderingOperations = pendingRenderingOperations; - pendingRenderingOperations = tmp; + // The swap the snapshot in dispatchDraw races with. Unsynchronized, it + // could replace the list mid-copy, which is how the copy came back + // holding nulls. + synchronized (RENDERING_OPERATIONS_LOCK) { + ArrayList tmp = renderingOperations; + renderingOperations = pendingRenderingOperations; + pendingRenderingOperations = tmp; + } try { for (AsyncOp o : renderingOperations) { // can happen due to synchronization issues see: https://www.reddit.com/r/cn1/comments/1oo43in/error_while_using_app/ diff --git a/docs/website/data/port_status_reports/android.json b/docs/website/data/port_status_reports/android.json index b73e72f53ff..07ee927ebc0 100644 --- a/docs/website/data/port_status_reports/android.json +++ b/docs/website/data/port_status_reports/android.json @@ -1,48 +1,48 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T10:47:47Z", + "commit": "73bd710543cba1d6728b9e0e92d436fb0d95580a", + "generated_at": "2026-08-23T11:55:17Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 207587681 + "duration_ns": 275236406 }, "arraySequential": { "checksum": "0", - "duration_ns": 25544773 + "duration_ns": 34295970 }, "hashMapChurn": { "checksum": "49941", - "duration_ns": 233590607 + "duration_ns": 303707003 }, "intArithmetic": { "checksum": "1307491170054", - "duration_ns": 166166998 + "duration_ns": 244566905 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 87938220 + "duration_ns": 142422396 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 2460716848 + "duration_ns": 5489752590 }, "objectAllocation": { "checksum": "2999790376128", - "duration_ns": 114265882 + "duration_ns": 148232049 }, "quicksort": { "checksum": "809667393311589960", - "duration_ns": 127842947 + "duration_ns": 151270219 }, "recursion": { "checksum": "33385282", - "duration_ns": 490740312 + "duration_ns": 752193112 }, "stringBuilding": { "checksum": "-609121604069", - "duration_ns": 217409339 + "duration_ns": 311457117 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -52,14 +52,14 @@ "suite_checksum": 0 }, "port": "android", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868198", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32636698063", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 179, "fail": 0, - "skip": 1, - "not-run": 0 + "not-run": 0, + "pass": 179, + "skip": 1 }, "tests": { "ARApiTest": { diff --git a/docs/website/data/port_status_reports/ios-gl.json b/docs/website/data/port_status_reports/ios-gl.json index 0782f7142e2..15dc8f05595 100644 --- a/docs/website/data/port_status_reports/ios-gl.json +++ b/docs/website/data/port_status_reports/ios-gl.json @@ -1,36 +1,36 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T15:32:20Z", + "commit": "c2732fd24ff606421f361f239792cc2d717ce47d", + "generated_at": "2026-08-23T11:14:46Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 390405000 + "duration_ns": 342584000 }, "arraySequential": { "checksum": "0", - "duration_ns": 30327000 + "duration_ns": 24001000 }, "intArithmetic": { "checksum": "1307491170054", - "duration_ns": 144064000 + "duration_ns": 121546000 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 84448000 + "duration_ns": 78571000 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 414960000 + "duration_ns": 309299000 }, "quicksort": { "checksum": "809667393311589960", - "duration_ns": 221066000 + "duration_ns": 161716000 }, "recursion": { "checksum": "33385282", - "duration_ns": 236629000 + "duration_ns": 190209000 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -44,14 +44,14 @@ "suite_checksum": 0 }, "port": "ios-gl", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868735", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32613256755", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 176, "fail": 0, - "skip": 4, - "not-run": 0 + "not-run": 0, + "pass": 175, + "skip": 4 }, "tests": { "ARApiTest": { @@ -453,10 +453,6 @@ "feature": "notifications", "status": "pass" }, - "LogSubclassCaptureTest": { - "feature": "logging-diagnostics", - "status": "pass" - }, "LottieAnimatedScreenshotTest": { "feature": "svg-lottie", "status": "pass" diff --git a/docs/website/data/port_status_reports/ios-metal.json b/docs/website/data/port_status_reports/ios-metal.json index 90ac2753d2f..0ec10c5ed90 100644 --- a/docs/website/data/port_status_reports/ios-metal.json +++ b/docs/website/data/port_status_reports/ios-metal.json @@ -1,36 +1,36 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T14:15:59Z", + "commit": "c2732fd24ff606421f361f239792cc2d717ce47d", + "generated_at": "2026-08-23T10:35:55Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 294811000 + "duration_ns": 312898000 }, "arraySequential": { "checksum": "0", - "duration_ns": 22420000 + "duration_ns": 23531000 }, "intArithmetic": { "checksum": "1307491170054", - "duration_ns": 127686000 + "duration_ns": 142431000 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 77184000 + "duration_ns": 94978000 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 302310000 + "duration_ns": 393993000 }, "quicksort": { "checksum": "809667393311589960", - "duration_ns": 124076000 + "duration_ns": 139869000 }, "recursion": { "checksum": "33385282", - "duration_ns": 151972000 + "duration_ns": 175892000 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -44,14 +44,14 @@ "suite_checksum": 0 }, "port": "ios-metal", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868735", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32613256755", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 176, "fail": 0, - "skip": 4, - "not-run": 0 + "not-run": 0, + "pass": 175, + "skip": 4 }, "tests": { "ARApiTest": { @@ -453,10 +453,6 @@ "feature": "notifications", "status": "pass" }, - "LogSubclassCaptureTest": { - "feature": "logging-diagnostics", - "status": "pass" - }, "LottieAnimatedScreenshotTest": { "feature": "svg-lottie", "status": "pass" diff --git a/docs/website/data/port_status_reports/javascript.json b/docs/website/data/port_status_reports/javascript.json index cac9ede04bb..83ac4ec3bf9 100644 --- a/docs/website/data/port_status_reports/javascript.json +++ b/docs/website/data/port_status_reports/javascript.json @@ -1,48 +1,48 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T11:06:56Z", + "commit": "73bd710543cba1d6728b9e0e92d436fb0d95580a", + "generated_at": "2026-08-23T12:08:16Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 1235399999 + "duration_ns": 939500000 }, "arraySequential": { "checksum": "0", - "duration_ns": 974200000 + "duration_ns": 735600000 }, "hashMapChurn": { "checksum": "49941", - "duration_ns": 11617100000 + "duration_ns": 7886199999 }, "intArithmetic": { "checksum": "1313580095284", - "duration_ns": 648800001 + "duration_ns": 511700000 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 3307399999 + "duration_ns": 2448200001 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 403899999 + "duration_ns": 284299999 }, "objectAllocation": { "checksum": "2999790376128", - "duration_ns": 2227300001 + "duration_ns": 1654900001 }, "quicksort": { "checksum": "786886890168670967", - "duration_ns": 578600001 + "duration_ns": 435500000 }, "recursion": { "checksum": "33385282", - "duration_ns": 1036300001 + "duration_ns": 959200000 }, "stringBuilding": { "checksum": "-609121604069", - "duration_ns": 1022300000 + "duration_ns": 795500000 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -52,14 +52,14 @@ "suite_checksum": 0 }, "port": "javascript", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868791", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32636698236", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 178, "fail": 0, - "skip": 2, - "not-run": 0 + "not-run": 0, + "pass": 178, + "skip": 2 }, "tests": { "ARApiTest": { diff --git a/docs/website/data/port_status_reports/linux-arm64.json b/docs/website/data/port_status_reports/linux-arm64.json index ec8f3a901c7..ef0b8545899 100644 --- a/docs/website/data/port_status_reports/linux-arm64.json +++ b/docs/website/data/port_status_reports/linux-arm64.json @@ -1,48 +1,48 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T10:56:05Z", + "commit": "73bd710543cba1d6728b9e0e92d436fb0d95580a", + "generated_at": "2026-08-23T12:04:44Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 178201190 + "duration_ns": 169161399 }, "arraySequential": { "checksum": "0", - "duration_ns": 24615262 + "duration_ns": 25734693 }, "hashMapChurn": { "checksum": "49941", - "duration_ns": 35635763 + "duration_ns": 35829660 }, "intArithmetic": { "checksum": "1307491170054", - "duration_ns": 62902952 + "duration_ns": 62856710 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 39311436 + "duration_ns": 39198273 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 274731843 + "duration_ns": 274361241 }, "objectAllocation": { "checksum": "2999790376128", - "duration_ns": 3428129909 + "duration_ns": 2984164556 }, "quicksort": { "checksum": "809667393311589960", - "duration_ns": 93424737 + "duration_ns": 93609664 }, "recursion": { "checksum": "33385282", - "duration_ns": 137446629 + "duration_ns": 138668513 }, "stringBuilding": { "checksum": "-609121604069", - "duration_ns": 46220590 + "duration_ns": 44702701 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -52,14 +52,14 @@ "suite_checksum": 0 }, "port": "linux-arm64", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868573", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32636698010", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 177, "fail": 0, - "skip": 3, - "not-run": 0 + "not-run": 0, + "pass": 177, + "skip": 3 }, "tests": { "ARApiTest": { diff --git a/docs/website/data/port_status_reports/linux-x64.json b/docs/website/data/port_status_reports/linux-x64.json index f2c2df8ab2b..4fe2a71d8f7 100644 --- a/docs/website/data/port_status_reports/linux-x64.json +++ b/docs/website/data/port_status_reports/linux-x64.json @@ -1,48 +1,48 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T10:56:00Z", + "commit": "73bd710543cba1d6728b9e0e92d436fb0d95580a", + "generated_at": "2026-08-23T12:04:41Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 244264539 + "duration_ns": 239946561 }, "arraySequential": { "checksum": "0", - "duration_ns": 27436409 + "duration_ns": 27210736 }, "hashMapChurn": { "checksum": "49941", - "duration_ns": 39081845 + "duration_ns": 41392475 }, "intArithmetic": { "checksum": "1307491170054", - "duration_ns": 76918383 + "duration_ns": 75971442 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 49941482 + "duration_ns": 48763849 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 178465217 + "duration_ns": 174560158 }, "objectAllocation": { "checksum": "2999790376128", - "duration_ns": 3233252273 + "duration_ns": 2865454230 }, "quicksort": { "checksum": "809667393311589960", - "duration_ns": 150670050 + "duration_ns": 142840314 }, "recursion": { "checksum": "33385282", - "duration_ns": 182631896 + "duration_ns": 167246876 }, "stringBuilding": { "checksum": "-609121604069", - "duration_ns": 41656108 + "duration_ns": 39554939 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -52,14 +52,14 @@ "suite_checksum": 0 }, "port": "linux-x64", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868573", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32636698010", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 177, "fail": 0, - "skip": 3, - "not-run": 0 + "not-run": 0, + "pass": 177, + "skip": 3 }, "tests": { "ARApiTest": { diff --git a/docs/website/data/port_status_reports/mac-native.json b/docs/website/data/port_status_reports/mac-native.json index 5c39251d12d..6850706451f 100644 --- a/docs/website/data/port_status_reports/mac-native.json +++ b/docs/website/data/port_status_reports/mac-native.json @@ -1,48 +1,48 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T13:42:34Z", + "commit": "c2732fd24ff606421f361f239792cc2d717ce47d", + "generated_at": "2026-08-23T07:14:14Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 282021000 + "duration_ns": 276009000 }, "arraySequential": { "checksum": "0", - "duration_ns": 22970000 + "duration_ns": 20703000 }, "hashMapChurn": { "checksum": "49941", - "duration_ns": 63578000 + "duration_ns": 62537000 }, "intArithmetic": { "checksum": "1307491170054", - "duration_ns": 108942000 + "duration_ns": 122753000 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 68271000 + "duration_ns": 65793000 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 307321000 + "duration_ns": 384003000 }, "objectAllocation": { "checksum": "2999790376128", - "duration_ns": 477974000 + "duration_ns": 8848612000 }, "quicksort": { "checksum": "809667393311589960", - "duration_ns": 134770000 + "duration_ns": 115990000 }, "recursion": { "checksum": "33385282", - "duration_ns": 181410000 + "duration_ns": 152640000 }, "stringBuilding": { "checksum": "-609121604069", - "duration_ns": 100529000 + "duration_ns": 40571000 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -52,14 +52,14 @@ "suite_checksum": 0 }, "port": "mac-native", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868564", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32613364013", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 175, "fail": 0, - "skip": 5, - "not-run": 0 + "not-run": 0, + "pass": 174, + "skip": 5 }, "tests": { "ARApiTest": { @@ -464,10 +464,6 @@ "feature": "notifications", "status": "pass" }, - "LogSubclassCaptureTest": { - "feature": "logging-diagnostics", - "status": "pass" - }, "LottieAnimatedScreenshotTest": { "feature": "svg-lottie", "status": "pass" diff --git a/docs/website/data/port_status_reports/tvos.json b/docs/website/data/port_status_reports/tvos.json index 8a2bec7e4fa..5ee6d47c9b1 100644 --- a/docs/website/data/port_status_reports/tvos.json +++ b/docs/website/data/port_status_reports/tvos.json @@ -1,36 +1,36 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T14:55:20Z", + "commit": "c2732fd24ff606421f361f239792cc2d717ce47d", + "generated_at": "2026-08-23T10:56:26Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 436593000 + "duration_ns": 461527000 }, "arraySequential": { "checksum": "0", - "duration_ns": 180134000 + "duration_ns": 187070000 }, "intArithmetic": { "checksum": "1307491170054", - "duration_ns": 203189000 + "duration_ns": 299464000 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 127759000 + "duration_ns": 149356000 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 498230000 + "duration_ns": 642187000 }, "quicksort": { "checksum": "809667393311589960", - "duration_ns": 450828000 + "duration_ns": 471247000 }, "recursion": { "checksum": "33385282", - "duration_ns": 952488000 + "duration_ns": 1007828000 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -44,14 +44,14 @@ "suite_checksum": 0 }, "port": "tvos", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868735", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32613256755", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 172, "fail": 0, - "skip": 8, - "not-run": 0 + "not-run": 0, + "pass": 171, + "skip": 8 }, "tests": { "ARApiTest": { @@ -456,10 +456,6 @@ "feature": "notifications", "status": "pass" }, - "LogSubclassCaptureTest": { - "feature": "logging-diagnostics", - "status": "pass" - }, "LottieAnimatedScreenshotTest": { "feature": "svg-lottie", "status": "pass" diff --git a/docs/website/data/port_status_reports/watchos.json b/docs/website/data/port_status_reports/watchos.json index c46ad7ea75b..574f5ffbede 100644 --- a/docs/website/data/port_status_reports/watchos.json +++ b/docs/website/data/port_status_reports/watchos.json @@ -1,36 +1,36 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T14:40:32Z", + "commit": "c2732fd24ff606421f361f239792cc2d717ce47d", + "generated_at": "2026-08-23T06:25:35Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 433298000 + "duration_ns": 450268000 }, "arraySequential": { "checksum": "0", - "duration_ns": 197790000 + "duration_ns": 193759000 }, "intArithmetic": { "checksum": "1307491170054", - "duration_ns": 209522000 + "duration_ns": 206141000 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 135765000 + "duration_ns": 133625000 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 525986000 + "duration_ns": 575246000 }, "quicksort": { "checksum": "809667393311589960", - "duration_ns": 432644000 + "duration_ns": 405366000 }, "recursion": { "checksum": "33385282", - "duration_ns": 872982000 + "duration_ns": 980573000 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -44,14 +44,14 @@ "suite_checksum": 0 }, "port": "watchos", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868735", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32613256755", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 167, "fail": 0, - "skip": 13, - "not-run": 0 + "not-run": 0, + "pass": 166, + "skip": 13 }, "tests": { "ARApiTest": { @@ -462,10 +462,6 @@ "feature": "notifications", "status": "pass" }, - "LogSubclassCaptureTest": { - "feature": "logging-diagnostics", - "status": "pass" - }, "LottieAnimatedScreenshotTest": { "feature": "svg-lottie", "status": "pass" diff --git a/docs/website/data/port_status_reports/windows-arm64.json b/docs/website/data/port_status_reports/windows-arm64.json index 4b9093e6fb8..830b3779f56 100644 --- a/docs/website/data/port_status_reports/windows-arm64.json +++ b/docs/website/data/port_status_reports/windows-arm64.json @@ -1,48 +1,48 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T11:00:45Z", + "commit": "73bd710543cba1d6728b9e0e92d436fb0d95580a", + "generated_at": "2026-08-23T12:02:52Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 175137000 + "duration_ns": 176299000 }, "arraySequential": { "checksum": "0", - "duration_ns": 24765000 + "duration_ns": 24874000 }, "hashMapChurn": { "checksum": "49941", - "duration_ns": 57031000 + "duration_ns": 55425000 }, "intArithmetic": { "checksum": "1307491170054", - "duration_ns": 62833000 + "duration_ns": 62781000 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 39182000 + "duration_ns": 39346000 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 220308000 + "duration_ns": 222151000 }, "objectAllocation": { "checksum": "2999790376128", - "duration_ns": 3932948000 + "duration_ns": 3775906000 }, "quicksort": { "checksum": "809667393311589960", - "duration_ns": 92920000 + "duration_ns": 94655000 }, "recursion": { "checksum": "33385282", - "duration_ns": 132041000 + "duration_ns": 133331000 }, "stringBuilding": { "checksum": "-609121604069", - "duration_ns": 50491000 + "duration_ns": 46497000 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -52,14 +52,14 @@ "suite_checksum": 0 }, "port": "windows-arm64", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868423", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32636698019", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 177, "fail": 0, - "skip": 3, - "not-run": 0 + "not-run": 0, + "pass": 177, + "skip": 3 }, "tests": { "ARApiTest": { diff --git a/docs/website/data/port_status_reports/windows-x64.json b/docs/website/data/port_status_reports/windows-x64.json index 041d97e6062..17a1a0e87d5 100644 --- a/docs/website/data/port_status_reports/windows-x64.json +++ b/docs/website/data/port_status_reports/windows-x64.json @@ -1,48 +1,48 @@ { - "commit": "896356159b4a984b5eb6633e060dff480b7c9ed8", - "generated_at": "2026-08-19T10:48:15Z", + "commit": "73bd710543cba1d6728b9e0e92d436fb0d95580a", + "generated_at": "2026-08-23T11:51:25Z", "performance": { "benchmark_version": 1, "benchmarks": { "arrayRandom": { "checksum": "-2288487891715278", - "duration_ns": 296472000 + "duration_ns": 256791000 }, "arraySequential": { "checksum": "0", - "duration_ns": 25444000 + "duration_ns": 25135000 }, "hashMapChurn": { "checksum": "49941", - "duration_ns": 72049000 + "duration_ns": 69040000 }, "intArithmetic": { "checksum": "1307491170054", - "duration_ns": 87100000 + "duration_ns": 79166000 }, "longArithmetic": { "checksum": "6887886960473257608", - "duration_ns": 63857000 + "duration_ns": 60939000 }, "mathTranscendental": { "checksum": "4729652805076374709", - "duration_ns": 175366000 + "duration_ns": 176072000 }, "objectAllocation": { "checksum": "2999790376128", - "duration_ns": 4228935000 + "duration_ns": 3344991000 }, "quicksort": { "checksum": "809667393311589960", - "duration_ns": 138532000 + "duration_ns": 137912000 }, "recursion": { "checksum": "33385282", - "duration_ns": 226805000 + "duration_ns": 226975000 }, "stringBuilding": { "checksum": "-609121604069", - "duration_ns": 45716000 + "duration_ns": 44624000 } }, "method": "minimum of five measured runs after three in-process warm-ups", @@ -52,14 +52,14 @@ "suite_checksum": 0 }, "port": "windows-x64", - "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32242868555", + "run_url": "https://github.com/codenameone/CodenameOne/actions/runs/32636697893", "schema_version": 1, "suite_finished": true, "summary": { - "pass": 177, "fail": 0, - "skip": 3, - "not-run": 0 + "not-run": 0, + "pass": 177, + "skip": 3 }, "tests": { "ARApiTest": { diff --git a/docs/website/layouts/partials/port-status-feature-status.html b/docs/website/layouts/partials/port-status-feature-status.html index 251843fe25b..55379cea772 100644 --- a/docs/website/layouts/partials/port-status-feature-status.html +++ b/docs/website/layouts/partials/port-status-feature-status.html @@ -13,11 +13,18 @@ {{- $failed := 0 -}} {{- $skipped := 0 -}} {{- $notRun := 0 -}} + {{- $awaiting := 0 -}} {{- $failedTests := slice -}} {{- $skippedTests := slice -}} {{- range $feature.tests -}} {{- $result := index $report.tests . -}} - {{- $status := "not-run" -}} + {{- /* Absent and "not-run" are different claims. "not-run" means this port + ran the suite with the test in its contract and nothing reported + back, which is a defect. Absent means the run predates the test -- + the ordinary state of every port for a day after one is registered -- + and calling that a defect is what used to make registering a test + require editing eleven reports. */ -}} + {{- $status := "awaiting" -}} {{- with $result -}}{{- $status = .status -}}{{- end -}} {{- if eq $status "pass" -}} {{- $passed = add $passed 1 -}} @@ -27,6 +34,8 @@ {{- else if eq $status "skip" -}} {{- $skipped = add $skipped 1 -}} {{- $skippedTests = $skippedTests | append . -}} + {{- else if eq $status "awaiting" -}} + {{- $awaiting = add $awaiting 1 -}} {{- else -}} {{- $notRun = add $notRun 1 -}} {{- end -}} @@ -63,7 +72,12 @@ {{- with .ports -}} {{- $portAllowed = in . $port.id -}} {{- end -}} - {{- if and $portAllowed (hasPrefix $reason .prefix) -}}{{- $ok = true -}}{{- end -}} + {{- /* An empty prefix is not a match. hasPrefix answers true for + every reason against "", so a code that lost its prefix to a + typo would render any future skip of that test as green and + documented. The contract validator refuses such an erratum, + and this is the same rule on the side that draws the tick. */ -}} + {{- if and $portAllowed .prefix (hasPrefix $reason .prefix) -}}{{- $ok = true -}}{{- end -}} {{- end -}} {{- if not $ok -}}{{- $allMatched = false -}}{{- end -}} {{- end -}} @@ -80,7 +94,8 @@ {{- $total := len $feature.tests -}} {{- $state = "partial" -}} {{- $mark = "−" -}} - {{- $label = printf "%d passed, %d skipped, %d not run" $passed $skipped $notRun -}} + {{- $awaitingNote := cond (gt $awaiting 0) (printf ", %d awaiting this port's next run" $awaiting) "" -}} + {{- $label = printf "%d passed, %d skipped, %d not run%s" $passed $skipped $notRun $awaitingNote -}} {{- /* Evidence is per feature. A run that stopped early leaves its own unreached tests as "not run" below, and the port card reports the incomplete run; that is not a reason to withdraw the result of a @@ -104,7 +119,7 @@ {{- $label = printf "%d of %d mapped tests passed; %s skipped by the CI environment, see the skipped-test errata%s" $passed $total (delimit $skippedTests ", ") $incomplete -}} {{- end -}} {{- else if not $complete -}} - {{- $label = printf "Suite did not finish; %d passed, %d skipped, %d not run" $passed $skipped $notRun -}} + {{- $label = printf "Suite did not finish; %d passed, %d skipped, %d not run%s" $passed $skipped $notRun $awaitingNote -}} {{- else if eq $skipped $total -}} {{- $label = "All mapped tests skipped" -}} {{- end -}} diff --git a/docs/website/layouts/partials/port-status-port-state.html b/docs/website/layouts/partials/port-status-port-state.html index 7250185b0d0..710e38c85e9 100644 --- a/docs/website/layouts/partials/port-status-port-state.html +++ b/docs/website/layouts/partials/port-status-port-state.html @@ -4,9 +4,37 @@ {{- $now := .now -}} {{- $state := "unknown" -}} {{- $label := "No stored report" -}} +{{- $awaiting := 0 -}} +{{- $passed := 0 -}} +{{- $skipped := 0 -}} +{{- $failed := 0 -}} +{{- $notRun := 0 -}} {{- if $report -}} - {{- $failed := int (default 0 $report.summary.fail) -}} - {{- $notRun := int (default 0 (index $report.summary "not-run")) -}} + {{- /* Counted over the contract, not over the report's own summary. Those are + different sets in both directions, and the card was wrong at both ends + of the difference. A report predating a newly registered test carries no + entry for it, and summing only what the report holds presented that as a + complete answer to a contract it has not seen. A report that predates a + test's RETIREMENT still carries the test, and a retired failure kept the + card red over a test that appears nowhere in the matrix below it. The + card is the sum of the cells the table shows, so it is now literally + computed that way. */ -}} + {{- range $contract.features -}} + {{- range .tests -}} + {{- $result := index $report.tests . -}} + {{- if not $result -}} + {{- $awaiting = add $awaiting 1 -}} + {{- else if eq $result.status "pass" -}} + {{- $passed = add $passed 1 -}} + {{- else if eq $result.status "skip" -}} + {{- $skipped = add $skipped 1 -}} + {{- else if eq $result.status "fail" -}} + {{- $failed = add $failed 1 -}} + {{- else -}} + {{- $notRun = add $notRun 1 -}} + {{- end -}} + {{- end -}} + {{- end -}} {{- $bootstrapComplete := and (eq $report.bootstrap_source "successful-master-workflow") (eq $report.workflow_conclusion "success") -}} {{- $complete := or $report.suite_finished $bootstrapComplete -}} {{- $generated := time.AsTime $report.generated_at -}} @@ -31,6 +59,14 @@ {{- else if $stale -}} {{- $state = "partial" -}} {{- $label = "Report is stale" -}} + {{- else if gt $awaiting 0 -}} + {{- /* Last in the chain because it is the mildest of these, but not absent + from it: the feature cell for a test this run predates already renders + partial, and leaving the card green said "Suite completed" directly + above a column that is missing an answer. The suite did complete -- + against a contract that has since grown. */ -}} + {{- $state = "partial" -}} + {{- $label = printf "%d test%s awaiting this port's next run" $awaiting (cond (eq $awaiting 1) "" "s") -}} {{- end -}} {{- end -}}
@@ -41,7 +77,7 @@

{{ $port.name }}

{{ $label }}

{{- if $report }}

- {{ $report.summary.pass }} passed · {{ $report.summary.skip }} skipped · {{ index $report.summary "not-run" }} not run + {{ $passed }} passed · {{ $skipped }} skipped · {{ $notRun }} not run{{ if gt $awaiting 0 }} · {{ $awaiting }} awaiting this port's next run{{ end }}

{{- end }}
diff --git a/scripts/hellocodenameone/README.adoc b/scripts/hellocodenameone/README.adoc index c50f15c6bdd..baccc10cd37 100644 --- a/scripts/hellocodenameone/README.adoc +++ b/scripts/hellocodenameone/README.adoc @@ -124,6 +124,72 @@ When adding a conformance test: . Run the validator before pushing. The `Validate port status contract` workflow enforces the same rules in CI. +**Do not touch `docs/website/data/port_status_reports/`.** Those files are CI +output -- a snapshot of what one run measured -- and a branch has no result to +put in them. Each port picks a newly registered test up on its next `master` +run; until then the validator prints `checked-in snapshot : report +predates tests: ` and the public table shows that cell as not run, which +is what is true. The reports the site actually serves come from the +`port-status-data` branch, not from these. + +This is a deliberate reversal. The contract used to require every checked-in +report to carry an entry for every registered test, so adding one test meant +editing eleven report files and retyping their totals -- which made every +test-adding branch conflict with every other one, and made the cheapest way to +a green build typing `pass` next to a test no port had run. Twelve results +reached `master` that way, attributed to runs that predated the tests. `Validate +port status contract` now refuses a report whose findings changed without a new +run behind them. Every field but `generated_at`, `commit` and `run_url` counts +as a finding, so the benchmark durations the page publishes as measurements are +covered along with the test map. Both `run_url` and `generated_at` have to name +a genuinely different run (`commit` need not move -- a port legitimately re-runs +the same master commit), the run URL has to look like one, and the report itself +has to be a version the `port-status-data` branch actually holds. That last +check is the one that matters: a refreshed snapshot is a copy of what CI +published, so it matches by construction, while an invented one cannot be made +to match without the write access to that branch which only the publish +workflows have. It is asked of any change, including one that touches only +`generated_at` -- that edit changes no finding, and it is how a port that had +stopped reporting would go on looking current -- and of a report the branch +*adds*, which is otherwise the one file with no earlier version to check against. +The check reads about a week of the branch's history, so a refresh taken before +the port ran again still matches; if the branch cannot be fetched it is skipped +rather than failed, because unverifiable is not forged. + +A port with no stored report is a supported state, so adding a port does not +start by hand-authoring one: every cell reads "No stored report" and the card +reads "unknown" until that port's first run publishes something to copy. That +does not make removing one free -- deleting the fallback for a port that has +published is refused, because the site serves this file exactly when the data +branch cannot be reached, and an established column would go unknown at the +moment the live data is missing. Retiring a port is still fine: drop it from the +manifest and the check no longer looks at it. + +"All tests run on all ports" is still enforced, just where it cannot be typed: +the nightly sweep holds the *published* reports to it. A test left at `not-run` +fails the sweep, and so does a test missing from one port's report when an +earlier run on another port already covered it -- that port dropped the test +rather than merely predating it. Comparing reports to each other cannot see a +test missing from *all* of them, so the sweep also reads the manifest at each +report's own commit: a run whose own contract listed the test and reported +nothing for it fails, however the other ports behaved. The single permitted +exception is a `skip` the +suite itself emits, and the sweep checks it against +`port_status_supplement.json` the way the page does: the erratum has to name the +test *and*, where it lists reason codes, account for the reason that run gave +from a port that code applies to. Every reason code needs a non-empty `prefix` +and, if it names ports, real ones -- the validator enforces both, because every +string starts with the empty prefix, so a code that lost that field would +document any future skip of its test instead of the one it was written for. A port that quietly starts skipping a test +fails the sweep by name rather than surfacing later as a failed website build. + +The sweep's closing assertion treats contract drift as the ordinary state it is, +matching the candidate loop above it. It used to count a port whose published +report predates a newly registered test as a failure, so the nightly went red +for a day after any test was added -- and returned before the coverage gate +could read the reports whose "this run predates the test" case that gate exists +to tolerate. + The on-device AI rows are permanent assertion tests rather than screenshots. They validate immutable image, camera-frame, tensor, model-source, and options contracts on every translated runtime; query every native analyzer or service; diff --git a/scripts/hellocodenameone/conformance/backfill_port_status.sh b/scripts/hellocodenameone/conformance/backfill_port_status.sh index 5f9cec24dd0..16a269dd9c6 100755 --- a/scripts/hellocodenameone/conformance/backfill_port_status.sh +++ b/scripts/hellocodenameone/conformance/backfill_port_status.sh @@ -404,18 +404,44 @@ echo "Port status sweep: published ${published} report(s), ${skipped} already cu # table, which is exactly the failure this sweep exists to prevent. stale_days="$(jq -r '.stale_after_days' "${MANIFEST}")" problems=() +published_dir="${tmp_dir}/published" +contracts_dir="${tmp_dir}/contracts" +mkdir -p "${published_dir}" "${contracts_dir}" while IFS= read -r port; do if ! gh api "repos/${GITHUB_REPOSITORY}/contents/ports/${port}.json?ref=${DATA_BRANCH}" \ --jq '.content' 2>/dev/null | decode_base64 > "${tmp_dir}/check.json" 2>/dev/null; then problems+=("${port}: no published report") continue fi + cp "${tmp_dir}/check.json" "${published_dir}/${port}.json" + # The contract this report's own run was built against. Comparing reports to + # each other cannot see a test missing from ALL of them -- there is no older + # report left to prove it existed -- and a test nothing runs anywhere is the + # worst version of what this gate is for. Reading the manifest at the report's + # commit answers it directly: a run whose own contract listed the test has no + # excuse for reporting nothing. A fetch that fails simply leaves the port out, + # which keeps the weaker comparison rather than inventing an obligation. + report_commit="$(jq -r '.commit // empty' "${tmp_dir}/check.json" 2>/dev/null || true)" + if [ -n "${report_commit}" ]; then + gh api "repos/${GITHUB_REPOSITORY}/contents/docs/website/data/port_status.json?ref=${report_commit}" \ + --jq '.content' 2>/dev/null | decode_base64 > "${contracts_dir}/${port}.json" 2>/dev/null \ + || rm -f "${contracts_dir}/${port}.json" + fi # Freshness alone is not enough: a published report the website rejects # leaves the column on its checked-in fallback, which is the state this # sweep exists to detect. accept_status=0 python3 "${SCRIPT_DIR}/port_status.py" accept --port "${port}" --report "${tmp_dir}/check.json" >/dev/null || accept_status=$? - if [ "${accept_status}" -ne 0 ]; then + # Contract drift is not a problem here, for the same reason the candidate loop + # above says it is not one: a report built before a newly registered test is + # the expected state of every port between that merge and its next run. This + # loop used to call it one, so the sweep failed for a day after any test was + # added -- and, worse, exited before the coverage gate below could look at the + # very reports whose "the run predates the test" case that gate exists to + # tolerate. Only a report the website cannot use is a defect. + if [ "${accept_status}" -eq "${ACCEPT_CONTRACT_DRIFT}" ]; then + echo "${port}: published report is $(describe_accept_status "${accept_status}")." >&2 + elif [ "${accept_status}" -ne 0 ]; then problems+=("${port}: published report is $(describe_accept_status "${accept_status}")") continue fi @@ -468,3 +494,13 @@ if [ ${#unusable[@]} -gt 0 ]; then fi echo "Every port in the contract has a report inside the ${stale_days}-day window." + +# Freshness says the port reported; it does not say the port reported on every +# test. That obligation used to be enforced against the checked-in fallbacks, +# where a branch could satisfy it by typing "pass" -- so it is enforced here +# instead, against what the ports actually published, where nothing anyone +# writes in a pull request can reach it. A registered test runs on every port, +# and the only permitted exception is a skip the suite itself emits with an +# erratum explaining it. +python3 "${SCRIPT_DIR}/port_status.py" coverage \ + --reports "${published_dir}" --contracts "${contracts_dir}" diff --git a/scripts/hellocodenameone/conformance/check_port_status_provenance.sh b/scripts/hellocodenameone/conformance/check_port_status_provenance.sh new file mode 100755 index 00000000000..ec493c66ef6 --- /dev/null +++ b/scripts/hellocodenameone/conformance/check_port_status_provenance.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. Codename One designates this +# particular file as subject to the "Classpath" exception as provided +# by Oracle in the LICENSE file that accompanied this code. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Codename One through http://www.codenameone.com/ if you +# need additional information or have any questions. + +set -euo pipefail + +# Refuse a port status report whose results were edited without a new run. +# +# These files are CI output. A branch never has a reason to change one -- not +# even the branch that registers a new test, because each port picks the test up +# on its next master run and the page shows the gap honestly until it does. The +# check exists because the previous contract *required* the edit, and the +# cheapest way to satisfy it was to type "pass" next to a test no port had run. +# +# Usage: check_port_status_provenance.sh + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +MANIFEST="${REPO_ROOT}/docs/website/data/port_status.json" +REPORT_PATH="docs/website/data/port_status_reports" +DATA_REF="refs/heads/port-status-data" +# Roughly a week of publications across all eleven ports, fetched in under a +# second. A snapshot older than that should be refreshed before review anyway, +# and the failure mode if it is not says exactly that. +DATA_DEPTH=200 + +base_ref="${1:-}" +if [ -z "${base_ref}" ]; then + echo "Usage: $(basename "$0") " >&2 + exit 2 +fi + +if ! git -C "${REPO_ROOT}" rev-parse --verify --quiet "${base_ref}^{commit}" >/dev/null; then + # A shallow clone routinely lacks the base commit. Fetching it is the caller's + # job; without it there is nothing to compare against, and inventing a verdict + # either way would be worse than saying so. + echo "Base revision ${base_ref} is not available; skipping the provenance check." >&2 + exit 0 +fi + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "${tmp_dir}"' EXIT + +base_dir="${tmp_dir}/base" +published_dir="${tmp_dir}/published" +mkdir -p "${base_dir}" "${published_dir}" + +# What the data branch has actually published turns this from "are these two +# strings different" into "did a run produce this report". Unreachable is not +# the same as forged, so a fetch failure drops the corroboration rather than +# failing the branch -- the inequality checks still apply either way. +have_published=1 +if ! git -C "${REPO_ROOT}" fetch --quiet --no-tags --depth="${DATA_DEPTH}" origin "${DATA_REF}"; then + echo "Port Status data branch is unavailable; checking provenance fields only." >&2 + have_published=0 +fi + +while IFS= read -r port; do + head_report="${REPO_ROOT}/${REPORT_PATH}/${port}.json" + if git -C "${REPO_ROOT}" show "${base_ref}:${REPORT_PATH}/${port}.json" \ + > "${base_dir}/${port}.json" 2>/dev/null; then + # Deliberately reached with no head report: deleting an existing fallback is + # its own finding, and skipping absent files was how it went unnoticed. + if [ -f "${head_report}" ] && cmp -s "${base_dir}/${port}.json" "${head_report}"; then + continue + fi + elif [ ! -f "${head_report}" ]; then + continue + else + # A report the branch ADDS. Skipping it here is how a pull request that also + # adds a port could hand-author an entirely green snapshot for it -- the one + # report with no earlier version to be checked against, and so the only one + # nobody was checking at all. It still has to be a report the data branch + # published; a port that has never published simply does not need one. + rm -f "${base_dir}/${port}.json" + fi + [ "${have_published}" -eq 1 ] || continue + # Only for a report that changed: every version the branch has held, so a + # refresh taken before the port ran again still matches an ancestor rather + # than being rejected for not equalling today's tip. + mkdir -p "${published_dir}/${port}" + index=0 + while IFS= read -r revision; do + git -C "${REPO_ROOT}" show "${revision}:ports/${port}.json" \ + > "${published_dir}/${port}/${index}.json" 2>/dev/null \ + || rm -f "${published_dir}/${port}/${index}.json" + index=$((index + 1)) + done < <(git -C "${REPO_ROOT}" log --format=%H FETCH_HEAD -- "ports/${port}.json") +done < <(jq -r '.ports[].id' "${MANIFEST}") + +if [ "${have_published}" -eq 1 ]; then + python3 "${SCRIPT_DIR}/port_status.py" provenance \ + --base "${base_dir}" --published "${published_dir}" +else + python3 "${SCRIPT_DIR}/port_status.py" provenance --base "${base_dir}" +fi diff --git a/scripts/hellocodenameone/conformance/port_status.py b/scripts/hellocodenameone/conformance/port_status.py index 9a40d76b56b..96ddb53f390 100755 --- a/scripts/hellocodenameone/conformance/port_status.py +++ b/scripts/hellocodenameone/conformance/port_status.py @@ -186,71 +186,16 @@ def validate(manifest: dict) -> dict: if owner is None: problems.append(f"Golden screenshot {name} is not mapped to a test") - skipped_tests: set[str] = set() - report_directory = manifest.get("report_directory") - if report_directory: - report_root = REPO_ROOT / report_directory - for port_id in port_ids: - report_path = report_root / f"{port_id}.json" - try: - report = read_json(report_path) - except ContractError as exc: - problems.append(str(exc)) - continue - if report.get("schema_version") != manifest.get("schema_version"): - problems.append(f"Stored report {report_path} has the wrong schema version") - if report.get("port") != port_id: - problems.append(f"Stored report {report_path} identifies port {report.get('port')}") - report_tests = report.get("tests") - if not isinstance(report_tests, dict): - problems.append(f"Stored report {report_path} has no test result map") - continue - unknown_tests = sorted(set(report_tests) - set(mapped)) - if unknown_tests: - problems.append( - f"Stored report {report_path} contains unknown tests: " - + ", ".join(unknown_tests) - ) - missing_tests = sorted(set(mapped) - set(report_tests)) - if missing_tests: - problems.append( - f"Stored report {report_path} is missing tests: " - + ", ".join(missing_tests) - ) - unrun_tests = [] - for test, result in report_tests.items(): - if not isinstance(result, dict) or result.get("status") not in { - "pass", "fail", "skip", "not-run" - }: - problems.append(f"Stored report {report_path} has an invalid result for {test}") - elif result.get("status") == "skip": - skipped_tests.add(test) - elif result.get("status") == "not-run": - unrun_tests.append(test) - if unrun_tests: - # A registered test that never started is indistinguishable, on the page, from one - # that runs and passes -- nothing here objected to it, so a test could be - # published and quietly never executed on any port. A port that genuinely cannot - # do something reports "skip", from the suite itself, and is unaffected; "not-run" - # is the absence of evidence, and the answer to it is to run the suite and check - # the report in rather than to record the absence. - problems.append( - f"Stored report {report_path} reports tests that never ran: " - + ", ".join(sorted(unrun_tests)) - ) - actual_summary = Counter( - result.get("status") - for result in report_tests.values() - if isinstance(result, dict) - ) - expected_summary = { - key: actual_summary.get(key, 0) - for key in ("pass", "fail", "skip", "not-run") - } - if report.get("summary") != expected_summary: - problems.append( - f"Stored report {report_path} summary does not match its test results" - ) + # The checked-in reports are a snapshot of what CI measured, not a second + # copy of the contract. Requiring them to carry exactly the manifest's test + # set made every test-adding PR hand-edit eleven files, and the cheapest way + # to satisfy that was to invent a result -- twelve "pass" entries reached + # master attributed to runs that never executed the test. Classify them with + # the same drift / malformed split publication uses: a snapshot that predates + # a test is drift and is reported, never fatal; a snapshot nothing can render + # is still a defect. + skipped_tests, snapshot_drift, snapshot_malformed = stored_report_problems(manifest) + problems.extend(snapshot_malformed) manual_feature_count = 0 try: @@ -273,7 +218,10 @@ def validate(manifest: dict) -> dict: "Skip errata references unknown tests: " + ", ".join(unknown_skip_reasons) ) - missing_skip_reasons = sorted(skipped_tests - set(skip_reason_tests)) + # Only tests the contract still defines. A snapshot taken before a test was + # retired still carries its skip, and demanding an erratum for something + # nobody can run again would be unfixable except by editing the snapshot. + missing_skip_reasons = sorted((skipped_tests & set(mapped)) - set(skip_reason_tests)) if missing_skip_reasons: problems.append("Skipped tests without errata: " + ", ".join(missing_skip_reasons)) for item in supplement.get("skip_reasons", []): @@ -282,6 +230,30 @@ def validate(manifest: dict) -> dict: problems.append( "Every skip erratum needs test, reason, platform_support, and verification" ) + # A reason code with no prefix documents everything. Both matchers ask + # whether the reason starts with it, and every string starts with the + # empty one -- so an erratum that lost this field by a typo would turn + # any future skip of that test green, on the nightly gate and on the + # page alike, which is the opposite of what writing an erratum is for. + for code in item.get("reason_codes") or []: + prefix = code.get("prefix") + if not isinstance(prefix, str) or not prefix: + problems.append( + f"Skip erratum {item.get('test')} has a reason code with no prefix" + ) + # Deliberately not named `ports`: that is the manifest's port list, + # in scope for the whole of validate(), and rebinding it here left + # the returned port count reading whichever erratum happened to be + # last. The counts test caught it, which is what it is for. + code_ports = code.get("ports") + if code_ports is not None and ( + not isinstance(code_ports, list) + or not code_ports + or any(port not in port_ids for port in code_ports) + ): + problems.append( + f"Skip erratum {item.get('test')} has a reason code naming unknown ports" + ) manual_features = supplement.get("features", []) manual_feature_count = len(manual_features) @@ -387,6 +359,7 @@ def validate(manifest: dict) -> dict: if problems: raise ContractError("\n".join(problems)) return { + "drift": snapshot_drift, "ports": len(ports), "features": len(features), "tests": len(mapped), @@ -398,6 +371,422 @@ def validate(manifest: dict) -> dict: } +def stored_report_problems(manifest: dict) -> tuple[set[str], list[str], list[str]]: + """Classify the checked-in fallback reports. + + Returns (skipped tests, drift, malformed). The reports under + ``report_directory`` are produced by the port workflows and refreshed from + the data branch before Hugo runs; nothing about a pull request is supposed + to touch them. Read them exactly the way publication reads a persisted + report, so "this snapshot predates a test the branch just registered" is the + ordinary, expected state it already is everywhere else in this pipeline + rather than a build failure a human resolves by inventing a result. + """ + skipped: set[str] = set() + drift: list[str] = [] + malformed: list[str] = [] + report_directory = manifest.get("report_directory") + if not report_directory: + return skipped, drift, malformed + root = REPO_ROOT / report_directory + for port in manifest.get("ports", []): + port_id = port.get("id") + if not port_id: + continue + path = root / f"{port_id}.json" + if not path.is_file(): + # A port with no published report renders as "No stored report" on + # every one of its cells, which is what is true of a port CI has + # never heard from. Demanding a file here is what made adding a port + # start by hand-authoring one, and a hand-authored report is the + # thing this whole contract is trying to stop existing. + drift.append(f"{port_id}: no stored report yet") + continue + try: + report = read_json(path) + except ContractError as exc: + malformed.append(str(exc)) + continue + port_drift, port_malformed = publishable_report_problems(manifest, port_id, report) + drift.extend(f"{port_id}: {item}" for item in port_drift) + malformed.extend(f"{port_id}: {item}" for item in port_malformed) + tests = report.get("tests") + if not isinstance(tests, dict): + continue + for name, result in tests.items(): + if isinstance(result, dict) and result.get("status") == "skip": + skipped.add(name) + return skipped, drift, malformed + + +def report_stamp(report: dict) -> datetime | None: + raw = report.get("generated_at") + if not isinstance(raw, str) or not raw: + return None + try: + stamp = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError: + return None + return stamp if stamp.tzinfo else None + + +def skip_is_documented(supplement: dict, port_id: str, test: str, reasons: list) -> bool: + """The page's rule for a green documented skip, applied to a report. + + Mirrors port-status-feature-status.html deliberately: an erratum documents a + skip only when it names the test AND, where it lists reason codes, every + reason the run gave matches one of them from a port that code applies to. + Matching on the test name alone would let any future skip of a named test + read as documented -- an encoder that regressed would render green under an + erratum written about a simulator. + """ + for item in supplement.get("skip_reasons", []): + if item.get("test") != test: + continue + codes = item.get("reason_codes") + if not codes: + return True + if not reasons: + continue + if all( + any( + code.get("prefix") + and (not code.get("ports") or port_id in code["ports"]) + and isinstance(reason, str) + and reason.startswith(code["prefix"]) + for code in codes + ) + for reason in reasons + ): + return True + return False + + +def coverage_problems( + manifest: dict, + reports: dict[str, dict], + contracts: dict[str, set[str]] | None = None, +) -> list[str]: + """Hold the *published* reports to "every registered test runs on every port". + + This is the gate that used to live, badly, in the checked-in snapshots. Two + rules, both decidable from the reports themselves: + + ``not-run`` is always a defect. The suite reached that port, the test was in + its contract, and nothing reported back. + + A test absent from a report is normally just a run that predates it -- the + port has not merged past the commit that registered the test yet. It becomes + a defect the moment some *older* report carries that test: a run that + happened earlier already knew about it, so a later run that does not is a + test the port has dropped rather than one it has not reached. No history + lookup and no grace period to tune; the reports date themselves. + + That comparison is blind to a test missing from *every* report, because then + no report is the older one that proves it existed -- and a test nothing runs + anywhere is the worst version of the failure this gate is for, not a + tolerable one. ``contracts`` closes it: the set of tests each report's own + commit defined, which the caller reads at that commit. A report whose + contract already listed the test has no excuse for omitting it, whatever the + other ports did. Offline callers pass None and keep the weaker comparison. + + A ``skip`` is the one permitted exception, and only with an erratum that + accounts for the reason the run actually gave. Reading skips out of the + checked-in fallbacks instead -- which is all validate() can see -- would let + a port start skipping a test, publish it, and pass this gate, with the + undocumented skip surfacing later as a failed website build rather than as + the name of the port that started skipping. + """ + problems: list[str] = [] + supplement = read_json(SUPPLEMENT) + mapped = test_to_feature(manifest) + stamps = {port: report_stamp(report) for port, report in reports.items()} + + # The earliest run that proves a test was in the contract. Anything younger + # than this has no excuse for missing it. + known_since: dict[str, datetime] = {} + for port, report in reports.items(): + stamp = stamps.get(port) + tests = report.get("tests") + if stamp is None or not isinstance(tests, dict): + continue + for name in tests: + if name in mapped and (name not in known_since or stamp < known_since[name]): + known_since[name] = stamp + + for port in sorted(reports): + report = reports[port] + tests = report.get("tests") + if not isinstance(tests, dict): + problems.append(f"{port}: report has no test result map") + continue + # `name in mapped`, the same filter the skip check below uses. Scanning + # every entry meant a report that predates a test's retirement and + # carries it as not-run failed this gate over a test nobody can run any + # more -- and the same report is tolerated as drift everywhere else, so + # the sweep stayed red until that port happened to rerun. + unrun = sorted( + name + for name, result in tests.items() + if isinstance(result, dict) + and result.get("status") == "not-run" + and name in mapped + ) + if unrun: + problems.append(f"{port}: reported no result for " + ", ".join(unrun)) + undocumented = sorted( + name + for name, result in tests.items() + if isinstance(result, dict) + and result.get("status") == "skip" + and name in mapped + and not skip_is_documented( + supplement, port, name, result.get("reasons") or [] + ) + ) + if undocumented: + problems.append( + f"{port}: skipped without an erratum that explains the reason given: " + + ", ".join(undocumented) + ) + stamp = stamps.get(port) + if stamp is None: + problems.append(f"{port}: report has no usable generated_at") + continue + own_contract = (contracts or {}).get(port) + absent = set(mapped) - set(tests) + dropped = sorted( + name + for name in absent + if name in known_since and known_since[name] < stamp + ) + if dropped: + problems.append( + f"{port}: ran at {report.get('generated_at')} without " + + ", ".join(dropped) + + ", which an earlier run on another port already covered" + ) + if own_contract is not None: + unreported = sorted(name for name in absent - set(dropped) if name in own_contract) + if unreported: + problems.append( + f"{port}: ran at {report.get('commit') or 'an unknown commit'}, which " + "defines " + ", ".join(unreported) + ", and reported nothing for them" + ) + return problems + + +PROVENANCE_FIELDS = ("generated_at", "commit", "run_url") +# What has to be *new* before changed findings are believable. `commit` is not +# among them: a port legitimately re-runs the same master commit, and its second +# run is a different run. `run_url` is the run's identity and `generated_at` is +# when it reported; a real snapshot carries new values for both. +RUN_IDENTITY_FIELDS = ("generated_at", "run_url") +# A run URL names a run on this forge. Shape alone proves nothing about whether +# the run happened -- the data branch below is what establishes that -- but it +# costs nothing and rejects a field filled in with a placeholder. +RUN_URL_RE = re.compile( + r"^https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/\d+(?:/[A-Za-z0-9_/-]*)?$" +) + + +def provenance_problems( + port_id: str, + before: dict | None, + after: dict | None, + published: list[dict] | None = None, +) -> list[str]: + """Refuse a hand-edited report. + + A report says "at this commit, this run, at this time, this is what the port + did". Editing what it did while leaving that provenance alone does not + correct the record, it forges it -- which is how twelve tests came to be + published as passing on ports that had never executed them. Changing the + findings is legitimate only as part of taking a new snapshot, and a new + snapshot carries a new stamp. + + Everything except the provenance fields counts as a finding, not just the + test map: ``performance`` is the ten benchmark durations the page publishes + as measurements of that run, and ``suite_finished`` is what makes a port card + say the suite completed. Naming a subset here would leave the numbers most + worth doubting -- the ones nobody can check by reading them -- as the one + thing a branch could still rewrite in place. + + ``published`` is what turns this from a shape check into a verification. It + is every version of this port's report the ``port-status-data`` branch has + held recently, and a changed snapshot has to *be* one of them -- which it + will be, because the only way to refresh one is to copy what CI published. + A new ``run_url`` and stamp are then not two strings a branch can invent; + they have to belong to a report that a run really produced, and producing one + needs the write access to the data branch that only the publish workflows + have. Pass None when the branch could not be reached: unverifiable is not the + same as forged, and failing a pull request because a fetch flaked would teach + people to route around this. + """ + if after is None: + if before is None: + return [] + # Absent because it never existed and absent because someone removed it + # are different things, and only the first is harmless. The site serves + # this file whenever the data branch cannot be reached or its newest + # report predates the contract, so deleting one turns an established + # port's whole column unknown at exactly the moment the live data is + # missing -- which is the moment the fallback exists for. Retiring a + # port is still fine: drop it from the manifest and this check never + # looks at it. + return [ + f"{port_id}: the checked-in report was deleted. It is the fallback " + "the site serves when the data branch is unreachable, so an " + "established port would render as unknown. Only a port that has " + "never published needs no report." + ] + + if before == after: + return [] + + advice = ( + "A checked-in report is a copy of what CI put on the port-status-data " + "branch, so refresh it from there rather than editing it; adding a test " + "needs no report change at all." + ) + + # Checked on any change to the field, not only alongside changed findings. + if before is None or before.get("run_url") != after.get("run_url"): + run_url = after.get("run_url") + if not isinstance(run_url, str) or not RUN_URL_RE.match(run_url): + return [ + f"{port_id}: run_url {run_url!r} does not name a workflow run. " + + advice + ] + + # Every identity field, not any provenance field. Accepting a change to one + # of the three left the gate open to the easier version of the same forgery: + # invent a result, type today's date into `generated_at`, and leave the + # `commit` and `run_url` still naming the run that never produced it. A + # snapshot that came from a run has a new run behind it. + findings = tuple( + {key: value for key, value in (report or {}).items() if key not in PROVENANCE_FIELDS} + for report in (before, after) + ) + if before is not None and findings[0] != findings[1] and not all( + before.get(field) != after.get(field) and after.get(field) + for field in RUN_IDENTITY_FIELDS + ): + changed = sorted( + key + for key in set(findings[0]) | set(findings[1]) + if findings[0].get(key) != findings[1].get(key) + ) + stale = sorted( + field + for field in RUN_IDENTITY_FIELDS + if not (before.get(field) != after.get(field) and after.get(field)) + ) + return [ + f"{port_id}: {', '.join(changed)} changed without a new run behind it -- " + f"{', '.join(stale)} still name{'s' if len(stale) == 1 else ''} the " + "previous one. These reports are CI output, and a branch never needs to " + "edit one. Adding a test needs no report change at all; each port picks " + "it up on its next master run." + ] + + # Asked of *any* change, including one that touches only the provenance + # fields. Retyping generated_at alone changes no finding, and the page reads + # that field to decide whether a column is stale -- so the edit nothing else + # objected to was the one that made a port which had stopped reporting look + # like it was still running. + if published is None: + return [] + if any(candidate == after for candidate in published): + return [] + if not published: + return [ + f"{port_id}: this port has never published a report, so there is " + "nothing for a checked-in one to be a copy of. Leave it out -- every " + "cell reads 'No stored report' until the port's first run, which is " + "what is true. " + advice + ] + return [ + f"{port_id}: this report is not a version the port-status-data branch " + "ever held. " + advice + ] + + +def documented_skip_goldens( + manifest: dict, + port_id: str, + logs: list[Path], + reference: Path, + comparisons: list[Path] | None = None, +) -> tuple[list[str], list[str]]: + """Golden names whose test reported a documented skip, and why. + + The screenshot count guard fails a run when a golden is not re-produced, + because a test that hangs or crashes leaves no per-test record and the + missing file is the only evidence there is. A test that prints + ``status=SKIPPED reason=...`` is the opposite of that: it left a record, and + the errata already say the reason is expected on this port. GoogleWebMap + skips on android and both iOS renderers when the Google Maps tiles never + load, which is a network the run cannot reach rather than anything about the + port -- and the guard failed the whole job over it anyway, so that skip path + could never actually succeed on a port that owns a golden. + + Only a skip that is *documented for this port* counts. Silence still fails, + an unexplained skip still fails, and a reason code written about another + port still fails, which is what keeps this from being a hole. + + And only goldens that are actually absent. The caller subtracts this count + from the number of uncovered goldens, so naming one the run did compare + would subtract a golden nothing was missing -- and the spare subtraction + would then hide a genuinely uncovered golden belonging to some other test. + A test that owns several screenshots and captures a few before skipping is + exactly that case. + """ + supplement = read_json(SUPPLEMENT) + skipped: dict[str, list[str]] = {} + for path in logs: + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for line in text.splitlines(): + match = SKIP_RE.search(line) + if not match: + continue + name, reason = match.group(1), match.group(2) + owner = name if name in test_to_feature(manifest) else screenshot_test(manifest, name) + if owner: + skipped.setdefault(owner, []).append(reason or "") + + compared: set[str] = set() + for path in comparisons or []: + if not path.is_file(): + continue + try: + payload = read_json(path) + except ContractError: + continue + for result in payload.get("results", []): + if isinstance(result, dict) and result.get("status") in {"equal", "different"}: + name = result.get("test") + if name: + compared.add(name) + + accounted: list[str] = [] + notes: list[str] = [] + if not reference.is_dir(): + return accounted, notes + for golden in sorted(reference.glob("*.png")): + owner = screenshot_test(manifest, golden.stem) + if owner is None or owner not in skipped or golden.stem in compared: + continue + reasons = [reason for reason in skipped[owner] if reason] + if skip_is_documented(supplement, port_id, owner, reasons): + accounted.append(golden.stem) + notes.append(f"{golden.stem}: {owner} skipped ({', '.join(reasons)})") + return accounted, notes + + def add_reason(entry: dict, reason: str) -> None: reasons = entry.setdefault("reasons", []) if reason and reason not in reasons: @@ -911,6 +1300,53 @@ def build_parser() -> argparse.ArgumentParser: accept_parser.add_argument("--port", required=True) accept_parser.add_argument("--report", required=True, type=Path) + coverage_parser = subparsers.add_parser( + "coverage", + help="hold published reports to running every registered test on every port", + ) + coverage_parser.add_argument( + "--reports", + required=True, + type=Path, + help="directory of published .json reports", + ) + coverage_parser.add_argument( + "--contracts", + type=Path, + help=( + "directory of .json manifests read at each report's own commit; " + "omit when they cannot be fetched" + ), + ) + + provenance_parser = subparsers.add_parser( + "provenance", + help="refuse a report whose results were edited without a new run", + ) + provenance_parser.add_argument( + "--base", + required=True, + type=Path, + help="directory holding the base revision's reports", + ) + provenance_parser.add_argument( + "--published", + type=Path, + help=( + "directory of /*.json holding every version of each report the " + "data branch has held recently; omit when it could not be fetched" + ), + ) + + skips_parser = subparsers.add_parser( + "documented-skips", + help="goldens whose test reported a skip this port's errata explain", + ) + skips_parser.add_argument("--port", required=True) + skips_parser.add_argument("--log", action="append", type=Path, default=[]) + skips_parser.add_argument("--reference", required=True, type=Path) + skips_parser.add_argument("--compare", action="append", type=Path, default=[]) + normalize_parser = subparsers.add_parser("normalize", help="write a normalized port report") normalize_parser.add_argument("--port", required=True) normalize_parser.add_argument("--log", action="append", type=Path, default=[]) @@ -939,6 +1375,85 @@ def main() -> int: f"{counts['tests']} tests, {counts['features']} features, " f"{counts['ports']} ports, {counts['goldens']} golden names." ) + for item in counts["drift"]: + # Information, not a warning to be silenced. Every port reaches + # a newly registered test on its next master run, and the page + # shows the gap as "not run" until it does. + print(f"port-status: checked-in snapshot {item}") + return 0 + if args.command == "documented-skips": + accounted, notes = documented_skip_goldens( + manifest, args.port, args.log, args.reference, args.compare + ) + for note in notes: + print(f"port-status: documented skip accounts for {note}", file=sys.stderr) + print(len(accounted)) + return 0 + if args.command == "coverage": + reports = {} + for port in manifest.get("ports", []): + port_id = port.get("id") + path = args.reports / f"{port_id}.json" + if not path.is_file(): + print(f"port-status: no published report for {port_id}", file=sys.stderr) + return 1 + reports[port_id] = read_json(path) + contracts = None + if args.contracts is not None: + contracts = {} + for port_id in reports: + path = args.contracts / f"{port_id}.json" + if not path.is_file(): + continue + try: + contracts[port_id] = set(test_to_feature(read_json(path))) + except ContractError: + # A manifest we cannot read proves nothing. Leaving the + # port out keeps the weaker comparison rather than + # inventing an obligation or excusing one. + continue + problems = coverage_problems(manifest, reports, contracts) + for problem in problems: + print(f"port-status coverage: {problem}", file=sys.stderr) + if problems: + print( + "Every registered test runs on every port unless the suite itself " + "reports a skip with an erratum. Fix the port or record the skip.", + file=sys.stderr, + ) + return 1 + print(f"Every registered test reported a result on all {len(reports)} ports.") + return 0 + if args.command == "provenance": + report_directory = manifest.get("report_directory") + problems = [] + for port in manifest.get("ports", []): + port_id = port.get("id") + base_path = args.base / f"{port_id}.json" + head_path = REPO_ROOT / report_directory / f"{port_id}.json" + if not head_path.is_file() and not base_path.is_file(): + # A port that has never published. Nothing to check. + continue + published = None + if args.published is not None: + candidates = args.published / port_id + published = [ + read_json(item) + for item in sorted(candidates.glob("*.json")) + ] if candidates.is_dir() else [] + problems.extend( + provenance_problems( + port_id, + read_json(base_path) if base_path.is_file() else None, + read_json(head_path) if head_path.is_file() else None, + published, + ) + ) + for problem in problems: + print(f"port-status: {problem}", file=sys.stderr) + if problems: + return 1 + print("No port status report was edited by hand.") return 0 if args.command == "accept": drift, malformed = publishable_report_problems( diff --git a/scripts/hellocodenameone/conformance/test_port_status.py b/scripts/hellocodenameone/conformance/test_port_status.py index e589a901f76..654d5644244 100755 --- a/scripts/hellocodenameone/conformance/test_port_status.py +++ b/scripts/hellocodenameone/conformance/test_port_status.py @@ -16,15 +16,43 @@ def setUpClass(cls): cls.manifest = port_status.read_json(port_status.DEFAULT_MANIFEST) def test_contract_covers_registered_tests_and_goldens(self): + # Deliberately no literal totals. Every one of these used to be a magic + # number that each test-adding branch had to retype, so two branches + # adding a test conflicted here by construction -- on a line whose only + # content was a number neither author had a reason to think about. What + # is worth asserting is the relationship: the suite and the contract + # describe the same set of tests, and nothing is counted twice. Which + # test belongs to which capability is a separate question, and a real + # one; it is asserted below rather than dropped. counts = port_status.validate(self.manifest) - self.assertEqual(180, counts["tests"]) - self.assertEqual(1, counts["performance_tests"]) - self.assertGreaterEqual(counts["features"], 54) - self.assertEqual(11, counts["ports"]) - self.assertEqual(20, counts["manual_features"]) + registered = port_status.registered_tests() + mapped = port_status.test_to_feature(self.manifest) + performance = self.manifest["performance_tests"] + + self.assertEqual(sorted(registered), sorted(set(registered))) + self.assertEqual(set(registered), set(mapped) | set(performance)) + self.assertEqual(set(), set(mapped) & set(performance)) + self.assertEqual(len(mapped), counts["tests"]) + self.assertEqual(len(performance), counts["performance_tests"]) + self.assertEqual(len(self.manifest["ports"]), counts["ports"]) + self.assertEqual(len(self.manifest["features"]), counts["features"]) + self.assertTrue(all(feature["tests"] for feature in self.manifest["features"])) + + # Floors, not equalities: these guard against a collapse -- a manifest + # that lost its features, a golden directory that stopped resolving -- + # and a branch that adds to any of them never has to touch this file. + self.assertGreater(counts["features"], 50) + self.assertGreater(counts["goldens"], 100) + self.assertGreater(counts["manual_features"], 15) self.assertEqual(8, counts["deployment_platforms"]) self.assertEqual(3, counts["browser_engines"]) - self.assertGreaterEqual(counts["goldens"], 100) + + def test_load_bearing_tests_stay_under_the_capability_they_prove(self): + # A spot check, not a registry: these are the mappings where landing a result under the + # wrong row would publish a specific capability claim the test never made. Nothing above + # catches that -- validate() only requires each test to sit under exactly one feature, + # and any feature satisfies it. Adding a feature does not oblige anyone to extend this + # list; it is the literal totals that every branch had to retype, not these. features = {feature["id"]: feature["tests"] for feature in self.manifest["features"]} self.assertEqual(["ARApiTest", "MotionSensorDeviceTest"], features["ar-motion-sensors"]) self.assertEqual(["CameraApiTest"], features["camera-access"]) @@ -313,21 +341,325 @@ def test_validate_rejects_inconsistent_stored_report_summary(self): ) with self.assertRaisesRegex( port_status.ContractError, - "summary does not match its test results", + "summary does not match the test results", ): port_status.validate(manifest) - def test_validate_rejects_a_test_that_never_ran(self): + def stored_reports(self): + directory = port_status.REPO_ROOT / self.manifest["report_directory"] + return { + port["id"]: port_status.read_json(directory / (port["id"] + ".json")) + for port in self.manifest["ports"] + } + + def test_coverage_rejects_a_test_that_never_ran(self): # A registered test left at "not-run" reads on the page exactly like one that runs and - # passes. Seven database tests were published that way -- added to the manifest, never - # run in any stored report -- and nothing here objected. + # passes. This used to be asserted against the checked-in snapshots, which is why every + # branch that registered a test had to edit eleven of them -- and why the cheapest way + # to go green was to type "pass" for a run that never happened. The obligation belongs + # to the reports the ports actually publish, where nothing a branch writes can satisfy it. + reports = self.stored_reports() + victim = next( + name + for name, result in reports["android"]["tests"].items() + if result.get("status") == "pass" + ) + reports["android"]["tests"][victim]["status"] = "not-run" + problems = port_status.coverage_problems(self.manifest, reports) + self.assertTrue( + any("android" in problem and victim in problem for problem in problems), + problems, + ) + + def test_coverage_ignores_a_retired_test_left_at_not_run(self): + # A report that predates a test's retirement still carries the test, and if that run + # never reached it the entry is "not-run". Reporting that is holding a port to an + # obligation the contract has withdrawn -- the same report is tolerated as drift + # everywhere else, so the sweep would have stayed red until the port happened to rerun. + reports = self.stored_reports() + reports["android"]["tests"]["RetiredApiTest"] = { + "feature": "crypto", + "status": "not-run", + } + self.assertEqual([], port_status.coverage_problems(self.manifest, reports)) + + def test_coverage_accepts_a_documented_skip(self): + # The distinction the rule turns on: a port that genuinely cannot do something reports + # "skip" from the suite itself, which is evidence rather than the absence of it -- but + # only where an erratum accounts for the reason the run gave. The published reports + # carry these already, so the shipped data is the fixture. + reports = self.stored_reports() + documented = [ + (port, name) + for port, report in reports.items() + for name, result in report["tests"].items() + if result.get("status") == "skip" + ] + self.assertTrue(documented) + self.assertEqual([], port_status.coverage_problems(self.manifest, reports)) + + def test_coverage_rejects_an_undocumented_skip(self): + # Otherwise a port can simply stop running a test: mark it skipped, publish, and this + # gate calls it a satisfactory result. validate() cannot catch it either -- it reads + # the checked-in fallbacks, not what the ports published -- so the first symptom would + # be a failed website build rather than the name of the port that started skipping. + reports = self.stored_reports() + victim = next( + name + for name, result in reports["android"]["tests"].items() + if result.get("status") == "pass" + ) + reports["android"]["tests"][victim] = { + "feature": reports["android"]["tests"][victim]["feature"], + "status": "skip", + "reasons": ["something-nobody-wrote-down"], + } + problems = port_status.coverage_problems(self.manifest, reports) + self.assertTrue( + any("android" in problem and victim in problem for problem in problems), + problems, + ) + + def test_coverage_rejects_a_skip_reason_scoped_to_another_port(self): + # Matching the test name alone would let any future skip of a named test read as + # documented. CameraApiTest has errata, but the missing-webcam code is written about + # Windows; the same code from Linux says something nobody has explained. + reports = self.stored_reports() + reports["linux-x64"]["tests"]["CameraApiTest"] = { + "feature": "camera-access", + "status": "skip", + "reasons": ["no-host-webcam-capture-on-win"], + } + problems = port_status.coverage_problems(self.manifest, reports) + self.assertTrue( + any("linux-x64" in problem and "CameraApiTest" in problem for problem in problems), + problems, + ) + + def test_a_reason_code_with_no_prefix_documents_nothing(self): + # Both matchers ask whether the reason starts with the prefix, and every string starts + # with the empty one. An erratum that lost this field to a typo would therefore turn any + # future skip of that test green -- the exact opposite of what writing one is for. + supplement = { + "skip_reasons": [ + {"test": "CameraApiTest", "reason_codes": [{"ports": ["android"]}]} + ] + } + self.assertFalse( + port_status.skip_is_documented( + supplement, "android", "CameraApiTest", ["something-entirely-unrelated"] + ) + ) + self.assertFalse( + port_status.skip_is_documented( + supplement, "android", "CameraApiTest", ["needs-runtime-permission-on-and"] + ) + ) + + def test_validate_rejects_a_reason_code_with_no_prefix(self): + # And the configuration error is caught where it is made, rather than only failing to + # match later. Both halves matter: the matcher cannot be the only guard, because the + # page draws its own tick from its own copy of this rule. + supplement = { + "skip_reasons": [{"test": "CameraApiTest", "reason_codes": [{"prefix": ""}]}] + } + self.assertFalse( + port_status.skip_is_documented(supplement, "android", "CameraApiTest", ["anything"]) + ) + + def documented_skip_count(self, port, marker, reference): + with tempfile.TemporaryDirectory() as tmp: + log = Path(tmp) / "suite.log" + log.write_text(marker, encoding="utf-8") + accounted, _ = port_status.documented_skip_goldens( + self.manifest, port, [log], port_status.REPO_ROOT / reference + ) + return accounted + + def test_a_documented_skip_accounts_for_its_golden(self): + # The screenshot count guard reads an unproduced golden as a test that hung, crashed or + # never delivered its frame -- which it is, when nothing else was said. A test that + # prints status=SKIPPED said something. GoogleWebMap takes that path when the Maps tiles + # never load, the errata document it on android, and the guard failed the whole job on + # the uncovered golden anyway -- so the skip path could never succeed on a port that + # owns a golden. + self.assertEqual( + ["GoogleWebMap"], + self.documented_skip_count( + "android", + "CN1SS:INFO:test=GoogleWebMap status=SKIPPED reason=map-tiles-never-loaded\n", + "scripts/android/screenshots", + ), + ) + + def test_silence_accounts_for_nothing(self): + # The case the guard exists for, and the one this must not soften: a test that hangs or + # crashes leaves no record, and the missing golden is the only evidence there is. + self.assertEqual( + [], self.documented_skip_count("android", "", "scripts/android/screenshots") + ) + + def test_an_undocumented_skip_accounts_for_nothing(self): + self.assertEqual( + [], + self.documented_skip_count( + "android", + "CN1SS:INFO:test=GoogleWebMap status=SKIPPED reason=something-nobody-wrote-down\n", + "scripts/android/screenshots", + ), + ) + + def test_a_skip_documented_for_another_port_accounts_for_nothing(self): + # map-tiles-never-loaded is written about android and the two iOS renderers. The same + # reason arriving from Linux, where the errata expect no-api-key instead, is a port + # behaving unexpectedly rather than a network nobody can reach. + self.assertEqual( + [], + self.documented_skip_count( + "linux-x64", + "CN1SS:INFO:test=GoogleWebMap status=SKIPPED reason=map-tiles-never-loaded\n", + "scripts/linux/screenshots", + ), + ) + + def test_only_goldens_the_run_did_not_produce_are_discounted(self): + # The caller subtracts this count from the number of UNCOVERED goldens, so naming one the + # run did compare would subtract a golden nothing was missing -- and that spare + # subtraction would then hide a genuinely uncovered golden belonging to some other test, + # which is the regression the guard exists to catch. A test owning several screenshots + # that captures a few before skipping is exactly the case. + marker = "CN1SS:INFO:test=CenteredDialogTitle status=SKIPPED reason=phone-dialog-on-watch\n" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = root / "ref" + reference.mkdir() + for name in ("CenteredDialogTitle_dark", "CenteredDialogTitle_light"): + (reference / (name + ".png")).write_bytes(b"") + log = root / "suite.log" + log.write_text(marker, encoding="utf-8") + nothing_compared = root / "none.json" + nothing_compared.write_text(json.dumps({"results": []}), encoding="utf-8") + one_compared = root / "one.json" + one_compared.write_text( + json.dumps( + {"results": [{"test": "CenteredDialogTitle_light", "status": "equal"}]} + ), + encoding="utf-8", + ) + self.assertEqual( + ["CenteredDialogTitle_dark", "CenteredDialogTitle_light"], + port_status.documented_skip_goldens( + self.manifest, "watchos", [log], reference, [nothing_compared] + )[0], + ) + self.assertEqual( + ["CenteredDialogTitle_dark"], + port_status.documented_skip_goldens( + self.manifest, "watchos", [log], reference, [one_compared] + )[0], + ) + + def test_coverage_rejects_a_skip_carrying_no_reason(self): + # An erratum with reason codes documents the reasons it lists, not the test. A skip + # that names none matches nothing, which is what the page already decides. + reports = self.stored_reports() + reports["android"]["tests"]["CameraApiTest"] = { + "feature": "camera-access", + "status": "skip", + } + problems = port_status.coverage_problems(self.manifest, reports) + self.assertTrue( + any("android" in problem and "CameraApiTest" in problem for problem in problems), + problems, + ) + + def test_coverage_catches_a_test_absent_from_every_report(self): + # Comparing reports to each other cannot see this: with the test missing everywhere, + # there is no older report left to prove it existed, so every port is excused and the + # gate prints success over a test nothing runs anywhere -- the worst version of the + # failure this gate is for. Each report's own contract answers it directly. + reports = self.stored_reports() + victim = "CryptoApiTest" + for report in reports.values(): + report["tests"].pop(victim, None) + self.assertEqual([], port_status.coverage_problems(self.manifest, reports)) + + contract = set(port_status.test_to_feature(self.manifest)) + problems = port_status.coverage_problems( + self.manifest, reports, {port: contract for port in reports} + ) + self.assertEqual(len(reports), len(problems), problems) + self.assertTrue(all(victim in problem for problem in problems), problems) + + def test_coverage_tolerates_a_report_whose_own_contract_predates_the_test(self): + # The state every port is in for a few hours after a test is registered, and the one + # this must never fail: the run happened against a manifest that did not define the + # test, so there was nothing to report. + reports = self.stored_reports() + victim = "CryptoApiTest" + for report in reports.values(): + report["tests"].pop(victim, None) + # Each port's own contract is exactly what its run reported, which is what "the run + # predates the test" means. Handing every port the CURRENT contract minus one test + # would instead accuse the five Apple ports of dropping LogSubclassCaptureTest, which + # their reports really do predate -- and the gate would be right to say so. + self.assertEqual( + [], + port_status.coverage_problems( + self.manifest, + reports, + {port: set(report["tests"]) for port, report in reports.items()}, + ), + ) + + def test_coverage_leaves_a_port_alone_when_its_contract_is_unknown(self): + # A manifest the sweep could not fetch proves nothing, so that port keeps the weaker + # report-to-report comparison rather than being excused or accused. + reports = self.stored_reports() + victim = "CryptoApiTest" + for report in reports.values(): + report["tests"].pop(victim, None) + contract = set(port_status.test_to_feature(self.manifest)) + problems = port_status.coverage_problems( + self.manifest, reports, {"android": contract} + ) + self.assertEqual(1, len(problems), problems) + self.assertIn("android", problems[0]) + + def test_coverage_accepts_a_report_older_than_the_test(self): + # The state every port is in between the commit that registers a test and that port's + # next master run. Failing here would put the old ritual straight back: the only way to + # merge a test would be to make eleven reports claim a result for it first. + reports = self.stored_reports() + newest = max(reports, key=lambda port: reports[port]["generated_at"]) + oldest = min(reports, key=lambda port: reports[port]["generated_at"]) + self.assertNotEqual(newest, oldest) + victim = next(iter(reports[newest]["tests"])) + del reports[oldest]["tests"][victim] + self.assertEqual([], port_status.coverage_problems(self.manifest, reports)) + + def test_coverage_rejects_a_test_a_later_run_dropped(self): + # The other half of the same comparison. A run that happened after another run which + # already covered the test has no "my contract predates it" excuse left. + reports = self.stored_reports() + newest = max(reports, key=lambda port: reports[port]["generated_at"]) + oldest = min(reports, key=lambda port: reports[port]["generated_at"]) + victim = next(iter(reports[oldest]["tests"])) + del reports[newest]["tests"][victim] + problems = port_status.coverage_problems(self.manifest, reports) + self.assertTrue( + any(newest in problem and victim in problem for problem in problems), + problems, + ) + + def test_validate_tolerates_a_snapshot_that_predates_a_test(self): + # Registering a test must not require touching a single report. This is the assertion + # that keeps it that way. original_directory = self.manifest["report_directory"] with tempfile.TemporaryDirectory(dir=port_status.REPO_ROOT) as tmp: report_root = Path(tmp) for port in self.manifest["ports"]: - source = port_status.REPO_ROOT / original_directory / ( - port["id"] + ".json" - ) + source = port_status.REPO_ROOT / original_directory / (port["id"] + ".json") (report_root / source.name).write_text( source.read_text(encoding="utf-8"), encoding="utf-8" ) @@ -338,60 +670,222 @@ def test_validate_rejects_a_test_that_never_ran(self): for name, result in android["tests"].items() if result.get("status") == "pass" ) - android["tests"][victim]["status"] = "not-run" + del android["tests"][victim] android["summary"]["pass"] -= 1 - android["summary"]["not-run"] += 1 android_path.write_text( - json.dumps(android, indent=2, sort_keys=True) + "\n", - encoding="utf-8", + json.dumps(android, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) manifest = dict(self.manifest) - manifest["report_directory"] = str( - report_root.relative_to(port_status.REPO_ROOT) - ) - with self.assertRaisesRegex( - port_status.ContractError, - "reports tests that never ran: " + victim, - ): - port_status.validate(manifest) + manifest["report_directory"] = str(report_root.relative_to(port_status.REPO_ROOT)) + counts = port_status.validate(manifest) + self.assertTrue( + any("android" in item and victim in item for item in counts["drift"]), + counts["drift"], + ) - def test_validate_accepts_a_test_the_port_skipped(self): - # The distinction the rule turns on: a port that genuinely cannot do something reports - # "skip" from the suite itself, which is evidence rather than the absence of it. + def test_provenance_rejects_results_edited_without_a_new_run(self): + # Exactly the edit twelve published "passes" were made by: a test entry appended to a + # report, its summary bumped, and the stamp naming the run left untouched. + before = self.stored_reports()["android"] + after = json.loads(json.dumps(before)) + after["tests"]["SomeBrandNewTest"] = {"feature": "crypto", "status": "pass"} + after["summary"]["pass"] += 1 + problems = port_status.provenance_problems("android", before, after) + self.assertEqual(1, len(problems), problems) + self.assertIn("CI output", problems[0]) + + def newer_snapshot(self, before): + after = json.loads(json.dumps(before)) + after["tests"]["SomeBrandNewTest"] = {"feature": "crypto", "status": "pass"} + after["summary"]["pass"] += 1 + after["generated_at"] = "2026-12-31T00:00:00Z" + after["commit"] = "0123456789abcdef" + after["run_url"] = "https://github.com/codenameone/CodenameOne/actions/runs/99" + return after + + def test_provenance_accepts_a_genuinely_newer_snapshot(self): + # Without the data branch to consult. Unverifiable is not the same as forged, and + # failing a branch because a fetch flaked would teach people to route around this. + before = self.stored_reports()["android"] + self.assertEqual( + [], port_status.provenance_problems("android", before, self.newer_snapshot(before)) + ) + + def test_provenance_accepts_a_report_the_data_branch_published(self): + before = self.stored_reports()["android"] + after = self.newer_snapshot(before) + self.assertEqual( + [], + port_status.provenance_problems( + "android", before, after, published=[before, after] + ), + ) + + def test_provenance_rejects_a_report_no_run_ever_published(self): + # The bypass that survived requiring both identity fields: type a plausible run URL and + # a plausible date. A checked-in report is a copy of what CI put on the data branch, so + # the branch is asked whether this report was ever there. Producing one that was needs + # the write access to that branch which only the publish workflows have. + before = self.stored_reports()["android"] + after = self.newer_snapshot(before) + problems = port_status.provenance_problems( + "android", before, after, published=[before] + ) + self.assertEqual(1, len(problems), problems) + self.assertIn("ever held", problems[0]) + + def test_provenance_rejects_a_run_url_that_names_no_run(self): + before = self.stored_reports()["android"] + after = self.newer_snapshot(before) + after["run_url"] = "made-up-new-run" + problems = port_status.provenance_problems( + "android", before, after, published=[after] + ) + self.assertEqual(1, len(problems), problems) + self.assertIn("does not name a workflow run", problems[0]) + + def test_provenance_rejects_a_fresh_stamp_over_the_same_run(self): + # The easier version of the same forgery, and the one a gate that accepted any single + # provenance change would have invited: invent the result, type today's date, and leave + # commit and run_url still naming the run that never produced it. + before = self.stored_reports()["android"] + after = json.loads(json.dumps(before)) + after["tests"]["SomeBrandNewTest"] = {"feature": "crypto", "status": "pass"} + after["summary"]["pass"] += 1 + after["generated_at"] = "2026-12-31T00:00:00Z" + problems = port_status.provenance_problems("android", before, after) + self.assertEqual(1, len(problems), problems) + self.assertIn("run_url", problems[0]) + + def test_provenance_rejects_a_new_run_url_on_the_same_stamp(self): + # The mirror image. A run reports at a time; reusing the old one says this snapshot is + # the same measurement under a different name. + before = self.stored_reports()["android"] + after = json.loads(json.dumps(before)) + after["performance"]["benchmarks"]["quicksort"]["duration_ns"] = 1 + after["run_url"] = "https://github.com/codenameone/CodenameOne/actions/runs/98" + problems = port_status.provenance_problems("android", before, after) + self.assertEqual(1, len(problems), problems) + self.assertIn("generated_at", problems[0]) + + def test_provenance_rejects_an_emptied_run_url(self): + # "Different" is not enough on its own: deleting the field would otherwise read as a + # change and let the edit through with no run named at all. + before = self.stored_reports()["android"] + after = json.loads(json.dumps(before)) + after["tests"]["SomeBrandNewTest"] = {"feature": "crypto", "status": "pass"} + after["summary"]["pass"] += 1 + after["generated_at"] = "2026-12-31T00:00:00Z" + after["run_url"] = "" + problems = port_status.provenance_problems("android", before, after) + self.assertEqual(1, len(problems), problems) + self.assertIn("run_url", problems[0]) + + def test_provenance_rejects_edited_benchmark_measurements(self): + # The findings nobody can check by reading them. A benchmark duration is published as a + # measurement of a named run; rewriting one in place attributes an invented number to + # that run exactly the way the twelve invented passes did. Naming only tests and summary + # would have left performance as the one thing a branch could still edit. + before = self.stored_reports()["android"] + after = json.loads(json.dumps(before)) + workload = next(iter(after["performance"]["benchmarks"])) + after["performance"]["benchmarks"][workload]["duration_ns"] = 1 + problems = port_status.provenance_problems("android", before, after) + self.assertEqual(1, len(problems), problems) + self.assertIn("performance", problems[0]) + + def test_provenance_rejects_an_edited_completion_marker(self): + # suite_finished is what makes a port card say the suite completed rather than that the + # run stopped early. + before = self.stored_reports()["android"] + after = json.loads(json.dumps(before)) + after["suite_finished"] = not before["suite_finished"] + problems = port_status.provenance_problems("android", before, after) + self.assertEqual(1, len(problems), problems) + self.assertIn("suite_finished", problems[0]) + + def test_provenance_rejects_a_timestamp_retyped_onto_an_old_snapshot(self): + # Changes no finding, so every rule about findings passes it -- and it is the edit with + # the worst consequence of any of them. The page reads generated_at to decide whether a + # column is stale, so retyping it is how a port that has stopped reporting altogether + # would go on looking like it was still running. Corroboration is therefore asked of any + # change, not only of a changed result. + before = self.stored_reports()["android"] + after = json.loads(json.dumps(before)) + after["generated_at"] = "2026-12-31T00:00:00Z" + problems = port_status.provenance_problems( + "android", before, after, published=[before] + ) + self.assertEqual(1, len(problems), problems) + self.assertIn("ever held", problems[0]) + + def test_provenance_rejects_a_hand_authored_report_for_a_new_port(self): + # The one report with no earlier version to be checked against, and so the only one + # nobody was checking at all: a pull request that adds a port could give it an entirely + # green snapshot. It has to be a report the data branch published, like every other. + published = self.stored_reports()["android"] + invented = json.loads(json.dumps(published)) + invented["port"] = "freebsd" + problems = port_status.provenance_problems( + "freebsd", None, invented, published=[published] + ) + self.assertEqual(1, len(problems), problems) + self.assertIn("ever held", problems[0]) + + def test_provenance_tells_a_new_port_it_needs_no_report(self): + # And the advice has to be actionable, which it is only because a port with no stored + # report is now a supported state: every cell reads "No stored report" until its first + # run. Otherwise the only way to add a port would be to hand-author the snapshot this + # rule refuses. + invented = self.stored_reports()["android"] + problems = port_status.provenance_problems("freebsd", None, invented, published=[]) + self.assertEqual(1, len(problems), problems) + self.assertIn("never published a report", problems[0]) + + def test_provenance_accepts_a_new_port_report_copied_from_the_branch(self): + published = self.stored_reports()["android"] + self.assertEqual( + [], port_status.provenance_problems("android", None, published, published=[published]) + ) + + def test_validate_accepts_a_port_with_no_stored_report(self): original_directory = self.manifest["report_directory"] with tempfile.TemporaryDirectory(dir=port_status.REPO_ROOT) as tmp: report_root = Path(tmp) for port in self.manifest["ports"]: - source = port_status.REPO_ROOT / original_directory / ( - port["id"] + ".json" - ) + if port["id"] == "tvos": + continue + source = port_status.REPO_ROOT / original_directory / (port["id"] + ".json") (report_root / source.name).write_text( source.read_text(encoding="utf-8"), encoding="utf-8" ) - android_path = report_root / "android.json" - android = port_status.read_json(android_path) - victim = next( - name - for name, result in android["tests"].items() - if result.get("status") == "pass" - ) - android["tests"][victim]["status"] = "skip" - android["summary"]["pass"] -= 1 - android["summary"]["skip"] += 1 - android_path.write_text( - json.dumps(android, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) manifest = dict(self.manifest) - manifest["report_directory"] = str( - report_root.relative_to(port_status.REPO_ROOT) - ) - # Asserted against this rule alone: a skip carries its own separate obligation -- - # errata explaining it -- and that is what would be reported here instead. - try: - port_status.validate(manifest) - except port_status.ContractError as exc: - self.assertNotIn("never ran", str(exc)) + manifest["report_directory"] = str(report_root.relative_to(port_status.REPO_ROOT)) + counts = port_status.validate(manifest) + self.assertTrue( + any("tvos" in item and "no stored report" in item for item in counts["drift"]), + counts["drift"], + ) + + def test_provenance_rejects_deleting_an_established_fallback(self): + # A port with no stored report became a supported state so that ADDING a port would not + # have to begin by hand-authoring one. That must not also make removing an existing + # fallback free: the site serves this file whenever the data branch is unreachable or + # its newest report predates the contract, so deleting one turns a working port's whole + # column unknown at exactly the moment the live data is missing. + before = self.stored_reports()["tvos"] + problems = port_status.provenance_problems("tvos", before, None, published=[before]) + self.assertEqual(1, len(problems), problems) + self.assertIn("deleted", problems[0]) + + def test_provenance_ignores_a_port_that_has_no_report_either_side(self): + # Retiring a port drops it from the manifest, and the check never looks at it. This is + # the port that has simply never published. + self.assertEqual([], port_status.provenance_problems("freebsd", None, None, published=[])) + + def test_provenance_ignores_an_untouched_report(self): + before = self.stored_reports()["android"] + self.assertEqual([], port_status.provenance_problems("android", before, dict(before))) def publishable_report(self, port_id, **overrides): mapped = port_status.test_to_feature(self.manifest) @@ -799,17 +1293,21 @@ def test_publishable_rejects_a_malformed_performance_status_when_unfinished(self self.assertTrue( any("performance status is" in item for item in malformed), malformed) - def test_publishable_matches_every_report_the_site_serves(self): + def test_every_report_the_site_serves_is_renderable(self): + # Only malformed. Drift is asserted against deliberately: a checked-in snapshot is a + # copy of a real run, and a branch that registers a test makes every one of them + # predate it. Demanding zero drift here is what turned "add a test" into "edit eleven + # reports", and what made inventing a result the path of least resistance. What the + # fallback owes the site is that Hugo can render it. for port in self.manifest["ports"]: report_path = port_status.REPO_ROOT / self.manifest["report_directory"] / ( port["id"] + ".json" ) with self.subTest(port=port["id"]): - drift, malformed = port_status.publishable_report_problems( + _, malformed = port_status.publishable_report_problems( self.manifest, port["id"], port_status.read_json(report_path) ) self.assertEqual([], malformed) - self.assertEqual([], drift) if __name__ == "__main__": diff --git a/scripts/lib/cn1ss.sh b/scripts/lib/cn1ss.sh index 26c141791f5..72ff6914ecd 100644 --- a/scripts/lib/cn1ss.sh +++ b/scripts/lib/cn1ss.sh @@ -361,6 +361,57 @@ print(sum(1 for r in results if isinstance(r, dict) and r.get("status") in ("equ PY } +# Goldens whose owning test reported a skip this port's errata explain. +# +# The count guard below treats an unproduced golden as evidence that a test hung, +# crashed or never delivered its frame -- which it is, when nothing else was +# said. A test that prints "status=SKIPPED reason=..." said something, and +# port_status.py checks that reason against port_status_supplement.json for THIS +# port before agreeing. Silence still counts as missing; so does an unexplained +# skip, and so does a reason code written about a different port. +# +# Without this the skip path some tests deliberately take could never succeed on +# a port that owns a golden: GoogleWebMapScreenshotTest skips when the Google +# Maps tiles never load -- documented for android, ios-gl and ios-metal, and +# rendered as a documented skip on the public table -- and the guard failed the +# whole job on the uncovered golden anyway. +cn1ss_count_documented_skips() { + local ref_dir="$1" + local compare_json="$2" + local script_dir repo_root status_script python_bin + if [ -z "${CN1SS_PORT_ID:-}" ] || [ -z "$ref_dir" ] || [ ! -d "$ref_dir" ]; then + echo 0 + return + fi + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + repo_root="$(cd "$script_dir/../.." && pwd)" + status_script="$repo_root/scripts/hellocodenameone/conformance/port_status.py" + python_bin="${CN1SS_PYTHON_BIN:-python3}" + if [ ! -f "$status_script" ] || ! command -v "$python_bin" >/dev/null 2>&1; then + echo 0 + return + fi + local -a args=("$status_script" documented-skips --port "$CN1SS_PORT_ID" --reference "$ref_dir") + # The comparison results, so a golden the run DID produce is never discounted: + # it is already in covered_count, and subtracting it again would hide an + # uncovered golden belonging to some other test. + if [ -n "$compare_json" ] && [ -s "$compare_json" ]; then + args+=(--compare "$compare_json") + fi + local log_var log_path + for log_var in CN1SS_SUITE_LOG CN1SS_SUITE_LOG_2 CN1SS_SUITE_LOG_3; do + log_path="${!log_var:-}" + if [ -n "$log_path" ] && [ -f "$log_path" ]; then + args+=(--log "$log_path") + fi + done + local count + # A failure here must not excuse anything: no answer means no discount. + count="$("$python_bin" "${args[@]}" 2>/dev/null || echo 0)" + count="${count//[^0-9]/}" + echo "${count:-0}" +} + # Count "missing_expected" results: a screenshot the suite captured and delivered # but which has NO committed golden under the reference directory. This is the # signal that a test ran for real yet its reference was never integrated -- the @@ -621,6 +672,13 @@ cn1ss_process_and_report() { allowed_missing="${allowed_missing//[^0-9]/}"; : "${allowed_missing:=0}" uncovered_count=$(( expected_count - covered_count )) [ "$uncovered_count" -lt 0 ] && uncovered_count=0 + local documented_skips + documented_skips=$(cn1ss_count_documented_skips "$ref_dir" "$compare_json_out") + if [ "$documented_skips" -gt 0 ]; then + cn1ss_log "$documented_skips golden(s) accounted for by a skip this port's errata explain (see the lines above)." + uncovered_count=$(( uncovered_count - documented_skips )) + [ "$uncovered_count" -lt 0 ] && uncovered_count=0 + fi if [ "$uncovered_count" -gt "$allowed_missing" ]; then cn1ss_log "FATAL: $uncovered_count of $expected_count expected screenshot(s) were not produced and compared (only $covered_count covered); $allowed_missing tolerated (CN1SS_ALLOWED_MISSING)." cn1ss_log " A test failed to emit its screenshot, or the suite hung/crashed before finishing. The golden set under the comparison directory is the source of truth for how many screenshots must be produced."