ui-smoke phase 3: screenshots, writable config mirrors, homing settle

.github/workflows/ci.yml

@@ -51,6 +51,17 @@ jobs:
       run: |
         set -x
         scripts/rip-environment runtests -p
+    - name: Upload UI smoke screenshots
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: ui-smoke-screenshots-gcc
+        path: |
+          tests/ui-smoke/**/screenshot.png
+          tests/ui-smoke/**/confirm.png
+          tests/ui-smoke/**/diff.png
+        if-no-files-found: ignore
+        retention-days: 14
     - name: Verify no untracked or modified files after test
       run: |
         .github/scripts/verify-clean-repo.sh
@@ -112,6 +123,17 @@ jobs:
       run: |
         set -x
         scripts/rip-environment runtests -p
+    - name: Upload UI smoke screenshots
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: ui-smoke-screenshots-clang
+        path: |
+          tests/ui-smoke/**/screenshot.png
+          tests/ui-smoke/**/confirm.png
+          tests/ui-smoke/**/diff.png
+        if-no-files-found: ignore
+        retention-days: 14
     - name: Verify no untracked or modified files after test
       run: |
         .github/scripts/verify-clean-repo.sh

debian/control.top.in

@@ -47,6 +47,7 @@ Build-Depends:
     tk@TCLTK_VERSION@-dev,
     xvfb <!nocheck>,
     x11-xserver-utils <!nocheck>,
+    imagemagick <!nocheck>,
     python3-opengl <!nocheck>,
     python3-pyqt5 <!nocheck>,
     python3-pyqt5.qsci <!nocheck>,

tests/ui-smoke/.gitignore

@@ -5,3 +5,9 @@ linuxcnc.err
 linuxcnc.pid
 ui-smoke.out
 ui-smoke.err
+# confirm.png is the per-run shot; diff.png is its comparison to the
+# committed reference.png (which IS tracked, so it is not listed here).
+screenshot.png
+confirm.png
+ui-smoke-qt.png
+diff.png

tests/ui-smoke/README

@@ -19,6 +19,35 @@ Shared helpers live in _lib/:
   checkresult.sh      shared pass/fail predicate
   skip-if-missing.sh  shared skip predicate
 
+Failure diagnostics (failure path only, no cost on a green run):
+  crashdump.sh   arms a core dump and prints a native backtrace if a GUI
+                 segfaults (the Qt/dbus/GL frames PYTHONFAULTHANDLER misses)
+  screenshot.sh  photographs the Xvfb root window before teardown. On a
+                 failure it captures the cause (a GUI hung on a blocking
+                 modal leaves no core and no traceback); on a clean Phase 2
+                 run it captures confirm.png, the GUI in its post-movement
+                 idle state (final DRO / toolpath) for visual confirmation.
+                 CI uploads both as build artifacts (ui-smoke-screenshots-*
+                 in ci.yml).
+  compare.sh     on a clean run, compares confirm.png to the committed
+                 known-good reference.png (ImageMagick) and writes a
+                 highlighted diff.png, also uploaded as an artifact.
+
+Reference images:
+  reference.png  committed per-GUI known-good shot (in a --run-program test
+                 directory); diff.png is confirm.png compared against it.
+                 The comparison NEVER fails a test: freetype/font versions
+                 differ across distros, so some drift is expected. The diff
+                 is only recorded, not used to gate the test.
+
+  Create or refresh the references on a built run-in-place tree:
+      . scripts/rip-environment
+      tests/ui-smoke/_lib/make-references.sh           # all run-program GUIs
+      tests/ui-smoke/_lib/make-references.sh axis      # just one
+  This sets UI_SMOKE_UPDATE_REFERENCE=1, which makes compare.sh save each
+  clean confirm shot as that test's reference.png. Review the PNGs before
+  committing; they are baselines from the generating machine's fonts.
+
 Skip vs fail policy: the only condition we skip on is xvfb-run absence
 (rare local dev env). Python and gi typelib deps the GUIs need are
 declared in debian/control under !nocheck so apt-get build-dep

tests/ui-smoke/_lib/compare.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# Known-good image comparison for the UI smoke confirm shots. Complements
+# screenshot.sh: that grabs confirm.png on a clean run; this compares it to
+# a committed reference.png and writes a visual diff.png. Like screenshot.sh
+# it carries no state and is a logged no-op whenever it cannot run, so it can
+# never turn a pass into a fail.
+#
+# Policy: we never fail a test on the image difference. freetype/font
+# versions differ across distros, so some drift is expected; the diff is
+# here to record what changed, not to gate. The function always returns 0.
+#
+# Local "make a known-good image" workflow: run a test with
+# UI_SMOKE_UPDATE_REFERENCE=1 (see make-references.sh) and the freshly
+# captured shot is saved as the committed reference instead of compared.
+
+# Pick the ImageMagick compare entry point: IM7 "magick compare", else IM6
+# "compare". Echoes nothing and returns 1 if neither is present.
+_compare_cmd() {
+    if command -v magick >/dev/null 2>&1; then
+        echo "magick compare"
+    elif command -v compare >/dev/null 2>&1; then
+        echo "compare"
+    else
+        return 1
+    fi
+}
+
+# compare_to_reference <shot> <reference> <diff>
+# Compare the captured shot to the committed reference, writing a highlighted
+# diff image. Always returns 0.
+compare_to_reference() {
+    shot="$1"
+    reference="$2"
+    diff="$3"
+
+    if [ ! -s "$shot" ]; then
+        echo "compare: no shot at $shot, skipping"
+        return 0
+    fi
+
+    # Update mode: adopt this shot as the new known-good reference.
+    if [ "${UI_SMOKE_UPDATE_REFERENCE:-}" = "1" ]; then
+        if cp -f "$shot" "$reference"; then
+            echo "compare: saved reference $reference (UI_SMOKE_UPDATE_REFERENCE=1)"
+        else
+            echo "compare: failed to save reference $reference"
+        fi
+        return 0
+    fi
+
+    if [ ! -s "$reference" ]; then
+        echo "compare: no reference at $reference yet, skipping (run with UI_SMOKE_UPDATE_REFERENCE=1 to create one)"
+        return 0
+    fi
+
+    cmd=$(_compare_cmd) || {
+        echo "compare: no ImageMagick compare available, skipping"
+        return 0
+    }
+
+    # -metric AE: count of differing pixels (interpretable); -fuzz absorbs
+    # anti-aliasing jitter. compare exits 0 (identical), 1 (differ) or 2
+    # (error, e.g. the shots are different sizes). We log the outcome and
+    # always succeed.
+    metric=$($cmd -metric AE -fuzz 5% "$reference" "$shot" "$diff" 2>&1)
+    rc=$?
+    case "$rc" in
+        0) echo "compare: $shot matches $reference (AE=$metric)" ;;
+        1) echo "compare: $shot differs from $reference (AE=$metric differing pixels); diff at $diff" ;;
+        *) echo "compare: could not compare $shot to $reference (rc=$rc): $metric" ;;
+    esac
+    return 0
+}

tests/ui-smoke/_lib/drive.py

@@ -32,6 +32,13 @@
 # stability check catches that.
 STATE_STABILITY_S = 0.5
 STATE_RETRY_BUDGET = 6
+# Pause after homing before requesting AUTO. gmoccapy only enables AUTO
+# once it has processed the all-homed signal in its own event loop (and
+# re-asserts MANUAL itself on that signal). Requesting AUTO before then is
+# rejected: it bounces back to MANUAL with an "It is not possible to
+# change to Auto Mode" warning. ensure_mode would retry and win, but the
+# warning lingers on screen; this settle lets the GUI catch up first.
+POST_HOME_SETTLE_S = 2.0
 
 # linuxcnc launcher PID, written to linuxcnc.pid by the launcher and read
 # once at startup. The driver watches it so a GUI crash, which tears
@@ -297,6 +304,11 @@ def run_program(cmd, stat, ngc_path, expect_delta_mm, tol, run_timeout):
     if not home_all(cmd, stat, timeout=60.0):
         return False
 
+    # Let the GUI react to the all-homed transition before requesting AUTO,
+    # so it does not reject the mode change (see POST_HOME_SETTLE_S).
+    time.sleep(POST_HOME_SETTLE_S)
+    stat.poll()
+
     if not ensure_mode(cmd, stat, linuxcnc.MODE_AUTO, "MODE_AUTO"):
         return False
 

tests/ui-smoke/_lib/gmoccapy-prepare.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Sourced by the gmoccapy ui-smoke tests (smoke and quit) to run gmoccapy
+# against a writable copy of its sim config. Sets GMOCCAPY_INI to the
+# mirrored ini path; the caller then execs run-gui.sh or quit-launch.sh
+# with "$GMOCCAPY_INI". Must be sourced with LIB_DIR already set.
+#
+# gmoccapy writes its preferences file next to the config: with no
+# PREFERENCE_FILE_PATH in the ini, getiniinfo falls back to
+# <config-dir>/<MACHINE>.pref. CI mounts the workspace read-only for the
+# runtime user, so that write raises PermissionError partway through
+# __init__ (during _get_pref_data, before the MDIHistory widget's
+# _hal_init runs). gmoccapy pops an error dialog and limps on in a
+# half-initialised state: the interp-idle handler then hits a widget with
+# no .stat and throws a second dialog. Both vanish once the config dir is
+# writable. Mirror it to tmp, same fix qtdragon-prepare.sh uses.
+
+: "${LIB_DIR:?gmoccapy-prepare.sh must be sourced with LIB_DIR set}"
+
+SRC_DIR="$(cd "$LIB_DIR/../../../configs/sim/gmoccapy" && pwd)"
+
+WORK_DIR="$(mktemp -d -t ui-smoke-gmoccapy.XXXXXX)"
+trap 'rm -rf "$WORK_DIR"' EXIT
+cp -r "$SRC_DIR/." "$WORK_DIR/"
+
+# Seed the preference file (config dir + <MACHINE>.pref; MACHINE=gmoccapy)
+# so the first-run "Important change(s)" modal stays hidden. That dialog
+# runs a nested gtk loop, so under xvfb it never gets dismissed: it sits
+# on top of the UI in the confirm shot and, worse, swallows the SIGTERM
+# in the quit test (the loop keeps running after main_quit). A real user
+# ticks "Don't show this again" once; hide_startup_messsage replicates
+# that. The triple-s key matches gmoccapy's own (sic).
+cat >"$WORK_DIR/gmoccapy.pref" <<'PREF'
+[DEFAULT]
+hide_startup_messsage = 99
+PREF
+
+# Consumed by the sourcing test.sh, which execs the launcher with it.
+# shellcheck disable=SC2034
+GMOCCAPY_INI="$WORK_DIR/gmoccapy.ini"

tests/ui-smoke/_lib/launch-env.sh

@@ -29,3 +29,10 @@ export SDL_AUDIODRIVER=dummy
 # names the line; for a C/C++ crash (Qt, dbus, GL) it shows the Python
 # frame that called in. The native side is captured by crashdump.sh.
 export PYTHONFAULTHANDLER=1
+
+# Xvfb virtual screen for the launchers' xvfb-run. There is no window
+# manager under xvfb-run, so a GUI's maximize() is a no-op and it renders
+# at its natural size; a screen smaller than the window clips the grab
+# (the failure/confirm screenshot then misses panels). 1920x1080 fits
+# every sim GUI so the whole window is captured.
+export UI_SMOKE_XVFB_SCREEN="${UI_SMOKE_XVFB_SCREEN:-1920x1080x24}"

tests/ui-smoke/_lib/launch.sh

@@ -43,6 +43,12 @@ DRIVER_TIMEOUT=180
 . "$LIB_DIR/crashdump.sh"
 crashdump_arm
 
+# Absolute path the offscreen-Qt self-grab writes to. An offscreen GUI
+# runs with its cwd at the (writable) config mirror, not the test dir, so
+# a relative name would land out of reach; pin it to the test dir, which
+# is this shell's cwd. Harmless for the GTK GUIs, which ignore it.
+export UI_SMOKE_QT_SHOT="$PWD/ui-smoke-qt.png"
+
 # Export the per-invocation values so the inner bash -c receives them
 # as proper env vars (avoids embedding paths into the inner script
 # via quoting, which breaks on apostrophes / spaces).
@@ -52,7 +58,7 @@ export CONFIG_INI LIB_DIR DRIVER_TIMEOUT
 # LIB_DIR and DRIVER_TIMEOUT are expanded by the inner bash (which sees
 # them via the exported env), not by the outer shell.
 # shellcheck disable=SC2016
-xvfb-run -a --server-args="-screen 0 1024x768x24" \
+xvfb-run -a --server-args="-screen 0 $UI_SMOKE_XVFB_SCREEN" \
     timeout "$LINUXCNC_TIMEOUT" \
     bash -c '
         # Run linuxcnc in its own process group so we can signal the
@@ -70,6 +76,29 @@ xvfb-run -a --server-args="-screen 0 1024x768x24" \
         timeout "$DRIVER_TIMEOUT" python3 "$LIB_DIR/drive.py" "$@" >ui-smoke.out 2>ui-smoke.err
         DRIVE_RC=$?
 
+        # Photograph the root window before teardown, while DISPLAY is the
+        # Xvfb server and the GUI is still up. On failure the picture shows
+        # the cause (a hung GUI on a blocking modal leaves no core for
+        # crashdump.sh and no Python traceback). On a clean Phase 2 run it
+        # is a confirmation shot of the GUI in its post-movement idle state
+        # (final DRO / toolpath), so a reviewer can eyeball the result. The
+        # short settle lets the GUI repaint the final position first.
+        . "$LIB_DIR/screenshot.sh"
+        if [ "$DRIVE_RC" -ne 0 ]; then
+            screenshot_grab screenshot.png
+        else
+            case " $* " in
+                *" --run-program "*)
+                    sleep 0.5
+                    screenshot_grab confirm.png
+                    # Compare the confirm shot to the committed known-good
+                    # reference and write a visual diff. Never fails the test.
+                    . "$LIB_DIR/compare.sh"
+                    compare_to_reference confirm.png reference.png diff.png
+                    ;;
+            esac
+        fi
+
         # Clean shutdown: GUI-specific quit first (lets linuxcnc end
         # its own SIGTERM trap run Cleanup which unloads halrun and
         # reaps shared memory). axis-remote works only for axis but is
@@ -109,4 +138,10 @@ echo "=== ui-smoke.err ==="
 # If the GUI dumped a core, print its native backtrace.
 crashdump_report
 
+# Note any screenshot so the CI artifact step and reviewer know it is
+# there to download: screenshot.png on failure, confirm.png on a clean run.
+[ -f screenshot.png ] && echo "=== screenshot: $TEST_DIR/screenshot.png ==="
+[ -f confirm.png ] && echo "=== confirm: $TEST_DIR/confirm.png ==="
+[ -f diff.png ] && echo "=== diff: $TEST_DIR/diff.png ==="
+
 exit "$RC"

tests/ui-smoke/_lib/make-references.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Generate (or refresh) the committed known-good reference.png images by
+# running the ui-smoke tests with UI_SMOKE_UPDATE_REFERENCE=1, which makes
+# compare.sh save each clean confirm shot as that test's reference.png
+# instead of comparing against it.
+#
+# Run from a built run-in-place tree with the rip environment sourced:
+#   . scripts/rip-environment
+#   tests/ui-smoke/_lib/make-references.sh            # all run-program GUIs
+#   tests/ui-smoke/_lib/make-references.sh axis       # just one
+#
+# Only the GUIs that capture a confirm shot (the --run-program tests) have a
+# reference. Review the resulting PNGs before committing; they are baselines
+# from this machine's fonts and will drift on other distros (which is why the
+# comparison never fails a test, only records a diff).
+
+set -u
+
+LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SMOKE_DIR="$(cd "$LIB_DIR/.." && pwd)"
+ROOT="$(cd "$SMOKE_DIR/../.." && pwd)"
+
+if ! command -v runtests >/dev/null 2>&1; then
+    echo "make-references: 'runtests' not found; source scripts/rip-environment first" >&2
+    exit 1
+fi
+
+# GUIs whose tests grab a confirm shot (i.e. run a program).
+GUIS=("$@")
+if [ "${#GUIS[@]}" -eq 0 ]; then
+    GUIS=(axis gmoccapy touchy qtdragon)
+fi
+
+export UI_SMOKE_UPDATE_REFERENCE=1
+
+rc=0
+for gui in "${GUIS[@]}"; do
+    dir="$SMOKE_DIR/$gui"
+    if [ ! -x "$dir/test.sh" ]; then
+        echo "make-references: no test at $dir, skipping"
+        continue
+    fi
+    echo "=== make-references: $gui ==="
+    runtests "$dir" || rc=1
+    if [ -s "$dir/reference.png" ]; then
+        echo "make-references: wrote $dir/reference.png"
+    else
+        echo "make-references: WARNING no reference.png produced for $gui" >&2
+        rc=1
+    fi
+done
+
+exit "$rc"