Revision control
Copy as Markdown
type: Group
instructions:
- type: WriteFile
filename: "{{.AgentWorkingDir}}/coin_vxworks_qemu_runner.sh"
fileMode: 493
fileContents: |
#!/bin/bash
# Seconds of serial silence before triggering an SSH probe; also the cadence for repeated probes
EMPTY_READ_GRACE=60
# Maximum time (s) with no output while waiting; exceeding this triggers a timeout failure.
# This is a *consecutive-silence* budget (reset by every serial line), not a total test-duration
# cap, so long-but-chatty tests (e.g. tst_qfiledialog ~536s total) are unaffected. Lowered
# 700 -> 300 (== 5 x EMPTY_READ_GRACE): the observed 700s timeouts were all stuck RTPs
# (probe==RUNNING), so a shorter budget reclaims ~400s per stuck occurrence without killing
# healthy tests.
MAX_WAIT_TIME=300
ALIVE_WAIT_TIME=600
# The test executable (guest path) is the first argument, passed by ctest.
# Everything below keys off it -- 'rtp exec' launches it and
# ssh_probe_status() matches its basename ($RTP_NAME) in 'rtp list' to
# tell RUNNING from FINISHED -- so assert its validity up front instead
# of discovering a malformed value mid-run.
testexecutable="$1"
testdir="$(dirname "$testexecutable")"
RTP_NAME="$(basename "$testexecutable")"
if [[ -z "$testexecutable" || "$RTP_NAME" == "/" || "$RTP_NAME" == "." ]]; then
echo "Invalid test executable argument: '$1'"
exit 1
fi
quoted_args=`python3 -c 'import sys, shlex; print(shlex.join(sys.argv[2:]))' "$@"`
# The XML log grows on the NFS share while the test runs; a closing
# </TestCase> is the one reliable finished signal (used to veto FINISHED).
XML_LOG=""
prev=""
for a in "$@"; do
if [[ "$prev" == "-o" && "$a" == *.xml,xml ]]; then
XML_LOG="${a%,xml}"
break
fi
prev="$a"
done
xml_complete() {
[[ -n "$XML_LOG" ]] && /usr/bin/grep -qs '</TestCase>' "$XML_LOG"
}
# Acquire a free emulator slot. ctest runs this script once per test with
# up to VXWORKS_QEMU_INSTANCES jobs in parallel; each job takes exclusive
# ownership of one emulator instance via an flock held for its lifetime.
N=${VXWORKS_QEMU_INSTANCES:-1}
SLOT=""
slot_fd=""
for s in $(seq 0 $((N - 1))); do
exec {fd}>"/tmp/vxworks_slot_${s}.lock"
if flock -n "$fd"; then
SLOT="$s"
slot_fd="$fd"
break
fi
exec {fd}>&-
done
if [[ -z "$SLOT" ]]; then
echo "No free VxWorks emulator slot"
exit 1
fi
echo "Using VxWorks emulator slot ${SLOT}"
# Per-slot resources.
GUEST_IN="/tmp/guest${SLOT}.in"
GUEST_OUT="/tmp/guest${SLOT}.out"
VX_SCRIPT="/home/qt/work/vx${SLOT}.sh"
GUEST_IP="172.31.1.$((10 + SLOT))"
if [[ "$VXWORKS_SSH" == *"@"* ]]; then
SSH_TARGET="${VXWORKS_SSH%@*}@${GUEST_IP}"
else
SSH_TARGET="${GUEST_IP}"
fi
# Pattern uniquely matching this slot's QEMU process: the slot's tap
# interface name appears on the QEMU command line. The [,] keeps
# pgrep/pkill from matching the (sudo) pkill command line itself, which
# contains the same literal.
QEMU_PROC_PATTERN="ifname=tap${SLOT}[,]"
# Kill only this slot's emulator (matched by $QEMU_PROC_PATTERN),
# never the emulators owned by the parallel jobs. SIGTERM alone is not
# enough: a wedged QEMU can ignore the TERM relayed through its sudo
# wrapper and keep the tap device busy, so the relaunched instance dies
# at startup and the new serial FIFOs never get a reader. Verify death
# and escalate to SIGKILL; QEMU runs as root (sudo chrt), so the -9 has
# to go through sudo.
kill_qemu() {
pkill -f "$QEMU_PROC_PATTERN"
for _ in $(seq 1 10); do
pgrep -f "$QEMU_PROC_PATTERN" >/dev/null || return 0
sleep 1
done
sudo pkill -9 -f "$QEMU_PROC_PATTERN"
for _ in $(seq 1 5); do
pgrep -f "$QEMU_PROC_PATTERN" >/dev/null || return 0
sleep 1
done
echo "Emulator for slot ${SLOT} survived SIGKILL"
return 1
}
# Bounded write to the guest serial FIFO. A plain 'echo > "$GUEST_IN"' opens
# the FIFO for writing and blocks until the guest is reading it; if the guest
# has wedged, that block is unbounded. Cap it: on timeout the emulator is
# dead, so kill it (the next run relaunches via the health check) and fail
# this test rather than hanging the slot.
serial_send() {
if ! timeout 10 bash -c 'printf "%s\n" "$1" > "$2"' _ "$1" "$GUEST_IN"; then
echo "Serial write to slot ${SLOT} timed out; treating emulator as dead"
kill_qemu
exit 1
fi
}
ssh_cmd() {
# A hard wall-clock cap ('timeout') plus ServerAlive* bound the *whole*
# probe. ConnectTimeout only limits TCP connect: if SSH connects but the
# guest command never returns (emulator wedged under heavy TCG load), the
# ssh call would otherwise block forever, so ssh_probe_status never
# returns and the MAX_WAIT_TIME watchdog below can never fire. That is
# exactly how a stuck test could deadlock the whole ctest run until the
# 4h CI ceiling.
# Clean library path: ctest's environment carries the provisioned OpenSSL,
# which the system ssh refuses to load ("OpenSSL version mismatch").
env -u LD_LIBRARY_PATH timeout -k 5 20 ssh -n -q -T \
-o BatchMode=yes -o ConnectTimeout=1 \
-o ServerAliveInterval=3 -o ServerAliveCountMax=2 \
-o HostKeyAlgorithms=+ssh-rsa \
-o PreferredAuthentications=publickey \
-o NumberOfPasswordPrompts=0 \
${SSH_TARGET} "$@" </dev/null
}
ssh_probe_status() {
# Returns one of: FINISHED | RUNNING | UNRESPONSIVE
# FINISHED is not authoritative: 'rtp list' may truncate long launch
# paths or return partial output; callers corroborate with
# xml_complete().
local out
out="$(ssh_cmd 'cmd rtp list; exit' 2>&1 || true)"
# If we don't see the normal SSH epilogue ("Au revoir!"), treat as UNRESPONSIVE
# (system is probably under a heavy load)
if ! printf '%s\n' "$out" | /usr/bin/grep -q 'Au revoir!'; then
echo "UNRESPONSIVE"
return 0
fi
# Is our test's RTP still listed? Match the actual test-binary name
# ($RTP_NAME, set at the top from the executable path) rather than a generic
# "tests" substring: the RTP shows up in 'rtp list' by its launch path,
# and keying on the real basename works regardless of where the test is
# installed. A generic "tests" match only happened to work for the CI
# install path and would mis-report a still-running test as FINISHED
# (=> premature exit 0 / false pass) for any other layout.
if printf '%s\n' "$out" | /usr/bin/grep -Fq "$RTP_NAME"; then
echo "RUNNING"
else
echo "FINISHED"
fi
}
echo RUNNING via serial: "$quoted_args"
echo "cmd rtp exec -p 200 -t 0x01000000 -u 0x1000000 $testexecutable -- $quoted_args" > "$VX_SCRIPT"
echo "cmd echo \"qtest_in_vxworks_complete: $?\"" >> "$VX_SCRIPT"
#rtp exec Execute a process.
# -p : specify the initial task priority
# 200 is high, 255 max
# -t : task options for the RTPs initial task
# 0x01000000 turn on coprocessor
# -u : specify the size of the process task stack
# -- : mark the end of "rtp exec" options.
# qtest_in_vxworks_complete echo is used to detect test process completion as it will
# be echoed after test process ends normally or crashes when normal log lines are not present.
# Check that emulator is functioning
( echo "cmd echo \"health check\"" > "$GUEST_IN" ) & pid=$!
( sleep 5 && kill -HUP $pid ) 2>/dev/null & watcher=$!
wait $pid 2>/dev/null && pkill -HUP -P $watcher
if [[ "$?" -eq "0" ]]; then
read -t 5 echoline < "$GUEST_OUT"
read -t 1 testline < "$GUEST_OUT"
fi
probe0="$(ssh_probe_status)"
if [[ -z "$testline" || "$probe0" != "FINISHED" ]]; then
if [[ "$probe0" == "UNRESPONSIVE" ]]; then
echo "Restarting stuck emulator (SSH down)"
elif [[ "$probe0" == "RUNNING" ]]; then
echo "Restarting stuck emulator (RTP still running)"
else
echo "Restarting stuck emulator (serial down)"
fi
if ! kill_qemu; then
# Can't reclaim the tap device; relaunching would just produce a
# dead QEMU and readerless FIFOs. Fail the test instead of
# wedging the slot; the next run retries the whole teardown.
exit 1
fi
rm -f "$GUEST_IN" "$GUEST_OUT"
mkfifo "$GUEST_IN" "$GUEST_OUT"
# Close the slot lock fd for the spawned processes so the long-lived
# emulator does not inherit and hold this job's slot lock.
# Non-zero exit: the relaunched guest's SSH never came up. Fail the
# attempt so qt-testrunner retries on a clean slot. (Older launchers
# always exit 0.)
if ! /bin/bash /home/qt/vxworks_qemu_launcher.sh "$VXWORKS_EMULATOR_TYPE" "$SLOT" {slot_fd}>&-; then
echo "Relaunch of emulator for slot ${SLOT} failed"
kill_qemu
exit 1
fi
sleep 1
# If the relaunched QEMU died at startup, the new FIFOs have no
# reader and every serial write below would block forever.
if ! pgrep -f "$QEMU_PROC_PATTERN" >/dev/null; then
echo "Relaunched emulator for slot ${SLOT} is not running"
exit 1
fi
# Guest configuration writes to the serial FIFO with no bound of its
# own; if the emulator wedges mid-boot those writes block forever.
# This exact hang deadlocked both ARM slots and two of four x86
# slots until the 4h CI ceiling (2026-07-10 runs). Normal duration
# is ~40s (sleep 10 + ~25 throttled writes); cap it and fail the
# test on overrun so the slot stays usable.
if ! timeout 120 /bin/bash /home/qt/work/coin_vxworks_vars.sh "$SLOT" {slot_fd}>&-; then
echo "Guest configuration for slot ${SLOT} timed out; treating emulator as dead"
kill_qemu
exit 1
fi
sleep 1
else
echo "VxWorks is functional"
fi
# Empty out any pending serial output to start clean
while read -t 1 line; do
echo $line
done < "$GUEST_OUT"
serial_send "cmd cd $testdir; echo done"
# Empty line before result of cd
read -t 1 line_cmd < "$GUEST_OUT"
if read -t 1 cdline < "$GUEST_OUT"; then
cdline_clean=$(printf '%s' "$cdline" | tr -d '\r')
if printf '%s' "$cdline_clean" | /usr/bin/grep -Eq 'error\s*='; then
printf 'entering %s: %s\n' "$testdir" "$cdline_clean"
exit 1
fi
fi
serial_send "<$VX_SCRIPT"
silent_seconds=0
last_probe_result="NONE"
while true; do
if ! read -t $EMPTY_READ_GRACE line; then
silent_seconds=$((silent_seconds + EMPTY_READ_GRACE))
probe="$(ssh_probe_status)"
last_probe_result="$probe"
wait_budget=$MAX_WAIT_TIME
if [[ "$probe" == "UNRESPONSIVE" ]]; then
# Unresponsive guest that never created its XML log wedged at
# launch and won't recover; give a few probes for NFS/SSH
# lag, then give up.
if [[ -n "$XML_LOG" && ! -e "$XML_LOG" ]] && (( silent_seconds >= 3 * EMPTY_READ_GRACE )); then
echo "SSH unresponsive and XML log never created after ${silent_seconds}s: guest wedged at launch"
kill_qemu
exit 1
fi
echo "system is under heavy load after ${silent_seconds}s of silence (SSH unresponsive)"
elif [[ "$probe" == "FINISHED" ]]; then
if [[ -z "$XML_LOG" ]] || xml_complete; then
echo "RTP finished after ${silent_seconds}s of silence, ending"
exit 0
fi
if [[ ! -e "$XML_LOG" ]]; then
# Not listed and no XML log: launch failed (a running
# silent test has an XML file from QTest startup). One
# repeat probe rides out NFS lag.
if (( silent_seconds >= 2 * EMPTY_READ_GRACE )); then
echo "RTP not running and XML log never created after ${silent_seconds}s: launch failed"
kill_qemu
exit 1
fi
echo "probe reports FINISHED after ${silent_seconds}s of silence and XML log not created yet - waiting for launch"
else
echo "probe reports FINISHED after ${silent_seconds}s of silence, but XML log is incomplete - still waiting"
wait_budget=$ALIVE_WAIT_TIME
fi
else
wait_budget=$ALIVE_WAIT_TIME
fi
if (( silent_seconds >= wait_budget )); then
echo "Timeout: no output for ${silent_seconds}s (last SSH probe: ${last_probe_result})"
# Tear down this slot's wedged emulator now rather than leaving it
# burning host CPU (TCG) until the next test's health check finds
# it; the next run relaunches a fresh instance.
kill_qemu
exit 1
fi
continue
fi
# We received serial output
silent_seconds=0
echo "$line"
if echo "$line" | /usr/bin/grep -q "qtest_in_vxworks_complete"
then
read -t 1 line < "$GUEST_OUT"
echo "$line"
exitcode=$(echo "$line" | sed -nr 's/qtest_in_vxworks_complete: (-?[0-9]+)/\1/gp' | tr -d '\r')
exit $exitcode
fi
# Handle crashes
# VxWorks reports RTP crashes on the serial console as e.g.
# "RTP Signal 11" / "RTP Signal 6" (11=SIGSEGV, 6=SIGABRT, ...)
# in addition to the symbolic SIG* names and S_rtpLib_* errors, so
# match all of these forms.
if echo "$line" | /usr/bin/grep -qE "(SIGSEGV)|(SIGABRT)|([Rr][Tt][Pp][[:space:]]+[Ss]ignal[[:space:]]+[0-9]+)|(S_rtpLib_[A-Z_]+)"
then
# Drain the pipe for context, mark as crashed, and restart emulator next run.
while read -t 1 line; do
echo $line
done < "$GUEST_OUT"
echo "Test crashed"
kill_qemu
exit 1
fi
done < "$GUEST_OUT"