From eb2b48f2dcd1394b6449adc845d36bf2d1e41b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 19 Feb 2025 03:56:52 +0100 Subject: [PATCH] automation: add jobs running tests from tools/tests/* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are a bunch of tests in tools/tests/, let them run in CI. For each subdirectory expect "make run" will run the test, and observe its exit code. This way, adding new tests is easy, and they will be automatically picked up. For better visibility, log test output to junit xml format, and let gitlab ingest it. Set SUT_ADDR variable with name/address of the system under test, so a network can be used to extract the file. The actual address is set using DHCP. And for the test internal network, still add the 192.168.0.1 IP (but don't replace the DHCP-provided one). Signed-off-by: Marek Marczykowski-Górecki Reviewed-by: Stefano Stabellini (cherry picked from commit fc10d6ecdd64c42dc4e3318ac31cd544a8d84959) --- automation/gitlab-ci/test.yaml | 23 +++++++++++++++ automation/scripts/build | 1 + automation/scripts/qubes-x86-64.sh | 28 ++++++++++++++++++ automation/scripts/run-tools-tests | 47 ++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100755 automation/scripts/run-tools-tests diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml index c8eefc7e01..a2dcf27b1f 100644 --- a/automation/gitlab-ci/test.yaml +++ b/automation/gitlab-ci/test.yaml @@ -130,6 +130,7 @@ PCIDEV: "03:00.0" PCIDEV_INTR: "MSI-X" CONSOLE_OPTS: "console=com1 com1=115200,8n1" + SUT_ADDR: test-2.testnet artifacts: paths: - smoke.serial @@ -258,6 +259,28 @@ adl-pvshim-x86-64-gcc-debug: - *x86-64-test-needs - alpine-3.18-gcc-debug +adl-tools-tests-pv-x86-64-gcc-debug: + extends: .adl-x86-64 + script: + - ./automation/scripts/qubes-x86-64.sh tools-tests-pv 2>&1 | tee ${LOGFILE} + artifacts: + reports: + junit: tests-junit.xml + needs: + - *x86-64-test-needs + - alpine-3.18-gcc-debug + +adl-tools-tests-pvh-x86-64-gcc-debug: + extends: .adl-x86-64 + script: + - ./automation/scripts/qubes-x86-64.sh tools-tests-pvh 2>&1 | tee ${LOGFILE} + artifacts: + reports: + junit: tests-junit.xml + needs: + - *x86-64-test-needs + - alpine-3.18-gcc-debug + zen2-smoke-x86-64-gcc-debug: extends: .zen2-x86-64 script: diff --git a/automation/scripts/build b/automation/scripts/build index 952599cc25..522efe774e 100755 --- a/automation/scripts/build +++ b/automation/scripts/build @@ -109,5 +109,6 @@ else # even though dist/ contains everything, while some containers don't even # build Xen cp -r dist binaries/ + cp -r tools/tests binaries/ collect_xen_artefacts fi diff --git a/automation/scripts/qubes-x86-64.sh b/automation/scripts/qubes-x86-64.sh index 7eb3ce1bf7..7c80e0c233 100755 --- a/automation/scripts/qubes-x86-64.sh +++ b/automation/scripts/qubes-x86-64.sh @@ -10,6 +10,8 @@ set -ex # - pci-pv PV dom0, PV domU + PCI Passthrough # - pvshim PV dom0, PVSHIM domU # - s3 PV dom0, S3 suspend/resume +# - tools-tests-pv PV dom0, run tests from tools/tests/* +# - tools-tests-pvh PVH dom0, run tests from tools/tests/* test_variant=$1 ### defaults @@ -19,6 +21,7 @@ timeout=120 domU_type="pvh" domU_vif="'bridge=xenbr0'," domU_extra_config= +retrieve_xml= case "${test_variant}" in ### test: smoke test & smoke test PVH & smoke test HVM & smoke test PVSHIM @@ -126,6 +129,21 @@ done " ;; + ### tests: tools-tests-pv, tools-tests-pvh + "tools-tests-pv"|"tools-tests-pvh") + retrieve_xml=1 + passed="test passed" + domU_check="" + dom0_check=" +/tests/run-tools-tests /tests /tmp/tests-junit.xml && echo \"${passed}\" +nc -l -p 8080 < /tmp/tests-junit.xml >/dev/null & +" + if [ "${test_variant}" = "tools-tests-pvh" ]; then + extra_xen_opts="dom0=pvh" + fi + + ;; + *) echo "Unrecognised test_variant '${test_variant}'" >&2 exit 1 @@ -178,6 +196,8 @@ mkdir srv mkdir sys rm var/run cp -ar ../binaries/dist/install/* . +cp -ar ../binaries/tests . +cp -a ../automation/scripts/run-tools-tests tests/ echo "#!/bin/bash @@ -192,6 +212,10 @@ ifconfig xenbr0 192.168.0.1 " > etc/local.d/xen.start +if [ -n "$retrieve_xml" ]; then + echo "timeout 30s udhcpc -i xenbr0" >> etc/local.d/xen.start +fi + if [ -n "$domU_check" ]; then echo " # get domU console content into test log @@ -272,6 +296,10 @@ if [ $timeout -le 0 ]; then exit 1 fi +if [ -n "$retrieve_xml" ]; then + nc -w 10 "$SUT_ADDR" 8080 > tests-junit.xml \n' > "$xml_out" +printf '\n' >> "$xml_out" +printf ' \n' >> "$xml_out" +failed= +for dir in "$1"/*; do + [ -d "$dir" ] || continue + echo "Running test in $dir" + printf ' \n' "$dir" >> "$xml_out" + ret= + for f in "$dir"/*; do + [ -f "$f" ] || continue + [ -x "$f" ] || continue + "$f" 2>&1 | tee /tmp/out + ret=$? + if [ "$ret" -ne 0 ]; then + echo "FAILED: $ret" + failed+=" $dir" + printf ' \n' "$f" "$ret" >> "$xml_out" + # TODO: could use xml escaping... but current tests seems to + # produce sane output + cat /tmp/out >> "$xml_out" + printf ' \n' >> "$xml_out" + else + echo "PASSED" + fi + done + if [ -z "$ret" ]; then + printf ' \n' "$dir" >> "$xml_out" + fi + printf ' \n' >> "$xml_out" +done +printf ' \n' >> "$xml_out" +printf '\n' >> "$xml_out" + +if [ -n "$failed" ]; then + exit 1 +fi