xen/arm: gic: Add GIC SGI boot/self tests in Xen

Boot self-tests (also referred to as boot-time tests or power-on
self-tests) are intended to check that Xen has configured the hardware
correctly before bringing up any domains.

Introduce tests to confirm that, using a dedicated SGI (GIC_SGI_TEST):
1. A cpu can send the SGI to itself
2. A cpu can send the SGI to another specific CPU (CPU0)
3. A cpu can send the SGI to all the other CPUs

Each CPU counts the test SGIs it takes. A sender samples those counters
before sending and then waits for the count of every CPU it targeted to
change, which is how it tells that the SGI was really delivered. The
counters are never reset, so comparing against a sample rather than an
absolute value keeps concurrent senders from disturbing each other. The
snapshot is allocated rather than kept on the stack, as NR_CPUS can be
large enough for such an array to overflow it.

A test reports a failure by panic(), so Xen never continues on a
platform where SGI delivery is broken. When the tests pass, Xen carries
on booting normally.

Also introduce a config CONFIG_BOOT_SELFTEST which enables these
tests. It depends on DEBUG and is off unless explicitly enabled.

Also introduce a boolean command line parameter "gic-test", so that a
build with CONFIG_BOOT_SELFTEST enabled can be shipped but the tests
selected at boot. It is documented in
docs/misc/xen-command-line.pandoc.

In order to keep all the boot self-tests together in the binary, a
separate section "initcallboottest" is introduced. Tests are registered
using __initcallboottest() and run once on every CPU by
do_init_boottests(), bracketed by begin/end messages. They run before
any domain is created on the primary core, and before the idle loop is
entered on a secondary core.

Signed-off-by: Ayan Kumar Halder <ayan.kumar.halder@amd.com>
Signed-off-by: Michal Orzel <michal.orzel@amd.com>
Reviewed-by: Julien Grall <julien@xen.org>

automation/gitlab-ci/build.yaml

@@ -420,6 +420,14 @@ alpine-3.24-arm64-gcc-debug:
       CONFIG_UBSAN=y
       CONFIG_UBSAN_FATAL=y
 
+alpine-3.24-arm64-gcc-debug-boot-selftest:
+  extends: .gcc-arm64-build-debug
+  <<: *build-test
+  variables:
+    CONTAINER: alpine:3.24-arm64v8
+    EXTRA_XEN_CONFIG: |
+      CONFIG_BOOT_SELFTEST=y
+
 alpine-3.24-arm64-gcc-randconfig:
   extends: .gcc-arm64-build
   variables:

automation/gitlab-ci/test.yaml

@@ -605,6 +605,14 @@ qemu-smoke-dom0less-arm64-gcc-debug-gicv3:
     - *arm64-test-needs
     - alpine-3.24-arm64-gcc-debug
 
+qemu-smoke-boot-selftest-arm64-gcc-debug:
+  extends: .qemu-arm64
+  script:
+    - ./automation/scripts/qemu-boot-selftest-arm64.sh 2>&1 | tee ${LOGFILE}
+  needs:
+    - *arm64-test-needs
+    - alpine-3.24-arm64-gcc-debug-boot-selftest
+
 qemu-smoke-dom0less-arm64-gcc-debug-staticmem:
   extends: .qemu-arm64
   script:

automation/scripts/qemu-boot-selftest-arm64.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -ex -o pipefail
+
+# Boot Xen under QEMU with gic-test in xen,xen-bootargs and check that every
+# self-test reported OK and that Xen carried on booting.
+
+XEN=binaries/xen
+# qemu-system-aarch64 comes from the debian:13-arm64v8 test container, the
+# same way the other qemu-smoke-*-arm64 scripts get it.
+QEMU=qemu-system-aarch64
+DTB_RAW=binaries/virt.dtb
+DTB=binaries/virt-bootselftest.dtb
+LOG=smoke.serial
+
+NR_CPUS=4
+
+test -f ${XEN}
+
+${QEMU} \
+    -machine virt,virtualization=true,gic-version=3,dumpdtb=${DTB_RAW} \
+    -cpu cortex-a57 -m 1024 -smp ${NR_CPUS} -display none -net none
+
+cp ${DTB_RAW} ${DTB}
+fdtput -t s ${DTB} /chosen xen,xen-bootargs \
+    "gic-test console=dtuart sync_console"
+
+rm -f ${LOG}
+timeout 60 ${QEMU} \
+    -machine virt,virtualization=true,gic-version=3 \
+    -cpu cortex-a57 -m 1024 -smp ${NR_CPUS} \
+    -serial file:${LOG} \
+    -monitor none -display none -no-reboot -net none \
+    -dtb ${DTB} \
+    -kernel ${XEN} || true
+
+fail=0
+
+check() {
+    local what=$1
+    local expected=$2
+    local got
+
+    got=$(grep -c -- "${what}" ${LOG} || true)
+    if [ "${got}" -ne "${expected}" ]; then
+        echo "FAIL: '${what}': expected ${expected}, got ${got}"
+        fail=1
+        return
+    fi
+
+    echo "OK: '${what}' x${expected}"
+}
+
+# Every CPU sends an SGI to itself...
+check "GIC selftest: CPU[0-9]*: SGI to self: OK" ${NR_CPUS}
+# ...every secondary CPU sends one to CPU0...
+check "GIC selftest: CPU[0-9]*: SGI to CPU0: OK" $((NR_CPUS - 1))
+# ...and whichever CPU runs last sends one to all the others.
+check "GIC selftest: CPU[0-9]*: SGI to all but self: OK" 1
+
+check "boot self-tests done" ${NR_CPUS}
+check "GIC selftest: .*did not receive" 0
+
+# A passing self-test must leave Xen booting normally.
+check "LOADING DOMAIN 0\|Xen dom0less mode detected" 1
+
+if [ ${fail} -ne 0 ]; then
+    echo "FAILED"
+    exit 1
+fi
+
+echo "PASSED"

docs/misc/xen-command-line.pandoc

@@ -1282,6 +1282,16 @@ available on Intel Panther Lake and Diamond Rapids CPUs, and AMD Zen6 CPUs.
 FRED is fully supported on AMD hardware.  On Intel hardware it is still tech
 preview, and in particular not security supported.
 
+### gic-test (arm)
+> `= <boolean>`
+
+> Default: `false`
+
+Only available when `CONFIG_BOOT_SELFTEST` is enabled.
+
+Run the GIC SGI boot self-tests while each CPU is brought up.  Xen panics if
+an SGI is not delivered; otherwise it carries on booting normally.
+
 ### gnttab
 > `= List of [ max-ver:<integer>, transitive=<bool>, transfer=<bool> ]`
 

xen/arch/arm/Kconfig

@@ -499,6 +499,19 @@ config ARM64_HARDEN_BRANCH_PREDICTOR
 config ARM32_HARDEN_BRANCH_PREDICTOR
     def_bool y if ARM_32 && HARDEN_BRANCH_PREDICTOR
 
+config BOOT_SELFTEST
+	bool "Enable boot self-tests"
+	depends on DEBUG
+	help
+	  This option enables boot self-tests. They are intended to check that
+	  Xen has configured the hardware correctly before bringing up any
+	  domains. A failure is reported by panic(); when the tests pass, Xen
+	  boots normally.
+
+	  Selected at boot with the "gic-test" command line option.
+
+	  If unsure, say N.
+
 source "arch/arm/platforms/Kconfig"
 
 source "common/Kconfig"

xen/arch/arm/Makefile

@@ -24,6 +24,7 @@ obj-y += domctl.o
 obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
 obj-y += efi/
 obj-y += gic.o
+obj-$(CONFIG_BOOT_SELFTEST) += gic-test.o
 obj-$(CONFIG_GICV2) += gic-v2.o
 obj-$(CONFIG_GICV3) += gic-v3.o
 obj-$(CONFIG_HAS_ITS) += gic-v3-its.o

xen/arch/arm/gic-test.c

@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#include <xen/atomic.h>
+#include <xen/cpumask.h>
+#include <xen/init.h>
+#include <xen/lib.h>
+#include <xen/param.h>
+#include <xen/percpu.h>
+#include <xen/smp.h>
+#include <xen/time.h>
+#include <xen/xmalloc.h>
+#include <asm/gic.h>
+#include <asm/processor.h>
+#include <asm/setup.h>
+
+static bool __initdata opt_gic_test;
+boolean_param("gic-test", opt_gic_test);
+
+static DEFINE_PER_CPU(unsigned int, sgi_test_count);
+
+void gic_sgi_test_interrupt(void)
+{
+    ACCESS_ONCE(this_cpu(sgi_test_count))++;
+}
+
+static unsigned int __init sgi_count(unsigned int cpu)
+{
+    return ACCESS_ONCE(per_cpu(sgi_test_count, cpu));
+}
+
+static void __init snapshot_sgi(unsigned int *before)
+{
+    unsigned int cpu;
+
+    for_each_online_cpu ( cpu )
+        before[cpu] = sgi_count(cpu);
+}
+
+/*
+ * Wait for every CPU in @mask to take one more GIC_SGI_TEST than the count
+ * recorded in @before.
+ */
+static void __init expect_sgi(const cpumask_t *mask,
+                              const unsigned int *before, const char *what)
+{
+    s_time_t deadline = NOW() + MILLISECS(100);
+    unsigned int cpu;
+
+    for_each_cpu ( cpu, mask )
+    {
+        while ( sgi_count(cpu) == before[cpu] )
+        {
+            if ( NOW() > deadline )
+                panic("GIC selftest: %s: CPU%u did not receive GIC_SGI_TEST\n",
+                      what, cpu);
+            cpu_relax();
+        }
+    }
+
+    printk("GIC selftest: CPU%u: %s: OK\n", smp_processor_id(), what);
+}
+
+/*
+ * "All but self" is only meaningful once every CPU can take an SGI, so it is
+ * run by whichever CPU observes that it is the last one to get here.
+ */
+static int __init gic_sgi_selftest(void)
+{
+    static atomic_t __initdata seen = ATOMIC_INIT(0);
+    unsigned int cpu = smp_processor_id();
+    unsigned int *before;
+
+    if ( !opt_gic_test )
+        return 0;
+
+    before = xzalloc_array(unsigned int, nr_cpu_ids);
+    if ( !before )
+        panic("GIC selftest: CPU%u: cannot allocate %u counters\n",
+              cpu, nr_cpu_ids);
+
+    snapshot_sgi(before);
+    send_SGI_self(GIC_SGI_TEST);
+    expect_sgi(cpumask_of(cpu), before, "SGI to self");
+
+    if ( cpu != 0 )
+    {
+        snapshot_sgi(before);
+        send_SGI_one(0, GIC_SGI_TEST);
+        expect_sgi(cpumask_of(0), before, "SGI to CPU0");
+    }
+
+    if ( atomic_add_return(1, &seen) == num_online_cpus() )
+    {
+        cpumask_t target;
+
+        cpumask_andnot(&target, &cpu_online_map, cpumask_of(cpu));
+
+        if ( !cpumask_empty(&target) )
+        {
+            snapshot_sgi(before);
+            send_SGI_allbutself(GIC_SGI_TEST);
+            expect_sgi(&target, before, "SGI to all but self");
+        }
+    }
+
+    xfree(before);
+
+    return 0;
+}
+__initcallboottest(gic_sgi_selftest);

xen/arch/arm/gic.c

@@ -330,6 +330,11 @@ static void do_static_sgi(struct cpu_user_regs *regs, enum gic_sgi sgi)
     case GIC_SGI_CALL_FUNCTION:
         smp_call_function_interrupt();
         break;
+#ifdef CONFIG_BOOT_SELFTEST
+    case GIC_SGI_TEST:
+        gic_sgi_test_interrupt();
+        break;
+#endif
     default:
         panic("Unhandled SGI %d on CPU%d\n", sgi, smp_processor_id());
         break;

xen/arch/arm/include/asm/gic.h

@@ -306,6 +306,9 @@ enum gic_sgi {
     GIC_SGI_EVENT_CHECK,
     GIC_SGI_DUMP_STATE,
     GIC_SGI_CALL_FUNCTION,
+#ifdef CONFIG_BOOT_SELFTEST
+    GIC_SGI_TEST,
+#endif
     GIC_SGI_STATIC_MAX,
 };
 
@@ -321,6 +324,11 @@ extern void send_SGI_one(unsigned int cpu, enum gic_sgi sgi);
 extern void send_SGI_self(enum gic_sgi sgi);
 extern void send_SGI_allbutself(enum gic_sgi sgi);
 
+#ifdef CONFIG_BOOT_SELFTEST
+/* Record a GIC_SGI_TEST delivered to this CPU. */
+void gic_sgi_test_interrupt(void);
+#endif
+
 /* print useful debug info */
 extern void gic_dump_info(struct vcpu *v);
 extern void gic_dump_vgic_info(struct vcpu *v);

xen/arch/arm/include/asm/setup.h

@@ -48,6 +48,15 @@ void setup_mm(void);
 extern uint32_t hyp_traps_vector[];
 void init_traps(void);
 
+#ifdef CONFIG_BOOT_SELFTEST
+#define __initcallboottest(fn) \
+    static const initcall_t __initcall_##fn __init_call("boottest") = (fn)
+
+void do_init_boottests(void);
+#else
+static inline void do_init_boottests(void) {}
+#endif
+
 int handle_device(struct domain *d, struct dt_device_node *dev, p2m_type_t p2mt,
                   struct rangeset *iomem_ranges, struct rangeset *irq_ranges);
 

xen/arch/arm/setup.c

@@ -81,6 +81,24 @@ static void __init init_idle_domain(void)
     /* TODO: setup_idle_pagetable(); */
 }
 
+#ifdef CONFIG_BOOT_SELFTEST
+extern const initcall_t __initcall_boot_test_start[],
+    __initcall_boot_test_end[];
+
+void do_init_boottests(void)
+{
+    const initcall_t *call;
+
+    printk("CPU%u: boot self-tests start\n", smp_processor_id());
+
+    for ( call = __initcall_boot_test_start; call < __initcall_boot_test_end;
+          call++ )
+        (*call)();
+
+    printk("CPU%u: boot self-tests done\n", smp_processor_id());
+}
+#endif /* CONFIG_BOOT_SELFTEST */
+
 static const char * __initdata processor_implementers[] = {
     ['A'] = "ARM Limited",
     ['B'] = "Broadcom Corporation",
@@ -471,6 +489,8 @@ void asmlinkage __init noreturn start_xen(unsigned long fdt_paddr)
     enable_errata_workarounds();
     enable_cpu_features();
 
+    do_init_boottests();
+
     /* Create initial domain 0. */
     if ( !is_dom0less_mode() )
         create_dom0();

xen/arch/arm/smpboot.c

@@ -30,6 +30,7 @@
 #include <asm/psci.h>
 #include <asm/acpi.h>
 #include <asm/tee/tee.h>
+#include <asm/setup.h>
 
 /* Override macros from asm/page.h to make them work with mfn_t */
 #undef virt_to_mfn
@@ -413,6 +414,8 @@ void asmlinkage noreturn start_secondary(void)
 
     printk(XENLOG_DEBUG "CPU %u booted.\n", smp_processor_id());
 
+    do_init_boottests();
+
     startup_cpu_idle_loop();
 }
 

xen/arch/arm/xen.lds.S

@@ -137,6 +137,10 @@ SECTIONS
 
        SETUP_DATA
 
+       __initcall_boot_test_start = .;
+       *(.initcallboottest.init)
+       __initcall_boot_test_end = .;
+
        . = ALIGN(4);
        __alt_instructions = .;
        *(.altinstructions)