x86/IRQ: re-use legacy vector ranges on APs

The legacy vectors have been actively used on CPU 0 only. CPUs not
sharing vector space with CPU 0 can easily re-use them, slightly
increasing the relatively scarce resource of total vectors available in
the system. As a result the legacy vector range simply becomes a
sub-range of the dynamic one, with an extra check performed in
_assign_irq_vector() (we can't rely on the
"per_cpu(vector_irq, new_cpu)[vector] >= 0" check in the subsequent
loop, as we need to also exclude vectors of disabled legacy IRQs).

Suggested-by: Andrew Cooper <andrew.cooper3@citrix.com>
Signed-off-by: Jan Beulich <jbeulich@suse.com>
Acked-by: Andrew Cooper <andrew.cooper3@citrix.com>

xen/arch/x86/io_apic.c

@@ -2389,8 +2389,7 @@ int ioapic_guest_write(unsigned long physbase, unsigned int reg, u32 val)
     }
 
     if ( desc->arch.vector <= 0 || desc->arch.vector > LAST_DYNAMIC_VECTOR ||
-         (desc->arch.vector >= FIRST_LEGACY_VECTOR &&
-          desc->arch.vector <= LAST_LEGACY_VECTOR) )
+         desc->handler->enable == enable_8259A_irq )
     {
         int vector = desc->arch.vector;
 
@@ -2617,7 +2616,7 @@ void __init init_ioapic_mappings(void)
 
     if ( nr_irqs == 0 )
         nr_irqs = cpu_has_apic ?
-                  max(16U + num_present_cpus() * NR_DYNAMIC_VECTORS,
+                  max(0U + num_present_cpus() * NR_DYNAMIC_VECTORS,
                       8 * nr_irqs_gsi) :
                   nr_irqs_gsi;
     else if ( nr_irqs < 16 )

xen/arch/x86/irq.c

@@ -361,17 +361,25 @@ void destroy_irq(unsigned int irq)
 int irq_to_vector(int irq)
 {
     int vector = IRQ_VECTOR_UNASSIGNED;
+    const struct irq_desc *desc;
 
     BUG_ON(irq >= nr_irqs || irq < 0);
+    desc = irq_to_desc(irq);
 
     if (IO_APIC_IRQ(irq))
     {
-        vector = irq_to_desc(irq)->arch.vector;
-        if (vector >= FIRST_LEGACY_VECTOR && vector <= LAST_LEGACY_VECTOR)
+        vector = desc->arch.vector;
+        /*
+         * Both parts of the condition are needed here during early boot, as
+         * at that time IRQ0 in particular may still have the 8259A chip set,
+         * but has already got its special IRQ0_VECTOR.
+         */
+        if ( desc->handler->enable == enable_8259A_irq &&
+             vector >= FIRST_LEGACY_VECTOR && vector <= LAST_LEGACY_VECTOR )
             vector = 0;
     }
     else if (MSI_IRQ(irq))
-        vector = irq_to_desc(irq)->arch.vector;
+        vector = desc->arch.vector;
     else
         vector = LEGACY_VECTOR(irq);
 
@@ -568,6 +576,10 @@ next:
             && test_bit(vector, irq_used_vectors) )
             goto next;
 
+        if ( cpumask_test_cpu(0, vec_mask) &&
+             vector >= FIRST_LEGACY_VECTOR && vector <= LAST_LEGACY_VECTOR )
+            goto next;
+
         for_each_cpu(new_cpu, vec_mask)
             if (per_cpu(vector_irq, new_cpu)[vector] >= 0)
                 goto next;
@@ -713,6 +725,10 @@ void irq_move_cleanup_interrupt(struct cpu_user_regs *regs)
 {
     unsigned vector, me;
 
+    /* This interrupt should not nest inside others. */
+    BUILD_BUG_ON(APIC_PRIO_CLASS(IRQ_MOVE_CLEANUP_VECTOR) !=
+                 APIC_PRIO_CLASS(FIRST_DYNAMIC_VECTOR));
+
     ack_APIC_irq();
 
     me = smp_processor_id();
@@ -730,14 +746,15 @@ void irq_move_cleanup_interrupt(struct cpu_user_regs *regs)
         if ((int)irq < 0)
             continue;
 
-        if ( vector >= FIRST_LEGACY_VECTOR && vector <= LAST_LEGACY_VECTOR )
-            continue;
-
         desc = irq_to_desc(irq);
         if (!desc)
             continue;
 
         spin_lock(&desc->lock);
+
+        if (desc->handler->enable == enable_8259A_irq)
+            goto unlock;
+
         if (!desc->arch.move_cleanup_count)
             goto unlock;
 
@@ -1894,6 +1911,7 @@ void do_IRQ(struct cpu_user_regs *regs)
                 kind = "";
             if ( !(vector >= FIRST_LEGACY_VECTOR &&
                    vector <= LAST_LEGACY_VECTOR &&
+                   !smp_processor_id() &&
                    bogus_8259A_irq(vector - FIRST_LEGACY_VECTOR)) )
             {
                 printk("CPU%u: No irq handler for vector %02x (IRQ %d%s)\n",

xen/include/asm-x86/apicdef.h

@@ -119,6 +119,9 @@
 /* Only available in x2APIC mode */
 #define		APIC_SELF_IPI	0x3F0
 
+/* Applicable to vectors, TPR, and PPR. */
+#define		APIC_PRIO_CLASS(v)	((v) & 0xF0)
+
 #define APIC_BASE __fix_to_virt(FIX_APIC_BASE)
 
 /* It's only used in x2APIC mode of an x2APIC unit. */

xen/include/asm-x86/mach-default/irq_vectors.h

@@ -19,22 +19,27 @@
 #define IRQ0_VECTOR             0xf0
 
 /* Legacy PIC uses vectors 0x20-0x2f. */
-#define FIRST_LEGACY_VECTOR     0x20
+#define FIRST_LEGACY_VECTOR     FIRST_DYNAMIC_VECTOR
 #define LAST_LEGACY_VECTOR      (FIRST_LEGACY_VECTOR + 0xf)
 
 #define HYPERCALL_VECTOR	0x82
 #define LEGACY_SYSCALL_VECTOR   0x80
 
-/* Dynamically-allocated vectors available to any driver. */
-#define FIRST_DYNAMIC_VECTOR    (LAST_LEGACY_VECTOR + 1)
+/*
+ * Dynamically-allocated vectors available to any driver. Note that the
+ * legacy vector range is a sub-range of this one, re-used on CPUs not
+ * sharing vectors with CPU 0.
+ */
+#define FIRST_DYNAMIC_VECTOR    0x20
 #define LAST_DYNAMIC_VECTOR     0xef
 #define NR_DYNAMIC_VECTORS	(LAST_DYNAMIC_VECTOR - FIRST_DYNAMIC_VECTOR + 1)
 
-#define IRQ_MOVE_CLEANUP_VECTOR FIRST_DYNAMIC_VECTOR
+/* There's no IRQ2 at the PIC. */
+#define IRQ_MOVE_CLEANUP_VECTOR (FIRST_LEGACY_VECTOR + 2)
 
 #define NR_VECTORS 256
 
-#define FIRST_IRQ_VECTOR        FIRST_LEGACY_VECTOR
+#define FIRST_IRQ_VECTOR        FIRST_DYNAMIC_VECTOR
 #define LAST_IRQ_VECTOR         LAST_HIPRIORITY_VECTOR
 
 #endif /* _ASM_IRQ_VECTORS_H */