diff --git a/xen/arch/riscv/Makefile b/xen/arch/riscv/Makefile index 511ced09ec..665ec26933 100644 --- a/xen/arch/riscv/Makefile +++ b/xen/arch/riscv/Makefile @@ -1,6 +1,7 @@ obj-y += aia.o obj-y += aplic.o obj-y += cpufeature.o +obj-y += device.init.o obj-y += domain.o obj-y += domain-build.init.o obj-$(CONFIG_DOM0LESS_BOOT) += dom0less-build.init.o diff --git a/xen/arch/riscv/aplic.c b/xen/arch/riscv/aplic.c index c2d7183e18..17c96177a3 100644 --- a/xen/arch/riscv/aplic.c +++ b/xen/arch/riscv/aplic.c @@ -325,9 +325,40 @@ static const hw_irq_controller aplic_xen_irq_type = { .set_affinity = aplic_set_irq_affinity, }; +static unsigned int cf_check aplic_guest_irq_startup(struct irq_desc *desc) +{ + BUG_ON("unimplemented"); +} + +/* + * Shared by ->shutdown(), ->enable(), ->disable() and ->end(), which have + * no state. + */ +static void cf_check aplic_guest_irq_stub(struct irq_desc *desc) +{ + BUG_ON("unimplemented"); +} + +static void cf_check aplic_guest_set_irq_affinity(struct irq_desc *desc, + const cpumask_t *mask) +{ + BUG_ON("unimplemented"); +} + +static const hw_irq_controller aplic_guest_irq_type = { + .typename = "aplic-guest", + .startup = aplic_guest_irq_startup, + .shutdown = aplic_guest_irq_stub, + .enable = aplic_guest_irq_stub, + .disable = aplic_guest_irq_stub, + .end = aplic_guest_irq_stub, + .set_affinity = aplic_guest_set_irq_affinity, +}; + static const struct intc_hw_operations aplic_ops = { .info = &aplic_info, .host_irq_type = &aplic_xen_irq_type, + .guest_irq_type = &aplic_guest_irq_type, .handle_interrupt = aplic_handle_interrupt, .set_irq_type = aplic_set_irq_type, }; diff --git a/xen/arch/riscv/device.c b/xen/arch/riscv/device.c new file mode 100644 index 0000000000..fc41c075c7 --- /dev/null +++ b/xen/arch/riscv/device.c @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include +#include +#include + +#include + +int __overlay_init map_irq_to_domain(struct domain *d, unsigned int irq, + bool need_mapping, const char *devname) +{ + int res; + + res = irq_permit_access(d, irq); + if ( res ) + { + printk(XENLOG_ERR "Unable to permit %pd access to IRQ %u\n", d, irq); + return res; + } + + if ( need_mapping ) + { + /* + * -EEXIST merely means that the IRQ has already been reserved, which + * legitimately happens when the IRQ is shared between devices. Any + * other failure has to be fatal: the IRQ would otherwise be routed to + * the domain without domain_vintc_deinit() ever releasing it again. + */ + res = vintc_reserve_virq(d, irq); + if ( res && (res != -EEXIST) ) + { + printk(XENLOG_ERR "Unable to reserve vIRQ %u for %pd\n", irq, d); + return res; + } + + res = route_irq_to_guest(d, irq, irq, devname); + if ( res < 0 ) + { + printk(XENLOG_ERR "Unable to map IRQ%u to %pd\n", irq, d); + return res; + } + } + + dt_dprintk(" - IRQ: %u\n", irq); + + return 0; +} + +int __overlay_init map_device_irqs_to_domain(struct domain *d, + struct dt_device_node *dev, + bool need_mapping, + struct rangeset *irq_ranges) +{ + unsigned int i, nirq = dt_number_of_irq(dev); + + if ( irq_ranges ) + return -EOPNOTSUPP; + + /* Give permission and map IRQs */ + for ( i = 0; i < nirq; i++ ) + { + int res, irq; + struct dt_raw_irq rirq; + + res = dt_device_get_raw_irq(dev, i, &rirq); + if ( res ) + { + printk(XENLOG_ERR "Unable to retrieve irq %u for %s\n", + i, dt_node_full_name(dev)); + return res; + } + + /* + * Don't map IRQs that have no physical meaning + * ie: IRQs whose controller is not APLIC/IMSIC/PLIC. + */ + if ( rirq.controller != dt_interrupt_controller ) + { + dt_dprintk("irq %u not connected to primary controller. Connected to %s\n", + i, dt_node_full_name(rirq.controller)); + continue; + } + + irq = platform_get_irq(dev, i); + if ( irq < 0 ) + { + printk("Unable to get irq %u for %s\n", i, dt_node_full_name(dev)); + return irq; + } + + res = map_irq_to_domain(d, irq, need_mapping, dt_node_name(dev)); + if ( res ) + return res; + } + + return 0; +} diff --git a/xen/arch/riscv/include/asm/intc.h b/xen/arch/riscv/include/asm/intc.h index 6fc0e620e9..1bfba7c615 100644 --- a/xen/arch/riscv/include/asm/intc.h +++ b/xen/arch/riscv/include/asm/intc.h @@ -15,6 +15,7 @@ enum intc_variant { }; struct cpu_user_regs; +struct domain; struct irq_desc; struct kernel_info; struct vcpu; @@ -34,6 +35,9 @@ struct intc_hw_operations { /* hw_irq_controller to enable/disable/eoi host irq */ const struct hw_interrupt_type *host_irq_type; + /* hw_irq_controller to enable/disable/eoi guest irq */ + const struct hw_interrupt_type *guest_irq_type; + /* Set IRQ type */ void (*set_irq_type)(struct irq_desc *desc, unsigned int type); /* Set IRQ priority */ @@ -63,6 +67,8 @@ struct vintc_ops { }; struct vintc { + unsigned int nr_virqs; + unsigned long *used_irqs; /* Callbacks invoked during domain construction only. */ const struct vintc_init_ops *init_ops; /* Runtime callbacks used for the lifetime of the guest. */ @@ -76,10 +82,13 @@ void register_intc_ops(const struct intc_hw_init_ops *init_ops); void intc_init(void); void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority); +int intc_route_irq_to_guest(struct irq_desc *desc, unsigned int priority); void intc_handle_external_irqs(struct cpu_user_regs *regs); int domain_vintc_init(struct domain *d); void domain_vintc_deinit(struct domain *d); +int vintc_reserve_virq(const struct domain *d, unsigned int virq); + #endif /* ASM__RISCV__INTERRUPT_CONTOLLER_H */ diff --git a/xen/arch/riscv/include/asm/irq.h b/xen/arch/riscv/include/asm/irq.h index 62648bdc42..57e814d90c 100644 --- a/xen/arch/riscv/include/asm/irq.h +++ b/xen/arch/riscv/include/asm/irq.h @@ -52,6 +52,11 @@ void init_IRQ(void); void do_IRQ(struct cpu_user_regs *regs, unsigned int irq); +int route_irq_to_guest(struct domain *d, unsigned int virq, + unsigned int irq, const char *devname); + +int release_guest_irq(const struct domain *d, unsigned int virq); + #endif /* ASM__RISCV__IRQ_H */ /* diff --git a/xen/arch/riscv/intc.c b/xen/arch/riscv/intc.c index f5c8af6dde..810d126e26 100644 --- a/xen/arch/riscv/intc.c +++ b/xen/arch/riscv/intc.c @@ -3,11 +3,15 @@ #include #include #include +#include +#include #include #include #include #include +#include #include +#include #include #include @@ -78,6 +82,22 @@ void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority) intc_set_irq_priority(desc, priority); } +int intc_route_irq_to_guest(struct irq_desc *desc, + unsigned int priority) +{ + ASSERT(spin_is_locked(&desc->lock)); + + ASSERT(intc_hw_ops->guest_irq_type); + + desc->handler = intc_hw_ops->guest_irq_type; + desc->status |= IRQ_GUEST; + + intc_set_irq_type(desc, desc->arch.type); + intc_set_irq_priority(desc, priority); + + return 0; +} + int __init make_intc_domU_node(struct kernel_info *kinfo) { const struct vintc *vintc = kinfo->bd.d->arch.vintc; @@ -101,6 +121,15 @@ int domain_vintc_init(struct domain *d) break; } + if ( !ret ) + { + d->arch.vintc->used_irqs = + xvzalloc_array(unsigned long, + BITS_TO_LONGS(d->arch.vintc->nr_virqs)); + if ( !d->arch.vintc->used_irqs ) + ret = -ENOMEM; + } + return ret; } @@ -108,6 +137,20 @@ void domain_vintc_deinit(struct domain *d) { const enum intc_variant variant = intc_hw_ops->info->hw_variant; + if ( !d->arch.vintc ) + return; + + if ( d->arch.vintc->used_irqs ) + { + unsigned int virq; + + for ( virq = 0; virq < d->arch.vintc->nr_virqs; virq++ ) + if ( test_bit(virq, d->arch.vintc->used_irqs) ) + release_guest_irq(d, virq); + + XVFREE(d->arch.vintc->used_irqs); + } + switch ( variant ) { case INTC_APLIC: @@ -118,3 +161,20 @@ void domain_vintc_deinit(struct domain *d) break; } } + +/* + * Mark @virq as used by @d so that domain_vintc_deinit() knows that it has to + * be released. + * + * Returns 0 on success, -EEXIST if @virq has already been reserved, which + * legitimately happens when an IRQ is shared between devices, and -ERANGE if + * @virq is outside the range of the interrupt sources the vINTC provides. + */ +int __overlay_init vintc_reserve_virq(const struct domain *d, + unsigned int virq) +{ + if ( virq >= d->arch.vintc->nr_virqs ) + return -ERANGE; + + return test_and_set_bit(virq, d->arch.vintc->used_irqs) ? -EEXIST : 0; +} diff --git a/xen/arch/riscv/irq.c b/xen/arch/riscv/irq.c index b5066fc3e9..9fc5a1bb70 100644 --- a/xen/arch/riscv/irq.c +++ b/xen/arch/riscv/irq.c @@ -12,11 +12,27 @@ #include #include #include +#include #include +#include #include #include +/* Describe an IRQ assigned to a guest */ +struct irq_guest +{ + struct domain *d; + unsigned int virq; + /* + * The action of a guest IRQ has the same lifetime as this structure, so + * embed it here to have both covered by a single allocation. Consequently + * it must not be freed on its own, which is why free_on_release is left + * false for it (see irq_release_action()). + */ + struct irqaction action; +}; + static irq_desc_t irq_desc[NR_IRQS]; struct irq_desc *irq_to_desc(unsigned int irq) @@ -198,6 +214,14 @@ void do_IRQ(struct cpu_user_regs *regs, unsigned int irq) if ( desc->handler->ack ) desc->handler->ack(desc); + if ( desc->status & IRQ_GUEST ) + /* + * With APLIC + IMSIC, guest interrupts bypass Xen and are delivered + * directly to the guest. Without IMSIC, interrupts would be trapped + * by Xen and would need injecting into the guest here. + */ + panic("unimplemented"); + if ( desc->status & IRQ_DISABLED ) goto out; @@ -227,3 +251,239 @@ void do_IRQ(struct cpu_user_regs *regs, unsigned int irq) spin_unlock(&desc->lock); irq_exit(); } + +static struct irq_guest *irq_get_guest_info(struct irq_desc *desc) +{ + ASSERT(spin_is_locked(&desc->lock)); + ASSERT(desc->status & IRQ_GUEST); + ASSERT(desc->action); + + return desc->action->dev_id; +} + +/* + * Detach the action registered with 'dev_id' from 'desc' and, if it was the + * last one, shut the interrupt down. + * + * To be called with desc->lock held, which is still held upon return. The + * detached action is returned (NULL if 'dev_id' had no action registered) and + * has to be handed to irq_release_action() once the lock has been dropped. + */ +static struct irqaction *irq_detach_action(struct irq_desc *desc, + const void *dev_id) +{ + struct irqaction *action, **action_ptr = &desc->action; + + ASSERT(spin_is_locked(&desc->lock)); + +#ifdef CONFIG_IRQ_HAS_MULTIPLE_ACTION + for ( ;; ) + { + action = *action_ptr; + if ( !action || (action->dev_id == dev_id) ) + break; + + action_ptr = &action->next; + } +#else + action = *action_ptr; +#endif + + if ( !action ) + { + printk(XENLOG_WARNING "Trying to free already-free IRQ %u\n", + desc->irq); + return NULL; + } + + /* Found it - remove it from the action list */ +#ifdef CONFIG_IRQ_HAS_MULTIPLE_ACTION + *action_ptr = action->next; +#else + *action_ptr = NULL; +#endif + + /* If this was the last action, shut down the IRQ */ + if ( !desc->action ) + { + desc->handler->shutdown(desc); + desc->status &= ~IRQ_GUEST; + } + + return action; +} + +/* + * Complete the release of an action detached by irq_detach_action(). + * + * To be called with desc->lock dropped: the lock cannot be held all the way + * through, as waiting for a handler still running on another CPU to complete + * requires do_IRQ() to be able to acquire the very same lock. + * + * Once this function has returned, the action (and hence any object embedding + * it) is no longer referenced by anyone and may be freed. + */ +static void irq_release_action(const struct irq_desc *desc, + struct irqaction *action) +{ + /* Wait to make sure it's not being used on another CPU. */ + while ( test_bit(_IRQ_INPROGRESS, &desc->status) ) + cpu_relax(); + + /* + * IRQ_INPROGRESS is cleared in do_IRQ() after re-acquiring desc->lock, + * and lock acquisition implies a full barrier, so the handler's accesses + * are ordered before the clearing becomes visible here. The barrier below + * adds the missing load-load ordering (the loop's exit branch already + * prevents the store in xvfree() from becoming visible early), so that + * having observed the bit cleared we also see whatever the handler did on + * that CPU. Only then is it safe to free the action. + */ + smp_rmb(); + + if ( action->free_on_release ) + xvfree(action); +} + +void release_irq(unsigned int irq, const void *dev_id) +{ + struct irq_desc *desc = irq_to_desc(irq); + struct irqaction *action; + unsigned long flags; + + spin_lock_irqsave(&desc->lock, flags); + action = irq_detach_action(desc, dev_id); + spin_unlock_irqrestore(&desc->lock, flags); + + if ( action ) + irq_release_action(desc, action); +} + +int release_guest_irq(const struct domain *d, unsigned int virq) +{ + struct irq_desc *desc = irq_to_desc(virq); + struct irqaction *action; + struct irq_guest *info; + unsigned long flags; + int ret = -EINVAL; + + spin_lock_irqsave(&desc->lock, flags); + + if ( !(desc->status & IRQ_GUEST) ) + goto unlock_err; + + info = irq_get_guest_info(desc); + if ( d != info->d ) + goto unlock_err; + + /* + * Detaching the action happens with desc->lock still held, so that a + * concurrent release_guest_irq() for the same IRQ will see IRQ_GUEST + * already cleared and bail out, rather than capturing the same 'info' and + * double-freeing it below. + */ + action = irq_detach_action(desc, info); + + spin_unlock_irqrestore(&desc->lock, flags); + + if ( action ) + irq_release_action(desc, action); + + xvfree(info); + + return 0; + + unlock_err: + spin_unlock_irqrestore(&desc->lock, flags); + return ret; +} + +/* Route an IRQ to a specific guest */ +int route_irq_to_guest(struct domain *d, unsigned int virq, + unsigned int irq, const char *devname) +{ + struct irq_guest *info; + struct irq_desc *desc = irq_to_desc(irq); + unsigned long flags; + int retval = 0; + + if ( d->is_dying ) + return -EINVAL; + + info = xvzalloc(struct irq_guest); + if ( !info ) + return -ENOMEM; + + info->d = d; + info->virq = virq; + + info->action.dev_id = info; + info->action.name = devname; + + spin_lock_irqsave(&desc->lock, flags); + + /* + * If the IRQ is already used by someone + * - If it's the same domain -> Xen doesn't need to update the IRQ desc. + * For safety check if we are not trying to assign the IRQ to a + * different vIRQ. + * - Otherwise -> For now, don't allow the IRQ to be shared between + * Xen and domains. + */ + if ( desc->action ) + { + if ( desc->status & IRQ_GUEST ) + { + struct domain *ad = irq_get_guest_info(desc)->d; + + if ( d != ad ) + { + printk(XENLOG_G_ERR "%pd: IRQ %u is already used by %pd\n", + d, irq, ad); + retval = -EBUSY; + } + else if ( irq_get_guest_info(desc)->virq != virq ) + { + printk(XENLOG_G_ERR + "%pd: IRQ %u is already assigned to vIRQ %u\n", + d, irq, irq_get_guest_info(desc)->virq); + retval = -EBUSY; + } + } + else + { + printk(XENLOG_G_ERR "%pd: IRQ %u is already used by Xen\n", + d, irq); + retval = -EBUSY; + } + goto out; + } + + retval = _setup_irq(desc, 0, &info->action); + if ( retval ) + goto out; + + retval = intc_route_irq_to_guest(desc, IRQ_NO_PRIORITY); + if ( retval ) + { + struct irqaction *action = irq_detach_action(desc, info); + + spin_unlock_irqrestore(&desc->lock, flags); + + if ( action ) + irq_release_action(desc, action); + + goto free_info; + } + + spin_unlock_irqrestore(&desc->lock, flags); + + return 0; + + out: + spin_unlock_irqrestore(&desc->lock, flags); + free_info: + xvfree(info); + + return retval; +} diff --git a/xen/arch/riscv/vaplic.c b/xen/arch/riscv/vaplic.c index a529a5b1dc..6fbdc07805 100644 --- a/xen/arch/riscv/vaplic.c +++ b/xen/arch/riscv/vaplic.c @@ -113,6 +113,16 @@ int domain_vaplic_init(struct domain *d) vaplic->regs.domaincfg = APLIC_DOMAINCFG_RO; + /* + * APLIC source 0 is reserved; sources are numbered + * 1..guest_aplic_num_sources and used directly as indices into used_irqs. + * Size the bitmap to guest_aplic_num_sources + 1 so the highest source has + * a valid slot (index 0 stays unused). Without the +1, + * vintc_reserve_virq() can't record the top source, so + * domain_vintc_deinit() never releases it. + */ + d->arch.vintc->nr_virqs = guest_aplic_num_sources + 1; + return 0; }