diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c index aa3df3b3ba..6627be2922 100644 --- a/xen/arch/arm/domain.c +++ b/xen/arch/arm/domain.c @@ -72,7 +72,11 @@ void idle_loop(void) /* Are we here for running vcpu context tasklets, or for idling? */ if ( unlikely(tasklet_work_to_do(cpu)) ) + { do_tasklet(); + /* Livepatch work is always kicked off via a tasklet. */ + check_for_livepatch_work(); + } /* * Test softirqs twice --- first to see if should even try scrubbing * and then, after it is done, whether softirqs became pending @@ -83,11 +87,6 @@ void idle_loop(void) do_idle(); do_softirq(); - /* - * We MUST be last (or before dsb, wfi). Otherwise after we get the - * softirq we would execute dsb,wfi (and sleep) and not patch. - */ - check_for_livepatch_work(); } } diff --git a/xen/arch/arm/traps.c b/xen/arch/arm/traps.c index 6f9bec22d3..30c4c1830b 100644 --- a/xen/arch/arm/traps.c +++ b/xen/arch/arm/traps.c @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -2239,11 +2238,6 @@ static void check_for_pcpu_work(void) { local_irq_enable(); do_softirq(); - /* - * Must be the last one - as the IPI will trigger us to come here - * and we want to patch the hypervisor with almost no stack. - */ - check_for_livepatch_work(); local_irq_disable(); } } diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index fe63c23676..caf2ecad7e 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -141,7 +141,11 @@ static void idle_loop(void) /* Are we here for running vcpu context tasklets, or for idling? */ if ( unlikely(tasklet_work_to_do(cpu)) ) + { do_tasklet(); + /* Livepatch work is always kicked off via a tasklet. */ + check_for_livepatch_work(); + } /* * Test softirqs twice --- first to see if should even try scrubbing * and then, after it is done, whether softirqs became pending @@ -151,11 +155,6 @@ static void idle_loop(void) !softirq_pending(cpu) ) pm_idle(); do_softirq(); - /* - * We MUST be last (or before pm_idle). Otherwise after we get the - * softirq we would execute pm_idle (and sleep) and not patch. - */ - check_for_livepatch_work(); } } diff --git a/xen/arch/x86/hvm/svm/svm.c b/xen/arch/x86/hvm/svm/svm.c index b7f67f9f03..32d8d847f2 100644 --- a/xen/arch/x86/hvm/svm/svm.c +++ b/xen/arch/x86/hvm/svm/svm.c @@ -1032,7 +1032,7 @@ static void noreturn svm_do_resume(struct vcpu *v) hvm_do_resume(v); - reset_stack_and_jump(svm_asm_do_resume); + reset_stack_and_jump_nolp(svm_asm_do_resume); } void svm_vmenter_helper(const struct cpu_user_regs *regs) diff --git a/xen/arch/x86/hvm/vmx/vmcs.c b/xen/arch/x86/hvm/vmx/vmcs.c index 65445afeb0..4c23645454 100644 --- a/xen/arch/x86/hvm/vmx/vmcs.c +++ b/xen/arch/x86/hvm/vmx/vmcs.c @@ -1890,7 +1890,7 @@ void vmx_do_resume(struct vcpu *v) if ( host_cr4 != read_cr4() ) __vmwrite(HOST_CR4, read_cr4()); - reset_stack_and_jump(vmx_asm_do_vmentry); + reset_stack_and_jump_nolp(vmx_asm_do_vmentry); } static inline unsigned long vmr(unsigned long field) diff --git a/xen/arch/x86/pv/domain.c b/xen/arch/x86/pv/domain.c index 0b37653b12..70fae43965 100644 --- a/xen/arch/x86/pv/domain.c +++ b/xen/arch/x86/pv/domain.c @@ -62,7 +62,7 @@ custom_runtime_param("pcid", parse_pcid); static void noreturn continue_nonidle_domain(struct vcpu *v) { check_wakeup_from_wait(); - reset_stack_and_jump(ret_from_intr); + reset_stack_and_jump_nolp(ret_from_intr); } static int setup_compat_l4(struct vcpu *v) diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c index 81e40ce3de..c87040c890 100644 --- a/xen/arch/x86/setup.c +++ b/xen/arch/x86/setup.c @@ -632,7 +632,7 @@ static void __init noreturn reinit_bsp_stack(void) stack_base[0] = stack; memguard_guard_stack(stack); - reset_stack_and_jump(init_done); + reset_stack_and_jump_nolp(init_done); } /* diff --git a/xen/common/livepatch.c b/xen/common/livepatch.c index 5e09dc990b..861a227dbd 100644 --- a/xen/common/livepatch.c +++ b/xen/common/livepatch.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -69,6 +70,7 @@ static struct livepatch_work livepatch_work; * Having an per-cpu lessens the load. */ static DEFINE_PER_CPU(bool_t, work_to_do); +static DEFINE_PER_CPU(struct tasklet, livepatch_tasklet); static int get_name(const struct xen_livepatch_name *name, char *n) { @@ -1582,17 +1584,16 @@ static int schedule_work(struct payload *data, uint32_t cmd, uint32_t timeout) smp_wmb(); livepatch_work.do_work = 1; - this_cpu(work_to_do) = 1; + tasklet_schedule_on_cpu(&this_cpu(livepatch_tasklet), smp_processor_id()); put_cpu_maps(); return 0; } -static void reschedule_fn(void *unused) +static void tasklet_fn(void *unused) { this_cpu(work_to_do) = 1; - raise_softirq(SCHEDULE_SOFTIRQ); } static int livepatch_spin(atomic_t *counter, s_time_t timeout, @@ -1652,7 +1653,7 @@ void check_for_livepatch_work(void) if ( atomic_inc_and_test(&livepatch_work.semaphore) ) { struct payload *p; - unsigned int cpus; + unsigned int cpus, i; bool action_done = false; p = livepatch_work.data; @@ -1682,7 +1683,9 @@ void check_for_livepatch_work(void) { dprintk(XENLOG_DEBUG, LIVEPATCH "%s: CPU%u - IPIing the other %u CPUs\n", p->name, cpu, cpus); - smp_call_function(reschedule_fn, NULL, 0); + for_each_online_cpu ( i ) + if ( i != cpu ) + tasklet_schedule_on_cpu(&per_cpu(livepatch_tasklet, i), i); } timeout = livepatch_work.timeout + NOW(); @@ -2116,8 +2119,34 @@ static void livepatch_printall(unsigned char key) spin_unlock(&payload_lock); } +static int cpu_callback( + struct notifier_block *nfb, unsigned long action, void *hcpu) +{ + unsigned int cpu = (unsigned long)hcpu; + + if ( action == CPU_UP_PREPARE ) + tasklet_init(&per_cpu(livepatch_tasklet, cpu), tasklet_fn, NULL); + + return NOTIFY_DONE; +} + +static struct notifier_block cpu_nfb = { + .notifier_call = cpu_callback +}; + static int __init livepatch_init(void) { + unsigned int cpu; + + for_each_online_cpu ( cpu ) + { + void *hcpu = (void *)(long)cpu; + + cpu_callback(&cpu_nfb, CPU_UP_PREPARE, hcpu); + } + + register_cpu_notifier(&cpu_nfb); + register_keyhandler('x', livepatch_printall, "print livepatch info", 1); arch_livepatch_init();