diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index 24a521138a..8a9fe8fa9a 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -2477,6 +2477,7 @@ static int __get_page_type(struct page_info *page, unsigned long type, int rc = 0, iommu_ret = 0; ASSERT(!(type & ~(PGT_type_mask | PGT_pae_xen_l2))); + ASSERT(!in_irq()); for ( ; ; ) { @@ -2509,20 +2510,21 @@ static int __get_page_type(struct page_info *page, unsigned long type, * may be unnecessary (e.g., page was GDT/LDT) but those * circumstances should be very rare. */ - cpumask_t mask; + cpumask_t *mask = this_cpu(scratch_cpumask); - cpumask_copy(&mask, d->domain_dirty_cpumask); + BUG_ON(in_irq()); + cpumask_copy(mask, d->domain_dirty_cpumask); /* Don't flush if the timestamp is old enough */ - tlbflush_filter(&mask, page->tlbflush_timestamp); + tlbflush_filter(mask, page->tlbflush_timestamp); - if ( unlikely(!cpumask_empty(&mask)) && + if ( unlikely(!cpumask_empty(mask)) && /* Shadow mode: track only writable pages. */ (!shadow_mode_enabled(page_get_owner(page)) || ((nx & PGT_type_mask) == PGT_writable_page)) ) { perfc_incr(need_flush_tlb_flush); - flush_tlb_mask(&mask); + flush_tlb_mask(mask); } /* We lose existing type and validity. */ @@ -3403,22 +3405,22 @@ long do_mmuext_op( case MMUEXT_TLB_FLUSH_MULTI: case MMUEXT_INVLPG_MULTI: { - cpumask_t pmask; + cpumask_t *mask = this_cpu(scratch_cpumask); if ( unlikely(d != pg_owner) ) rc = -EPERM; else if ( unlikely(vcpumask_to_pcpumask(d, guest_handle_to_param(op.arg2.vcpumask, const_void), - &pmask)) ) + mask)) ) rc = -EINVAL; if ( unlikely(rc) ) break; if ( op.cmd == MMUEXT_TLB_FLUSH_MULTI ) - flush_tlb_mask(&pmask); + flush_tlb_mask(mask); else if ( __addr_ok(op.arg1.linear_addr) ) - flush_tlb_one_mask(&pmask, op.arg1.linear_addr); + flush_tlb_one_mask(mask, op.arg1.linear_addr); break; } @@ -3456,14 +3458,14 @@ long do_mmuext_op( else if ( likely(cache_flush_permitted(d)) ) { unsigned int cpu; - cpumask_t mask; + cpumask_t *mask = this_cpu(scratch_cpumask); - cpumask_clear(&mask); + cpumask_clear(mask); for_each_online_cpu(cpu) - if ( !cpumask_intersects(&mask, + if ( !cpumask_intersects(mask, per_cpu(cpu_sibling_mask, cpu)) ) - __cpumask_set_cpu(cpu, &mask); - flush_mask(&mask, FLUSH_CACHE); + __cpumask_set_cpu(cpu, mask); + flush_mask(mask, FLUSH_CACHE); } else { @@ -4458,7 +4460,7 @@ static int __do_update_va_mapping( struct page_info *gl1pg; l1_pgentry_t *pl1e; unsigned long bmap_ptr, gl1mfn; - cpumask_t pmask; + cpumask_t *mask = NULL; int rc; perfc_incr(calls_to_update_va); @@ -4504,15 +4506,17 @@ static int __do_update_va_mapping( flush_tlb_local(); break; case UVMF_ALL: - flush_tlb_mask(d->domain_dirty_cpumask); + mask = d->domain_dirty_cpumask; break; default: + mask = this_cpu(scratch_cpumask); rc = vcpumask_to_pcpumask(d, const_guest_handle_from_ptr(bmap_ptr, void), - &pmask); - flush_tlb_mask(&pmask); + mask); break; } + if ( mask ) + flush_tlb_mask(mask); break; case UVMF_INVLPG: @@ -4522,15 +4526,17 @@ static int __do_update_va_mapping( paging_invlpg(v, va); break; case UVMF_ALL: - flush_tlb_one_mask(d->domain_dirty_cpumask, va); + mask = d->domain_dirty_cpumask; break; default: + mask = this_cpu(scratch_cpumask); rc = vcpumask_to_pcpumask(d, const_guest_handle_from_ptr(bmap_ptr, void), - &pmask); - flush_tlb_one_mask(&pmask, va); + mask); break; } + if ( mask ) + flush_tlb_one_mask(mask, va); break; } diff --git a/xen/arch/x86/msi.c b/xen/arch/x86/msi.c index 2c38adb1b1..6756a14f1e 100644 --- a/xen/arch/x86/msi.c +++ b/xen/arch/x86/msi.c @@ -40,7 +40,6 @@ static void __pci_disable_msix(struct msi_desc *); /* bitmap indicate which fixed map is free */ static DEFINE_SPINLOCK(msix_fixmap_lock); static DECLARE_BITMAP(msix_fixmap_pages, FIX_MSIX_MAX_PAGES); -static DEFINE_PER_CPU(cpumask_var_t, scratch_mask); static int msix_fixmap_alloc(void) { @@ -167,7 +166,7 @@ void msi_compose_msg(unsigned vector, const cpumask_t *cpu_mask, struct msi_msg if ( cpu_mask ) { - cpumask_t *mask = this_cpu(scratch_mask); + cpumask_t *mask = this_cpu(scratch_cpumask); if ( !cpumask_intersects(cpu_mask, &cpu_online_map) ) return; @@ -1458,43 +1457,12 @@ int pci_restore_msi_state(struct pci_dev *pdev) return 0; } -static int msi_cpu_callback( - struct notifier_block *nfb, unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long)hcpu; - - switch ( action ) - { - case CPU_UP_PREPARE: - if ( !alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) ) - return notifier_from_errno(ENOMEM); - break; - case CPU_UP_CANCELED: - case CPU_DEAD: - free_cpumask_var(per_cpu(scratch_mask, cpu)); - break; - default: - break; - } - - return NOTIFY_DONE; -} - -static struct notifier_block msi_cpu_nfb = { - .notifier_call = msi_cpu_callback -}; - void __init early_msi_init(void) { if ( use_msi < 0 ) use_msi = !(acpi_gbl_FADT.boot_flags & ACPI_FADT_NO_MSI); if ( !use_msi ) return; - - register_cpu_notifier(&msi_cpu_nfb); - if ( msi_cpu_callback(&msi_cpu_nfb, CPU_UP_PREPARE, NULL) & - NOTIFY_STOP_MASK ) - BUG(); } static void dump_msi(unsigned char key) diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c index 3a9dd3ea9c..ba651fb038 100644 --- a/xen/arch/x86/smpboot.c +++ b/xen/arch/x86/smpboot.c @@ -56,6 +56,8 @@ DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_mask); /* representing HT and core siblings of each logical CPU */ DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_mask); +DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, scratch_cpumask); + cpumask_t cpu_online_map __read_mostly; EXPORT_SYMBOL(cpu_online_map); @@ -646,6 +648,7 @@ static void cpu_smpboot_free(unsigned int cpu) free_cpumask_var(per_cpu(cpu_sibling_mask, cpu)); free_cpumask_var(per_cpu(cpu_core_mask, cpu)); + free_cpumask_var(per_cpu(scratch_cpumask, cpu)); if ( per_cpu(stubs.addr, cpu) ) { @@ -734,7 +737,8 @@ static int cpu_smpboot_alloc(unsigned int cpu) goto oom; if ( zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) && - zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) ) + zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) && + alloc_cpumask_var(&per_cpu(scratch_cpumask, cpu)) ) return 0; oom: @@ -791,7 +795,8 @@ void __init smp_prepare_cpus(unsigned int max_cpus) panic("No memory for socket CPU siblings map"); if ( !zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, 0)) || - !zalloc_cpumask_var(&per_cpu(cpu_core_mask, 0)) ) + !zalloc_cpumask_var(&per_cpu(cpu_core_mask, 0)) || + !alloc_cpumask_var(&per_cpu(scratch_cpumask, 0)) ) panic("No memory for boot CPU sibling/core maps"); set_cpu_sibling_map(0); diff --git a/xen/include/asm-x86/smp.h b/xen/include/asm-x86/smp.h index e3782bbef1..95d6cd29fe 100644 --- a/xen/include/asm-x86/smp.h +++ b/xen/include/asm-x86/smp.h @@ -25,6 +25,7 @@ */ DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_mask); DECLARE_PER_CPU(cpumask_var_t, cpu_core_mask); +DECLARE_PER_CPU(cpumask_var_t, scratch_cpumask); void smp_send_nmi_allbutself(void);