diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile index 7792bff597..e4ad1ce851 100644 --- a/xen/arch/arm/Makefile +++ b/xen/arch/arm/Makefile @@ -39,7 +39,6 @@ obj-$(CONFIG_MEM_ACCESS) += mem_access.o obj-y += mm.o obj-y += monitor.o obj-y += p2m.o -obj-y += percpu.o obj-y += platform.o obj-y += platform_hypercall.o obj-y += physdev.o diff --git a/xen/arch/arm/include/asm/smp.h b/xen/arch/arm/include/asm/smp.h index e99a3a3f53..8f765ed12a 100644 --- a/xen/arch/arm/include/asm/smp.h +++ b/xen/arch/arm/include/asm/smp.h @@ -2,6 +2,7 @@ #define __ASM_SMP_H #ifndef __ASSEMBLY__ +#include #include #include #endif @@ -12,12 +13,6 @@ extern unsigned long smp_up_cpu; DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_mask); DECLARE_PER_CPU(cpumask_var_t, cpu_core_mask); -/* - * Do we, for platform reasons, need to actually keep CPUs online when we - * would otherwise prefer them to be off? - */ -#define park_offline_cpus false - extern void noreturn stop_cpu(void); extern int arch_smp_init(void); diff --git a/xen/arch/arm/percpu.c b/xen/arch/arm/percpu.c deleted file mode 100644 index 87fe960330..0000000000 --- a/xen/arch/arm/percpu.c +++ /dev/null @@ -1,85 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include -#include -#include -#include -#include - -unsigned long __per_cpu_offset[NR_CPUS]; -#define INVALID_PERCPU_AREA (-(long)__per_cpu_start) -#define PERCPU_ORDER (get_order_from_bytes(__per_cpu_data_end-__per_cpu_start)) - -void __init percpu_init_areas(void) -{ - unsigned int cpu; - for ( cpu = 1; cpu < NR_CPUS; cpu++ ) - __per_cpu_offset[cpu] = INVALID_PERCPU_AREA; -} - -static int init_percpu_area(unsigned int cpu) -{ - char *p; - if ( __per_cpu_offset[cpu] != INVALID_PERCPU_AREA ) - return -EBUSY; - if ( (p = alloc_xenheap_pages(PERCPU_ORDER, 0)) == NULL ) - return -ENOMEM; - memset(p, 0, __per_cpu_data_end - __per_cpu_start); - __per_cpu_offset[cpu] = p - __per_cpu_start; - return 0; -} - -struct free_info { - unsigned int cpu; - struct rcu_head rcu; -}; -static DEFINE_PER_CPU(struct free_info, free_info); - -static void _free_percpu_area(struct rcu_head *head) -{ - struct free_info *info = container_of(head, struct free_info, rcu); - unsigned int cpu = info->cpu; - char *p = __per_cpu_start + __per_cpu_offset[cpu]; - free_xenheap_pages(p, PERCPU_ORDER); - __per_cpu_offset[cpu] = INVALID_PERCPU_AREA; -} - -static void free_percpu_area(unsigned int cpu) -{ - struct free_info *info = &per_cpu(free_info, cpu); - info->cpu = cpu; - call_rcu(&info->rcu, _free_percpu_area); -} - -static int cpu_percpu_callback( - struct notifier_block *nfb, unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long)hcpu; - int rc = 0; - - switch ( action ) - { - case CPU_UP_PREPARE: - rc = init_percpu_area(cpu); - break; - case CPU_UP_CANCELED: - case CPU_DEAD: - free_percpu_area(cpu); - break; - default: - break; - } - - return notifier_from_errno(rc); -} - -static struct notifier_block cpu_percpu_nfb = { - .notifier_call = cpu_percpu_callback, - .priority = 100 /* highest priority */ -}; - -static int __init percpu_presmp_init(void) -{ - register_cpu_notifier(&cpu_percpu_nfb); - return 0; -} -presmp_initcall(percpu_presmp_init); diff --git a/xen/arch/ppc/include/asm/smp.h b/xen/arch/ppc/include/asm/smp.h index 7b1517ce18..2b872218be 100644 --- a/xen/arch/ppc/include/asm/smp.h +++ b/xen/arch/ppc/include/asm/smp.h @@ -7,10 +7,4 @@ DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_mask); DECLARE_PER_CPU(cpumask_var_t, cpu_core_mask); -/* - * Do we, for platform reasons, need to actually keep CPUs online when we - * would otherwise prefer them to be off? - */ -#define park_offline_cpus false - #endif diff --git a/xen/arch/ppc/stubs.c b/xen/arch/ppc/stubs.c index bdb5f8c66d..fff82f5cf3 100644 --- a/xen/arch/ppc/stubs.c +++ b/xen/arch/ppc/stubs.c @@ -141,7 +141,6 @@ void smp_send_state_dump(unsigned int cpu) /* domain.c */ DEFINE_PER_CPU(struct vcpu *, curr_vcpu); -unsigned long __per_cpu_offset[NR_CPUS]; void context_switch(struct vcpu *prev, struct vcpu *next) { diff --git a/xen/arch/riscv/include/asm/smp.h b/xen/arch/riscv/include/asm/smp.h index a824be8e78..092b101ff4 100644 --- a/xen/arch/riscv/include/asm/smp.h +++ b/xen/arch/riscv/include/asm/smp.h @@ -10,12 +10,6 @@ DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_mask); DECLARE_PER_CPU(cpumask_var_t, cpu_core_mask); -/* - * Do we, for platform reasons, need to actually keep CPUs online when we - * would otherwise prefer them to be off? - */ -#define park_offline_cpus false - /* * Mapping between Xen logical cpu index and hartid. */ diff --git a/xen/arch/riscv/stubs.c b/xen/arch/riscv/stubs.c index 2aa245f272..5951b0ce91 100644 --- a/xen/arch/riscv/stubs.c +++ b/xen/arch/riscv/stubs.c @@ -133,7 +133,6 @@ void smp_send_state_dump(unsigned int cpu) /* domain.c */ DEFINE_PER_CPU(struct vcpu *, curr_vcpu); -unsigned long __per_cpu_offset[NR_CPUS]; void context_switch(struct vcpu *prev, struct vcpu *next) { diff --git a/xen/arch/x86/Makefile b/xen/arch/x86/Makefile index 1fa12ed4aa..b35fd5196c 100644 --- a/xen/arch/x86/Makefile +++ b/xen/arch/x86/Makefile @@ -54,7 +54,6 @@ obj-y += mpparse.o obj-y += nmi.o obj-y += numa.o obj-y += pci.o -obj-y += percpu.o obj-y += physdev.o obj-$(CONFIG_COMPAT) += x86_64/physdev.o obj-$(CONFIG_X86_PSR) += psr.o diff --git a/xen/arch/x86/include/asm/Makefile b/xen/arch/x86/include/asm/Makefile index daab34ff0a..2c27787d31 100644 --- a/xen/arch/x86/include/asm/Makefile +++ b/xen/arch/x86/include/asm/Makefile @@ -1,3 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only generic-y += div64.h -generic-y += percpu.h diff --git a/xen/arch/x86/include/asm/percpu.h b/xen/arch/x86/include/asm/percpu.h new file mode 100644 index 0000000000..34f589f990 --- /dev/null +++ b/xen/arch/x86/include/asm/percpu.h @@ -0,0 +1,14 @@ +#ifndef __X86_PERCPU_H__ +#define __X86_PERCPU_H__ + +#define PARK_OFFLINE_CPUS_VAR + +/* + * Force uses of per_cpu() with an invalid area to attempt to access the + * middle of the non-canonical address space resulting in a #GP, rather than a + * possible #PF at (NULL + a little) which has security implications in the + * context of PV guests. + */ +#define INVALID_PERCPU_AREA (0x8000000000000000UL - (unsigned long)__per_cpu_start) + +#endif /* __X86_PERCPU_H__ */ diff --git a/xen/common/Makefile b/xen/common/Makefile index fc52e0857d..f90bb00d23 100644 --- a/xen/common/Makefile +++ b/xen/common/Makefile @@ -31,6 +31,7 @@ obj-y += notifier.o obj-$(CONFIG_NUMA) += numa.o obj-y += page_alloc.o obj-y += pdx.o +obj-y += percpu.o obj-$(CONFIG_PERF_COUNTERS) += perfc.o obj-bin-$(CONFIG_HAS_PMAP) += pmap.init.o obj-y += preempt.o diff --git a/xen/arch/x86/percpu.c b/xen/common/percpu.c similarity index 85% rename from xen/arch/x86/percpu.c rename to xen/common/percpu.c index 3205eacea6..e4e8b7bcab 100644 --- a/xen/arch/x86/percpu.c +++ b/xen/common/percpu.c @@ -1,20 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ #include #include #include #include #include +#include -unsigned long __per_cpu_offset[NR_CPUS]; +#ifndef INVALID_PERCPU_AREA +#define INVALID_PERCPU_AREA (-(long)__per_cpu_start) +#endif -/* - * Force uses of per_cpu() with an invalid area to attempt to access the - * middle of the non-canonical address space resulting in a #GP, rather than a - * possible #PF at (NULL + a little) which has security implications in the - * context of PV guests. - */ -#define INVALID_PERCPU_AREA (0x8000000000000000UL - (unsigned long)__per_cpu_start) #define PERCPU_ORDER get_order_from_bytes(__per_cpu_data_end - __per_cpu_start) +extern char __per_cpu_start[]; +extern const char __per_cpu_data_end[]; + +unsigned long __read_mostly __per_cpu_offset[NR_CPUS]; + void __init percpu_init_areas(void) { unsigned int cpu; @@ -28,7 +30,7 @@ static int init_percpu_area(unsigned int cpu) char *p; if ( __per_cpu_offset[cpu] != INVALID_PERCPU_AREA ) - return 0; + return park_offline_cpus ? 0 : -EBUSY; if ( (p = alloc_xenheap_pages(PERCPU_ORDER, 0)) == NULL ) return -ENOMEM; @@ -74,16 +76,21 @@ static int cf_check cpu_percpu_callback( case CPU_UP_PREPARE: rc = init_percpu_area(cpu); break; + case CPU_UP_CANCELED: case CPU_DEAD: case CPU_RESUME_FAILED: if ( !park_offline_cpus && system_state != SYS_STATE_suspend ) free_percpu_area(cpu); break; + case CPU_REMOVE: if ( park_offline_cpus ) free_percpu_area(cpu); break; + + default: + break; } return notifier_from_errno(rc); diff --git a/xen/include/asm-generic/percpu.h b/xen/include/asm-generic/percpu.h index 60af4f9ff9..3fdb3a2a02 100644 --- a/xen/include/asm-generic/percpu.h +++ b/xen/include/asm-generic/percpu.h @@ -2,29 +2,6 @@ #ifndef __ASM_GENERIC_PERCPU_H__ #define __ASM_GENERIC_PERCPU_H__ -#ifndef __ASSEMBLY__ - -#include -#include - -extern char __per_cpu_start[]; -extern const char __per_cpu_data_end[]; -extern unsigned long __per_cpu_offset[NR_CPUS]; -void percpu_init_areas(void); - -#define per_cpu(var, cpu) \ - (*RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu])) - -#define this_cpu(var) \ - (*RELOC_HIDE(&per_cpu__##var, get_per_cpu_offset())) - -#define per_cpu_ptr(var, cpu) \ - (*RELOC_HIDE(var, __per_cpu_offset[cpu])) -#define this_cpu_ptr(var) \ - (*RELOC_HIDE(var, get_per_cpu_offset())) - -#endif - #endif /* __ASM_GENERIC_PERCPU_H__ */ /* diff --git a/xen/include/xen/percpu.h b/xen/include/xen/percpu.h index 57522f346b..e7f585c7ed 100644 --- a/xen/include/xen/percpu.h +++ b/xen/include/xen/percpu.h @@ -29,6 +29,36 @@ #include +#ifndef __ASSEMBLY__ + +#include +#include + +#ifndef PARK_OFFLINE_CPUS_VAR +/* + * Do we, for platform reasons, need to actually keep CPUs online when we + * would otherwise prefer them to be off? + */ +#define park_offline_cpus false +#endif + +extern unsigned long __per_cpu_offset[]; + +#define per_cpu(var, cpu) \ + (*RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu])) + +#define this_cpu(var) \ + (*RELOC_HIDE(&per_cpu__##var, get_per_cpu_offset())) + +#define per_cpu_ptr(var, cpu) \ + (*RELOC_HIDE(var, __per_cpu_offset[cpu])) +#define this_cpu_ptr(var) \ + (*RELOC_HIDE(var, get_per_cpu_offset())) + +void percpu_init_areas(void); + +#endif /* __ASSEMBLY__ */ + /* Linux compatibility. */ #define get_cpu_var(var) this_cpu(var) #define put_cpu_var(var)