diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c index a77b31071e..7ad870e382 100644 --- a/xen/arch/arm/setup.c +++ b/xen/arch/arm/setup.c @@ -256,9 +256,11 @@ void __init init_pdx(void) { const struct membanks *mem = bootinfo_get_mem(); paddr_t bank_start, bank_size, bank_end, ram_end = 0; - int bank; + unsigned int bank; #ifndef CONFIG_PDX_NONE + for ( bank = 0 ; bank < mem->nr_banks; bank++ ) + pfn_pdx_add_region(mem->bank[bank].start, mem->bank[bank].size); /* * Arm does not have any restrictions on the bits to compress. Pass 0 to * let the common code further restrict the mask. @@ -266,26 +268,24 @@ void __init init_pdx(void) * If the logic changes in pfn_pdx_hole_setup we might have to * update this function too. */ - uint64_t mask = pdx_init_mask(0x0); + pfn_pdx_compression_setup(0); for ( bank = 0 ; bank < mem->nr_banks; bank++ ) { - bank_start = mem->bank[bank].start; - bank_size = mem->bank[bank].size; + const struct membank *m = &mem->bank[bank]; - mask |= bank_start | pdx_region_mask(bank_start, bank_size); + if ( !pdx_is_region_compressible(m->start, + PFN_UP(m->start + m->size) - + PFN_DOWN(m->start)) ) + { + pfn_pdx_compression_reset(); + printk(XENLOG_WARNING + "PFN compression disabled, RAM region [%#" PRIpaddr ", %#" + PRIpaddr "] not covered\n", + m->start, m->start + m->size - 1); + break; + } } - - for ( bank = 0 ; bank < mem->nr_banks; bank++ ) - { - bank_start = mem->bank[bank].start; - bank_size = mem->bank[bank].size; - - if (~mask & pdx_region_mask(bank_start, bank_size)) - mask = 0; - } - - pfn_pdx_hole_setup(mask >> PAGE_SHIFT); #endif for ( bank = 0 ; bank < mem->nr_banks; bank++ ) diff --git a/xen/arch/x86/srat.c b/xen/arch/x86/srat.c index 688f410287..747607439f 100644 --- a/xen/arch/x86/srat.c +++ b/xen/arch/x86/srat.c @@ -261,8 +261,6 @@ acpi_numa_memory_affinity_init(const struct acpi_srat_mem_affinity *ma) void __init acpi_numa_arch_fixup(void) {} -static uint64_t __initdata srat_region_mask; - static int __init cf_check srat_parse_region( struct acpi_subtable_header *header, const unsigned long end) { @@ -282,15 +280,13 @@ static int __init cf_check srat_parse_region( printk(KERN_INFO "SRAT: %013"PRIx64"-%013"PRIx64"\n", ma->base_address, ma->base_address + ma->length - 1); - srat_region_mask |= ma->base_address | - pdx_region_mask(ma->base_address, ma->length); + pfn_pdx_add_region(ma->base_address, ma->length); return 0; } void __init srat_parse_regions(paddr_t addr) { - u64 mask; unsigned int i; if (acpi_disabled || acpi_numa < 0 || @@ -299,19 +295,29 @@ void __init srat_parse_regions(paddr_t addr) /* Set "PXM" as early as feasible. */ numa_fw_nid_name = "PXM"; - srat_region_mask = pdx_init_mask(addr); acpi_table_parse_srat(ACPI_SRAT_TYPE_MEMORY_AFFINITY, srat_parse_region, 0); - for (mask = srat_region_mask, i = 0; mask && i < e820.nr_map; i++) { + pfn_pdx_compression_setup(addr); + + /* Ensure all RAM ranges in the e820 are covered. */ + for (i = 0; i < e820.nr_map; i++) { if (e820.map[i].type != E820_RAM) continue; - if (~mask & pdx_region_mask(e820.map[i].addr, e820.map[i].size)) - mask = 0; + if (!pdx_is_region_compressible( + e820.map[i].addr, + PFN_UP(e820.map[i].addr + e820.map[i].size) - + PFN_DOWN(e820.map[i].addr))) { + pfn_pdx_compression_reset(); + printk(XENLOG_WARNING + "PFN compression disabled, RAM region [%#" PRIx64 + ", %#" PRIx64 "] not covered\n", + e820.map[i].addr, + e820.map[i].addr + e820.map[i].size - 1); + return; + } } - - pfn_pdx_hole_setup(mask >> PAGE_SHIFT); } unsigned int numa_node_to_arch_nid(nodeid_t n) diff --git a/xen/common/pdx.c b/xen/common/pdx.c index 00aa7e4300..c5ea58873c 100644 --- a/xen/common/pdx.c +++ b/xen/common/pdx.c @@ -19,6 +19,7 @@ #include #include #include +#include #include /** @@ -55,6 +56,44 @@ void set_pdx_range(unsigned long smfn, unsigned long emfn) __set_bit(idx, pdx_group_valid); } +#ifndef CONFIG_PDX_NONE + +#ifdef CONFIG_X86 +# include +# define MAX_PFN_RANGES E820MAX +#elif defined(CONFIG_HAS_DEVICE_TREE_DISCOVERY) +# include +# define MAX_PFN_RANGES NR_MEM_BANKS +#endif + +#ifndef MAX_PFN_RANGES +# error "Missing architecture maximum number of RAM ranges" +#endif + +/* Generic PFN compression helpers. */ +static struct pfn_range { + unsigned long base_pfn, pages; +} ranges[MAX_PFN_RANGES] __initdata; +static unsigned int __initdata nr_ranges; + +void __init pfn_pdx_add_region(paddr_t base, paddr_t size) +{ + if ( !size ) + return; + + if ( nr_ranges >= ARRAY_SIZE(ranges) ) + { + ASSERT((nr_ranges + 1) > nr_ranges); + nr_ranges++; + return; + } + + ranges[nr_ranges].base_pfn = PFN_DOWN(base); + ranges[nr_ranges++].pages = PFN_UP(base + size) - PFN_DOWN(base); +} + +#endif /* !CONFIG_PDX_NONE */ + #ifdef CONFIG_PDX_MASK_COMPRESSION /* @@ -115,20 +154,25 @@ static uint64_t fill_mask(uint64_t mask) return mask; } -bool pdx_is_region_compressible(paddr_t base, unsigned long npages) -{ - return !(paddr_to_pfn(base) & pfn_hole_mask) && - !(pdx_region_mask(base, npages * PAGE_SIZE) & ~ma_va_bottom_mask); -} - -/* We don't want to compress the low MAX_ORDER bits of the addresses. */ -uint64_t __init pdx_init_mask(uint64_t base_addr) -{ - return fill_mask(max(base_addr, - (uint64_t)1 << (MAX_ORDER + PAGE_SHIFT)) - 1); -} - -uint64_t pdx_region_mask(uint64_t base, uint64_t len) +/** + * Calculates a mask covering "moving" bits of all addresses of a region + * + * The i-th bit of the mask must be set if there's 2 different addresses + * in the region that have different j-th bits. where j >= i. + * + * e.g: + * base=0x1B00000000 + * len+base=0x1B00042000 + * + * ought to return 0x000007FFFF, which implies that every bit position + * with a zero in the mask remains unchanged in every address of the + * region. + * + * @param base Base address of the region + * @param len Size in octets of the region + * @return Mask of moving bits at the bottom of all the region addresses + */ +static uint64_t pdx_region_mask(uint64_t base, uint64_t len) { /* * We say a bit "moves" in a range if there exist 2 addresses in that @@ -143,9 +187,45 @@ uint64_t pdx_region_mask(uint64_t base, uint64_t len) return fill_mask(base ^ (base + len - 1)); } -void __init pfn_pdx_hole_setup(unsigned long mask) +bool pdx_is_region_compressible(paddr_t base, unsigned long npages) +{ + return !(paddr_to_pfn(base) & pfn_hole_mask) && + !(pdx_region_mask(base, npages * PAGE_SIZE) & ~ma_va_bottom_mask); +} + +/** + * Creates the mask to start from when calculating non-compressible bits + * + * This function is intimately related to pdx_region_mask(), and together + * they are meant to calculate the mask of non-compressible bits given the + * current memory map. + * + * @param base_addr Address of the first maddr in the system + * @return An integer of the form 2^n - 1 + */ +static uint64_t __init pdx_init_mask(uint64_t base_addr) +{ + return fill_mask(max(base_addr, + /* Don't compress the low MAX_ORDER bits. */ + (uint64_t)1 << (MAX_ORDER + PAGE_SHIFT)) - 1); +} + +void __init pfn_pdx_compression_setup(paddr_t base) { unsigned int i, j, bottom_shift = 0, hole_shift = 0; + unsigned long mask = pdx_init_mask(base) >> PAGE_SHIFT; + + if ( nr_ranges > ARRAY_SIZE(ranges) ) + { + printk(XENLOG_WARNING + "Too many PFN ranges (%u > %zu), not attempting PFN compression\n", + nr_ranges, ARRAY_SIZE(ranges)); + return; + } + + for ( i = 0; i < nr_ranges; i++ ) + mask |= ranges[i].base_pfn | + pdx_region_mask(ranges[i].base_pfn, ranges[i].pages); /* * We skip the first MAX_ORDER bits, as we never want to compress them. @@ -184,6 +264,18 @@ void __init pfn_pdx_hole_setup(unsigned long mask) ma_top_mask = pfn_top_mask << PAGE_SHIFT; } +void __init pfn_pdx_compression_reset(void) +{ + pfn_pdx_bottom_mask = ~0UL; + ma_va_bottom_mask = ~0UL; + pfn_top_mask = 0; + ma_top_mask = 0; + pfn_hole_mask = 0; + pfn_pdx_hole_shift = 0; + + nr_ranges = 0; +} + #endif /* CONFIG_PDX_COMPRESSION */ /* diff --git a/xen/include/xen/pdx.h b/xen/include/xen/pdx.h index 8e373cac8b..10153da98b 100644 --- a/xen/include/xen/pdx.h +++ b/xen/include/xen/pdx.h @@ -108,38 +108,6 @@ extern unsigned int pfn_pdx_hole_shift; extern unsigned long pfn_hole_mask; extern unsigned long pfn_top_mask, ma_top_mask; -/** - * Calculates a mask covering "moving" bits of all addresses of a region - * - * The i-th bit of the mask must be set if there's 2 different addresses - * in the region that have different j-th bits. where j >= i. - * - * e.g: - * base=0x1B00000000 - * len+base=0x1B00042000 - * - * ought to return 0x000007FFFF, which implies that every bit position - * with a zero in the mask remains unchanged in every address of the - * region. - * - * @param base Base address of the region - * @param len Size in octets of the region - * @return Mask of moving bits at the bottom of all the region addresses - */ -uint64_t pdx_region_mask(uint64_t base, uint64_t len); - -/** - * Creates the mask to start from when calculating non-compressible bits - * - * This function is intimately related to pdx_region_mask(), and together - * they are meant to calculate the mask of non-compressible bits given the - * current memory map. - * - * @param base_addr Address of the first maddr in the system - * @return An integer of the form 2^n - 1 - */ -uint64_t pdx_init_mask(uint64_t base_addr); - /** * Map pfn to its corresponding pdx * @@ -189,17 +157,6 @@ static inline paddr_t directmapoff_to_maddr(unsigned long offset) (offset & ma_va_bottom_mask)); } -/** - * Initializes global variables with information about the compressible - * range of the current memory regions. - * - * @param mask This mask is the biggest pdx_mask of every region in the - * system ORed with all base addresses of every region in the - * system. This results in a mask where every zero in a bit - * position marks a potentially compressible bit. - */ -void pfn_pdx_hole_setup(unsigned long mask); - #endif /* CONFIG_PDX_MASK_COMPRESSION */ #ifdef CONFIG_PDX_NONE @@ -220,17 +177,15 @@ static inline bool pdx_is_region_compressible(paddr_t base, return true; } -static inline uint64_t pdx_init_mask(uint64_t base_addr) +static inline void pfn_pdx_add_region(paddr_t base, paddr_t size) { - return 0; } -static inline uint64_t pdx_region_mask(uint64_t base, uint64_t len) +static inline void pfn_pdx_compression_setup(paddr_t base) { - return 0; } -static inline void pfn_pdx_hole_setup(unsigned long mask) +static inline void pfn_pdx_compression_reset(void) { } @@ -247,6 +202,28 @@ static inline void pfn_pdx_hole_setup(unsigned long mask) */ bool pdx_is_region_compressible(paddr_t base, unsigned long npages); +/** + * Register a RAM region with the PFN compression logic. + * + * @param base Start of the region in bytes. + * @param size Length of the region in bytes. + */ +void pfn_pdx_add_region(paddr_t base, paddr_t size); + +/** + * Initializes global variables with information about the compressible + * range of the current memory regions. + * + * @param base address to start compression from. + */ +void pfn_pdx_compression_setup(paddr_t base); + +/** + * Reset the global variables to it's default values, thus disabling PFN + * compression. + */ +void pfn_pdx_compression_reset(void); + #endif /* !CONFIG_PDX_NONE */ #endif /* __XEN_PDX_H__ */