sparsemem memory model
Subsystems (4)
PATCH 3/6 sparsemem memory model
Sparsemem abstracts the use of discontiguous mem_maps[]. This kind of mem_map[] is needed by discontiguous memory machines (like in the old CONFIG_DISCONTIGMEM case) as well as memory hotplug systems. Sparsemem replaces DISCONTIGMEM when enabled, and it is hoped that it can eventually become a complete replacement.
A significant advantage over DISCONTIGMEM is that it's completely separated from CONFIG_NUMA. When producing this patch, it became apparent in that NUMA and DISCONTIG are often confused.
Another advantage is that sparse doesn't require each NUMA node's ranges to be contiguous. It can handle overlapping ranges between nodes with no problems, where DISCONTIGMEM currently throws away that memory.
Sparsemem uses an array to provide different pfn_to_page() translations for each SECTION_SIZE area of physical memory. This is what allows the mem_map[] to be chopped up.
In order to do quick pfn_to_page() operations, the section number of the page is encoded in page->flags. Part of the sparsemem infrastructure enables sharing of these bits more dynamically (at compile-time) between the page_zone() and sparsemem operations. However, on 32-bit architectures, the number of bits is quite limited, and may require growing the size of the page->flags type in certain conditions. Several things might force this to
occur: a decrease in the SECTION_SIZE (if you want to hotplug smaller areas of memory), an increase in the physical address space, or an increase in the number of used page->flags.
One thing to note is that, once sparsemem is present, the NUMA node information no longer needs to be stored in the page->flags. It might provide speed increases on certain platforms and will be stored there if there is room. But, if out of room, an alternate (theoretically slower) mechanism is used.
This patch introduces CONFIG_FLATMEM. It is used in almost all cases where there used to be an #ifndef DISCONTIG, because SPARSEMEM and DISCONTIGMEM often have to compile out the same areas of code.
--- include/linux/mm.h | 86 ++++++++++++++++++++++++++++++++++++------- include/linux/mmzone.h | 96 +++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/numa.h | 2 - mm/Kconfig | 38 +++++++++++++++++-- mm/Makefile | 1 mm/bootmem.c | 8 +++- mm/memory.c | 2 - mm/page_alloc.c | 39 +++++++++++++++---- mm/sparse.c | 85 +++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 327 insertions(+), 30 deletions(-)
include/linux/mm.h +72 -14
@@ -401,39 +401,91 @@ static inline void put_page(struct page* sets it, so none of the operations on it need to be atomic. */ -/* Page flags: | NODE | ZONE | ... | FLAGS | */ -#define NODES_PGOFF ((sizeof(page_flags_t)*8) - NODES_SHIFT) -#define ZONES_PGOFF (NODES_PGOFF - ZONES_SHIFT) + +/* +* page->flags layout: +* +* There are three possibilities for how page->flags get +* laid out. The first is for the normal case, without +* sparsemem. The second is for sparsemem when there is +* plenty of space for node and section. The last is when +* we have run out of space and have to fall back to an +* alternate (slower) way of determining the node. +* +* No sparsemem: | NODE | ZONE | ... | FLAGS | +* with space for node: | SECTION | NODE | ZONE | ... | FLAGS | +* no space for node: | SECTION | ZONE | ... | FLAGS | +*/ +#if SECTIONS_SHIFT +NODES_SHIFT +ZONES_SHIFT <=FLAGS_RESERVED +#define NODES_WIDTH NODES_SHIFT +#else +#define NODES_WIDTH 0 +#endif + +#ifdef CONFIG_SPARSEMEM +#define SECTIONS_WIDTH SECTIONS_SHIFT +#else +#define SECTIONS_WIDTH 0 +#endif + +#define ZONES_WIDTH ZONES_SHIFT + +/* Page flags: | [SECTION] | [NODE] | ZONE | ... | FLAGS | */ +#define SECTIONS_PGOFF ((sizeof(page_flags_t)*8) - SECTIONS_WIDTH) +#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) +#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) + +/* +* We are going to use the flags for the page to node mapping if its in +* there. This includes the case where there is no node, so it is implicit. +*/ +#define FLAGS_HAS_NODE (NODES_WIDTH > 0 || NODES_SHIFT == 0) + +#ifndef PFN_SECTION_SHIFT +#define PFN_SECTION_SHIFT 0 +#endif /* * Define the bit shifts to access each section. For non-existant * sections we define the shift as 0; that plus a 0 mask ensures * the compiler will optimise away reference to them. */ -#define NODES_PGSHIFT (NODES_PGOFF * (NODES_SHIFT != 0)) -#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_SHIFT != 0)) +#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) +#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) +#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) -/* NODE:ZONE is used to lookup the zone from a page. */ +/* NODE:ZONE or SECTION:ZONE is used to lookup the zone from a page. */ +#if FLAGS_HAS_NODE #define ZONETABLE_SHIFT (NODES_SHIFT + ZONES_SHIFT) +#else +#define ZONETABLE_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT) +#endif #define ZONETABLE_PGSHIFT ZONES_PGSHIFT -#if NODES_SHIFT +ZONES_SHIFT >FLAGS_RESERVED -#error NODES_SHIFT+ZONES_SHIFT > FLAGS_RESERVED +#if SECTIONS_WIDTH +NODES_WIDTH +ZONES_WIDTH >FLAGS_RESERVED +#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > FLAGS_RESERVED#endif -#define NODEZONE (node, zone) ((node << ZONES_SHIFT) | zone) - -#define ZONES_MASK ((1UL << ZONES_SHIFT) - 1) -#define NODES_MASK ((1UL << NODES_SHIFT) - 1) +#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) +#define NODES_MASK ((1UL << NODES_WIDTH) - 1) +#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)#define ZONETABLE_MASK ((1UL << ZONETABLE_SHIFT) - 1)static inline unsigned long page_zonenum (struct page *page) {return (page->flags >>ZONES_PGSHIFT ) &ZONES_MASK ; } +static inline struct zone *page_zone (struct page *page);static inline unsigned long page_to_nid (struct page *page) { -return (page->flags >>NODES_PGSHIFT ) &NODES_MASK ; +if (FLAGS_HAS_NODE ) +return (page->flags >>NODES_PGSHIFT ) &NODES_MASK ; +else +return page_zone (page)->zone_pgdat->node_id; +} +static inline unsigned long page_to_section (struct page *page) +{ +return (page->flags >>SECTIONS_PGSHIFT ) &SECTIONS_MASK ; }struct zone ;
@@ -455,12 +507,18 @@ static inline void set_page_node(struct page->flags &= ~(NODES_MASK <<NODES_PGSHIFT ); page->flags |= (node &NODES_MASK ) <<NODES_PGSHIFT ; } +static inline void set_page_section (struct page *page,unsigned long section) +{ + page->flags &= ~(SECTIONS_MASK <<SECTIONS_PGSHIFT ); + page->flags |= (section &SECTIONS_MASK ) <<SECTIONS_PGSHIFT ; +}static inline void set_page_links (struct page *page,unsigned long zone, -unsigned long node) +unsigned long node,unsigned long pfn) {set_page_zone (page, zone);set_page_node (page, node); +set_page_section (page,pfn_to_section_nr (pfn)); }#ifndef CONFIG_DISCONTIGMEM
include/linux/mmzone.h +96 -0
122 lines not on this page · raw message
include/linux/numa.h +1 -1
@@ -3,7 +3,7 @@#include <linux/config.h> -#ifdef CONFIG_DISCONTIGMEM +#ifndef CONFIG_FLATMEM #include <asm/numnodes.h> #endif
mm/bootmem.c +6 -2
@@ -256,6 +256,7 @@ found:static unsigned long __init free_all_bootmem_core (pg_data_t *pgdat) {struct page *page; +unsigned long pfn;bootmem_data_t *bdata = pgdat->bdata;unsigned long i, count, total =0 ;unsigned long idx;
@@ -266,7 +267,7 @@ static unsigned long __init free_all_boo count =0 ;/* first extant page of the node */ - page =virt_to_page (phys_to_virt (bdata->node_boot_start)); + pfn = bdata->node_boot_start >>PAGE_SHIFT ; idx = bdata->node_low_pfn - (bdata->node_boot_start >>PAGE_SHIFT ); map = bdata->node_bootmem_map;/* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */
@@ -275,6 +276,9 @@ static unsigned long __init free_all_boo gofast =1 ;for (i =0 ; i < idx; ) {unsigned long v = ~map[i /BITS_PER_LONG ]; + + page =pfn_to_page (pfn); +if (gofast && v == ~0UL ) {int j, order;
@@ -302,8 +306,8 @@ static unsigned long __init free_all_boo } }else { i+=BITS_PER_LONG ; - page +=BITS_PER_LONG ; } + pfn +=BITS_PER_LONG ; } total += count;
mm/Kconfig +35 -3
@@ -6,6 +6,7 @@ choice prompt "Memory model" depends on SELECT_MEMORY_MODEL default DISCONTIGMEM_MANUAL if ARCH_DISCONTIGMEM_DEFAULT + default SPARSEMEM_MANUAL if ARCH_SPARSEMEM_DEFAULT default FLATMEM_MANUAL config FLATMEM_MANUAL
@@ -17,7 +18,15 @@ config FLATMEM_MANUAL only have one option here: FLATMEM. This is normal and a correct option. - If unsure, choose this option over any other. + Some users of more advanced features like NUMA and + memory hotplug may have different options here. + DISCONTIGMEM is an more mature, better tested system, + but is incompatible with memory hotplug and may suffer + decreased performance over SPARSEMEM. If unsure between + "Sparse Memory" and "Discontiguous Memory", choose + "Discontiguous Memory". + + If unsure, choose this option (Flat Memory) over any other. config DISCONTIGMEM_MANUAL bool "Discontigious Memory"
@@ -35,15 +44,38 @@ config DISCONTIGMEM_MANUAL If unsure, choose "Flat Memory" over this option. +config SPARSEMEM_MANUAL + bool "Sparse Memory" + depends on ARCH_SPARSEMEM_ENABLE + help + This will be the only option for some systems, including + memory hotplug systems. This is normal. + + For many other systems, this will be an alternative to + "Discontigious Memory". This option provides some potential + performance benefits, along with decreased code complexity, + but it is newer, and more experimental. + + If unsure, choose "Discontiguous Memory" or "Flat Memory" + over this option. + endchoice config DISCONTIGMEM def_bool y depends on (!SELECT_MEMORY_MODEL && ARCH_DISCONTIGMEM_ENABLE) || DISCONTIGMEM_MANUAL +config SPARSEMEM + def_bool y + depends on SPARSEMEM_MANUAL + config FLATMEM def_bool y - depends on !DISCONTIGMEM || FLATMEM_MANUAL + depends on (!DISCONTIGMEM && !SPARSEMEM) || FLATMEM_MANUAL + +config FLAT_NODE_MEM_MAP + def_bool y + depends on !SPARSEMEM # # Both the NUMA code and DISCONTIGMEM use arrays of pg_data_t's
@@ -56,4 +88,4 @@ config NEED_MULTIPLE_NODES config HAVE_MEMORY_PRESENT def_bool y - depends on ARCH_HAVE_MEMORY_PRESENT + depends on ARCH_HAVE_MEMORY_PRESENT || SPARSEMEM
mm/Makefile +1 -0
@@ -15,6 +15,7 @@ obj-y := bootmem.o filemap.o mempool.o obj-$(CONFIG_SWAP) += page_io.o swap_state.o swapfile.o thrash.o obj-$(CONFIG_HUGETLBFS) += hugetlb.o obj-$(CONFIG_NUMA) += mempolicy.o +obj-$(CONFIG_SPARSEMEM) += sparse.o obj-$(CONFIG_SHMEM) += shmem.o obj-$(CONFIG_TINY_SHMEM) += tiny-shmem.o
mm/memory.c +1 -1
@@ -58,7 +58,7 @@#include <linux/swapops.h> #include <linux/elf.h> -#ifndef CONFIG_DISCONTIGMEM +#ifndef CONFIG_NEED_MULTIPLE_NODES /* use the per-pgdat data instead for discontigmem - mbligh */ unsigned long max_mapnr;struct page *mem_map;
mm/page_alloc.c +30 -9
@@ -66,7 +66,7 @@ EXPORT_SYMBOL(nr_swap_pages);* Used by page_zone() to look up the address of the struct zone whose * id is encoded in the upper bits of page->flags */ -struct zone *zone_table[1 << (ZONES_SHIFT +NODES_SHIFT )]; +struct zone *zone_table[1 <<ZONETABLE_SHIFT ];EXPORT_SYMBOL (zone_table);static char *zone_names[MAX_NR_ZONES ] = {"DMA" ,"Normal" ,"HighMem" };
@@ -1582,11 +1582,15 @@ static void __init calculate_zone_totalpvoid __init memmap_init_zone (unsigned long size,int nid,unsigned long zone,unsigned long start_pfn) { -struct page *start =pfn_to_page (start_pfn);struct page *page; +int end_pfn = start_pfn + size; +int pfn; -for (page = start; page < (start + size); page++) { -set_page_links (page, zone, nid); +for (pfn = start_pfn; pfn < end_pfn; pfn++, page++) { +if (!early_pfn_valid (pfn)) +continue ; + page =pfn_to_page (pfn); +set_page_links (page, zone, nid, pfn);set_page_count (page,0 );reset_page_mapcount (page);SetPageReserved (page);
@@ -1610,6 +1614,20 @@ void zone_init_free_lists(struct pglist_ } } +#define ZONETABLE_INDEX (x, zone_nr) ((x << ZONES_SHIFT) | zone_nr) +void zonetable_add (struct zone *zone,int nid,int zid,unsigned long pfn, +unsigned long size) +{ +unsigned long snum =pfn_to_section_nr (pfn); +unsigned long end =pfn_to_section_nr (pfn + size); + +if (FLAGS_HAS_NODE ) + zone_table[ZONETABLE_INDEX (nid, zid)] = zone; +else +for (; snum <= end; snum++) + zone_table[ZONETABLE_INDEX (snum, zid)] = zone; +} +#ifndef __HAVE_ARCH_MEMMAP_INIT#define memmap_init (size, nid, zone, start_pfn) \ memmap_init_zone((size), (nid), (zone), (start_pfn))
@@ -1638,7 +1656,6 @@ static void __init free_area_init_core(sunsigned long size, realsize;unsigned long batch; - zone_table[NODEZONE (nid, j)] = zone; realsize = size = zones_size[j];if (zholes_size) realsize -= zholes_size[j];
@@ -1723,6 +1740,8 @@ static void __init free_area_init_core(smemmap_init (size, nid, j, zone_start_pfn); +zonetable_add (zone, nid, j, zone_start_pfn, size); + zone_start_pfn += size;zone_init_free_lists (pgdat, zone, zone->spanned_pages);
@@ -1731,28 +1750,30 @@ static void __init free_area_init_core(sstatic void __init alloc_node_mem_map (struct pglist_data *pgdat) { -unsigned long size; -struct page *map; -/* Skip empty nodes */ if (!pgdat->node_spanned_pages)return ; +#ifdef CONFIG_FLAT_NODE_MEM_MAP /* ia64 gets its own node_mem_map, before this, without bootmem */ if (!pgdat->node_mem_map) { +unsigned long size; +struct page *map; + size = (pgdat->node_spanned_pages +1 ) *sizeof (struct page ); map =alloc_remap (pgdat->node_id, size);if (!map) map =alloc_bootmem_node (pgdat, size); pgdat->node_mem_map = map; } -#ifndef CONFIG_DISCONTIGMEM +#ifdef CONFIG_FLATMEM /* * With no DISCONTIG, the global mem_map is just set as node 0's */ if (pgdat ==NODE_DATA (0 )) mem_map =NODE_DATA (0 )->node_mem_map;#endif +#endif /* CONFIG_FLAT_NODE_MEM_MAP */ } void __init free_area_init_node(int nid, struct pglist_data *pgdat,
mm/sparse.c +85 -0
85 lines not on this page · raw message
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>