bootmem rewrite
Subsystems (12)
cover bootmem rewrite
Hi,
This is a complete overhaul of the bootmem allocator while preserving its original functionality, excluding bugs.
free_bootmem and reserve_bootmem become a bit stricter than they are right now, callsites have to make sure that the PFN range is contiguous but it might go across node boundaries.
alloc_bootmem satisfying the allocation goal is more likely as the routines will try to allocate on the node holding the goal first before falling back as opposed to the original behaviour that satisfies the goal only if it is on the first node.
All in all, I think the code has become simpler and cleaner. All public interfaces have been documented, too.
The first patch moves the bootmem node descriptor definitions into bootmem.c where they belong.
The second patch is the new allocator itself.
The third patch converts all users of ->node_boot_start to ->node_min_pfn as this is what they really use. It then removes the unused ->node_boot_start.
Compile and runtime tested on X86_32, therefor RFC only.
arch/alpha/mm/numa.c | 8 +- arch/arm/mm/discontig.c | 34 +- arch/arm/plat-omap/fb.c | 4 +- arch/avr32/mm/init.c | 3 +- arch/ia64/mm/discontig.c | 30 +- arch/m32r/mm/discontig.c | 4 +- arch/m32r/mm/init.c | 4 +- arch/m68k/mm/init.c | 4 +- arch/mips/sgi-ip27/ip27-memory.c | 3 +- arch/mn10300/mm/init.c | 6 +- arch/parisc/mm/init.c | 3 +- arch/powerpc/mm/numa.c | 3 +- arch/sh/mm/init.c | 2 +- arch/sh/mm/numa.c | 5 +- arch/sparc64/mm/init.c | 3 +- arch/x86/mm/discontig_32.c | 3 +- arch/x86/mm/numa_64.c | 6 +- include/linux/bootmem.h | 115 ++--- mm/bootmem.c | 914 +++++++++++++++++++------------------- mm/page_alloc.c | 4 +- 20 files changed, 560 insertions(+), 598 deletions(-)
RFC 1/3 mm: Move bootmem descriptors to a single place
There are a lot of places that define either a single bootmem descriptor or an array of them. Use only one central array with MAX_NUMNODES items instead.
---
Andrew, this patch is already in -mm together with:
- mm: Fix free_all_bootmem_core alignment check
- mm: Normalize internal argument passing of bootmem data
- mm: Unexport __alloc_bootmem_core()
They are obsoleted by this series. This patch stays the same.
arch/alpha/mm/numa.c +4 -4
@@ -19,7 +19,6 @@#include <asm/pgalloc.h> pg_data_t node_data[MAX_NUMNODES ]; -bootmem_data_t node_bdata[MAX_NUMNODES ];EXPORT_SYMBOL (node_data);#undef DEBUG_DISCONTIG
@@ -141,7 +140,7 @@ setup_memory_node(int nid, void *kernel_printk (" not enough mem to reserve NODE_DATA" );return ; } -NODE_DATA (nid)->bdata = &node_bdata[nid]; +NODE_DATA (nid)->bdata = &bootmem_node_data[nid];printk (" Detected node memory: start %8lu, end %8lu\n" , node_min_pfn, node_max_pfn);
@@ -304,8 +303,9 @@ void __init paging_init(void) dma_local_pfn =virt_to_phys ((char *)MAX_DMA_ADDRESS ) >>PAGE_SHIFT ;for_each_online_node (nid) { -unsigned long start_pfn = node_bdata[nid].node_boot_start >>PAGE_SHIFT ; -unsigned long end_pfn = node_bdata[nid].node_low_pfn; +bootmem_data_t *bdata = &bootmem_node_data[nid]; +unsigned long start_pfn = bdata->node_boot_start >>PAGE_SHIFT ; +unsigned long end_pfn = bdata->node_low_pfn;if (dma_local_pfn >= end_pfn - start_pfn) zones_size[ZONE_DMA ] = end_pfn - start_pfn;
arch/arm/mm/discontig.c +16 -18
@@ -21,26 +21,24 @@* Our node_data structure for discontiguous memory. */ -static bootmem_data_t node_bootmem_data[MAX_NUMNODES ]; -pg_data_t discontig_node_data[MAX_NUMNODES ] = { - { .bdata = &node_bootmem_data[0 ] }, - { .bdata = &node_bootmem_data[1 ] }, - { .bdata = &node_bootmem_data[2 ] }, - { .bdata = &node_bootmem_data[3 ] }, + { .bdata = &bootmem_node_data[0 ] }, + { .bdata = &bootmem_node_data[1 ] }, + { .bdata = &bootmem_node_data[2 ] }, + { .bdata = &bootmem_node_data[3 ] }, #if MAX_NUMNODES == 16 - { .bdata = &node_bootmem_data[4 ] }, - { .bdata = &node_bootmem_data[5 ] }, - { .bdata = &node_bootmem_data[6 ] }, - { .bdata = &node_bootmem_data[7 ] }, - { .bdata = &node_bootmem_data[8 ] }, - { .bdata = &node_bootmem_data[9 ] }, - { .bdata = &node_bootmem_data[10 ] }, - { .bdata = &node_bootmem_data[11 ] }, - { .bdata = &node_bootmem_data[12 ] }, - { .bdata = &node_bootmem_data[13 ] }, - { .bdata = &node_bootmem_data[14 ] }, - { .bdata = &node_bootmem_data[15 ] }, + { .bdata = &bootmem_node_data[4 ] }, + { .bdata = &bootmem_node_data[5 ] }, + { .bdata = &bootmem_node_data[6 ] }, + { .bdata = &bootmem_node_data[7 ] }, + { .bdata = &bootmem_node_data[8 ] }, + { .bdata = &bootmem_node_data[9 ] }, + { .bdata = &bootmem_node_data[10 ] }, + { .bdata = &bootmem_node_data[11 ] }, + { .bdata = &bootmem_node_data[12 ] }, + { .bdata = &bootmem_node_data[13 ] }, + { .bdata = &bootmem_node_data[14 ] }, + { .bdata = &bootmem_node_data[15 ] }, #endif };
arch/ia64/mm/discontig.c +5 -6
@@ -36,7 +36,6 @@ struct early_node_data {struct ia64_node_data *node_data;unsigned long pernode_addr;unsigned long pernode_size; -struct bootmem_data bootmem_data;unsigned long num_physpages;#ifdef CONFIG_ZONE_DMA unsigned long num_dma_physpages;
@@ -76,7 +75,7 @@ static int __init build_node_maps(unsign int node) {unsigned long cstart, epfn, end = start + len; -struct bootmem_data *bdp = &mem_data[node].bootmem_data; +struct bootmem_data *bdp = &bootmem_node_data[node]; epfn =GRANULEROUNDUP (end) >>PAGE_SHIFT ; cstart =GRANULEROUNDDOWN (start);
@@ -167,7 +166,7 @@ static void __init fill_pernode(int node {void *cpu_data;int cpus =early_nr_cpus_node (node); -struct bootmem_data *bdp = &mem_data[node].bootmem_data; +struct bootmem_data *bdp = &bootmem_node_data[node]; mem_data[node].pernode_addr = pernode; mem_data[node].pernode_size = pernodesize;
@@ -224,7 +223,7 @@ static int __init find_pernode_space(uns {unsigned long epfn;unsigned long pernodesize =0 , pernode, pages, mapsize; -struct bootmem_data *bdp = &mem_data[node].bootmem_data; +struct bootmem_data *bdp = &bootmem_node_data[node]; epfn = (start + len) >>PAGE_SHIFT ;
@@ -440,7 +439,7 @@ void __init find_memory(void)efi_memmap_walk (find_max_min_low_pfn,NULL );for_each_online_node (node) -if (mem_data[node].bootmem_data.node_low_pfn) { +if (bootmem_node_data[node].node_low_pfn) {node_clear (node, memory_less_mask); mem_data[node].min_pfn = ~0UL ; }
@@ -460,7 +459,7 @@ void __init find_memory(void)else if (node_isset (node, memory_less_mask))continue ; - bdp = &mem_data[node].bootmem_data; + bdp = &bootmem_node_data[node]; pernode = mem_data[node].pernode_addr; pernodesize = mem_data[node].pernode_size; map = pernode + pernodesize;
arch/m32r/mm/discontig.c +1 -3
@@ -20,7 +20,6 @@ extern char _end[];struct pglist_data *node_data[MAX_NUMNODES ];EXPORT_SYMBOL (node_data); -static bootmem_data_t node_bdata[MAX_NUMNODES ]__initdata ;pg_data_t m32r_node_data[MAX_NUMNODES ];
@@ -81,7 +80,7 @@ unsigned long __init setup_memory(void)for_each_online_node (nid) { mp = &mem_prof[nid];NODE_DATA (nid)=(pg_data_t *)&m32r_node_data[nid]; -NODE_DATA (nid)->bdata = &node_bdata[nid]; +NODE_DATA (nid)->bdata = &bootmem_node_data[nid]; min_pfn = mp->start_pfn; max_pfn = mp->start_pfn + mp->pages; bootmap_size = init_bootmem_node(NODE_DATA(nid), mp->free_pfn,
@@ -163,4 +162,3 @@ unsigned long __init zone_sizes_init(voireturn holes; } -
arch/m68k/mm/init.c +1 -3
@@ -32,8 +32,6 @@ DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); -static bootmem_data_t __initdata bootmem_data[MAX_NUMNODES ]; -pg_data_t pg_data_map[MAX_NUMNODES ];EXPORT_SYMBOL (pg_data_map);
@@ -58,7 +56,7 @@ void __init m68k_setup_node(int node) pg_data_table[i] = pg_data_map + node; }#endif - pg_data_map[node].bdata = bootmem_data + node; + pg_data_map[node].bdata = bootmem_node_data + node;node_set_online (node); }
arch/mips/sgi-ip27/ip27-memory.c +1 -2
@@ -37,7 +37,6 @@static short __initdata slot_lastfilled_cache[MAX_COMPACT_NODES ];static unsigned short __initdata slot_psize_cache[MAX_COMPACT_NODES ][MAX_MEM_SLOTS ]; -static struct bootmem_data __initdata plat_node_bdata[MAX_COMPACT_NODES ];struct node_data *__node_data[MAX_COMPACT_NODES ];
@@ -453,7 +452,7 @@ static void __init node_mem_init(cnodeid __node_data[node] =__va (slot_freepfn <<PAGE_SHIFT ); pd =NODE_DATA (node); - pd->bdata = &plat_node_bdata[node]; + pd->bdata = &bootmem_node_data[node];cpus_clear (hub_data (node)->h_cpus);
arch/parisc/mm/init.c +1 -2
@@ -36,7 +36,6 @@ extern int data_start;#ifdef CONFIG_DISCONTIGMEM struct node_map_data node_data[MAX_NUMNODES ]__read_mostly ; -bootmem_data_t bmem_data[MAX_NUMNODES ]__read_mostly ;unsigned char pfnnid_map[PFNNID_MAP_MAX ]__read_mostly ;#endif
@@ -262,7 +261,7 @@ static void __init setup_bootmem(void)#ifdef CONFIG_DISCONTIGMEM for (i =0 ; i <MAX_PHYSMEM_RANGES ; i++) {memset (NODE_DATA (i),0 ,sizeof (pg_data_t)); -NODE_DATA (i)->bdata = &bmem_data[i]; +NODE_DATA (i)->bdata = &bootmem_node_data[i]; }memset (pfnnid_map,0xff ,sizeof (pfnnid_map));
arch/powerpc/mm/numa.c +1 -2
@@ -39,7 +39,6 @@ EXPORT_SYMBOL(numa_cpu_lookup_table);EXPORT_SYMBOL (numa_cpumask_lookup_table);EXPORT_SYMBOL (node_data); -static bootmem_data_t __initdata plat_node_bdata[MAX_NUMNODES ];static int min_common_depth;static int n_mem_addr_cells, n_mem_size_cells;
@@ -685,7 +684,7 @@ void __init do_init_bootmem(void)dbg ("node %d\n" , nid);dbg ("NODE_DATA() = %p\n" ,NODE_DATA (nid)); -NODE_DATA (nid)->bdata = &plat_node_bdata[nid]; +NODE_DATA (nid)->bdata = &bootmem_node_data[nid];NODE_DATA (nid)->node_start_pfn = start_pfn;NODE_DATA (nid)->node_spanned_pages = end_pfn - start_pfn;
arch/sh/mm/numa.c +2 -3
@@ -14,7 +14,6 @@#include <linux/pfn.h> #include <asm/sections.h> -static bootmem_data_t plat_node_bdata[MAX_NUMNODES ];struct pglist_data *node_data[MAX_NUMNODES ]__read_mostly ;EXPORT_SYMBOL_GPL (node_data);
@@ -35,7 +34,7 @@ void __init setup_memory(void)NODE_DATA (0 ) =pfn_to_kaddr (free_pfn);memset (NODE_DATA (0 ),0 ,sizeof (struct pglist_data )); free_pfn +=PFN_UP (sizeof (struct pglist_data )); -NODE_DATA (0 )->bdata = &plat_node_bdata[0 ]; +NODE_DATA (0 )->bdata = &bootmem_node_data[0 ];/* Set up node 0 */ setup_bootmem_allocator (free_pfn);
@@ -66,7 +65,7 @@ void __init setup_bootmem_node(int nid, free_pfn +=PFN_UP (sizeof (struct pglist_data ));memset (NODE_DATA (nid),0 ,sizeof (struct pglist_data )); -NODE_DATA (nid)->bdata = &plat_node_bdata[nid]; +NODE_DATA (nid)->bdata = &bootmem_node_data[nid];NODE_DATA (nid)->node_start_pfn = start_pfn;NODE_DATA (nid)->node_spanned_pages = end_pfn - start_pfn;
arch/sparc64/mm/init.c +1 -2
@@ -788,7 +788,6 @@ int numa_cpu_lookup_table[NR_CPUS];cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES ]; #ifdef CONFIG_NEED_MULTIPLE_NODES -static bootmem_data_t plat_node_bdata[MAX_NUMNODES]; struct mdesc_mblock { u64 base;
@@ -871,7 +870,7 @@ static void __init allocate_node_data(inNODE_DATA (nid) =__va (paddr);memset (NODE_DATA (nid),0 ,sizeof (struct pglist_data )); -NODE_DATA (nid)->bdata = &plat_node_bdata[nid]; +NODE_DATA (nid)->bdata = &bootmem_node_data[nid];#endif p =NODE_DATA (nid);
arch/x86/mm/discontig_32.c +1 -2
@@ -41,7 +41,6 @@struct pglist_data *node_data[MAX_NUMNODES ]__read_mostly ;EXPORT_SYMBOL (node_data); -static bootmem_data_t node0_bdata;/* * numa interface - we expect the numa architecture specific code to have
@@ -382,7 +381,7 @@ unsigned long __init setup_memory(void)propagate_e820_map_node (nid);memset (NODE_DATA (0 ),0 ,sizeof (struct pglist_data )); -NODE_DATA (0 )->bdata = &node0_bdata; +NODE_DATA (0 )->bdata = &bootmem_node_data[0 ];setup_bootmem_allocator ();return max_low_pfn; }
arch/x86/mm/numa_64.c +1 -5
@@ -27,8 +27,6 @@struct pglist_data *node_data[MAX_NUMNODES ]__read_mostly ;EXPORT_SYMBOL (node_data); -bootmem_data_t plat_node_bdata[MAX_NUMNODES ]; -struct memnode memnode;#ifdef CONFIG_SMP
@@ -215,7 +213,7 @@ void __init setup_node_bootmem(int nodei nodedata_phys + pgdat_size - 1);memset (NODE_DATA (nodeid),0 ,sizeof (pg_data_t)); -NODE_DATA (nodeid)->bdata = &plat_node_bdata[nodeid]; +NODE_DATA (nodeid)->bdata = &bootmem_node_data[nodeid];NODE_DATA (nodeid)->node_start_pfn = start_pfn;NODE_DATA (nodeid)->node_spanned_pages = end_pfn - start_pfn;
@@ -671,5 +669,3 @@ void __init init_cpu_to_node(void)numa_set_node (i, node); } } - -
include/linux/bootmem.h +2 -0
@@ -38,6 +38,8 @@ typedef struct bootmem_data {struct list_head list; } bootmem_data_t; +extern bootmem_data_t bootmem_node_data[]; +extern unsigned long bootmem_bootmap_pages (unsigned long );extern unsigned long init_bootmem (unsigned long addr,unsigned long memend);extern void free_bootmem (unsigned long addr,unsigned long size);
mm/bootmem.c +2 -0
@@ -36,6 +36,8 @@ static LIST_HEAD(bdata_list);unsigned long saved_max_pfn;#endif +bootmem_data_t bootmem_node_data[MAX_NUMNODES ]__initdata ; +/* return the number of _pages_ that will be allocated for the boot bitmap */ unsigned long __init bootmem_bootmap_pages (unsigned long pages) {
mm/page_alloc.c +1 -3
@@ -3992,9 +3992,7 @@ void __init set_dma_reserve(unsigned lon }#ifndef CONFIG_NEED_MULTIPLE_NODES -static bootmem_data_t contig_bootmem_data; -struct pglist_data contig_page_data = { .bdata = &contig_bootmem_data }; - +struct pglist_data contig_page_data = { .bdata = &bootmem_node_data[0 ] };EXPORT_SYMBOL (contig_page_data);#endif
RFC 2/3 mm: Reimplement bootmem
In reply to Johannes WeinerThis is a complete overhaul of the bootmem allocator.
alloc_bootmem() will more likely respect the requested goal as the old logic tried allocation on the first node in the system and dropped the goal immediately if it not happened to reside on that node. Now we try the first allocation on the node holding the goal and then fall back.
free_bootmem() and reserve_bootmem() have become more strict. Users have to make sure that the PFNS of the range are contiguous.
__alloc_bootmem_node_low() now also falls back to other nodes as __alloc_bootmem_node() already does.
Otherwise, massive code cleanup and documentation on all public interfaces.
---
include/linux/bootmem.h | 112 ++--- mm/bootmem.c | 925 ++++++++++++++++++++++++------------------------ 2 files changed, 511 insertions(+), 526 deletions(-)
mm/bootmem.c +471 -454
@@ -1,12 +1,11 @@/* -* linux/mm/bootmem.c +* bootmem - Physical boot-time memory allocator and configurator * -* Copyright (C) 1999 Ingo Molnar -* Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 +* Copyright (C) 1999 Ingo Molnar +* 2008 Johannes Weiner * -* simple boot-time physical memory area allocator and -* free memory collector. It's used to deal with reserved -* system memory and memory holes as well. +* Access to this subsystem has to be serialized externally (which is +* true for the boot process anyway). */ #include <linux/init.h> #include <linux/pfn.h>
@@ -19,15 +18,10 @@#include "internal.h" -/* -* Access to this subsystem has to be serialized externally. (this is -* true for the boot process anyway) -*/ unsigned long max_low_pfn;unsigned long min_low_pfn;unsigned long max_pfn; -static LIST_HEAD (bdata_list);#ifdef CONFIG_CRASH_DUMP /* * If we have booted due to a crash, max_pfn will be a very low value. We need
@@ -38,574 +32,597 @@ unsigned long saved_max_pfn;bootmem_data_t bootmem_node_data[MAX_NUMNODES ]__initdata ; -/* return the number of _pages_ that will be allocated for the boot bitmap */ -unsigned long __init bootmem_bootmap_pages (unsigned long pages) -{ -unsigned long mapsize; +static struct list_head bdata_list__initdata =LIST_HEAD_INIT (bdata_list); - mapsize = (pages+7 )/8 ; - mapsize = (mapsize + ~PAGE_MASK ) &PAGE_MASK ; - mapsize >>=PAGE_SHIFT ; +static unsigned long __init bootmem_bootmap_bytes (unsigned long pages) +{ +unsigned long bytes = (pages +7 ) /8 ; -return mapsize; +/* +* Wordsize aligned, see __free_all_bootmem_node +* for the reason. +*/ +return ALIGN (bytes,sizeof (long )); }/* -* link bdata in order +* bootmem_bootmap_pages - bitmap size in pages +* @pages: number of pages the bitmap must represent */ -static void __init link_bootmem (bootmem_data_t *bdata) +unsigned long __init bootmem_bootmap_pages (unsigned long pages) { -bootmem_data_t *ent; +unsigned long bytes =bootmem_bootmap_bytes (pages); -if (list_empty (&bdata_list)) { -list_add (&bdata->list, &bdata_list); -return ; - } -/* insert in order */ -list_for_each_entry (ent, &bdata_list, list) { -if (bdata->node_boot_start < ent->node_boot_start) { -list_add_tail (&bdata->list, &ent->list); -return ; - } - } -list_add_tail (&bdata->list, &bdata_list); +return PAGE_ALIGN (bytes) /PAGE_SIZE ; } -/* -* Given an initialised bdata, it returns the size of the boot bitmap -*/ -static unsigned long __init get_mapsize (bootmem_data_t *bdata) +static void link_bdata (bootmem_data_t *bdata) { -unsigned long mapsize; -unsigned long start =PFN_DOWN (bdata->node_boot_start); -unsigned long end = bdata->node_low_pfn; +struct list_head *iter; + +list_for_each (iter, &bdata_list) { +bootmem_data_t *entry =list_entry (iter, bootmem_data_t, list); + +if (bdata->node_min_pfn < entry->node_min_pfn) +break ; + } - mapsize = ((end - start) +7 ) /8 ; -return ALIGN (mapsize,sizeof (long )); +list_add_tail (&bdata->list, iter); } -/* -* Called once to set up the allocator itself. -*/ -static unsigned long __init init_bootmem_core (pg_data_t *pgdat, -unsigned long mapstart,unsigned long start,unsigned long end) +static unsigned long __init __init_bootmem_node (int nid,unsigned long mappfn, +unsigned long startpfn, +unsigned long endpfn) { -bootmem_data_t *bdata = pgdat->bdata; -unsigned long mapsize; +unsigned long bytes; +bootmem_data_t *bdata = &bootmem_node_data[nid]; - bdata->node_bootmem_map =phys_to_virt (PFN_PHYS (mapstart)); - bdata->node_boot_start =PFN_PHYS (start); - bdata->node_low_pfn = end; -link_bootmem (bdata); + bdata->node_bootmem_map =phys_to_virt (mappfn <<PAGE_SHIFT ); + bdata->node_min_pfn = startpfn; + bdata->node_low_pfn = endpfn; -/* -* Initially all pages are reserved - setup_arch() has to -* register free RAM areas explicitly. -*/ - mapsize =get_mapsize (bdata); -memset (bdata->node_bootmem_map,0xff , mapsize); +/* XXX: To be dropped */ + bdata->node_boot_start = startpfn <<PAGE_SHIFT ; -return mapsize; + bytes =bootmem_bootmap_bytes (endpfn - startpfn); +memset (bdata->node_bootmem_map,0xff , bytes); +link_bdata (bdata); + +return bytes; }/* -* Marks a particular physical memory range as unallocatable. Usable RAM -* might be used for boot-time allocations - or it might get added -* to the free page pool later on. +* init_bootmem_node - Register node as boot memory +* @pgdat: node to register +* @mappfn: PFN where map can be placed +* @startpfn: first PFN on the node +* @endpfn: first PFN after the node +* +* Returns the number of bytes needed to hold the node's page bitmap. */ -static int __init can_reserve_bootmem_core (bootmem_data_t *bdata, -unsigned long addr,unsigned long size,int flags) +unsigned long __init init_bootmem_node (pg_data_t *pgdat,unsigned long mappfn, +unsigned long startpfn,unsigned long endpfn) { -unsigned long sidx, eidx; -unsigned long i; +return __init_bootmem_node (pgdat->node_id, mappfn, startpfn, endpfn); +} + + +/* +* init_bootmem - Register boot memory on UMA configurations +* @lowpfn: first _usable_ PFN +* @pages: number of pages to register +* +* Operates on the first/only node in the system. +* +* Returns the number of bytes needed to hold the page bitmap. +*/ +unsigned long __init init_bootmem (unsigned long lowpfn,unsigned long pages) +{ + max_low_pfn = pages; + min_low_pfn = lowpfn; +return __init_bootmem_node (0 , lowpfn,0 , pages); +} -BUG_ON (!size); +static unsigned long __init __free_all_bootmem_node (int nid) +{ +int aligned; +struct page *page; +unsigned long start, end, pages, count =0 ; +bootmem_data_t *bdata = &bootmem_node_data[nid]; -/* out of range, don't hold other */ -if (addr + size < bdata->node_boot_start || -PFN_DOWN (addr) > bdata->node_low_pfn) +if (!bdata->node_bootmem_map)return 0 ; + start = bdata->node_min_pfn; + end = bdata->node_low_pfn; +/* -* Round up to index to the range. +* If the start is aligned to the machine's wordsize, we might +* be able to free pages in bulks of that order. */ -if (addr > bdata->node_boot_start) - sidx=PFN_DOWN (addr - bdata->node_boot_start); -else - sidx =0 ; + aligned = !(start & (BITS_PER_LONG -1 )); - eidx =PFN_UP (addr + size - bdata->node_boot_start); -if (eidx > bdata->node_low_pfn -PFN_DOWN (bdata->node_boot_start)) - eidx = bdata->node_low_pfn -PFN_DOWN (bdata->node_boot_start); - -for (i = sidx; i < eidx; i++) { -if (test_bit (i, bdata->node_bootmem_map)) { -if (flags &BOOTMEM_EXCLUSIVE ) -return -EBUSY ; +while (start < end) { +unsigned long *map, idx, vec; + + map = bdata->node_bootmem_map; + idx = (start - bdata->node_min_pfn) /BITS_PER_LONG ; + vec = ~map[idx]; + +/* Free wordsize order blocks at once */ +if (aligned && vec == ~0UL && start +BITS_PER_LONG < end) { +int order =ilog2 (BITS_PER_LONG ); + +__free_pages_bootmem (pfn_to_page (start), order); + count +=BITS_PER_LONG ; + +/* Free the rest of the pages in this vector one by one. */ + }else { +unsigned long off =0 ; + +while (vec && off <BITS_PER_LONG ) { +if (vec &1 ) { + page =pfn_to_page (start + off); +__free_pages_bootmem (page,0 ); + count++; + } + vec >>=1 ; + off++; + } } + start +=BITS_PER_LONG ; } -return 0 ; +/* Bitmap can go now, too */ + page =virt_to_page (bdata->node_bootmem_map); + pages = bdata->node_low_pfn - bdata->node_min_pfn; + pages =bootmem_bootmap_pages (pages); + count += pages; +while (pages--) +__free_pages_bootmem (page++,0 ); + +return count; } -static void __init reserve_bootmem_core (bootmem_data_t *bdata, -unsigned long addr,unsigned long size,int flags) +/* +* free_all_bootmem_node - Release node's free memory to the buddy allocator +* @pgdat: node to release +* +* Returns the number of released pages. +*/ +unsigned long __init free_all_bootmem_node (pg_data_t *pgdat) { -unsigned long sidx, eidx; -unsigned long i; - -BUG_ON (!size); +register_page_bootmem_info_node (pgdat); +return __free_all_bootmem_node (pgdat->node_id); +} -/* out of range */ -if (addr + size < bdata->node_boot_start || -PFN_DOWN (addr) > bdata->node_low_pfn) -return ; +/* +* free_all_bootmem - Release free memory to the buddy allocator +* +* Operates on the first/only node in the system. +* Returns the number of released pages. +*/ +unsigned long __init free_all_bootmem (void ) +{ +return free_all_bootmem_node (NODE_DATA (0 )); +} -/* -* Round up to index to the range. -*/ -if (addr > bdata->node_boot_start) - sidx=PFN_DOWN (addr - bdata->node_boot_start); -else - sidx =0 ; +static void __init __free (void *map,unsigned long start,unsigned long end) +{ +unsigned long idx; - eidx =PFN_UP (addr + size - bdata->node_boot_start); -if (eidx > bdata->node_low_pfn -PFN_DOWN (bdata->node_boot_start)) - eidx = bdata->node_low_pfn -PFN_DOWN (bdata->node_boot_start); - -for (i = sidx; i < eidx; i++) { -if (test_and_set_bit (i, bdata->node_bootmem_map)) { -#ifdef CONFIG_DEBUG_BOOTMEM -printk ("hm, page %08lx reserved twice.\n" , i*PAGE_SIZE ); -#endif - } - } +for (idx = start; idx < end; idx++) +if (!test_and_clear_bit (idx, map)) +BUG (); } -static void __init free_bootmem_core (bootmem_data_t *bdata,unsigned long addr, -unsigned long size) +static int __init __reserve (void *map,unsigned long start, +unsigned long end,int flags) { -unsigned long sidx, eidx; -unsigned long i; +unsigned long idx; +int exclusive = flags &BOOTMEM_EXCLUSIVE ; -BUG_ON (!size); +for (idx = start; idx < end; idx++) +if (test_and_set_bit (idx, map)) +if (exclusive) { +__free (map, start, idx); +return -EBUSY ; + } +return 0 ; +} + +static int __init mark_bootmem (unsigned long start,unsigned long end, +int reserve,int flags) +{ +unsigned long pos; +bootmem_data_t *bdata; -/* out range */ -if (addr + size < bdata->node_boot_start || -PFN_DOWN (addr) > bdata->node_low_pfn) -return ;/* -* round down end of usable mem, partially free pages are -* considered reserved. +* If nodes span other nodes, the arch code must +* specify explicit nodes. If we mark one PFN free +* on two nodes, chaos and confusion will break lose. */ -if (addr >= bdata->node_boot_start && addr < bdata->last_success) - bdata->last_success = addr; + pos = start; +list_for_each_entry (bdata, &bdata_list, list) { +unsigned long sidx, eidx, max; -/* -* Round up to index to the range. -*/ -if (PFN_UP (addr) >PFN_DOWN (bdata->node_boot_start)) - sidx =PFN_UP (addr) -PFN_DOWN (bdata->node_boot_start); -else - sidx =0 ; +if (pos < bdata->node_min_pfn) { +/* Range spans non-contiguous nodes? */ +BUG_ON (pos != start); +continue ; + } - eidx =PFN_DOWN (addr + size - bdata->node_boot_start); -if (eidx > bdata->node_low_pfn -PFN_DOWN (bdata->node_boot_start)) - eidx = bdata->node_low_pfn -PFN_DOWN (bdata->node_boot_start); + max =min (bdata->node_low_pfn, end); + sidx = pos - bdata->node_min_pfn; + eidx = max - bdata->node_min_pfn; + +if (reserve) { +int err; + + err =__reserve (bdata->node_bootmem_map, + sidx, eidx, flags); +if (err) { +/* Unroll the reservation again */ +mark_bootmem (start, pos,0 ,0 ); +return err; + } + }else +__free (bdata->node_bootmem_map, sidx, eidx); -for (i = sidx; i < eidx; i++) { -if (unlikely (!test_and_clear_bit (i, bdata->node_bootmem_map))) -BUG (); +if (max == end) +return 0 ; + pos = bdata->node_low_pfn; } +BUG (); }/* -* We 'merge' subsequent allocations to save space. We might 'lose' -* some fraction of a page if allocations cannot be satisfied due to -* size constraints on boxes where there is physical RAM space -* fragmentation - in these cases (mostly large memory boxes) this -* is not a problem. -* -* On low memory boxes we get it right in 100% of the cases. -* -* alignment has to be a power of 2 value. -* -* NOTE: This function is _not_ reentrant. -*/ -void *__init -__alloc_bootmem_core (struct bootmem_data *bdata,unsigned long size, -unsigned long align,unsigned long goal,unsigned long limit) -{ -unsigned long areasize, preferred; -unsigned long i, start =0 , incr, eidx, end_pfn; -void *ret; -unsigned long node_boot_start; -void *node_bootmem_map; +* free_bootmem_node - Mark page range as usable +* @pgdat: node the range resides on +* @addr: starting address of the range +* @size: size of the range in bytes +*/ +void __init free_bootmem_node (pg_data_t *pgdat,unsigned long addr, +unsigned long size) +{ +unsigned long sidx, eidx; +bootmem_data_t *bdata = &bootmem_node_data[pgdat->node_id]; -if (!size) { -printk ("__alloc_bootmem_core(): zero-sized request\n" ); -BUG (); - } -BUG_ON (align & (align-1 )); +/* Partially free pages are considered reserved */ + sidx =PFN_UP (addr) - bdata->node_min_pfn; + eidx =PFN_DOWN (addr + size) - bdata->node_min_pfn; -/* on nodes without memory - bootmem_map is NULL */ -if (!bdata->node_bootmem_map) -return NULL ; +__free (bdata->node_bootmem_map, sidx, eidx); +} -/* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */ - node_boot_start = bdata->node_boot_start; - node_bootmem_map = bdata->node_bootmem_map; -if (align) { - node_boot_start =ALIGN (bdata->node_boot_start, align); -if (node_boot_start > bdata->node_boot_start) - node_bootmem_map = (unsigned long *)bdata->node_bootmem_map + -PFN_DOWN (node_boot_start - bdata->node_boot_start)/BITS_PER_LONG ; - } +/* +* free_bootmem - Mark page range as usable +* @addr: starting address of the range +* @size: size of the range in bytes +* +* The specified range must be contigous and can even span subsequent nodes. +* +* Usage on discontiguous area is a BUG! +*/ +void __init free_bootmem (unsigned long addr,unsigned long size) +{ +unsigned long start, end; -if (limit && node_boot_start >= limit) -return NULL ; +/* Partially free pages are considered reserved */ + start =PFN_UP (addr); + end =PFN_DOWN (addr + size); - end_pfn = bdata->node_low_pfn; - limit =PFN_DOWN (limit); -if (limit && end_pfn > limit) - end_pfn = limit; +mark_bootmem (start, end,0 ,0 ); +} - eidx = end_pfn -PFN_DOWN (node_boot_start); +/* +* reserve_bootmem_node - Mark page range as reserved +* @pgdat: node the range resides on +* @addr: starting address of the range +* @size: size of the range in bytes +* @flags: reservation flags +* +* If flags contains BOOTMEM_EXCLUSIVE, the function will return +* -EBUSY if the range is already reserved. +*/ +int __init reserve_bootmem_node (pg_data_t *pgdat,unsigned long addr, +unsigned long size,int flags) +{ +unsigned long sidx, eidx; +bootmem_data_t *bdata = &bootmem_node_data[pgdat->node_id]; -/* -* We try to allocate bootmem pages above 'goal' -* first, then we try to allocate lower pages. -*/ - preferred =0 ; -if (goal &&PFN_DOWN (goal) < end_pfn) { -if (goal > node_boot_start) - preferred = goal - node_boot_start; - -if (bdata->last_success > node_boot_start && - bdata->last_success - node_boot_start >= preferred) -if (!limit || (limit && limit > bdata->last_success)) - preferred = bdata->last_success - node_boot_start; - } +/* Partially reserved pages are considered reserved */ + sidx =PFN_DOWN (addr) - bdata->node_min_pfn; + eidx =PFN_UP (addr + size) - bdata->node_min_pfn; - preferred =PFN_DOWN (ALIGN (preferred, align)); - areasize = (size +PAGE_SIZE -1 ) /PAGE_SIZE ; - incr = align >>PAGE_SHIFT ? :1 ; - -restart_scan: -for (i = preferred; i < eidx;) { -unsigned long j; - - i =find_next_zero_bit (node_bootmem_map, eidx, i); - i =ALIGN (i, incr); -if (i >= eidx) -break ; -if (test_bit (i, node_bootmem_map)) { - i += incr; -continue ; - } -for (j = i +1 ; j < i + areasize; ++j) { -if (j >= eidx) - goto fail_block; -if (test_bit (j, node_bootmem_map)) - goto fail_block; - } - start = i; - goto found; - fail_block: - i =ALIGN (j, incr); -if (i == j) - i += incr; - } +return __reserve (bdata->node_bootmem_map, sidx, eidx, flags); +} -if (preferred >0 ) { - preferred =0 ; - goto restart_scan; - } -return NULL ; +/* +* reserve_bootmem - Mark page range as reserved +* @addr: starting address of the range +* @size: size of the range in bytes +* @flags: reservation flags +* +* The specified range must be contiguous and can even span subsequent nodes. +* +* Usage on discontiguous area is a BUG! +* +* If flags contains BOOTMEM_EXCLUSIVE, the function will return +* -EBUSY if the range is already reserved. +*/ +#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE +int __init reserve_bootmem (unsigned long addr,unsigned long size,int flags) +{ +unsigned long start, end; -found: - bdata->last_success =PFN_PHYS (start) + node_boot_start; -BUG_ON (start >= eidx); +/* Partially reserved pages are considered reserved */ + start =PFN_DOWN (addr); + end =PFN_UP (addr + size); -/* -* Is the next page of the previous allocation-end the start -* of this allocation's buffer? If yes then we can 'merge' -* the previous partial page with this allocation. -*/ -if (align <PAGE_SIZE && - bdata->last_offset && bdata->last_pos+1 == start) { -unsigned long offset, remaining_size; - offset =ALIGN (bdata->last_offset, align); -BUG_ON (offset >PAGE_SIZE ); - remaining_size =PAGE_SIZE - offset; -if (size < remaining_size) { - areasize =0 ; -/* last_pos unchanged */ - bdata->last_offset = offset + size; - ret =phys_to_virt (bdata->last_pos *PAGE_SIZE + - offset + node_boot_start); - }else { - remaining_size = size - remaining_size; - areasize = (remaining_size +PAGE_SIZE -1 ) /PAGE_SIZE ; - ret =phys_to_virt (bdata->last_pos *PAGE_SIZE + - offset + node_boot_start); - bdata->last_pos = start + areasize -1 ; - bdata->last_offset = remaining_size; - } - bdata->last_offset &= ~PAGE_MASK ; - }else { - bdata->last_pos = start + areasize -1 ; - bdata->last_offset = size & ~PAGE_MASK ; - ret =phys_to_virt (start *PAGE_SIZE + node_boot_start); - } +return mark_bootmem (start, end,1 , flags); +} +#endif -/* -* Reserve the area now: -*/ -for (i = start; i < start + areasize; i++) -if (unlikely (test_and_set_bit (i, node_bootmem_map))) -BUG (); -memset (ret,0 , size); -return ret; +static void *__init __do_alloc (bootmem_data_t *bdata,unsigned long align, +unsigned long start,unsigned long bytes) +{ +int merge; +void *region; +unsigned long new_start, new_end; + +/* Align at the previous allocation end if we are subsequent */ +if (bdata->alloc_off &&PFN_DOWN (bdata->alloc_off) +1 == start) + new_start =ALIGN (bdata->alloc_off, align); +else + new_start =PFN_PHYS (start); + + merge =PFN_DOWN (new_start) < start; + + new_end = new_start + bytes; + bdata->alloc_off = new_end; + +if (__reserve (bdata->node_bootmem_map, +PFN_DOWN (new_start) + merge, +PFN_UP (new_end), +BOOTMEM_EXCLUSIVE )) +BUG (); + + region =phys_to_virt (PFN_PHYS (bdata->node_min_pfn) + new_start); +memset (region,0 , bytes); +return region; } -static unsigned long __init free_all_bootmem_core (pg_data_t *pgdat) +static void *__init do_alloc (bootmem_data_t *bdata,unsigned long size, +unsigned long align,unsigned long goal, +unsigned long limit) { -struct page *page; -unsigned long pfn; -bootmem_data_t *bdata = pgdat->bdata; -unsigned long i, count, total =0 ; -unsigned long idx; -unsigned long *map; -int gofast =0 ; +unsigned long min, max, start, step; -BUG_ON (!bdata->node_bootmem_map); +if (!bdata->node_bootmem_map) +return NULL ; - count =0 ; -/* first extant page of the node */ - pfn =PFN_DOWN (bdata->node_boot_start); - idx = bdata->node_low_pfn - pfn; - map = bdata->node_bootmem_map; -/* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */ -if (bdata->node_boot_start ==0 || -ffs (bdata->node_boot_start) -PAGE_SHIFT >ffs (BITS_PER_LONG )) - gofast =1 ; -for (i =0 ; i < idx; ) { -unsigned long v = ~map[i /BITS_PER_LONG ]; +BUG_ON (align & (align -1 )); +BUG_ON (limit && goal + size > limit); -if (gofast && v == ~0UL ) { -int order; + min = bdata->node_min_pfn; + max = bdata->node_low_pfn; - page =pfn_to_page (pfn); - count +=BITS_PER_LONG ; - order =ffs (BITS_PER_LONG ) -1 ; -__free_pages_bootmem (page, order); - i +=BITS_PER_LONG ; - page +=BITS_PER_LONG ; - }else if (v) { -unsigned long m; - - page =pfn_to_page (pfn); -for (m =1 ; m && i < idx; m<<=1 , page++, i++) { -if (v & m) { - count++; -__free_pages_bootmem (page,0 ); - } - } - }else { - i +=BITS_PER_LONG ; - } - pfn +=BITS_PER_LONG ; - } - total += count; + goal >>=PAGE_SHIFT ; + limit >>=PAGE_SHIFT ; + +if (limit && max > limit) + max = limit; +if (max <= min) +return NULL ;/* -* Now free the allocator bitmap itself, it's not -* needed anymore: +* For the block search, alignment does only matter +* if it is bigger than the page size. */ - page =virt_to_page (bdata->node_bootmem_map); - count =0 ; - idx = (get_mapsize (bdata) +PAGE_SIZE -1 ) >>PAGE_SHIFT ; -for (i =0 ; i < idx; i++, page++) { -__free_pages_bootmem (page,0 ); - count++; - } - total += count; - bdata->node_bootmem_map =NULL ; + step =max (align >>PAGE_SHIFT ,1UL ); -return total; -} +if (goal && goal < max) + start =ALIGN (goal, step); +else + start =ALIGN (min, step); -unsigned long __init init_bootmem_node (pg_data_t *pgdat,unsigned long freepfn, -unsigned long startpfn,unsigned long endpfn) -{ -return init_bootmem_core (pgdat, freepfn, startpfn, endpfn); -} +/* From here on, all numbers are relative to the node */ + max -= bdata->node_min_pfn; + start -= bdata->node_min_pfn; -void __init reserve_bootmem_node (pg_data_t *pgdat,unsigned long physaddr, -unsigned long size,int flags) -{ -int ret; +while (1 ) { +unsigned long end, idx; - ret =can_reserve_bootmem_core (pgdat->bdata, physaddr, size, flags); -if (ret <0 ) -return ; -reserve_bootmem_core (pgdat->bdata, physaddr, size, flags); -} + start =find_next_zero_bit (bdata->node_bootmem_map, max, start); + start =ALIGN (start, step); + end = start +PFN_UP (size); -void __init free_bootmem_node (pg_data_t *pgdat,unsigned long physaddr, -unsigned long size) -{ -free_bootmem_core (pgdat->bdata, physaddr, size); -} +if (start >= max || end > max) +break ; -unsigned long __init free_all_bootmem_node (pg_data_t *pgdat) -{ -register_page_bootmem_info_node (pgdat); -return free_all_bootmem_core (pgdat); -} +/* +* XXX: Might require one page more than +* we actually use later due to merging... +*/ +for (idx = start; idx < end; idx++) +if (test_bit (idx, bdata->node_bootmem_map)) { +/* No luck, search on */ + start =ALIGN (idx, step); +if (start == idx) + start += step; +continue ; + } -unsigned long __init init_bootmem (unsigned long start,unsigned long pages) -{ - max_low_pfn = pages; - min_low_pfn = start; -return init_bootmem_core (NODE_DATA (0 ), start,0 , pages); +return __do_alloc (bdata, align, start, size); + } + +return NULL ; } -#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE -int __init reserve_bootmem (unsigned long addr,unsigned long size, -int flags) +static void *__init ___alloc_bootmem_nopanic (unsigned long size, +unsigned long align, +unsigned long goal, +unsigned long limit) {bootmem_data_t *bdata; -int ret; +restart:list_for_each_entry (bdata, &bdata_list, list) { - ret =can_reserve_bootmem_core (bdata, addr, size, flags); -if (ret <0 ) -return ret; +void *region; + +if (goal &&PFN_DOWN (goal) < bdata->node_min_pfn) +continue ; +if (limit &&PFN_DOWN (limit) < bdata->node_min_pfn) +continue ; + + region =do_alloc (bdata, size, align, goal, limit); +if (region) +return region; + } +if (goal) { + goal =0 ; + goto restart; } -list_for_each_entry (bdata, &bdata_list, list) -reserve_bootmem_core (bdata, addr, size, flags); -return 0 ; +return NULL ; } -#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ -void __init free_bootmem (unsigned long addr,unsigned long size) +/* +* __alloc_bootmem_nopanic - Allocate boot memory without panicking +* @size: size of the region +* @align: alignment of the region +* @goal: preferred starting address of the region +* +* The goal is dropped if it can not be satisfied. +* The allocation can be on any node. +*/ +void *__init __alloc_bootmem_nopanic (unsigned long size, +unsigned long align,unsigned long goal) { -bootmem_data_t *bdata; -list_for_each_entry (bdata, &bdata_list, list) -free_bootmem_core (bdata, addr, size); +return ___alloc_bootmem_nopanic (size, align, goal,0 ); } -unsigned long __init free_all_bootmem (void ) +static void *__init ___alloc_bootmem (unsigned long size,unsigned long align, +unsigned long goal,unsigned long limit) { -return free_all_bootmem_core (NODE_DATA (0 )); -} +void *region; -void *__init __alloc_bootmem_nopanic (unsigned long size,unsigned long align, -unsigned long goal) -{ -bootmem_data_t *bdata; -void *ptr; + region =___alloc_bootmem_nopanic (size, align, goal, limit); +if (region) +return region; -list_for_each_entry (bdata, &bdata_list, list) { - ptr =__alloc_bootmem_core (bdata, size, align, goal,0 ); -if (ptr) -return ptr; - } +printk (KERN_ALERT "bootmem alloc of %lu bytes failed!\n" , size); +panic ("Out of memory" );return NULL ; } +/* +* __alloc_bootmem - Allocate boot memory +* @size: size of the region +* @align: alignment of the region +* @goal: preferred starting address of the region +* +* The goal is dropped if it can not be satisfied. +* The allocation can be on any node. +* The function panics when allocation is impossible. +*/ void *__init __alloc_bootmem (unsigned long size,unsigned long align, -unsigned long goal) +unsigned long goal) { -void *mem =__alloc_bootmem_nopanic (size,align,goal); - -if (mem) -return mem; -/* -* Whoops, we cannot satisfy the allocation request. -*/ -printk (KERN_ALERT "bootmem alloc of %lu bytes failed!\n" , size); -panic ("Out of memory" ); -return NULL ; +return ___alloc_bootmem (size, align, goal,0 ); } - -void *__init __alloc_bootmem_node (pg_data_t *pgdat,unsigned long size, -unsigned long align,unsigned long goal) +static void *__init ___alloc_bootmem_node (pg_data_t *pgdat,unsigned long size, +unsigned long align,unsigned long goal, +unsigned long limit) { -void *ptr; +void *region; +bootmem_data_t *bdata = &bootmem_node_data[pgdat->node_id]; - ptr =__alloc_bootmem_core (pgdat->bdata, size, align, goal,0 ); -if (ptr) -return ptr; + region =do_alloc (bdata, size, align, goal, limit); +if (region) +return region; -return __alloc_bootmem (size, align, goal); +/* No success on this node, drop the goal too */ +return ___alloc_bootmem (size, align,0 , limit); } -#ifdef CONFIG_SPARSEMEM -void *__init alloc_bootmem_section (unsigned long size, -unsigned long section_nr) +/* +* __alloc_bootmem_node - Allocate boot memory +* @pgdat: node to allocate from +* @size: size of the region +* @align: alignment of the region +* @goal: preferred starting address of the region +* +* The goal is dropped if it can not be satisfied. +* The allocation can fall back to other nodes. +* The function panics when allocation is impossible. +*/ +void *__init __alloc_bootmem_node (pg_data_t *pgdat,unsigned long size, +unsigned long align,unsigned long goal) { -void *ptr; -unsigned long limit, goal, start_nr, end_nr, pfn; -struct pglist_data *pgdat; - - pfn =section_nr_to_pfn (section_nr); - goal =PFN_PHYS (pfn); - limit =PFN_PHYS (section_nr_to_pfn (section_nr +1 )) -1 ; - pgdat =NODE_DATA (early_pfn_to_nid (pfn)); - ptr =__alloc_bootmem_core (pgdat->bdata, size,SMP_CACHE_BYTES , goal, - limit); - -if (!ptr) -return NULL ; - - start_nr =pfn_to_section_nr (PFN_DOWN (__pa (ptr))); - end_nr =pfn_to_section_nr (PFN_DOWN (__pa (ptr) + size)); -if (start_nr != section_nr || end_nr != section_nr) { -printk (KERN_WARNING "alloc_bootmem failed on section %ld.\n" , - section_nr); -free_bootmem_core (pgdat->bdata,__pa (ptr), size); - ptr =NULL ; - } - -return ptr; +return ___alloc_bootmem_node (pgdat, size, align, goal,0 ); } -#endif #ifndef ARCH_LOW_ADDRESS_LIMIT #define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL#endif +/* +* __alloc_bootmem_low - Allocate low boot memory +* @size: size of the region +* @align: alignment of the region +* @goal: preferred starting address of the region +* +* The goal is dropped if it can not be satisfied. +* The allocation can be on any node. +* The function panics when allocation is impossible. +*/ +void *__init __alloc_bootmem_low (unsigned long size,unsigned long align,unsigned long goal) { -bootmem_data_t *bdata; -void *ptr; - -list_for_each_entry (bdata, &bdata_list, list) { - ptr =__alloc_bootmem_core (bdata, size, align, goal, -ARCH_LOW_ADDRESS_LIMIT ); -if (ptr) -return ptr; - } - -/* -* Whoops, we cannot satisfy the allocation request. -*/ -printk (KERN_ALERT "low bootmem alloc of %lu bytes failed!\n" , size); -panic ("Out of low memory" ); -return NULL ; +return ___alloc_bootmem (size, align, goal,ARCH_LOW_ADDRESS_LIMIT ); } +/* +* __alloc_bootmem_low_node - Allocate low boot memory +* @pgdat: node to allocate from +* @size: size of the region +* @align: alignment of the region +* @goal: preferred starting address of the region +* +* The goal is dropped if it can not be satisfied. +* The allocation can fall back on other nodes. +* The function panics when allocation is impossible. +*/ void *__init __alloc_bootmem_low_node (pg_data_t *pgdat,unsigned long size,unsigned long align,unsigned long goal) { -return __alloc_bootmem_core (pgdat->bdata, size, align, goal, -ARCH_LOW_ADDRESS_LIMIT ); +return ___alloc_bootmem_node (pgdat, size, align, + goal,ARCH_LOW_ADDRESS_LIMIT ); +} + +/* +* alloc_bootmem_section - Allocate memory from a section +* @size: size of memory region +* @section_nr: section where the region resides on +*/ +#ifdef CONFIG_SPARSEMEM +void *__init alloc_bootmem_section (unsigned long size, +unsigned long section_nr) +{ +bootmem_data_t *bdata; +unsigned long pfn, goal, limit; + + pfn =section_nr_to_pfn (section_nr); + goal =PFN_PHYS (pfn); + limit =PFN_PHYS (section_nr_to_pfn (section_nr +1 )); + bdata = &bootmem_node_data[early_pfn_to_nid (pfn)]; + +return do_alloc (bdata, size,SMP_CACHE_BYTES , goal, limit); } +#endif
include/linux/bootmem.h +40 -72
163 lines not on this page · raw message
RFC 3/3 mm: Remove node_boot_start from bootmem_data_t
In reply to Johannes WeinerDrop node_boot_start and convert users to node_min_pfn which is what they really mean.
---
arch/alpha/mm/numa.c | 2 +- arch/arm/plat-omap/fb.c | 4 +--- arch/avr32/mm/init.c | 3 +-- arch/ia64/mm/discontig.c | 19 ++++++++++--------- arch/m32r/mm/init.c | 4 +--- arch/mn10300/mm/init.c | 6 +++--- arch/sh/mm/init.c | 2 +- include/linux/bootmem.h | 1 - mm/bootmem.c | 3 --- 9 files changed, 18 insertions(+), 26 deletions(-)
arch/alpha/mm/numa.c +1 -1
@@ -304,7 +304,7 @@ void __init paging_init(void)for_each_online_node (nid) {bootmem_data_t *bdata = &bootmem_node_data[nid]; -unsigned long start_pfn = bdata->node_boot_start >>PAGE_SHIFT ; +unsigned long start_pfn = bdata->node_min_pfn;unsigned long end_pfn = bdata->node_low_pfn;if (dma_local_pfn >= end_pfn - start_pfn)
arch/arm/plat-omap/fb.c +1 -3
@@ -182,7 +182,7 @@ void __init omapfb_reserve_sdram(void)return ; bdata =NODE_DATA (0 )->bdata; - sdram_start = bdata->node_boot_start; + sdram_start = bdata->node_min_pfn <<PAGE_SHIFT ; sdram_size = (bdata->node_low_pfn <<PAGE_SHIFT ) - sdram_start; reserved =0 ;for (i =0 ; ; i++) {
@@ -340,5 +340,3 @@ unsigned long omapfb_reserve_sram(unsign#endif - -
arch/avr32/mm/init.c +1 -2
@@ -125,8 +125,7 @@ void __init paging_init(void)unsigned long zones_size[MAX_NR_ZONES ];unsigned long low, start_pfn; - start_pfn = pgdat->bdata->node_boot_start; - start_pfn >>=PAGE_SHIFT ; + start_pfn = pgdat->bdata->node_min_pfn; low = pgdat->bdata->node_low_pfn;memset (zones_size,0 ,sizeof (zones_size));
arch/ia64/mm/discontig.c +10 -9
@@ -74,17 +74,17 @@ pg_data_t *pgdat_list[MAX_NUMNODES];static int __init build_node_maps (unsigned long start,unsigned long len,int node) { -unsigned long cstart, epfn, end = start + len; +unsigned long spfn, epfn, end = start + len;struct bootmem_data *bdp = &bootmem_node_data[node]; epfn =GRANULEROUNDUP (end) >>PAGE_SHIFT ; - cstart =GRANULEROUNDDOWN (start); + spfn =GRANULEROUNDDOWN (start) >>PAGE_SHIFT ;if (!bdp->node_low_pfn) { - bdp->node_boot_start = cstart; + bdp->node_min_pfn = spfn; bdp->node_low_pfn = epfn; }else { - bdp->node_boot_start =min (cstart, bdp->node_boot_start); + bdp->node_min_pfn =min (spfn, bdp->node_min_pfn); bdp->node_low_pfn =max (epfn, bdp->node_low_pfn); }
@@ -221,20 +221,21 @@ static void __init fill_pernode(int nodestatic int __init find_pernode_space (unsigned long start,unsigned long len,int node) { -unsigned long epfn; +unsigned long spfn, epfn;unsigned long pernodesize =0 , pernode, pages, mapsize;struct bootmem_data *bdp = &bootmem_node_data[node]; + spfn = start >>PAGE_SHIFT ; epfn = (start + len) >>PAGE_SHIFT ; - pages = bdp->node_low_pfn - (bdp->node_boot_start >>PAGE_SHIFT ); + pages = bdp->node_low_pfn - bdp->node_min_pfn; mapsize =bootmem_bootmap_pages (pages) <<PAGE_SHIFT ;/* * Make sure this memory falls within this node's usable memory * since we may have thrown some away in build_maps(). */ -if (start < bdp->node_boot_start || epfn > bdp->node_low_pfn) +if (spfn < bdp->node_min_pfn || epfn > bdp->node_low_pfn)return 0 ;/* Don't setup this node's local space twice... */
@@ -296,7 +297,7 @@ static void __init reserve_pernode_space bdp = pdp->bdata;/* First the bootmem_map itself */ - pages = bdp->node_low_pfn - (bdp->node_boot_start>>PAGE_SHIFT ); + pages = bdp->node_low_pfn - bdp->node_min_pfn; size =bootmem_bootmap_pages (pages) <<PAGE_SHIFT ; base =__pa (bdp->node_bootmem_map);reserve_bootmem_node (pdp, base, size,BOOTMEM_DEFAULT );
@@ -466,7 +467,7 @@ void __init find_memory(void)init_bootmem_node (pgdat_list[node], map>>PAGE_SHIFT , - bdp->node_boot_start>>PAGE_SHIFT , + bdp->node_min_pfn, bdp->node_low_pfn); }
arch/m32r/mm/init.c +1 -3
@@ -93,8 +93,7 @@ void free_initrd_mem(unsigned long, unsi#endif /* It'd be good if these lines were in the standard header file. */ -#define START_PFN (nid) \ - (NODE_DATA(nid)->bdata->node_boot_start >> PAGE_SHIFT) +#define START_PFN (nid) (NODE_DATA(nid)->bdata->node_min_pfn)#define MAX_LOW_PFN (nid) (NODE_DATA(nid)->bdata->node_low_pfn)#ifndef CONFIG_DISCONTIGMEM
@@ -252,4 +251,3 @@ void free_initrd_mem(unsigned long start printk (KERN_INFO "Freeing initrd memory: %ldk freed\n", (end - start) >> 10); }#endif -
arch/mn10300/mm/init.c +3 -3
@@ -67,8 +67,8 @@ void __init paging_init(void)/* declare the sizes of the RAM zones (only use the normal zone) */ zones_size[ZONE_NORMAL ] = - (contig_page_data.bdata->node_low_pfn) - - (contig_page_data.bdata->node_boot_start >>PAGE_SHIFT ); + contig_page_data.bdata->node_low_pfn - + contig_page_data.bdata->node_min_pfn;/* pass the memory from the bootmem allocator to the main allocator */ free_area_init (zones_size);
@@ -87,7 +87,7 @@ void __init mem_init(void)if (!mem_map)BUG (); -#define START_PFN (contig_page_data.bdata->node_boot_start >> PAGE_SHIFT) +#define START_PFN (contig_page_data.bdata->node_min_pfn)#define MAX_LOW_PFN (contig_page_data.bdata->node_low_pfn) max_mapnr = num_physpages =MAX_LOW_PFN -START_PFN ;
arch/sh/mm/init.c +1 -1
@@ -191,7 +191,7 @@ void __init paging_init(void)pg_data_t *pgdat =NODE_DATA (nid);unsigned long low, start_pfn; - start_pfn = pgdat->bdata->node_boot_start >>PAGE_SHIFT ; + start_pfn = pgdat->bdata->node_min_pfn; low = pgdat->bdata->node_low_pfn;if (max_zone_pfns[ZONE_NORMAL ] < low)
mm/bootmem.c +0 -3
@@ -81,9 +81,6 @@ static unsigned long __init __init_bootm bdata->node_min_pfn = startpfn; bdata->node_low_pfn = endpfn; -/* XXX: To be dropped */ - bdata->node_boot_start = startpfn <<PAGE_SHIFT ; - bytes =bootmem_bootmap_bytes (endpfn - startpfn);memset (bdata->node_bootmem_map,0xff , bytes);link_bdata (bdata);
include/linux/bootmem.h +0 -1
@@ -13,7 +13,6 @@ extern unsigned long saved_max_pfn;#endif typedef struct bootmem_data { - unsigned long node_boot_start; unsigned long node_min_pfn; unsigned long node_low_pfn; void *node_bootmem_map;
Andrew Morton
maintainerQuoted 4 lines · On Wed, 21 May 2008 03:37:35 +0200 Johannes Weiner <[email protected]> wrote:
Hi,
This is a complete overhaul of the bootmem allocator while preserving its original functionality, excluding bugs.
Where angels fear to tread.
Quoted 35 lines
free_bootmemandreserve_bootmembecome a bit stricter than they are right now, callsites have to make sure that the PFN range is contiguous but it might go across node boundaries.
alloc_bootmemsatisfying the allocation goal is more likely as the routines will try to allocate on the node holding the goal first before falling back as opposed to the original behaviour that satisfies the goal only if it is on the first node.All in all, I think the code has become simpler and cleaner. All public interfaces have been documented, too.
The first patch moves the bootmem node descriptor definitions into
bootmem.cwhere they belong.The second patch is the new allocator itself.
The third patch converts all users of ->node_boot_start to ->node_min_pfn as this is what they really use. It then removes the unused ->node_boot_start.
Compile and runtime tested on
X86_32, therefor RFC only.arch/alpha/mm/numa.c | 8 +- arch/arm/mm/discontig.c | 34 +- arch/arm/plat-omap/fb.c | 4 +- arch/avr32/mm/init.c | 3 +- arch/ia64/mm/discontig.c | 30 +- arch/m32r/mm/discontig.c | 4 +- arch/m32r/mm/init.c | 4 +- arch/m68k/mm/init.c | 4 +- arch/mips/sgi-ip27/ip27-memory.c | 3 +- arch/mn10300/mm/init.c | 6 +- arch/parisc/mm/init.c | 3 +- arch/powerpc/mm/numa.c | 3 +-
Quoted 8 lines
arch/sh/mm/init.c | 2 +- arch/sh/mm/numa.c | 5 +- arch/sparc64/mm/init.c | 3 +- arch/x86/mm/discontig_32.c | 3 +- arch/x86/mm/numa_64.c | 6 +- include/linux/bootmem.h | 115 ++--- mm/bootmem.c | 914 +++++++++++++++++++------------------- mm/page_alloc.c | 4 +-
Oh gee.
bootmem is an area where large numbers of people have done hit-and-run jobs over a lot of years. Nobody owns it and I'm sure that you are now the world's expert. We just need to push ahead with this, I guess.
I expect there will be problems - so many architectures which do such different things, and all the configuration options churning things around.
So how to move ahead with this?
- I think I'd prefer not to drop
mm-fix-free_all_bootmem_core-alignment-check.patch mm-normalize-internal-argument-passing-of-bootmem-data.patch mm-unexport-__alloc_bootmem_core.patch
because those are small, simple things which are on track for 2.6.27 whereas a massive rewrite may take longer to get merged, and may never get there at all, in which case we lost those little fixes.
- It would suit my purposes to have these patches right at the tail of the -mm patch queue so that I can drop them easily if problems occur, and so that others can revert them easily when diagnosing problems.
- It would be nice to get some review attention from architecture guys, but I can understand them finding other things to do, when bootmem is presumably good-enough-for-now.
- Is x86_32 the only test platform which you have available? Awkward.
Anyway, if you can redo these patches against most-recent-mm or, better, against http://userweb.kernel.org/~akpm/mmotm then it would make things easier for me to handle. I can then at least test it all on my seven-odd test boxes. Please feel free to ping me if you want a single rolled-up patch - that's always trivial and I can do it in three minutes.
Finally, if you haven't done so, I'd encourage you to stuff as many handy debugging printks into this code as you possibly can. Just fill 'er up with them. So that when people start running it and it goes boom, they can send you their debug output _without_ having to go through another handful of email-email-patch-rebuild-retest cycles. We can pull them all out later on.
Hi Andrew,
Quoted 9 lines · Andrew Morton
Quoted 4 lines · On Wed, 21 May 2008 03:37:35 +0200 Johannes Weiner <[email protected]> wrote:
Hi,
This is a complete overhaul of the bootmem allocator while preserving its original functionality, excluding bugs.
Where angels fear to tread.
Anyone who has seen this code realizes that this kind of advertisment is essential to get people looking at patches for it ;)
Quoted 44 lines
Quoted 35 lines
free_bootmemandreserve_bootmembecome a bit stricter than they are right now, callsites have to make sure that the PFN range is contiguous but it might go across node boundaries.
alloc_bootmemsatisfying the allocation goal is more likely as the routines will try to allocate on the node holding the goal first before falling back as opposed to the original behaviour that satisfies the goal only if it is on the first node.All in all, I think the code has become simpler and cleaner. All public interfaces have been documented, too.
The first patch moves the bootmem node descriptor definitions into
bootmem.cwhere they belong.The second patch is the new allocator itself.
The third patch converts all users of ->node_boot_start to ->node_min_pfn as this is what they really use. It then removes the unused ->node_boot_start.
Compile and runtime tested on
X86_32, therefor RFC only.arch/alpha/mm/numa.c | 8 +- arch/arm/mm/discontig.c | 34 +- arch/arm/plat-omap/fb.c | 4 +- arch/avr32/mm/init.c | 3 +- arch/ia64/mm/discontig.c | 30 +- arch/m32r/mm/discontig.c | 4 +- arch/m32r/mm/init.c | 4 +- arch/m68k/mm/init.c | 4 +- arch/mips/sgi-ip27/ip27-memory.c | 3 +- arch/mn10300/mm/init.c | 6 +- arch/parisc/mm/init.c | 3 +- arch/powerpc/mm/numa.c | 3 +-Quoted 8 lines
arch/sh/mm/init.c | 2 +- arch/sh/mm/numa.c | 5 +- arch/sparc64/mm/init.c | 3 +- arch/x86/mm/discontig_32.c | 3 +- arch/x86/mm/numa_64.c | 6 +- include/linux/bootmem.h | 115 ++--- mm/bootmem.c | 914 +++++++++++++++++++------------------- mm/page_alloc.c | 4 +-
Quoted 1 line
Oh gee.
Okay, seeing it again it looks a bit brutal. But it's not. The basic principles are the same, it's not that I completely changed the implementation. Okay, perhaps I did.
And the arch changes are trivial.
Quoted 7 lines
bootmem is an area where large numbers of people have done hit-and-run jobs over a lot of years. Nobody owns it and I'm sure that you are now the world's expert. We just need to push ahead with this, I guess.
I expect there will be problems - so many architectures which do such different things, and all the configuration options churning things around.
I expect problems too. I just can not go any further with it on the resources I have.
Quoted 4 lines
So how to move ahead with this?
- I think I'd prefer not to drop
Quoted 8 lines
mm-fix-free_all_bootmem_core-alignment-check.patch mm-normalize-internal-argument-passing-of-bootmem-data.patch mm-unexport-__alloc_bootmem_core.patch
because those are small, simple things which are on track for 2.6.27 whereas a massive rewrite may take longer to get merged, and may never get there at all, in which case we lost those little fixes.
I can see that.
Quoted 4 lines
- It would suit my purposes to have these patches right at the tail of the -mm patch queue so that I can drop them easily if problems occur, and so that others can revert them easily when diagnosing problems.
Good.
Quoted 5 lines
- It would be nice to get some review attention from architecture guys, but I can understand them finding other things to do, when bootmem is presumably good-enough-for-now.
- Is
x86_32the only test platform which you have available? Awkward.
Yes, unfortunately. Hardware offers via private mail, please!
Quoted 6 lines
Anyway, if you can redo these patches against most-recent-mm or, better, against http://userweb.kernel.org/~akpm/mmotm then it would make things easier for me to handle. I can then at least test it all on my seven-odd test boxes. Please feel free to ping me if you want a single rolled-up patch - that's always trivial and I can do it in three minutes.
I guess I just apply the quilt series to the tree you forked off?
Quoted 6 lines
Finally, if you haven't done so, I'd encourage you to stuff as many handy debugging printks into this code as you possibly can. Just fill 'er up with them. So that when people start running it and it goes boom, they can send you their debug output _without_ having to go through another handful of email-email-patch-rebuild-retest cycles. We can pull them all out later on.
Okay, I will make it gossip and send you a -mmotm-based version of it.
Hannes
Andrew Morton
maintainerQuoted 8 lines · On Thu, 22 May 2008 02:33:45 +0200 Johannes Weiner <[email protected]> wrote:
Quoted 6 lines
Anyway, if you can redo these patches against most-recent-mm or, better, against http://userweb.kernel.org/~akpm/mmotm then it would make things easier for me to handle. I can then at least test it all on my seven-odd test boxes. Please feel free to ping me if you want a single rolled-up patch - that's always trivial and I can do it in three minutes.
I guess I just apply the quilt series to the tree you forked off?
yup. http://userweb.kernel.org/~akpm/mmotm/mmotm-readme.txt
Quoted 8 lines
Quoted 6 lines
Finally, if you haven't done so, I'd encourage you to stuff as many handy debugging printks into this code as you possibly can. Just fill 'er up with them. So that when people start running it and it goes boom, they can send you their debug output _without_ having to go through another handful of email-email-patch-rebuild-retest cycles. We can pull them all out later on.
Okay, I will make it gossip and send you a -mmotm-based version of it.
Good luck :)