Updates for memory dlpar and dynamic reconfiguration memory
cover Updates for memory dlpar and dynamic reconfiguration memory
The following patchset corrects a couple of issues with memory dlpar and adds support for the new memory layout in the device tree on the latest Power hardware and kernels.
On newer hardware and kernels, the device tree has changed so that all memory appears under the node ibm,dynamic-reconfiguration-memory. This node has several properties that describe lmbs, lmb-size and numa affinity.
The ibm,dynamic-memory property is a list of all possible lmbs, where each list entry contains the base address, drc index, associativity array index and a set of flags.
The ibm,associativity-lookup-arrays property contains a list of associativity arrays and the ibm,lmb-size contains the size of the lmbs
For reference, memory was previously represented with a node, of the form memory@XXXXXXXX, for each lmb assigned the system. Each of these memory nodes had properties such as ibm,my-drc-index for the drc index and regs for the base address and lmb size.
With the new device tree layout, code associated with memory dlpar needs to be updated to parse the new device tree layout and properly call the memory hotplug notifier chains. Additionally, the numa associativity roiutine called during hotplug add of memory needs a seperate version for parsing the new device tree layout.
PATCH 1/5 Allow drc indexes to be specified in formats other than decimal
Allow the drc index passed to the /proc/ppc64/ofdt file to be specified in formats other than decimal. This allows us to easily specify drc indexes in hex that would otherwise appears as negative integers.
---
arch/powerpc/platforms/pseries/reconfig.c +1 -1
@@ -365,7 +365,7 @@ *buf ='\0' ; buf++; - handle =simple_strtoul (handle_str,NULL ,10 ); + handle =simple_strtoul (handle_str,NULL ,0 ); *npp =of_find_node_by_phandle (handle);return buf;
Re: [PATCH 1/5] Allow drc indexes to be specified in formats other than decimal
Hello.
Quoted 14 lines · Nathan Fontenot
Allow the drc index passed to the
/proc/ppc64/ofdtfile to be specified in formats other than decimal. This allows us to easily specify drc indexes in hex that would otherwise appears as negative integers.Signed-off-by Nathan Fontenot <[email protected]>---
Index:
linux-2.6.git/arch/powerpc/platforms/pseries/reconfig.c
===================================================================
---linux-2.6.git.orig/arch/powerpc/platforms/pseries/reconfig.c2008-06-03 16:29:04.000000000 -0500 +++ linux-2.6.git/arch/powerpc/platforms/pseries/reconfig.c 2008-06-19 22:03:12.000000000 -0500
arch/powerpc/platforms/pseries/reconfig.c line 368
Quoted 5 lines
arch/powerpc/platforms/pseries/reconfig.c +1
-1@@ -365,7 +365,7 @@ *buf ='\0' ; buf++; - handle =simple_strtoul (handle_str,NULL ,10 ); + handle =simple_strtoul (handle_str,NULL ,0 );
I guess checkpatch.pl told you to consider switching tostrict_strtol()? :-)
WBR, Sergei
Re: [PATCH 1/5] Allow drc indexes to be specified in formats other than decimal
Quoted 5 lines · Sergei Shtylyov
I guess checkpatch.pl told you to consider switching to
strict_strtol()? :-)WBR, Sergei
Yes, it did warn about not switching. There other places in this file that use simple_strtoul so I was hoping to submit a followon patch that converts everything.
-Nathan
PATCH 2/5 Use the base address of lmbs instead of drc index for dlpar
In reply to Nathan FontenotUse the base address of the lmb to get the starting page frame number instead of trying to extract it from the drc index of the lmb.
In the previous representations of memory in the device tree, the drc index of a lmb would match up with the base address of the lmb. For example a lmb woulf have a drc index of 8000000a and a base address of a0000000. This allowed the starting page frame number to be extracted from the drc index.
The lmbs represented under the ibm,dynamic-reconfiguration-memory nodein the device tree do not always have a drc index that matches, thus we need to get the starting page frame number from the base address of an lmb instead of the drc index.
---
arch/powerpc/platforms/pseries/hotplug-memory.c +17 -20
@@ -18,8 +18,9 @@static int pseries_remove_memory (struct device_node *np) {const char *type; -const unsigned int *my_index;const unsigned int *regs; +unsigned long base; +unsigned int lmb_size;u64 start_pfn, start;struct zone *zone;int ret = -EINVAL ;
@@ -32,17 +33,16 @@return 0 ;/* -* Find the memory index and size of the removing section +* Find the base address and size of the lmb */ - my_index =of_get_property (np,"ibm,my-drc-index" ,NULL ); -if (!my_index) -return ret; - regs =of_get_property (np,"reg" ,NULL );if (!regs)return ret; - start_pfn =section_nr_to_pfn (*my_index &0xffff ); + base = *(unsigned long *)regs; + lmb_size = regs[3 ]; + + start_pfn = base >>PFN_SECTION_SHIFT ; zone =page_zone (pfn_to_page (start_pfn));/*
@@ -54,29 +54,29 @@* to sysfs "state" file and we can't remove sysfs entries * while writing to it. So we have to defer it to here. */ - ret =__remove_pages (zone, start_pfn, regs[3 ] >>PAGE_SHIFT ); + ret =__remove_pages (zone, start_pfn, lmb_size >>PAGE_SHIFT );if (ret)return ret;/* * Update memory regions for memory remove */ -lmb_remove (start_pfn <<PAGE_SHIFT , regs[3 ]); +lmb_remove (base, lmb_size);/* * Remove htab bolted mappings for this section of memory */ - start = (unsigned long )__va (start_pfn <<PAGE_SHIFT ); - ret =remove_section_mapping (start, start + regs[3 ]); + start = (unsigned long )__va (base); + ret =remove_section_mapping (start, start + lmb_size);return ret; }static int pseries_add_memory (struct device_node *np) {const char *type; -const unsigned int *my_index;const unsigned int *regs; -u64 start_pfn; +unsigned long base; +unsigned int lmb_size;int ret = -EINVAL ;/*
@@ -87,22 +87,19 @@return 0 ;/* -* Find the memory index and size of the added section +* Find the base and size of the lmb */ - my_index =of_get_property (np,"ibm,my-drc-index" ,NULL ); -if (!my_index) -return ret; - regs =of_get_property (np,"reg" ,NULL );if (!regs)return ret; - start_pfn =section_nr_to_pfn (*my_index &0xffff ); + base = *(unsigned long *)regs; + lmb_size = regs[3 ];/* * Update memory region to represent the memory add */ -lmb_add (start_pfn <<PAGE_SHIFT , regs[3 ]); +lmb_add (base, lmb_size);return 0 ; }
PATCH 3/5 Update device tree correctly for drconf memory add/remove
In reply to Nathan FontenotMemory dlpar updates to correctly update the device tree when an lmb is added/removed. The main difference for memory under the ibm,dynamic-reconfiguration-memory node is that dlpar adding or removing an lmb only requires an update to the ibm,dynamic-memory property. Previously, entire nodes were added or removed when a lmb was dlpar added/removed.
The patch updates the /proc/ppc64/ofdt interface to recognize when the ibm,dynamic-memory preoperty is updated and properly invokes the memory hotplug notifier chain.
This also updates the pseries hotplug notifier to be able to gather information for lmbs represented under the ibm,dynamic-reconfiguration-memory node and have the lmbs added/removed.
---
include/asm-powerpc/pSeries_reconfig.h +4 -2
@@ -9,8 +9,10 @@* added or removed on pSeries systems. */ -#define PSERIES_RECONFIG_ADD 0x0001 -#define PSERIES_RECONFIG_REMOVE 0x0002 +#define PSERIES_RECONFIG_ADD 0x0001 +#define PSERIES_RECONFIG_REMOVE 0x0002 +#define PSERIES_DRCONF_MEM_ADD 0x0003 +#define PSERIES_DRCONF_MEM_REMOVE 0x0004#ifdef CONFIG_PPC_PSERIES extern int pSeries_reconfig_notifier_register (struct notifier_block *);
arch/powerpc/platforms/pseries/reconfig.c +32 -4
@@ -422,8 +422,8 @@ {struct device_node *np;unsigned char *value; -char *name, *end; -int length; +char *name, *end, *next_prop; +int rc, length;struct property *newprop, *oldprop; buf =parse_node (buf, bufsize, &np); end = buf + bufsize;
@@ -431,7 +431,8 @@if (!np)return -ENODEV ; -if (parse_next_property (buf, end, &name, &length, &value) ==NULL ) + next_prop =parse_next_property (buf, end, &name, &length, &value); +if (!next_prop)return -EINVAL ; newprop =new_property (name, length, value,NULL );
@@ -442,7 +443,34 @@if (!oldprop)return -ENODEV ; -return prom_update_property (np, newprop, oldprop); + rc =prom_update_property (np, newprop, oldprop); +if (rc) +return rc; + +/* For memory under the ibm,dynamic-reconfiguration-memory node +* of the device tree, adding and removing memory is just an update +* to the ibm,dynamic-memory property instead of adding/removing a +* memory node in the device tree. For these cases we still need to +* involve the notifier chain. +*/ +if (!strcmp (name,"ibm,dynamic-memory" )) { +int action; + + next_prop =parse_next_property (next_prop, end, &name, + &length, &value); +if (!next_prop) +return -EINVAL ; + +if (!strcmp (name,"add" )) + action =PSERIES_DRCONF_MEM_ADD ; +else + action =PSERIES_DRCONF_MEM_REMOVE ; + +blocking_notifier_call_chain (&pSeries_reconfig_chain, + action, value); + } + +return 0 ; }/**
arch/powerpc/platforms/pseries/hotplug-memory.c +68 -26
122 lines not on this page · raw message
PATCH 4/5 Helper routines for parsing drconf memory
In reply to Nathan FontenotThis patch pulls several pieces of exisitng code for parsing memory under the ibm,dynamic-reconfiguration-memory node of the device tree into helper routines. This is in preparation for the next patch that will also use these same helper routines. There are no functional changes in this patch.
---
arch/powerpc/mm/numa.c +170 -40
@@ -268,6 +268,144 @@return result; } +struct of_drconf_cell { +u64 base_addr; +u32 drc_index; +u32 reserved; +u32 aa_index; +u32 flags; +}; + +#define DRCONF_MEM_ASSIGNED 0x00000008 +#define DRCONF_MEM_AI_INVALID 0x00000040 +#define DRCONF_MEM_RESERVED 0x00000080 + +/* +* Read the next lmb list entry from the ibm,dynamic-memory property +* and return the information in the provided of_drconf_cell structure. +*/ +static void read_drconf_cell (struct of_drconf_cell *drmem,const u32 **cellp) +{ +const u32 *cp; + + drmem->base_addr =read_n_cells (n_mem_addr_cells, cellp); + + cp = *cellp; + drmem->drc_index = cp[0 ]; + drmem->reserved = cp[1 ]; + drmem->aa_index = cp[2 ]; + drmem->flags = cp[3 ]; + + *cellp = cp +4 ; +} + +/* +* Retreive and validate the ibm,dynamic-memory property of the device tree. +* +* The layout of the ibm,dynamic-memory property is a number N of lmb +* list entries followed by N lmb list entries. Each lmb list entry +* contains information as layed out in the of_drconf_cell struct above. +*/ +static int of_get_drconf_memory (struct device_node *memory,const u32 **dm) +{ +const u32 *prop; +u32 len, entries; + + prop =of_get_property (memory,"ibm,dynamic-memory" , &len); +if (!prop || len <sizeof (unsigned int )) +return 0 ; + + entries = *prop++; + +/* Now that we know the number of entries, revalidate the size +* of the property read in to ensure we have everything +*/ +if (len < (entries * (n_mem_addr_cells +4 ) +1 ) *sizeof (unsigned int )) +return 0 ; + + *dm = prop; +return entries; +} + +/* +* Retreive and validate the ibm,lmb-size property for drconf memory +* from the device tree. +*/ +static u64 of_get_lmb_size (struct device_node *memory) +{ +const u32 *prop; +u32 len; + + prop =of_get_property (memory,"ibm,lmb-size" , &len); +if (!prop || len <sizeof (unsigned int )) +return 0 ; + +return read_n_cells (n_mem_size_cells, &prop); +} + +struct assoc_arrays { +u32 n_arrays; +u32 array_sz; +const u32 *arrays; +}; + +/* +* Retreive and validate the list of associativity arrays for drconf +* memory from the ibm,associativity-lookup-arrays property of the +* device tree.. +* +* The layout of the ibm,associativity-lookup-arrays property is a number N +* indicating the number of associativity arrays, followed by a number M +* indicating the size of each associativity array, followed by a list +* of N associativity arrays. +*/ +static int of_get_assoc_arrays (struct device_node *memory, +struct assoc_arrays *aa) +{ +const u32 *prop; +u32 len; + + prop =of_get_property (memory,"ibm,associativity-lookup-arrays" , &len); +if (!prop || len <2 *sizeof (unsigned int )) +return -1 ; + + aa->n_arrays = *prop++; + aa->array_sz = *prop++; + +/* Now that we know the number of arrrays and size of each array, +* revalidate the size of the property read in. +*/ +if (len < (aa->n_arrays * aa->array_sz +2 ) *sizeof (unsigned int )) +return -1 ; + + aa->arrays = prop; +return 0 ; +} + +/* +* This is like of_node_to_nid_single() for memory represented in the +* ibm,dynamic-reconfiguration-memory node. +*/ +static int of_drconf_to_nid_single (struct of_drconf_cell *drmem, +struct assoc_arrays *aa) +{ +int default_nid =0 ; +int nid = default_nid; +int index; + +if (min_common_depth >0 && min_common_depth <= aa->array_sz && + !(drmem->flags &DRCONF_MEM_AI_INVALID ) && + drmem->aa_index < aa->n_arrays) { + index = drmem->aa_index * aa->array_sz + min_common_depth -1 ; + nid = aa->arrays[index]; + +if (nid ==0xffff || nid >=MAX_NUMNODES ) + nid = default_nid; + } + +return nid; +} +/* * Figure out to which domain a cpu belongs and stick it there. * Return the id of the domain used.
@@ -355,57 +493,49 @@*/ static void __init parse_drconf_memory (struct device_node *memory) { -const unsigned int *lm, *dm, *aa; -unsigned int ls, ld, la; -unsigned int n, aam, aalen; -unsigned long lmb_size, size, start; -int nid, default_nid =0 ; -unsigned int ai, flags; - - lm =of_get_property (memory,"ibm,lmb-size" , &ls); - dm =of_get_property (memory,"ibm,dynamic-memory" , &ld); - aa =of_get_property (memory,"ibm,associativity-lookup-arrays" , &la); -if (!lm || !dm || !aa || - ls <sizeof (unsigned int ) || ld <sizeof (unsigned int ) || - la <2 *sizeof (unsigned int )) +const u32 *dm; +unsigned int n, rc; +unsigned long lmb_size, size; +int nid; +struct assoc_arrays aa; + + n =of_get_drconf_memory (memory, &dm); +if (!n)return ; - lmb_size =read_n_cells (n_mem_size_cells, &lm); - n = *dm++;/* number of LMBs */ - aam = *aa++;/* number of associativity lists */ - aalen = *aa++;/* length of each associativity list */ -if (ld < (n * (n_mem_addr_cells +4 ) +1 ) *sizeof (unsigned int ) || - la < (aam * aalen +2 ) *sizeof (unsigned int )) + lmb_size =of_get_lmb_size (memory); +if (!lmb_size) +return ; + + rc =of_get_assoc_arrays (memory, &aa); +if (rc)return ;for (; n !=0 ; --n) { - start =read_n_cells (n_mem_addr_cells, &dm); - ai = dm[2 ]; - flags = dm[3 ]; - dm +=4 ; -/* 0x80 == reserved, 0x8 = assigned to us */ -if ((flags &0x80 ) || !(flags &0x8 )) -continue ; - nid = default_nid; -/* flags & 0x40 means associativity index is invalid */ -if (min_common_depth >0 && min_common_depth <= aalen && - (flags &0x40 ) ==0 && ai < aam) { -/* this is like of_node_to_nid_single */ - nid = aa[ai * aalen + min_common_depth -1 ]; -if (nid ==0xffff || nid >=MAX_NUMNODES ) - nid = default_nid; - } +struct of_drconf_cell drmem; -fake_numa_create_new_node (((start + lmb_size) >>PAGE_SHIFT ), - &nid); +read_drconf_cell (&drmem, &dm); + +/* skip this block if the reserved bit is set in flags (0x80) +or if the block is not assigned to this partition (0x8) */ +if ((drmem.flags &DRCONF_MEM_RESERVED ) + || !(drmem.flags &DRCONF_MEM_ASSIGNED )) +continue ; + + nid =of_drconf_to_nid_single (&drmem, &aa); + +fake_numa_create_new_node ( + ((drmem.base_addr + lmb_size) >>PAGE_SHIFT ), + &nid);node_set_online (nid); - size =numa_enforce_memory_limit (start, lmb_size); + size =numa_enforce_memory_limit (drmem.base_addr, lmb_size);if (!size)continue ; -add_active_range (nid, start >>PAGE_SHIFT , - (start >>PAGE_SHIFT ) + (size >>PAGE_SHIFT )); +add_active_range (nid, drmem.base_addr >>PAGE_SHIFT , + (drmem.base_addr >>PAGE_SHIFT ) + + (size >>PAGE_SHIFT )); } }
Re: [PATCH 4/5] Helper routines for parsing drconf memory
Quoted 4 lines · Nathan Fontenot
This patch pulls several pieces of exisitng code for parsing memory under the ibm,dynamic-reconfiguration-memory node of the device tree into helper routines. This is in preparation for the next patch that will also use these same helper routines. There are no functional changes in this patch.
I went to apply this, but it is whitespace-damaged...
Paul.
Re: [PATCH 4/5] Helper routines for parsing drconf memory
Quoted 9 lines · Paul Mackerras
Quoted 4 lines · Nathan Fontenot
This patch pulls several pieces of exisitng code for parsing memory under the ibm,dynamic-reconfiguration-memory node of the device tree into helper routines. This is in preparation for the next patch that will also use these same helper routines. There are no functional changes in this patch.
I went to apply this, but it is whitespace-damaged...
Hmmm... I am having problems getting this to apply to the latest mainline cleanly. I'll update the patchset and re-submit.
-Nathan
PATCH 5/5 Add ability to associate drconf memory sections in hotplug memory add
In reply to Nathan FontenotUpdate the association of a memory section with a numa node that occurs during hotplug add of a memory section. This adds a check for the ibm,dyamic-reconfiguration-memory node in the device tree and calls the new hot_add_drconf_scn_to_nid that can properly parse the device tree and make the proper associations.
---
arch/powerpc/mm/numa.c +83 -18
@@ -900,6 +900,79 @@#ifdef CONFIG_MEMORY_HOTPLUG /* +* Validate the node associated with the memory section we are +* trying to add. +*/ +int valid_hot_add_scn (int *nid,unsigned long start,u32 lmb_size, +unsigned long scn_addr) +{ +nodemask_t nodes; + +if (*nid <0 || !node_online (*nid)) + *nid =any_online_node (NODE_MASK_ALL ); + +if ((scn_addr >= start) && (scn_addr < (start + lmb_size))) { +nodes_setall (nodes); +while (NODE_DATA (*nid)->node_spanned_pages ==0 ) { +node_clear (*nid, nodes); + *nid =any_online_node (nodes); + } + +return 1 ; + } + +return 0 ; +} + +/* +* Find the node associated with a hot added memory section represented +* by the ibm,dynamic-reconfiguration-memory node. +*/ +static int hot_add_drconf_scn_to_nid (struct device_node *memory, +unsigned long scn_addr) +{ +const u32 *dm; +unsigned int n, rc; +unsigned long lmb_size; +int default_nid =any_online_node (NODE_MASK_ALL ); +int nid; +struct assoc_arrays aa; + + n =of_get_drconf_memory (memory, &dm); +if (!n) +return default_nid;; + + lmb_size =of_get_lmb_size (memory); +if (!lmb_size) +return default_nid; + + rc =of_get_assoc_arrays (memory, &aa); +if (rc) +return default_nid; + +for (; n !=0 ; --n) { +struct of_drconf_cell drmem; + +read_drconf_cell (&drmem, &dm); + +/* skip this block if it is reserved or not assigned to +* this partition */ +if ((drmem.flags &DRCONF_MEM_RESERVED ) + || !(drmem.flags &DRCONF_MEM_ASSIGNED )) +continue ; + + nid =of_drconf_to_nid_single (&drmem, &aa); + +if (valid_hot_add_scn (&nid, drmem.base_addr, lmb_size, + scn_addr)) +return nid; + } + +BUG ();/* section address should be found above */ +return 0 ; +} + +/* * Find the node associated with a hot added memory section. Section * corresponds to a SPARSEMEM section, not an LMB. It is assumed that * sections are fully contained within a single LMB.
@@ -907,12 +980,17 @@int hot_add_scn_to_nid (unsigned long scn_addr) {struct device_node *memory =NULL ; -nodemask_t nodes; -int default_nid =any_online_node (NODE_MASK_ALL );int nid;if (!numa_enabled || (min_common_depth <0 )) -return default_nid; +return any_online_node (NODE_MASK_ALL ); + + memory =of_find_node_by_path ("/ibm,dynamic-reconfiguration-memory" ); +if (memory) { + nid =hot_add_drconf_scn_to_nid (memory, scn_addr); +of_node_put (memory); +return nid; + }while ((memory =of_find_node_by_type (memory,"memory" )) !=NULL ) {unsigned long start, size;
@@ -931,13 +1009,9 @@ size =read_n_cells (n_mem_size_cells, &memcell_buf); nid =of_node_to_nid_single (memory); -/* Domains not present at boot default to 0 */ -if (nid <0 || !node_online (nid)) - nid = default_nid; - -if ((scn_addr >= start) && (scn_addr < (start + size))) { +if (valid_hot_add_scn (&nid, start, size, scn_addr)) {of_node_put (memory); - goto got_nid; +return nid; }if (--ranges)/* process all ranges in cell */
@@ -945,14 +1019,5 @@ }BUG ();/* section address should be found above */ return 0 ; - -/* Temporary code to ensure that returned node is not empty */ -got_nid: -nodes_setall (nodes); -while (NODE_DATA (nid)->node_spanned_pages ==0 ) { -node_clear (nid, nodes); - nid =any_online_node (nodes); - } -return nid; }#endif /* CONFIG_MEMORY_HOTPLUG */