mm/demotion: Expose memory type details via sysfs
Subsystems (2)
RFC PATCH 1/2 mm/demotion: Expose memory type details via sysfs
This patch adds /sys/devices/virtual/memtier/ where all memory tier related details can be found. All allocated memory types will be listed there as /sys/devices/virtual/memtier/memtypeN/
The nodes which are part of a specific memory type can be listed via /sys/devices/system/memtier/memtypeN/nodes.
The adistance value of a specific memory type can be listed via /sys/devices/system/memtier/memtypeN/adistance.
A directory listing looks like:
:/sys/devices/virtual/memtier# tree memtype1 memtype1 ├── adistance ├── nodes ├── subsystem -> ../../../../bus/memtier └── uevent
Since we will be using struct device to expose details via sysfs, drop struct kref and use struct device for refcounting the memtype.
--- include/linux/memory-tiers.h | 3 +- mm/memory-tiers.c | 97 +++++++++++++++++++++++++++++++++--- 2 files changed, 91 insertions(+), 9 deletions(-)
include/linux/memory-tiers.h +2 -1
@@ -6,6 +6,7 @@#include <linux/nodemask.h> #include <linux/kref.h> #include <linux/mmzone.h> +#include <linux/device.h> /* * Each tier cover a abstrace distance chunk size of 128 */
@@ -28,7 +29,7 @@ struct memory_dev_type {int adistance;/* Nodes of same abstract distance */ nodemask_t nodes; -struct kref kref; +struct device dev; };#ifdef CONFIG_NUMA
mm/memory-tiers.c +89 -8
154 lines not on this page · raw message
Signature · 1 line
2.37.2
RFC PATCH 2/2 mm/demotion: Expose memory tier details via sysfs
All allocated memory tiers will be listed as/sys/devices/virtual/memtier/memtierN/
Each memtier directory contains symbolic link for the memory types that are part of the memory tier. A directory hierarchy looks like
:/sys/devices/virtual/memtier# tree memtier512/ memtier512/ ├── memtype1 -> ../memtype1 ├── memtype2 -> ../memtype2 ├── subsystem -> ../../../../bus/memtier └── uevent
The nodes which are part of a specific memory type can be listed via /sys/devices/system/memtier/memtypeN/nodes.
The adistance value of a specific memory type can be listed via /sys/devices/system/memtier/memtypeN/adistance.
--- mm/memory-tiers.c | 62 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 15 deletions(-)
mm/memory-tiers.c +47 -15
@@ -20,6 +20,7 @@ struct memory_tier {* adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE */ int adistance_start; +struct device dev;/* All the nodes that are part of all the lower memory tiers. */ nodemask_t lower_tier_mask; };
@@ -36,6 +37,7 @@ static struct memory_dev_type *default_dram_type;#define MAX_MEMORY_TYPE_ID 20static DEFINE_IDR (memory_type_idr);#define to_memory_type (device) container_of(device, struct memory_dev_type, dev) +#define to_memory_tier (device) container_of(device, struct memory_tier, dev)static struct bus_type memory_tier_subsys = { .name ="memtier" , .dev_name ="memtier" ,
@@ -103,8 +105,25 @@ static int top_tier_adistance;static struct demotion_nodes *node_demotion__read_mostly ;#endif /* CONFIG_MIGRATION */ +static void memory_tier_device_release (struct device *dev) +{ +struct memory_tier *tier =to_memory_tier (dev); +/* +* synchronize_rcu in clear_node_memory_tier makes sure +* we don't have rcu access to this memory tier. +*/ +kfree (tier); +} + +static void destroy_memory_tier (struct memory_tier *memtier) +{ +list_del (&memtier->list); +device_unregister (&memtier->dev); +} +static struct memory_tier *find_create_memory_tier (struct memory_dev_type *memtype) { +int ret;bool found_slot = false;struct memory_tier *memtier, *new_memtier;int adistance = memtype->adistance;
@@ -128,15 +147,14 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtylist_for_each_entry (memtier, &memory_tiers, list) {if (adistance == memtier->adistance_start) { -list_add (&memtype->tier_sibiling, &memtier->memory_types); -return memtier; + goto link_memtype; }else if (adistance < memtier->adistance_start) { found_slot = true;break ; } } - new_memtier =kmalloc (sizeof (struct memory_tier ),GFP_KERNEL ); + new_memtier =kzalloc (sizeof (struct memory_tier ),GFP_KERNEL );if (!new_memtier)return ERR_PTR (-ENOMEM );
@@ -147,8 +165,30 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtylist_add_tail (&new_memtier->list, &memtier->list); else list_add_tail(&new_memtier->list, &memory_tiers); -list_add (&memtype->tier_sibiling, &new_memtier->memory_types); -return new_memtier; + + new_memtier->dev.id = adistance; + new_memtier->dev.bus = &memory_tier_subsys; + new_memtier->dev.release = memory_tier_device_release; + + ret =device_register (&new_memtier->dev); +if (ret) { +list_del (&memtier->list); +put_device (&memtier->dev); +return ERR_PTR (ret); + } + memtier = new_memtier; + +link_memtype: +list_add (&memtype->tier_sibiling, &memtier->memory_types); +/* +* ignore error below because the driver creating the device can get +* unloaded and hence the below sysfs create link can fail. We continue +* with the in memory representation. +*/ + ret =sysfs_create_link (&memtier->dev.kobj, + &memtype->dev.kobj,kobject_name (&memtype->dev.kobj)); + +return memtier; }static struct memory_tier *__node_get_memory_tier (int node)
@@ -424,16 +464,6 @@ static struct memory_tier *set_node_memory_tier(int node)return memtier; } -static void destroy_memory_tier (struct memory_tier *memtier) -{ -list_del (&memtier->list); -/* -* synchronize_rcu in clear_node_memory_tier makes sure -* we don't have rcu access to this memory tier. -*/ -kfree (memtier); -} -static bool clear_node_memory_tier (int node) {bool cleared = false;
@@ -462,6 +492,8 @@ static bool clear_node_memory_tier(int node)node_clear (node, memtype->nodes);if (nodes_empty (memtype->nodes)) {list_del_init (&memtype->tier_sibiling); +sysfs_delete_link (&memtier->dev.kobj, + &memtype->dev.kobj,kobject_name (&memtype->dev.kobj));if (list_empty (&memtier->memory_types))destroy_memory_tier (memtier); }
Signature · 1 line
2.37.2
Re: [RFC PATCH 2/2] mm/demotion: Expose memory tier details via sysfs
Quoted 8 lines · Aneesh Kumar K.V
All allocated memory tiers will be listed as
/sys/devices/virtual/memtier/memtierN/Each memtier directory contains symbolic link for the memory types that are part of the memory tier. A directory hierarchy looks like
:/sys/devices/virtual/memtier# tree memtier512/ memtier512/
So you suggest to use abstract_distance_start as memory tier ID? That will make memory tier ID stable unless we change abstract distance chunk size or abstract distance division points. That is, we have at least 2 choices here
1. memory_tier0, memory_tier1, memory_tier2, ...
The ID will start from 0. This is easy to understand by users. The main drawback is that the memory tier ID may be changed when a NUMA node is onlined/offlined. That is, the memory tier ID is relatively unstable.
2. memory_tier<abstract_distance_start1>, memory_tier<abstract_distance_start2>, ...
The ID will be discontinuous. So it's not as intuitive as 0,1,2,.... The main advantage is that the memory tier ID will not change when a NUMA node is onlined/offlined. The ID will be changed only when we change abstract distance chunk size or abstract distance division points. That is considered relatively seldom.
Personally, I prefer the 2nd choice too. But I want to collect opinions from other people too.
Quoted 2 lines
├── memtype1 -> ../memtype1 ├── memtype2 -> ../memtype2
I think abstract_distance_start and abstract_distance_end is the key information of a memory tier too. So we should show them here.
Quoted 10 lines
├── subsystem -> ../../../../bus/memtier └── ueventThe nodes which are part of a specific memory type can be listed via
/sys/devices/system/memtier/memtypeN/nodes.The adistance value of a specific memory type can be listed via
/sys/devices/system/memtier/memtypeN/adistance.Signed-off-by Aneesh Kumar K.V <[email protected]>
Best Regards,
Huang, Ying
[snip]
Quoted 3 lines · Aneesh Kumar K.V
This patch adds
/sys/devices/virtual/memtier/where all memory tier related details can be found. All allocated memory types will be listed there as/sys/devices/virtual/memtier/memtypeN/
Another choice is to make memory types and memory tiers system devices. That is,
/sys/devices/system/memory_type/memory_typeN/sys/devices/system/memory_tier/memory_tierN
That looks more natural to me. Because we already have "node" and "memory" devices there. Why don't you put memory types and memory tiers there?
And, I think we shouldn't put "memory_type" in the "memory_tier" directory. "memory_type" isn't a part of "memory_tier".
Quoted 2 lines
The nodes which are part of a specific memory type can be listed via
/sys/devices/system/memtier/memtypeN/nodes.
How about create links to /sys/devices/system/node/nodeN in "memory_type". But I'm OK to have "nodes" file too.
Quoted 7 lines
The adistance value of a specific memory type can be listed via
/sys/devices/system/memtier/memtypeN/adistance.A directory listing looks like:
:/sys/devices/virtual/memtier# tree memtype1 memtype1 ├── adistance
Why not just use "abstract_distance"? This is user space interface, it's better to be intuitive.
Quoted 7 lines
├── nodes ├── subsystem -> ../../../../bus/memtier └── ueventSince we will be using struct device to expose details via sysfs, drop struct kref and use struct device for refcounting the memtype.
Best Regards,
Huang, Ying
Quoted 6 lines · Ying
Quoted 3 lines · Aneesh Kumar K.V
This patch adds
/sys/devices/virtual/memtier/where all memory tier related details can be found. All allocated memory types will be listed there as/sys/devices/virtual/memtier/memtypeN/
Quoted 6 lines
Another choice is to make memory types and memory tiers system devices. That is,
/sys/devices/system/memory_type/memory_typeN/sys/devices/system/memory_tier/memory_tierN
subsys_system_register() documentation says
* Do not use this interface for anything new, it exists for compatibility
* with bad ideas only. New subsystems should use plain subsystems; and
* add the subsystem-wide attributes should be added to the subsystem
* directory itself and not some create fake root-device placed in
* /sys/devices/system/<name>.
memtier being a virtual device, I was under the impression that /sys/devices/virtual is the recommended place.
Quoted 7 lines
That looks more natural to me. Because we already have "node" and "memory" devices there. Why don't you put memory types and memory tiers there?
And, I think we shouldn't put "
memory_type" in the "memory_tier" directory. "memory_type" isn't a part of "memory_tier".
I was looking consolidating both memory tier and memory type into the same sysfs subsystem. Your recommendation imply we create two subsystem memory_tier and memtype. I was trying to avoid that. May be a generic term like "memory_tiering" can help to consolidate all tiering related details there?
Quoted 27 lines
Quoted 2 lines
The nodes which are part of a specific memory type can be listed via
/sys/devices/system/memtier/memtypeN/nodes.How about create links to
/sys/devices/system/node/nodeNin "memory_type". But I'm OK to have "nodes" file too.Quoted 7 lines
The adistance value of a specific memory type can be listed via
/sys/devices/system/memtier/memtypeN/adistance.A directory listing looks like:
:/sys/devices/virtual/memtier# tree memtype1 memtype1 ├── adistanceWhy not just use "
abstract_distance"? This is user space interface, it's better to be intuitive.Quoted 7 lines
├── nodes ├── subsystem -> ../../../../bus/memtier └── ueventSince we will be using struct device to expose details via sysfs, drop struct kref and use struct device for refcounting the memtype.
Best Regards,
Huang, Ying
Quoted 30 lines · Aneesh Kumar K V
Quoted 6 lines · Ying
Quoted 3 lines · Aneesh Kumar K.V
This patch adds
/sys/devices/virtual/memtier/where all memory tier related details can be found. All allocated memory types will be listed there as/sys/devices/virtual/memtier/memtypeN/Quoted 6 lines
Another choice is to make memory types and memory tiers system devices. That is,
/sys/devices/system/memory_type/memory_typeN/sys/devices/system/memory_tier/memory_tierN
subsys_system_register()documentation says* Do not use this interface for anything new, it exists for compatibility
* with bad ideas only. New subsystems should use plain subsystems; and
* add the subsystem-wide attributes should be added to the subsystem
* directory itself and not some create fake root-device placed in
* /sys/devices/system/<name>.memtier being a virtual device, I was under the impression that
/sys/devices/virtualis the recommended place.Quoted 4 lines
That looks more natural to me. Because we already have "node" and "memory" devices there. Why don't you put memory types and memory tiers there?
Quoted 9 lines
Quoted 3 lines
And, I think we shouldn't put "
memory_type" in the "memory_tier" directory. "memory_type" isn't a part of "memory_tier".I was looking consolidating both memory tier and memory type into the same sysfs subsystem. Your recommendation imply we create two subsystem
memory_tierand memtype. I was trying to avoid that. May be a generic term like "memory_tiering" can help to consolidate all tiering related details there?
A generic term "memory_tiering" sounds good to me.
Given that this will be a user-facing, stable kernel API, I think we'd better to only add what is most useful for userspace and don't have to mirror the kernel internal data structures in this interface.
My understanding is that we haven't fully settled down on how to customize memory tiers from userspace. So we don't have to show memory_type yet, which is a kernel data structure at this point.
The userspace does need to know what are the memory tiers and which NUMA nodes are included in each memory tier. How about we provide the "nodelist" interface for each memory tier as in the original proposal?
The userspace would also like to know which memory tiers/nodes belong to the top tiers (the promotion targets). We can provide a "toptiers" or "toptiers_nodelist" interface to report that.
Both should still be useful even if we decide to add memory_type for memory tier customization.
Quoted 28 lines
Quoted 27 lines
Quoted 2 lines
The nodes which are part of a specific memory type can be listed via
/sys/devices/system/memtier/memtypeN/nodes.How about create links to
/sys/devices/system/node/nodeNin "memory_type". But I'm OK to have "nodes" file too.Quoted 7 lines
The adistance value of a specific memory type can be listed via
/sys/devices/system/memtier/memtypeN/adistance.A directory listing looks like:
:/sys/devices/virtual/memtier# tree memtype1 memtype1 ├── adistanceWhy not just use "
abstract_distance"? This is user space interface, it's better to be intuitive.Quoted 7 lines
├── nodes ├── subsystem -> ../../../../bus/memtier └── ueventSince we will be using struct device to expose details via sysfs, drop struct kref and use struct device for refcounting the memtype.
Best Regards,
Huang, Ying
Quoted 52 lines · Wei Xu
Quoted 30 lines · Aneesh Kumar K V
Quoted 6 lines · Ying
Quoted 3 lines · Aneesh Kumar K.V
This patch adds
/sys/devices/virtual/memtier/where all memory tier related details can be found. All allocated memory types will be listed there as/sys/devices/virtual/memtier/memtypeN/Quoted 6 lines
Another choice is to make memory types and memory tiers system devices. That is,
/sys/devices/system/memory_type/memory_typeN/sys/devices/system/memory_tier/memory_tierN
subsys_system_register()documentation says* Do not use this interface for anything new, it exists for compatibility
* with bad ideas only. New subsystems should use plain subsystems; and
* add the subsystem-wide attributes should be added to the subsystem
* directory itself and not some create fake root-device placed in
* /sys/devices/system/<name>.memtier being a virtual device, I was under the impression that
/sys/devices/virtualis the recommended place.Quoted 4 lines
That looks more natural to me. Because we already have "node" and "memory" devices there. Why don't you put memory types and memory tiers there?
Quoted 9 lines
Quoted 3 lines
And, I think we shouldn't put "
memory_type" in the "memory_tier" directory. "memory_type" isn't a part of "memory_tier".I was looking consolidating both memory tier and memory type into the same sysfs subsystem. Your recommendation imply we create two subsystem
memory_tierand memtype. I was trying to avoid that. May be a generic term like "memory_tiering" can help to consolidate all tiering related details there?A generic term "
memory_tiering" sounds good to me.Given that this will be a user-facing, stable kernel API, I think we'd better to only add what is most useful for userspace and don't have to mirror the kernel internal data structures in this interface.
My understanding is that we haven't fully settled down on how to customize memory tiers from userspace. So we don't have to show
memory_typeyet, which is a kernel data structure at this point.
Quoted 8 lines
The userspace does need to know what are the memory tiers and which NUMA nodes are included in each memory tier. How about we provide the "nodelist" interface for each memory tier as in the original proposal?
The userspace would also like to know which memory tiers/nodes belong to the top tiers (the promotion targets). We can provide a "toptiers" or "
toptiers_nodelist" interface to report that.
How about also including abstract distance range of a memory tier? That will be useful to derive the hierarchy.
Quoted 3 lines
Both should still be useful even if we decide to add
memory_typefor memory tier customization.
-aneesh
Quoted 58 lines · Aneesh Kumar K V
Quoted 56 lines · Wei Xu
Quoted 30 lines · Aneesh Kumar K V
Quoted 6 lines · Ying
Quoted 3 lines · Aneesh Kumar K.V
This patch adds
/sys/devices/virtual/memtier/where all memory tier related details can be found. All allocated memory types will be listed there as/sys/devices/virtual/memtier/memtypeN/Quoted 6 lines
Another choice is to make memory types and memory tiers system devices. That is,
/sys/devices/system/memory_type/memory_typeN/sys/devices/system/memory_tier/memory_tierN
subsys_system_register()documentation says* Do not use this interface for anything new, it exists for compatibility
* with bad ideas only. New subsystems should use plain subsystems; and
* add the subsystem-wide attributes should be added to the subsystem
* directory itself and not some create fake root-device placed in
* /sys/devices/system/<name>.memtier being a virtual device, I was under the impression that
/sys/devices/virtualis the recommended place.Quoted 4 lines
That looks more natural to me. Because we already have "node" and "memory" devices there. Why don't you put memory types and memory tiers there?
Quoted 9 lines
Quoted 3 lines
And, I think we shouldn't put "
memory_type" in the "memory_tier" directory. "memory_type" isn't a part of "memory_tier".I was looking consolidating both memory tier and memory type into the same sysfs subsystem. Your recommendation imply we create two subsystem
memory_tierand memtype. I was trying to avoid that. May be a generic term like "memory_tiering" can help to consolidate all tiering related details there?A generic term "
memory_tiering" sounds good to me.Given that this will be a user-facing, stable kernel API, I think we'd better to only add what is most useful for userspace and don't have to mirror the kernel internal data structures in this interface.
My understanding is that we haven't fully settled down on how to customize memory tiers from userspace. So we don't have to show
memory_typeyet, which is a kernel data structure at this point.The userspace does need to know what are the memory tiers and which NUMA nodes are included in each memory tier. How about we provide the "nodelist" interface for each memory tier as in the original proposal?
Quoted 7 lines
Quoted 4 lines
The userspace would also like to know which memory tiers/nodes belong to the top tiers (the promotion targets). We can provide a "toptiers" or "
toptiers_nodelist" interface to report that.How about also including abstract distance range of a memory tier? That will be useful to derive the hierarchy.
With the base abstract distance in the memtier name, do we need to show the abstract distance range if we don't customize memory tiers?
Quoted 5 lines
Quoted 3 lines
Both should still be useful even if we decide to add
memory_typefor memory tier customization.-aneesh
Quoted 66 lines · Wei Xu
Quoted 63 lines · Aneesh Kumar K V
Quoted 60 lines · Wei Xu
Quoted 30 lines · Aneesh Kumar K V
Quoted 6 lines · Ying
Quoted 3 lines · Aneesh Kumar K.V
This patch adds
/sys/devices/virtual/memtier/where all memory tier related details can be found. All allocated memory types will be listed there as/sys/devices/virtual/memtier/memtypeN/Quoted 6 lines
Another choice is to make memory types and memory tiers system devices. That is,
/sys/devices/system/memory_type/memory_typeN/sys/devices/system/memory_tier/memory_tierN
subsys_system_register()documentation says* Do not use this interface for anything new, it exists for compatibility
* with bad ideas only. New subsystems should use plain subsystems; and
* add the subsystem-wide attributes should be added to the subsystem
* directory itself and not some create fake root-device placed in
* /sys/devices/system/<name>.memtier being a virtual device, I was under the impression that
/sys/devices/virtualis the recommended place.Quoted 4 lines
That looks more natural to me. Because we already have "node" and "memory" devices there. Why don't you put memory types and memory tiers there?
Quoted 9 lines
Quoted 3 lines
And, I think we shouldn't put "
memory_type" in the "memory_tier" directory. "memory_type" isn't a part of "memory_tier".I was looking consolidating both memory tier and memory type into the same sysfs subsystem. Your recommendation imply we create two subsystem
memory_tierand memtype. I was trying to avoid that. May be a generic term like "memory_tiering" can help to consolidate all tiering related details there?A generic term "
memory_tiering" sounds good to me.Given that this will be a user-facing, stable kernel API, I think we'd better to only add what is most useful for userspace and don't have to mirror the kernel internal data structures in this interface.
My understanding is that we haven't fully settled down on how to customize memory tiers from userspace. So we don't have to show
memory_typeyet, which is a kernel data structure at this point.The userspace does need to know what are the memory tiers and which NUMA nodes are included in each memory tier. How about we provide the "nodelist" interface for each memory tier as in the original proposal?
The userspace would also like to know which memory tiers/nodes belong to the top tiers (the promotion targets). We can provide a "toptiers" or "
toptiers_nodelist" interface to report that.
Quoted 6 lines
Quoted 2 lines
How about also including abstract distance range of a memory tier? That will be useful to derive the hierarchy.
With the base abstract distance in the memtier name, do we need to show the abstract distance range if we don't customize memory tiers?
IMHO it would simpler to let userspace find abstract distance by reading a file rather than parsing a file name.
-aneesh