incoming
Subsystems (91)
Andrew Morton
maintainerPATCH 106/146 mm: migrate: add more comments for selecting target node randomly
In reply to Andrew MortonFrom: Baolin Wang <[email protected]>
Subject: mm: migrate: add more comments for selecting target node randomly
As Yang Shi suggested [1], it will be helpful to explain why we should select target node randomly now if there are multiple target nodes.
[1] https://lore.kernel.org/all/CAHbLzkqSqCL+g7dfzeOw8fPyeEC0BBv13Ny1UVGHDkadnQdR=g@mail.gmail.com/
---
mm/migrate.c | 8 ++++++++ 1 file changed, 8 insertions(+)
mm/migrate.c renamed from mm/migrate.c~mm-migrate-add-more-comments-for-selecting-target-node-randomly +8 -0
@@ -1206,6 +1206,14 @@ int next_demotion_node(int node)/* * If there are multiple target nodes, just select one * target node randomly. +* +* In addition, we can also use round-robin to select +* target node, but we should introduce another variable +* for node_demotion[] to record last selected target node, +* that may cause cache ping-pong due to the changing of +* last target node. Or introducing per-cpu data to avoid +* caching issue, which seems more complicated. So selecting +* target node randomly seems better until now. */ index =get_random_int () % target_nr;break ;
_
Andrew Morton
maintainerPATCH 107/146 mm/migrate: move node demotion code to near its user
In reply to Andrew MortonFrom: Huang Ying <[email protected]>
Subject: mm/migrate: move node demotion code to near its user
Now, node_demotion and next_demotion_node() are placed between __unmap_and_move() and unmap_and_move(). This hurts code readability. So move them near their users in the file. There's no functionality change in this patch.
---
mm/migrate.c | 265 ++++++++++++++++++++++++------------------------- 1 file changed, 132 insertions(+), 133 deletions(-)
mm/migrate.c renamed from mm/migrate.c~mm-migrate-move-node-demotion-code-to-near-its-user +132 -133
@@ -1093,139 +1093,6 @@ out:return rc; } - -/* -* node_demotion[] example: -* -* Consider a system with two sockets. Each socket has -* three classes of memory attached: fast, medium and slow. -* Each memory class is placed in its own NUMA node. The -* CPUs are placed in the node with the "fast" memory. The -* 6 NUMA nodes (0-5) might be split among the sockets like -* this: -* -* Socket A: 0, 1, 2 -* Socket B: 3, 4, 5 -* -* When Node 0 fills up, its memory should be migrated to -* Node 1. When Node 1 fills up, it should be migrated to -* Node 2. The migration path start on the nodes with the -* processors (since allocations default to this node) and -* fast memory, progress through medium and end with the -* slow memory: -* -* 0 -> 1 -> 2 -> stop -* 3 -> 4 -> 5 -> stop -* -* This is represented in the node_demotion[] like this: -* -* { nr=1, nodes[0]=1 }, // Node 0 migrates to 1 -* { nr=1, nodes[0]=2 }, // Node 1 migrates to 2 -* { nr=0, nodes[0]=-1 }, // Node 2 does not migrate -* { nr=1, nodes[0]=4 }, // Node 3 migrates to 4 -* { nr=1, nodes[0]=5 }, // Node 4 migrates to 5 -* { nr=0, nodes[0]=-1 }, // Node 5 does not migrate -* -* Moreover some systems may have multiple slow memory nodes. -* Suppose a system has one socket with 3 memory nodes, node 0 -* is fast memory type, and node 1/2 both are slow memory -* type, and the distance between fast memory node and slow -* memory node is same. So the migration path should be: -* -* 0 -> 1/2 -> stop -* -* This is represented in the node_demotion[] like this: -* { nr=2, {nodes[0]=1, nodes[1]=2} }, // Node 0 migrates to node 1 and node 2 -* { nr=0, nodes[0]=-1, }, // Node 1 dose not migrate -* { nr=0, nodes[0]=-1, }, // Node 2 does not migrate -*/ - -/* -* Writes to this array occur without locking. Cycles are -* not allowed: Node X demotes to Y which demotes to X... -* -* If multiple reads are performed, a single rcu_read_lock() -* must be held over all reads to ensure that no cycles are -* observed. -*/ -#define DEFAULT_DEMOTION_TARGET_NODES 15 - -#if MAX_NUMNODES <DEFAULT_DEMOTION_TARGET_NODES -#define DEMOTION_TARGET_NODES (MAX_NUMNODES - 1) -#else -#define DEMOTION_TARGET_NODES DEFAULT_DEMOTION_TARGET_NODES -#endif - -struct demotion_nodes { -unsigned short nr; -short nodes[DEMOTION_TARGET_NODES ]; -}; - -static struct demotion_nodes *node_demotion__read_mostly ; - -/** -* next_demotion_node() - Get the next node in the demotion path -* @node: The starting node to lookup the next node -* -* Return: node id for next memory node in the demotion path hierarchy -* from @node; NUMA_NO_NODE if @node is terminal. This does not keep -* @node online or guarantee that it *continues* to be the next demotion -* target. -*/ -int next_demotion_node (int node) -{ -struct demotion_nodes *nd; -unsigned short target_nr, index; -int target; - -if (!node_demotion) -return NUMA_NO_NODE ; - - nd = &node_demotion[node]; - -/* -* node_demotion[] is updated without excluding this -* function from running. RCU doesn't provide any -* compiler barriers, so the READ_ONCE() is required -* to avoid compiler reordering or read merging. -* -* Make sure to use RCU over entire code blocks if -* node_demotion[] reads need to be consistent. -*/ -rcu_read_lock (); - target_nr =READ_ONCE (nd->nr); - -switch (target_nr) { -case 0 : - target =NUMA_NO_NODE ; - goto out; -case 1 : - index =0 ; -break ; -default : -/* -* If there are multiple target nodes, just select one -* target node randomly. -* -* In addition, we can also use round-robin to select -* target node, but we should introduce another variable -* for node_demotion[] to record last selected target node, -* that may cause cache ping-pong due to the changing of -* last target node. Or introducing per-cpu data to avoid -* caching issue, which seems more complicated. So selecting -* target node randomly seems better until now. -*/ - index =get_random_int () % target_nr; -break ; - } - - target =READ_ONCE (nd->nodes[index]); - -out: -rcu_read_unlock (); -return target; -} -/* * Obtain the lock on page, remove all ptes and migrate the page * to the newly allocated page in newpage.
@@ -3059,6 +2926,138 @@ void migrate_vma_finalize(struct migrateEXPORT_SYMBOL (migrate_vma_finalize);#endif /* CONFIG_DEVICE_PRIVATE */ +/* +* node_demotion[] example: +* +* Consider a system with two sockets. Each socket has +* three classes of memory attached: fast, medium and slow. +* Each memory class is placed in its own NUMA node. The +* CPUs are placed in the node with the "fast" memory. The +* 6 NUMA nodes (0-5) might be split among the sockets like +* this: +* +* Socket A: 0, 1, 2 +* Socket B: 3, 4, 5 +* +* When Node 0 fills up, its memory should be migrated to +* Node 1. When Node 1 fills up, it should be migrated to +* Node 2. The migration path start on the nodes with the +* processors (since allocations default to this node) and +* fast memory, progress through medium and end with the +* slow memory: +* +* 0 -> 1 -> 2 -> stop +* 3 -> 4 -> 5 -> stop +* +* This is represented in the node_demotion[] like this: +* +* { nr=1, nodes[0]=1 }, // Node 0 migrates to 1 +* { nr=1, nodes[0]=2 }, // Node 1 migrates to 2 +* { nr=0, nodes[0]=-1 }, // Node 2 does not migrate +* { nr=1, nodes[0]=4 }, // Node 3 migrates to 4 +* { nr=1, nodes[0]=5 }, // Node 4 migrates to 5 +* { nr=0, nodes[0]=-1 }, // Node 5 does not migrate +* +* Moreover some systems may have multiple slow memory nodes. +* Suppose a system has one socket with 3 memory nodes, node 0 +* is fast memory type, and node 1/2 both are slow memory +* type, and the distance between fast memory node and slow +* memory node is same. So the migration path should be: +* +* 0 -> 1/2 -> stop +* +* This is represented in the node_demotion[] like this: +* { nr=2, {nodes[0]=1, nodes[1]=2} }, // Node 0 migrates to node 1 and node 2 +* { nr=0, nodes[0]=-1, }, // Node 1 dose not migrate +* { nr=0, nodes[0]=-1, }, // Node 2 does not migrate +*/ + +/* +* Writes to this array occur without locking. Cycles are +* not allowed: Node X demotes to Y which demotes to X... +* +* If multiple reads are performed, a single rcu_read_lock() +* must be held over all reads to ensure that no cycles are +* observed. +*/ +#define DEFAULT_DEMOTION_TARGET_NODES 15 + +#if MAX_NUMNODES <DEFAULT_DEMOTION_TARGET_NODES +#define DEMOTION_TARGET_NODES (MAX_NUMNODES - 1) +#else +#define DEMOTION_TARGET_NODES DEFAULT_DEMOTION_TARGET_NODES +#endif + +struct demotion_nodes { +unsigned short nr; +short nodes[DEMOTION_TARGET_NODES ]; +}; + +static struct demotion_nodes *node_demotion__read_mostly ; + +/** +* next_demotion_node() - Get the next node in the demotion path +* @node: The starting node to lookup the next node +* +* Return: node id for next memory node in the demotion path hierarchy +* from @node; NUMA_NO_NODE if @node is terminal. This does not keep +* @node online or guarantee that it *continues* to be the next demotion +* target. +*/ +int next_demotion_node (int node) +{ +struct demotion_nodes *nd; +unsigned short target_nr, index; +int target; + +if (!node_demotion) +return NUMA_NO_NODE ; + + nd = &node_demotion[node]; + +/* +* node_demotion[] is updated without excluding this +* function from running. RCU doesn't provide any +* compiler barriers, so the READ_ONCE() is required +* to avoid compiler reordering or read merging. +* +* Make sure to use RCU over entire code blocks if +* node_demotion[] reads need to be consistent. +*/ +rcu_read_lock (); + target_nr =READ_ONCE (nd->nr); + +switch (target_nr) { +case 0 : + target =NUMA_NO_NODE ; + goto out; +case 1 : + index =0 ; +break ; +default : +/* +* If there are multiple target nodes, just select one +* target node randomly. +* +* In addition, we can also use round-robin to select +* target node, but we should introduce another variable +* for node_demotion[] to record last selected target node, +* that may cause cache ping-pong due to the changing of +* last target node. Or introducing per-cpu data to avoid +* caching issue, which seems more complicated. So selecting +* target node randomly seems better until now. +*/ + index =get_random_int () % target_nr; +break ; + } + + target =READ_ONCE (nd->nodes[index]); + +out: +rcu_read_unlock (); +return target; +} +#if defined(CONFIG_HOTPLUG_CPU )/* Disable reclaim-based migration. */ static void __disable_all_migrate_targets (void )
_
Andrew Morton
maintainerPATCH 108/146 mm/migrate: remove redundant variables used in a for-loop
In reply to Andrew MortonFrom: Colin Ian King <[email protected]>
Subject: mm/migrate: remove redundant variables used in a for-loop
The variable addr is being set and incremented in a for-loop but not actually being used. It is redundant and so addr and also variable start can be removed.
---
mm/migrate.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
mm/migrate.c renamed from mm/migrate.c~mm-migrate-remove-redundant-variables-used-in-a-for-loop +2 -3
@@ -2481,8 +2481,7 @@ static bool migrate_vma_check_page(strucstatic void migrate_vma_unmap (struct migrate_vma *migrate) {const unsigned long npages = migrate->npages; -const unsigned long start = migrate->start; -unsigned long addr, i, restore =0 ; +unsigned long i, restore =0 ;bool allow_drain = true;lru_add_drain ();
@@ -2528,7 +2527,7 @@ static void migrate_vma_unmap(struct mig } } -for (addr = start, i =0 ; i < npages && restore; addr +=PAGE_SIZE , i++) { +for (i =0 ; i < npages && restore; i++) {struct page *page =migrate_pfn_to_page (migrate->src[i]);if (!page || (migrate->src[i] &MIGRATE_PFN_MIGRATE ))
_
Andrew Morton
maintainerPATCH 109/146 mm/thp: drop unused trace events hugepage_[invalidate|splitting]
In reply to Andrew MortonFrom: Anshuman Khandual <[email protected]>
Subject: mm/thp: drop unused trace events hugepage_[invalidate|splitting]
The trace events hugepage_[invalidate|splitting], were added via the commit 9e813308a5c1 ("powerpc/thp: Add tracepoints to track hugepage invalidate"). Afterwards their call sites i.e trace_hugepage_[invalidate|splitting] were just dropped off, leaving these trace points unused.
---
include/trace/events/thp.h | 35 ----------------------------------- 1 file changed, 35 deletions(-)
include/trace/events/thp.h renamed from include/trace/events/thp.h~mm-thp-drop-unused-trace-events-hugepage_ +0 -35
@@ -8,24 +8,6 @@#include <linux/types.h> #include <linux/tracepoint.h> -TRACE_EVENT (hugepage_invalidate, - - TP_PROTO(unsigned long addr, unsigned long pte), -TP_ARGS (addr, pte), -TP_STRUCT__entry ( - __field(unsigned long, addr) - __field(unsigned long, pte) - ), - -TP_fast_assign ( - __entry->addr = addr; - __entry->pte = pte; - ), - -TP_printk ("hugepage invalidate at addr 0x%lx and pte = 0x%lx" , - __entry->addr, __entry->pte) -); - TRACE_EVENT(hugepage_set_pmd, TP_PROTO(unsigned long addr, unsigned long pmd),
@@ -65,23 +47,6 @@ TRACE_EVENT(hugepage_update,TP_printk ("hugepage update at addr 0x%lx and pte = 0x%lx clr = 0x%lx, set = 0x%lx" , __entry->addr, __entry->pte, __entry->clr, __entry->set) ); -TRACE_EVENT (hugepage_splitting, - - TP_PROTO(unsigned long addr, unsigned long pte), -TP_ARGS (addr, pte), -TP_STRUCT__entry ( - __field(unsigned long, addr) - __field(unsigned long, pte) - ), - -TP_fast_assign ( - __entry->addr = addr; - __entry->pte = pte; - ), - -TP_printk ("hugepage splitting at addr 0x%lx and pte = 0x%lx" , - __entry->addr, __entry->pte) -);#endif /* _TRACE_THP_H */
_
Andrew Morton
maintainerPATCH 110/146 mm: ksm: fix use-after-free kasan report in ksm_might_need_to_copy
In reply to Andrew MortonFrom: Nanyong Sun <[email protected]>
Subject: mm: ksm: fix use-after-free kasan report in ksm_might_need_to_copy
When under the stress of swapping in/out with KSM enabled, there is a low probability that kasan reports the BUG of use-after-free in ksm_might_need_to_copy() when do swap in. The freed object is the anon_vma got from page_anon_vma(page).
It is because a swapcache page associated with one anon_vma now needed for another anon_vma, but the page's original vma was unmapped and the anon_vma was freed. In this case the if condition below always return false and then alloc a new page to copy. Swapin process then use the new page and can continue to run well, so this is harmless actually.
} else if (anon_vma->root == vma->anon_vma->root &&
page->index == linear_page_index(vma, address)) {This patch exchange the order of above two judgment statement to avoid the kasan warning. Let cpu run "page->index == linear_page_index(vma, address)" firstly and return false basically to skip the read of anon_vma->root which may trigger the kasan use-after-free warning.
==================================================================
BUG: KASAN: use-after-free in ksm_might_need_to_copy+0x12e/0x5b0 Read of size 8 at addr ffff88be9977dbd0 by task khugepaged/694 CPU: 8 PID: 694 Comm: khugepaged Kdump: loaded Tainted: G OE - 4.18.0.x86_64 Hardware name: 1288H V5/BC11SPSC0, BIOS 7.93 01/14/2021 Call Trace: dump_stack+0xf1/0x19b print_address_description+0x70/0x360 kasan_report+0x1b2/0x330 ksm_might_need_to_copy+0x12e/0x5b0 do_swap_page+0x452/0xe70 __collapse_huge_page_swapin+0x24b/0x720 khugepaged_scan_pmd+0xcae/0x1ff0 khugepaged+0x8ee/0xd70 kthread+0x1a2/0x1d0 ret_from_fork+0x1f/0x40 Allocated by task 2306153: kasan_kmalloc+0xa0/0xd0 kmem_cache_alloc+0xc0/0x1c0 anon_vma_clone+0xf7/0x380 anon_vma_fork+0xc0/0x390 copy_process+0x447b/0x4810 _do_fork+0x118/0x620 do_syscall_64+0x112/0x360 entry_SYSCALL_64_after_hwframe+0x65/0xca Freed by task 2306242: __kasan_slab_free+0x130/0x180 kmem_cache_free+0x78/0x1d0 unlink_anon_vmas+0x19c/0x4a0 free_pgtables+0x137/0x1b0 exit_mmap+0x133/0x320 mmput+0x15e/0x390 do_exit+0x8c5/0x1210 do_group_exit+0xb5/0x1b0 __x64_sys_exit_group+0x21/0x30 do_syscall_64+0x112/0x360 entry_SYSCALL_64_after_hwframe+0x65/0xca The buggy address belongs to the object at ffff88be9977dba0 which belongs to the cache anon_vma_chain of size 64 The buggy address is located 48 bytes inside of 64-byte region [ffff88be9977dba0, ffff88be9977dbe0) The buggy address belongs to the page:
page:ffffea00fa65df40 count:1 mapcount:0 mapping:ffff888107717800 index:0x0
flags: 0x17ffffc0000100(slab)
==================================================================
---
mm/ksm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
mm/ksm.c renamed from mm/ksm.c~mm-ksm-fix-use-after-free-kasan-report-in-ksm_might_need_to_copy +2 -2
@@ -2576,8 +2576,8 @@ struct page *ksm_might_need_to_copy(strureturn page;/* no need to copy it */ }else if (!anon_vma) {return page;/* no need to copy it */ - }else if (anon_vma->root == vma->anon_vma->root && - page->index ==linear_page_index (vma, address)) { + }else if (page->index ==linear_page_index (vma, address) && + anon_vma->root == vma->anon_vma->root) {return page;/* still no need to copy it */ }if (!PageUptodate (page))
_
Andrew Morton
maintainerPATCH 111/146 mm/hwpoison: mf_mutex for soft offline and unpoison
In reply to Andrew MortonFrom: Naoya Horiguchi <[email protected]>
Subject: mm/hwpoison: mf_mutex for soft offline and unpoison
Patch series "mm/hwpoison: fix unpoison_memory()", v4.
Main purpose of this series is to sync unpoison code to recent changes around how hwpoison code takes page refcount. Unpoison should work or simply fail (without crash) if impossible.
The recent works of keeping hwpoison pages in shmem pagecache introduce a new state of hwpoisoned pages, but unpoison for such pages is not supported yet with this series.
It seems that soft-offline and unpoison can be used as general purpose page offline/online mechanism (not in the context of memory error). I think that we need some additional works to realize it because currently soft-offline and unpoison are assumed not to happen so frequently (print out too many messages for aggressive usecases). But anyway this could be another interesting next topic.
v1: https://lore.kernel.org/linux-mm/[email protected]/
v2: https://lore.kernel.org/linux-mm/[email protected]/
v3: https://lore.kernel.org/linux-mm/[email protected]/
This patch (of 3):
Originally mf_mutex is introduced to serialize multiple MCE events, but it is not that useful to allow unpoison to run in parallel with memory_failure() and soft offline. So apply mf_mutex to soft offline and unpoison. The memory failure handler and soft offline handler get simpler with this.
---
mm/memory-failure.c | 62 ++++++++++++------------------------------ 1 file changed, 18 insertions(+), 44 deletions(-)
mm/memory-failure.c renamed from mm/memory-failure.c~mm-hwpoison-mf_mutex-for-soft-offline-and-unpoison +18 -44
@@ -1502,14 +1502,6 @@ static int memory_failure_hugetlb(unsignlock_page (head); page_flags = head->flags; -if (!PageHWPoison (head)) { -pr_err ("Memory failure: %#lx: just unpoisoned\n" , pfn); -num_poisoned_pages_dec (); -unlock_page (head); -put_page (head); -return 0 ; - } -/* * TODO: hwpoison for pud-sized hugetlb doesn't work right now, so * simply disable it. In order to make it work properly, we need
@@ -1623,6 +1615,8 @@ out:return rc; } +static DEFINE_MUTEX (mf_mutex); +/** * memory_failure - Handle memory failure of a page. * @pfn: Page Number of the corrupted page
@@ -1649,7 +1643,6 @@ int memory_failure(unsigned long pfn, inint res =0 ;unsigned long page_flags;bool retry = true; -static DEFINE_MUTEX (mf_mutex);if (!sysctl_memory_failure_recovery)panic ("Memory failure on page %lx" , pfn);
@@ -1783,16 +1776,6 @@ try_again:*/ page_flags = p->flags; -/* -* unpoison always clear PG_hwpoison inside page lock -*/ -if (!PageHWPoison (p)) { -pr_err ("Memory failure: %#lx: just unpoisoned\n" , pfn); -num_poisoned_pages_dec (); -unlock_page (p); -put_page (p); - goto unlock_mutex; - }if (hwpoison_filter (p)) {if (TestClearPageHWPoison (p))num_poisoned_pages_dec ();
@@ -1973,6 +1956,7 @@ int unpoison_memory(unsigned long pfn)struct page *page;struct page *p;int freeit =0 ; +int ret =0 ;unsigned long flags =0 ; static DEFINE_RATELIMIT_STATE(unpoison_rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST);
@@ -1983,39 +1967,30 @@ int unpoison_memory(unsigned long pfn) p =pfn_to_page (pfn); page =compound_head (p); +mutex_lock (&mf_mutex); +if (!PageHWPoison (p)) {unpoison_pr_info ("Unpoison: Page was already unpoisoned %#lx\n" , pfn, &unpoison_rs); -return 0 ; + goto unlock_mutex; }if (page_count (page) >1 ) {unpoison_pr_info ("Unpoison: Someone grabs the hwpoison page %#lx\n" , pfn, &unpoison_rs); -return 0 ; + goto unlock_mutex; }if (page_mapped (page)) {unpoison_pr_info ("Unpoison: Someone maps the hwpoison page %#lx\n" , pfn, &unpoison_rs); -return 0 ; + goto unlock_mutex; }if (page_mapping (page)) {unpoison_pr_info ("Unpoison: the hwpoison page has non-NULL mapping %#lx\n" , pfn, &unpoison_rs); -return 0 ; - } - -/* -* unpoison_memory() can encounter thp only when the thp is being -* worked by memory_failure() and the page lock is not held yet. -* In such case, we yield to memory_failure() and make unpoison fail. -*/ -if (!PageHuge (page) &&PageTransHuge (page)) { -unpoison_pr_info ("Unpoison: Memory failure is now running on %#lx\n" , - pfn, &unpoison_rs); -return 0 ; + goto unlock_mutex; }if (!get_hwpoison_page (p, flags)) {
@@ -2023,29 +1998,23 @@ int unpoison_memory(unsigned long pfn)num_poisoned_pages_dec ();unpoison_pr_info ("Unpoison: Software-unpoisoned free page %#lx\n" , pfn, &unpoison_rs); -return 0 ; + goto unlock_mutex; } -lock_page (page); -/* -* This test is racy because PG_hwpoison is set outside of page lock. -* That's acceptable because that won't trigger kernel panic. Instead, -* the PG_hwpoison page will be caught and isolated on the entrance to -* the free buddy page pool. -*/ if (TestClearPageHWPoison (page)) {unpoison_pr_info ("Unpoison: Software-unpoisoned page %#lx\n" , pfn, &unpoison_rs);num_poisoned_pages_dec (); freeit =1 ; } -unlock_page (page);put_page (page);if (freeit && !(pfn ==my_zero_pfn (0 ) &&page_count (p) ==1 ))put_page (page); -return 0 ; +unlock_mutex: +mutex_unlock (&mf_mutex); +return ret; }EXPORT_SYMBOL (unpoison_memory);
@@ -2226,9 +2195,12 @@ int soft_offline_page(unsigned long pfn,return -EIO ; } +mutex_lock (&mf_mutex); +if (PageHWPoison (page)) {pr_info ("%s: %#lx page already poisoned\n" , __func__, pfn);put_ref_page (ref_page); +mutex_unlock (&mf_mutex);return 0 ; }
@@ -2247,5 +2219,7 @@ retry: } } +mutex_unlock (&mf_mutex); +return ret; }
_
Andrew Morton
maintainerPATCH 112/146 mm/hwpoison: remove MF_MSG_BUDDY_2ND and MF_MSG_POISONED_HUGE
In reply to Andrew MortonFrom: Naoya Horiguchi <[email protected]>
Subject: mm/hwpoison: remove MF_MSG_BUDDY_2ND and MF_MSG_POISONED_HUGE
These action_page_types are no longer used, so remove them.
---
include/linux/mm.h | 2 -- include/ras/ras_event.h | 2 -- mm/memory-failure.c | 2 -- 3 files changed, 6 deletions(-)
include/linux/mm.h renamed from include/linux/mm.h~mm-hwpoison-remove-mf_msg_buddy_2nd-and-mf_msg_poisoned_huge +0 -2
@@ -3201,7 +3201,6 @@ enum mf_action_page_type {MF_MSG_KERNEL_HIGH_ORDER ,MF_MSG_SLAB ,MF_MSG_DIFFERENT_COMPOUND , -MF_MSG_POISONED_HUGE ,MF_MSG_HUGE ,MF_MSG_FREE_HUGE , MF_MSG_NON_PMD_HUGE,
@@ -3216,7 +3215,6 @@ enum mf_action_page_type {MF_MSG_CLEAN_LRU ,MF_MSG_TRUNCATED_LRU ,MF_MSG_BUDDY , -MF_MSG_BUDDY_2ND ,MF_MSG_DAX ,MF_MSG_UNSPLIT_THP , MF_MSG_UNKNOWN,
include/ras/ras_event.h renamed from include/ras/ras_event.h~mm-hwpoison-remove-mf_msg_buddy_2nd-and-mf_msg_poisoned_huge +0 -2
@@ -358,7 +358,6 @@ TRACE_EVENT(aer_event,EM (MF_MSG_KERNEL_HIGH_ORDER ,"high-order kernel page" ) \ EM ( MF_MSG_SLAB, "kernel slab page" ) \ EM ( MF_MSG_DIFFERENT_COMPOUND, "different compound page after locking" ) \ - EM ( MF_MSG_POISONED_HUGE, "huge page already hardware poisoned" ) \ EM ( MF_MSG_HUGE, "huge page" ) \ EM ( MF_MSG_FREE_HUGE, "free huge page" ) \ EM ( MF_MSG_NON_PMD_HUGE, "non-pmd-sized huge page" ) \
@@ -373,7 +372,6 @@ TRACE_EVENT(aer_event,EM (MF_MSG_CLEAN_LRU ,"clean LRU page" ) \ EM ( MF_MSG_TRUNCATED_LRU, "already truncated LRU page" ) \ EM ( MF_MSG_BUDDY, "free buddy page" ) \ - EM ( MF_MSG_BUDDY_2ND, "free buddy page (2nd try)" ) \ EM ( MF_MSG_DAX, "dax page" ) \ EM ( MF_MSG_UNSPLIT_THP, "unsplit thp" ) \ EMe ( MF_MSG_UNKNOWN, "unknown page" )
mm/memory-failure.c renamed from mm/memory-failure.c~mm-hwpoison-remove-mf_msg_buddy_2nd-and-mf_msg_poisoned_huge +0 -2
@@ -723,7 +723,6 @@ static const char * const action_page_ty [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page", [MF_MSG_SLAB] = "kernel slab page", [MF_MSG_DIFFERENT_COMPOUND] = "different compound page after locking", - [MF_MSG_POISONED_HUGE] = "huge page already hardware poisoned", [MF_MSG_HUGE] = "huge page", [MF_MSG_FREE_HUGE] = "free huge page", [MF_MSG_NON_PMD_HUGE] = "non-pmd-sized huge page",
@@ -738,7 +737,6 @@ static const char * const action_page_ty [MF_MSG_CLEAN_LRU] = "clean LRU page", [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page", [MF_MSG_BUDDY] = "free buddy page", - [MF_MSG_BUDDY_2ND] = "free buddy page (2nd try)", [MF_MSG_DAX] = "dax page", [MF_MSG_UNSPLIT_THP] = "unsplit thp", [MF_MSG_UNKNOWN] = "unknown page",
_
Andrew Morton
maintainerPATCH 113/146 mm/hwpoison: fix unpoison_memory()
In reply to Andrew MortonFrom: Naoya Horiguchi <[email protected]>
Subject: mm/hwpoison: fix unpoison_memory()
After recent soft-offline rework, error pages can be taken off from buddy allocator, but the existing unpoison_memory() does not properly undo the operation. Moreover, due to the recent change on __get_hwpoison_page(), get_page_unless_zero() is hardly called for hwpoisoned pages. So __get_hwpoison_page() highly likely returns -EBUSY (meaning to fail to grab page refcount) and unpoison just clears PG_hwpoison without releasing a refcount. That does not lead to a critical issue like kernel panic, but unpoisoned pages never get back to buddy (leaked permanently), which is not good.
To (partially) fix this, we need to identify "taken off" pages from other types of hwpoisoned pages. We can't use refcount or page flags for this purpose, so a pseudo flag is defined by hacking ->private field. Someone might think that put_page() is enough to cancel taken-off pages, but the normal free path contains some operations not suitable for the current purpose, and can fire VM_BUG_ON().
Note that unpoison_memory() is now supposed to be cancel hwpoison events injected only by madvise() or /sys/devices/system/memory/{hard,soft}_offline_page, not by MCE injection, so please don't try to use unpoison when testing with MCE injection.
---
include/linux/mm.h | 1 include/linux/page-flags.h | 4 + mm/memory-failure.c | 109 ++++++++++++++++++++++++++++------- mm/page_alloc.c | 27 ++++++++ 4 files changed, 122 insertions(+), 19 deletions(-)
include/linux/mm.h renamed from include/linux/mm.h~mm-hwpoison-fix-unpoison_memory +1 -0
@@ -3174,6 +3174,7 @@ enum mf_flags {MF_ACTION_REQUIRED =1 <<1 ,MF_MUST_KILL =1 <<2 ,MF_SOFT_OFFLINE =1 <<3 , + MF_UNPOISON = 1 << 4, };extern int memory_failure (unsigned long pfn,int flags);extern void memory_failure_queue (unsigned long pfn,int flags);
include/linux/page-flags.h renamed from include/linux/page-flags.h~mm-hwpoison-fix-unpoison_memory +4 -0
@@ -522,7 +522,11 @@ PAGEFLAG_FALSE(Uncached, uncached) PAGEFLAG(HWPoison, hwpoison, PF_ANY) TESTSCFLAG(HWPoison, hwpoison, PF_ANY)#define __PG_HWPOISON (1UL << PG_hwpoison) +#define MAGIC_HWPOISON 0x48575053U/* HWPS */ +extern void SetPageHWPoisonTakenOff (struct page *page); +extern void ClearPageHWPoisonTakenOff (struct page *page);extern bool take_page_off_buddy (struct page *page); +extern bool put_page_back_buddy (struct page *page);#else PAGEFLAG_FALSE(HWPoison, hwpoison)#define __PG_HWPOISON 0
mm/memory-failure.c renamed from mm/memory-failure.c~mm-hwpoison-fix-unpoison_memory +90 -19
159 lines not on this page · raw message
mm/page_alloc.c renamed from mm/page_alloc.c~mm-hwpoison-fix-unpoison_memory +27 -0
@@ -19,6 +19,7 @@#include <linux/mm.h> #include <linux/highmem.h> #include <linux/swap.h> +#include <linux/swapops.h> #include <linux/interrupt.h> #include <linux/pagemap.h> #include <linux/jiffies.h>
@@ -9508,6 +9509,7 @@ bool take_page_off_buddy(struct page *padel_page_from_free_list (page_head, zone, page_order);break_down_buddy_pages (zone, page_head, page,0 , page_order, migratetype); +SetPageHWPoisonTakenOff (page);if (!is_migrate_isolate (migratetype))__mod_zone_freepage_state (zone,-1 , migratetype); ret = true;
@@ -9519,6 +9521,31 @@ bool take_page_off_buddy(struct page *paspin_unlock_irqrestore (&zone->lock, flags); return ret; } + +/* +* Cancel takeoff done by take_page_off_buddy(). +*/ +bool put_page_back_buddy (struct page *page) +{ +struct zone *zone =page_zone (page); +unsigned long pfn =page_to_pfn (page); +unsigned long flags; +int migratetype =get_pfnblock_migratetype (page, pfn); +bool ret = false; + +spin_lock_irqsave (&zone->lock, flags); +if (put_page_testzero (page)) { +ClearPageHWPoisonTakenOff (page); +__free_one_page (page, pfn, zone,0 , migratetype,FPI_NONE ); +if (TestClearPageHWPoison (page)) { +num_poisoned_pages_dec (); + ret = true; + } + } +spin_unlock_irqrestore (&zone->lock, flags); + +return ret; +}#endif #ifdef CONFIG_ZONE_DMA
_
Andrew Morton
maintainerPATCH 114/146 mm: memcg/percpu: account extra objcg space to memory cgroups
In reply to Andrew MortonFrom: Qi Zheng <[email protected]>
Subject: mm: memcg/percpu: account extra objcg space to memory cgroups
Similar to slab memory allocator, for each accounted percpu object there is an extra space which is used to store obj_cgroup membership. Charge it too.
---
mm/percpu-internal.h | 18 ++++++++++++++++++ mm/percpu.c | 10 +++++----- 2 files changed, 23 insertions(+), 5 deletions(-)
mm/percpu.c renamed from mm/percpu.c~mm-memcg-percpu-account-extra-objcg-space-to-memory-cgroups +5 -5
@@ -1635,7 +1635,7 @@ static bool pcpu_memcg_pre_alloc_hook(siif (!objcg)return true; -if (obj_cgroup_charge (objcg, gfp, size *num_possible_cpus ())) { +if (obj_cgroup_charge (objcg, gfp,pcpu_obj_full_size (size))) {obj_cgroup_put (objcg);return false; }
@@ -1656,10 +1656,10 @@ static void pcpu_memcg_post_alloc_hook(srcu_read_lock ();mod_memcg_state (obj_cgroup_memcg (objcg),MEMCG_PERCPU_B , - size *num_possible_cpus ()); +pcpu_obj_full_size (size));rcu_read_unlock (); }else { -obj_cgroup_uncharge (objcg, size *num_possible_cpus ()); +obj_cgroup_uncharge (objcg,pcpu_obj_full_size (size));obj_cgroup_put (objcg); } }
@@ -1676,11 +1676,11 @@ static void pcpu_memcg_free_hook(structreturn ; chunk->obj_cgroups[off >>PCPU_MIN_ALLOC_SHIFT ] =NULL ; -obj_cgroup_uncharge (objcg, size *num_possible_cpus ()); +obj_cgroup_uncharge (objcg,pcpu_obj_full_size (size));rcu_read_lock ();mod_memcg_state (obj_cgroup_memcg (objcg),MEMCG_PERCPU_B , - -(size *num_possible_cpus ())); + -pcpu_obj_full_size (size));rcu_read_unlock ();obj_cgroup_put (objcg);
mm/percpu-internal.h renamed from mm/percpu-internal.h~mm-memcg-percpu-account-extra-objcg-space-to-memory-cgroups +18 -0
@@ -113,6 +113,24 @@ static inline int pcpu_chunk_map_bits(streturn pcpu_nr_pages_to_map_bits (chunk->nr_pages); } +#ifdef CONFIG_MEMCG_KMEM +/** +* pcpu_obj_full_size - helper to calculate size of each accounted object +* @size: size of area to allocate in bytes +* +* For each accounted object there is an extra space which is used to store +* obj_cgroup membership. Charge it too. +*/ +static inline size_t pcpu_obj_full_size (size_t size) +{ +size_t extra_size; + + extra_size = size /PCPU_MIN_ALLOC_SIZE *sizeof (struct obj_cgroup *); + +return size *num_possible_cpus () + extra_size; +} +#endif /* CONFIG_MEMCG_KMEM */ +#ifdef CONFIG_PERCPU_STATS #include <linux/spinlock.h>
_
Andrew Morton
maintainerPATCH 115/146 mm/rmap: fix potential batched TLB flush race
In reply to Andrew MortonFrom: Huang Ying <[email protected]>
Subject: mm/rmap: fix potential batched TLB flush race
In theory, the following race is possible for batched TLB flushing.
CPU0 CPU1
---- ----shrink_page_list()
unmap
zap_pte_range()
flush_tlb_batched_pending()
flush_tlb_mm()
try_to_unmap()
set_tlb_ubc_flush_pending()
mm->tlb_flush_batched = true
mm->tlb_flush_batched = falseAfter the TLB is flushed on CPU1 via flush_tlb_mm() and before mm->tlb_flush_batched is set to false, some PTE is unmapped on CPU0 and the TLB flushing is pended. Then the pended TLB flushing will be lost. Although both set_tlb_ubc_flush_pending() and flush_tlb_batched_pending() are called with PTL locked, different PTL instances may be used.
Because the race window is really small, and the lost TLB flushing will cause problem only if a TLB entry is inserted before the unmapping in the race window, the race is only theoretical. But the fix is simple and cheap too.
Syzbot has reported this too as follows,
==================================================================
BUG: KCSAN: data-race in flush_tlb_batched_pending / try_to_unmap_one
write to 0xffff8881072cfbbc of 1 bytes by task 17406 on cpu 1:
flush_tlb_batched_pending+0x5f/0x80 mm/rmap.c:691 madvise_free_pte_range+0xee/0x7d0 mm/madvise.c:594 walk_pmd_range mm/pagewalk.c:128 [inline] walk_pud_range mm/pagewalk.c:205 [inline] walk_p4d_range mm/pagewalk.c:240 [inline] walk_pgd_range mm/pagewalk.c:277 [inline] __walk_page_range+0x981/0x1160 mm/pagewalk.c:379 walk_page_range+0x131/0x300 mm/pagewalk.c:475 madvise_free_single_vma mm/madvise.c:734 [inline] madvise_dontneed_free mm/madvise.c:822 [inline] madvise_vma mm/madvise.c:996 [inline] do_madvise+0xe4a/0x1140 mm/madvise.c:1202 __do_sys_madvise mm/madvise.c:1228 [inline] __se_sys_madvise mm/madvise.c:1226 [inline] __x64_sys_madvise+0x5d/0x70 mm/madvise.c:1226 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x44/0xd0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x44/0xae
write to 0xffff8881072cfbbc of 1 bytes by task 71 on cpu 0:
set_tlb_ubc_flush_pending mm/rmap.c:636 [inline] try_to_unmap_one+0x60e/0x1220 mm/rmap.c:1515 rmap_walk_anon+0x2fb/0x470 mm/rmap.c:2301 try_to_unmap+0xec/0x110 shrink_page_list+0xe91/0x2620 mm/vmscan.c:1719 shrink_inactive_list+0x3fb/0x730 mm/vmscan.c:2394 shrink_list mm/vmscan.c:2621 [inline] shrink_lruvec+0x3c9/0x710 mm/vmscan.c:2940 shrink_node_memcgs+0x23e/0x410 mm/vmscan.c:3129 shrink_node+0x8f6/0x1190 mm/vmscan.c:3252 kswapd_shrink_node mm/vmscan.c:4022 [inline] balance_pgdat+0x702/0xd30 mm/vmscan.c:4213 kswapd+0x200/0x340 mm/vmscan.c:4473 kthread+0x2c7/0x2e0 kernel/kthread.c:327 ret_from_fork+0x1f/0x30
value changed: 0x01 -> 0x00
Reported by Kernel Concurrency Sanitizer on:
CPU: 0 PID: 71 Comm: kswapd0 Not tainted 5.16.0-rc1-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 ==================================================================
---
include/linux/mm_types.h | 2 - mm/rmap.c | 43 ++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 8 deletions(-)
include/linux/mm_types.h renamed from include/linux/mm_types.h~mm-rmap-fix-potential-batched-tlb-flush-race +1 -1
@@ -647,7 +647,7 @@ struct mm_struct {atomic_t tlb_flush_pending;#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH /* See flush_tlb_batched_pending() */ -bool tlb_flush_batched; +atomic_t tlb_flush_batched;#endif struct uprobes_state uprobes_state;#ifdef CONFIG_PREEMPT_RT
mm/rmap.c renamed from mm/rmap.c~mm-rmap-fix-potential-batched-tlb-flush-race +36 -7
67 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 116/146 zpool: remove the list of pools_head
In reply to Andrew MortonFrom: Zhaoyu Liu <[email protected]>
Subject: zpool: remove the list of pools_head
The list of pools_head is no longer needed because the caller has been deleted in commit 479305fd7172 ("zpool: remove zpool_evict()").
---
mm/zpool.c | 12 ------------ 1 file changed, 12 deletions(-)
mm/zpool.c renamed from mm/zpool.c~zpool-remove-the-list-of-pools_head +0 -12
@@ -24,16 +24,11 @@ struct zpool {const struct zpool_ops *ops;bool evictable;bool can_sleep_mapped; - -struct list_head list; };static LIST_HEAD (drivers_head);static DEFINE_SPINLOCK (drivers_lock); -static LIST_HEAD (pools_head); -static DEFINE_SPINLOCK (pools_lock); -/** * zpool_register_driver() - register a zpool implementation. * @driver: driver to register
@@ -195,10 +190,6 @@ struct zpool *zpool_create_pool(const chpr_debug ("created pool type %s\n" , type); -spin_lock (&pools_lock); -list_add (&zpool->list, &pools_head); -spin_unlock (&pools_lock); -return zpool; }
@@ -217,9 +208,6 @@ void zpool_destroy_pool(struct zpool *zp {pr_debug ("destroying pool type %s\n" , zpool->driver->type); -spin_lock (&pools_lock); -list_del (&zpool->list); -spin_unlock (&pools_lock); zpool->driver->destroy (zpool->pool);zpool_put_driver (zpool->driver);kfree (zpool);
_
Andrew Morton
maintainerPATCH 117/146 zram: use ATTRIBUTE_GROUPS
In reply to Andrew MortonFrom: Luis Chamberlain <[email protected]>
Subject: zram: use ATTRIBUTE_GROUPS
Embrace ATTRIBUTE_GROUPS to avoid boiler plate code. This should not introduce any functional changes.
---
drivers/block/zram/zram_drv.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)
drivers/block/zram/zram_drv.c renamed from drivers/block/zram/zram_drv.c~zram-use-attribute_groups +2 -9
@@ -1903,14 +1903,7 @@ static struct attribute *zram_disk_attrs NULL, }; -static const struct attribute_group zram_disk_attr_group = { - .attrs = zram_disk_attrs, -}; - -static const struct attribute_group *zram_disk_attr_groups[] = { - &zram_disk_attr_group, -NULL , -}; +ATTRIBUTE_GROUPS (zram_disk);/* * Allocate and initialize new zram device. the function returns
@@ -1982,7 +1975,7 @@ static int zram_add(void)blk_queue_max_write_zeroes_sectors (zram->disk->queue,UINT_MAX );blk_queue_flag_set (QUEUE_FLAG_STABLE_WRITES , zram->disk->queue); - ret =device_add_disk (NULL , zram->disk, zram_disk_attr_groups); + ret =device_add_disk (NULL , zram->disk, zram_disk_groups);if (ret) goto out_cleanup_disk;
_
Andrew Morton
maintainerPATCH 118/146 mm: fix some comment errors
In reply to Andrew MortonFrom: Quanfa Fu <[email protected]>
Subject: mm: fix some comment errors
---
mm/khugepaged.c | 2 +- mm/memory-failure.c | 2 +- mm/slab_common.c | 2 +- mm/swap.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-)
mm/khugepaged.c renamed from mm/khugepaged.c~writeback-fix-some-comment-errors +1 -1
@@ -1303,7 +1303,7 @@ static int khugepaged_scan_pmd(struct mm/* * Record which node the original page is from and save this * information to khugepaged_node_load[]. -* Khupaged will allocate hugepage from the node has the max +* Khugepaged will allocate hugepage from the node has the max * hit record. */ node =page_to_nid (page);
mm/memory-failure.c renamed from mm/memory-failure.c~writeback-fix-some-comment-errors +1 -1
@@ -1306,7 +1306,7 @@ static int __get_unpoison_page(struct pa* * get_hwpoison_page() takes a page refcount of an error page to handle memory * error on it, after checking that the error page is in a well-defined state -* (defined as a page-type we can successfully handle the memor error on it, +* (defined as a page-type we can successfully handle the memory error on it, * such as LRU page and hugetlb page). * * Memory error handling could be triggered at any time on any type of page,
mm/slab_common.c renamed from mm/slab_common.c~writeback-fix-some-comment-errors +1 -1
@@ -819,7 +819,7 @@ void __init setup_kmalloc_cache_index_taif (KMALLOC_MIN_SIZE >=64 ) {/* -* The 96 byte size cache is not used if the alignment +* The 96 byte sized cache is not used if the alignment * is 64 byte. */ for (i =64 +8 ; i <=96 ; i +=8 )
mm/swap.c renamed from mm/swap.c~writeback-fix-some-comment-errors +1 -1
@@ -882,7 +882,7 @@ void lru_cache_disable(void)* all online CPUs so any calls of lru_cache_disabled wrapped by * local_lock or preemption disabled would be ordered by that. * The atomic operation doesn't need to have stronger ordering -* requirements because that is enforeced by the scheduling +* requirements because that is enforced by the scheduling * guarantees. */ __lru_add_drain_all (true);
_
Andrew Morton
maintainerPATCH 119/146 mm: make some vars and functions static or __init
In reply to Andrew MortonFrom: Ting Liu <[email protected]>
Subject: mm: make some vars and functions static or __init
"page_idle_ops" as a global var, but its scope of use within this document. So it should be static.
"page_ext_ops" is a var used in the kernel initial phase. And other functions are aslo used in the kernel initial phase. So they should be __init or __initdata to reclaim memory.
---
include/linux/page_idle.h | 1 - mm/page_ext.c | 4 ++-- mm/page_owner.c | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-)
include/linux/page_idle.h renamed from include/linux/page_idle.h~mm-make-some-vars-and-functions-static-or-__init +0 -1
@@ -13,7 +13,6 @@* If there is not enough space to store Idle and Young bits in page flags, use * page ext flags instead. */ -extern struct page_ext_operations page_idle_ops;static inline bool folio_test_young (struct folio *folio) {
mm/page_ext.c renamed from mm/page_ext.c~mm-make-some-vars-and-functions-static-or-__init +2 -2
@@ -64,12 +64,12 @@ static bool need_page_idle(void) {return true; } -struct page_ext_operations page_idle_ops = { +static struct page_ext_operations page_idle_ops__initdata = { .need = need_page_idle, };#endif -static struct page_ext_operations *page_ext_ops[] = { +static struct page_ext_operations *page_ext_ops[]__initdata = { #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif
mm/page_owner.c renamed from mm/page_owner.c~mm-make-some-vars-and-functions-static-or-__init +2 -2
@@ -46,7 +46,7 @@ static int __init early_page_owner_param }early_param ("page_owner" , early_page_owner_param); -static bool need_page_owner (void ) +static __init bool need_page_owner (void ) {return page_owner_enabled; }
@@ -75,7 +75,7 @@ static noinline void register_early_stac early_handle =create_dummy_stack (); } -static void init_page_owner (void ) +static __init void init_page_owner (void ) {if (!page_owner_enabled)return ;
_
Andrew Morton
maintainerPATCH 120/146 mm/hmm.c: allow VM_MIXEDMAP to work with hmm_range_fault
In reply to Andrew MortonFrom: Alistair Popple <[email protected]>
Subject: mm/hmm.c: allow VM_MIXEDMAP to work with hmm_range_fault
hmm_range_fault() can be used instead of get_user_pages() for devices which allow faulting however unlike get_user_pages() it will return an error when used on a VM_MIXEDMAP range.
To make hmm_range_fault() more closely match get_user_pages() remove this restriction. This requires dealing with the !ARCH_HAS_PTE_SPECIAL case in hmm_vma_handle_pte(). Rather than replicating the logic of vm_normal_page() call it directly and do a check for the zero pfn similar to what get_user_pages() currently does.
Also add a test to hmm selftest to verify functionality.
---
lib/test_hmm.c | 24 +++++++++++++ mm/hmm.c | 5 +- tools/testing/selftests/vm/hmm-tests.c | 42 +++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-)
lib/test_hmm.c renamed from lib/test_hmm.c~mm-hmmc-allow-vm_mixedmap-to-work-with-hmm_range_fault +24 -0
@@ -1086,9 +1086,33 @@ static long dmirror_fops_unlocked_ioctl(return 0 ; } +static int dmirror_fops_mmap (struct file *file,struct vm_area_struct *vma) +{ +unsigned long addr; + +for (addr = vma->vm_start; addr < vma->vm_end; addr +=PAGE_SIZE ) { +struct page *page; +int ret; + + page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!page) +return -ENOMEM ; + + ret =vm_insert_page (vma, addr, page); +if (ret) { +__free_page (page); +return ret; + } +put_page (page); + } + +return 0 ; +} +static const struct file_operations dmirror_fops = { .open = dmirror_fops_open, .release = dmirror_fops_release, + .mmap = dmirror_fops_mmap, .unlocked_ioctl = dmirror_fops_unlocked_ioctl, .llseek = default_llseek, .owner =THIS_MODULE ,
mm/hmm.c renamed from mm/hmm.c~mm-hmmc-allow-vm_mixedmap-to-work-with-hmm_range_fault +3 -2
@@ -300,7 +300,8 @@ static int hmm_vma_handle_pte(struct mm_* Since each architecture defines a struct page for the zero page, just * fall through and treat it like a normal page. */ -if (pte_special (pte) && !pte_devmap (pte) && +if (!vm_normal_page (walk->vma, addr, pte) && + !pte_devmap (pte) && !is_zero_pfn (pte_pfn (pte))) {if (hmm_pte_need_fault (hmm_vma_walk, pfn_req_flags,0 )) {pte_unmap (ptep);
@@ -518,7 +519,7 @@ static int hmm_vma_walk_test(unsigned lostruct hmm_range *range = hmm_vma_walk->range;struct vm_area_struct *vma = walk->vma; -if (!(vma->vm_flags & (VM_IO |VM_PFNMAP |VM_MIXEDMAP )) && +if (!(vma->vm_flags & (VM_IO |VM_PFNMAP )) && vma->vm_flags &VM_READ )return 0 ;
tools/testing/selftests/vm/hmm-tests.c renamed from tools/testing/selftests/vm/hmm-tests.c~mm-hmmc-allow-vm_mixedmap-to-work-with-hmm_range_fault +42 -0
48 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 121/146 mm/damon: unified access_check function naming rules
In reply to Andrew MortonFrom: Xin Hao <[email protected]>
Subject: mm/damon: unified access_check function naming rules
Patch series "mm/damon: Do some small changes", v4.
This patch (of 4):
In damon/paddr.c file, two functions names start with underscore,
static void __damon_pa_prepare_access_check(struct damon_ctx *ctx,
struct damon_region *r)
static void __damon_pa_prepare_access_check(struct damon_ctx *ctx,
struct damon_region *r)
In damon/vaddr.c file, there are also two functions with the same function,
static void damon_va_prepare_access_check(struct damon_ctx *ctx,
struct mm_struct *mm, struct damon_region *r)
static void damon_va_check_access(struct damon_ctx *ctx,
struct mm_struct *mm, struct damon_region *r)
It makes sense to keep consistent, and it is not easy to be confused with the function that call them.
---
mm/damon/vaddr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
mm/damon/vaddr.c renamed from mm/damon/vaddr.c~mm-damon-unified-access_check-function-naming-rules +4 -4
@@ -410,7 +410,7 @@ static void damon_va_mkold(struct mm_str* Functions for the access checking of the regions */ -static void damon_va_prepare_access_check (struct damon_ctx *ctx, +static void __damon_va_prepare_access_check (struct damon_ctx *ctx,struct mm_struct *mm,struct damon_region *r) { r->sampling_addr =damon_rand (r->ar.start, r->ar.end);
@@ -429,7 +429,7 @@ void damon_va_prepare_access_checks(struif (!mm)continue ;damon_for_each_region (r, t) -damon_va_prepare_access_check (ctx, mm, r); +__damon_va_prepare_access_check (ctx, mm, r);mmput (mm); } }
@@ -515,7 +515,7 @@ static bool damon_va_young(struct mm_str* mm 'mm_struct' for the given virtual address space * r the region to be checked */ -static void damon_va_check_access (struct damon_ctx *ctx, +static void __damon_va_check_access (struct damon_ctx *ctx,struct mm_struct *mm,struct damon_region *r) {static struct mm_struct *last_mm;
@@ -551,7 +551,7 @@ unsigned int damon_va_check_accesses(strif (!mm)continue ;damon_for_each_region (r, t) { -damon_va_check_access (ctx, mm, r); +__damon_va_check_access (ctx, mm, r); max_nr_accesses =max (r->nr_accesses, max_nr_accesses); }mmput (mm);
_
Andrew Morton
maintainerPATCH 122/146 mm/damon: add 'age' of region tracepoint support
In reply to Andrew MortonFrom: Xin Hao <[email protected]>
Subject: mm/damon: add 'age' of region tracepoint support
In Damon, we can get age information by analyzing the nr_access change, But short time sampling is not effective, we have to obtain enough data for analysis through long time trace, this also means that we need to consume more cpu resources and storage space.
Now the region add a new 'age' variable, we only need to get the change of age value through a little time trace, for example, age has been increasing to 141, but nr_access shows a value of 0 at the same time, Through this,we can conclude that the region has a very low nr_access value for a long time.
---
include/trace/events/damon.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
include/trace/events/damon.h renamed from include/trace/events/damon.h~mm-damon-add-age-of-region-tracepoint-support +5 -2
@@ -22,6 +22,7 @@ TRACE_EVENT(damon_aggregated, __field(unsigned long, start) __field(unsigned long, end) __field(unsigned int, nr_accesses) + __field(unsigned int, age) ), TP_fast_assign(
@@ -30,11 +31,13 @@ TRACE_EVENT(damon_aggregated, __entry->start = r->ar.start; __entry->end = r->ar.end; __entry->nr_accesses = r->nr_accesses; + __entry->age = r->age; ), -TP_printk ("target_id=%lu nr_regions=%u %lu-%lu: %u" , +TP_printk ("target_id=%lu nr_regions=%u %lu-%lu: %u %u" , __entry->target_id, __entry->nr_regions, - __entry->start, __entry->end, __entry->nr_accesses) + __entry->start, __entry->end, + __entry->nr_accesses, __entry->age) );#endif /* _TRACE_DAMON_H */
_
Andrew Morton
maintainerPATCH 123/146 mm/damon/core: use abs() instead of diff_of()
In reply to Andrew MortonFrom: Xin Hao <[email protected]>
Subject: mm/damon/core: use abs() instead of diff_of()
In kernel, we can use abs(a - b) to get the absolute value, So there is no need to redefine a new one.
---
mm/damon/core.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
mm/damon/core.c renamed from mm/damon/core.c~mm-damon-core-using-function-abs-instead-of-diff_of +2 -4
@@ -750,8 +750,6 @@ static void damon_merge_two_regions(strudamon_destroy_region (r, t); } -#define diff_of (a, b) (a > b ? a - b : b - a) -/* * Merge adjacent regions having similar access frequencies *
@@ -765,13 +763,13 @@ static void damon_merge_regions_of(strucstruct damon_region *r, *prev =NULL , *next;damon_for_each_region_safe (r, next, t) { -if (diff_of (r->nr_accesses, r->last_nr_accesses) > thres) +if (abs (r->nr_accesses - r->last_nr_accesses) > thres) r->age =0 ;else r->age++;if (prev && prev->ar.end == r->ar.start && -diff_of (prev->nr_accesses, r->nr_accesses) <= thres && +abs (prev->nr_accesses - r->nr_accesses) <= thres &&sz_damon_region (prev) +sz_damon_region (r) <= sz_limit)damon_merge_two_regions (t, prev, r);else
_
Andrew Morton
maintainerPATCH 124/146 mm/damon: remove some unneeded function definitions in damon.h
In reply to Andrew MortonFrom: Xin Hao <[email protected]>
Subject: mm/damon: remove some unneeded function definitions in damon.h
In damon.h some func definitions about VA & PA can only be used in its own file, so there no need to define in the header file, and the header file will look cleaner.
If other files later need these functions, the prototypes can be added to damon.h at that time.
---
include/linux/damon.h | 21 --------------------- mm/damon/paddr.c | 11 ++++++----- mm/damon/vaddr.c | 18 ++++++++++-------- 3 files changed, 16 insertions(+), 34 deletions(-)
include/linux/damon.h renamed from include/linux/damon.h~mm-damon-remove-some-no-need-func-definitions-in-damonh-file +0 -21
@@ -461,34 +461,13 @@ int damon_stop(struct damon_ctx **ctxs,#endif /* CONFIG_DAMON */ #ifdef CONFIG_DAMON_VADDR - -/* Monitoring primitives for virtual memory address spaces */ -void damon_va_init (struct damon_ctx *ctx); -void damon_va_update (struct damon_ctx *ctx); -void damon_va_prepare_access_checks (struct damon_ctx *ctx); -unsigned int damon_va_check_accesses (struct damon_ctx *ctx);bool damon_va_target_valid (void *t); -void damon_va_cleanup (struct damon_ctx *ctx); -int damon_va_apply_scheme (struct damon_ctx *context,struct damon_target *t, -struct damon_region *r,struct damos *scheme); -int damon_va_scheme_score (struct damon_ctx *context,struct damon_target *t, -struct damon_region *r,struct damos *scheme);void damon_va_set_primitives (struct damon_ctx *ctx); -#endif /* CONFIG_DAMON_VADDR */ #ifdef CONFIG_DAMON_PADDR - -/* Monitoring primitives for the physical memory address space */ -void damon_pa_prepare_access_checks (struct damon_ctx *ctx); -unsigned int damon_pa_check_accesses (struct damon_ctx *ctx);bool damon_pa_target_valid (void *t); -int damon_pa_apply_scheme (struct damon_ctx *context,struct damon_target *t, -struct damon_region *r,struct damos *scheme); -int damon_pa_scheme_score (struct damon_ctx *context,struct damon_target *t, -struct damon_region *r,struct damos *scheme);void damon_pa_set_primitives (struct damon_ctx *ctx); -#endif /* CONFIG_DAMON_PADDR */ #endif /* _DAMON_H */
mm/damon/paddr.c renamed from mm/damon/paddr.c~mm-damon-remove-some-no-need-func-definitions-in-damonh-file +6 -5
@@ -73,7 +73,7 @@ static void __damon_pa_prepare_access_chdamon_pa_mkold (r->sampling_addr); } -void damon_pa_prepare_access_checks (struct damon_ctx *ctx) +static void damon_pa_prepare_access_checks (struct damon_ctx *ctx) {struct damon_target *t;struct damon_region *r;
@@ -192,7 +192,7 @@ static void __damon_pa_check_access(stru last_addr = r->sampling_addr; } -unsigned int damon_pa_check_accesses (struct damon_ctx *ctx) +static unsigned int damon_pa_check_accesses (struct damon_ctx *ctx) {struct damon_target *t;struct damon_region *r;
@@ -213,7 +213,7 @@ bool damon_pa_target_valid(void *t)return true; } -int damon_pa_apply_scheme (struct damon_ctx *ctx,struct damon_target *t, +static int damon_pa_apply_scheme (struct damon_ctx *ctx,struct damon_target *t,struct damon_region *r,struct damos *scheme) {unsigned long addr;
@@ -246,8 +246,9 @@ int damon_pa_apply_scheme(struct damon_creturn 0 ; } -int damon_pa_scheme_score (struct damon_ctx *context,struct damon_target *t, -struct damon_region *r,struct damos *scheme) +static int damon_pa_scheme_score (struct damon_ctx *context, +struct damon_target *t,struct damon_region *r, +struct damos *scheme) {switch (scheme->action) {case DAMOS_PAGEOUT :
mm/damon/vaddr.c renamed from mm/damon/vaddr.c~mm-damon-remove-some-no-need-func-definitions-in-damonh-file +10 -8
@@ -272,7 +272,7 @@ static void __damon_va_init_regions(stru }/* Initialize '->regions_list' of every target (task) */ -void damon_va_init (struct damon_ctx *ctx) +static void damon_va_init (struct damon_ctx *ctx) {struct damon_target *t;
@@ -292,7 +292,8 @@ void damon_va_init(struct damon_ctx *ctx* * Returns true if it is. */ -static bool damon_intersect (struct damon_region *r,struct damon_addr_range *re) +static bool damon_intersect (struct damon_region *r, +struct damon_addr_range *re) {return !(r->ar.end <= re->start || re->end <= r->ar.start); }
@@ -356,7 +357,7 @@ static void damon_va_apply_three_regions/* * Update regions for current memory mappings */ -void damon_va_update (struct damon_ctx *ctx) +static void damon_va_update (struct damon_ctx *ctx) {struct damon_addr_range three_regions[3 ];struct damon_target *t;
@@ -418,7 +419,7 @@ static void __damon_va_prepare_access_chdamon_va_mkold (mm, r->sampling_addr); } -void damon_va_prepare_access_checks (struct damon_ctx *ctx) +static void damon_va_prepare_access_checks (struct damon_ctx *ctx) {struct damon_target *t;struct mm_struct *mm;
@@ -539,7 +540,7 @@ static void __damon_va_check_access(stru last_addr = r->sampling_addr; } -unsigned int damon_va_check_accesses (struct damon_ctx *ctx) +static unsigned int damon_va_check_accesses (struct damon_ctx *ctx) {struct damon_target *t;struct mm_struct *mm;
@@ -603,7 +604,7 @@ out: }#endif /* CONFIG_ADVISE_SYSCALLS */ -int damon_va_apply_scheme (struct damon_ctx *ctx,struct damon_target *t, +static int damon_va_apply_scheme (struct damon_ctx *ctx,struct damon_target *t,struct damon_region *r,struct damos *scheme) {int madv_action;
@@ -633,8 +634,9 @@ int damon_va_apply_scheme(struct damon_creturn damos_madvise (t, r, madv_action); } -int damon_va_scheme_score (struct damon_ctx *context,struct damon_target *t, -struct damon_region *r,struct damos *scheme) +static int damon_va_scheme_score (struct damon_ctx *context, +struct damon_target *t,struct damon_region *r, +struct damos *scheme) {switch (scheme->action) {
_
Andrew Morton
maintainerPATCH 125/146 mm/damon/vaddr: remove swap_ranges() and replace it with swap()
In reply to Andrew MortonFrom: Yihao Han <[email protected]>
Subject: mm/damon/vaddr: remove swap_ranges() and replace it with swap()
Remove 'swap_ranges()' and replace it with the macro 'swap()' defined in 'include/linux/minmax.h' to simplify code and improve efficiency
---
mm/damon/vaddr.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-)
mm/damon/vaddr.c renamed from mm/damon/vaddr.c~mm-damon-vaddr-remove-swap_ranges-and-replace-it-with-swap +3 -13
@@ -98,16 +98,6 @@ static unsigned long sz_range(struct damreturn r->end - r->start; } -static void swap_ranges (struct damon_addr_range *r1, -struct damon_addr_range *r2) -{ -struct damon_addr_range tmp; - - tmp = *r1; - *r1 = *r2; - *r2 = tmp; -} -/* * Find three regions separated by two biggest unmapped regions *
@@ -146,9 +136,9 @@ static int __damon_va_three_regions(stru gap.start = last_vma->vm_end; gap.end = vma->vm_start;if (sz_range (&gap) >sz_range (&second_gap)) { -swap_ranges (&gap, &second_gap); +swap (gap, second_gap);if (sz_range (&second_gap) >sz_range (&first_gap)) -swap_ranges (&second_gap, &first_gap); +swap (second_gap, first_gap); } next: last_vma = vma;
@@ -159,7 +149,7 @@ next:/* Sort the two biggest gaps by address */ if (first_gap.start > second_gap.start) -swap_ranges (&first_gap, &second_gap); +swap (first_gap, second_gap);/* Store the result */ regions[0 ].start =ALIGN (start,DAMON_MIN_REGION );
_
Andrew Morton
maintainerPATCH 126/146 mm/damon/schemes: add the validity judgment of thresholds
In reply to Andrew MortonFrom: Xin Hao <[email protected]>
Subject: mm/damon/schemes: add the validity judgment of thresholds
In dbgfs "schemes" interface, i do some test like this:
# cd /sys/kernel/debug/damon
# echo "2 1 2 1 10 1 3 10 1 1 1 1 1 1 1 1 2 3" > schemes
# cat schemes
# 2 1 2 1 10 1 3 10 1 1 1 1 1 1 1 1 2 3 0 0There have some unreasonable places, i set the valules of these variables "<min_sz, max_sz> <min_nr_a, max_nr_a>, <min_age, max_age>, <wmarks.high, wmarks.mid, wmarks.low>" as "<2, 1>, <2, 1>, <10, 1>, <1, 2, 3>.
So there add a validity judgment for these thresholds value.
---
mm/damon/dbgfs.c | 7 +++++++ 1 file changed, 7 insertions(+)
mm/damon/dbgfs.c renamed from mm/damon/dbgfs.c~mm-damon-schemes-add-the-validity-judgment-of-thresholds +7 -0
@@ -213,6 +213,13 @@ static struct damos **str_to_schemes(conif (!damos_action_valid (action)) goto fail; +if (min_sz > max_sz || min_nr_a > max_nr_a || min_age > max_age) + goto fail; + +if (wmarks.high < wmarks.mid || wmarks.high < wmarks.low || + wmarks.mid < wmarks.low) + goto fail; + pos += parsed; scheme =damon_new_scheme (min_sz, max_sz, min_nr_a, max_nr_a, min_age, max_age, action, "a, &wmarks);
_
Andrew Morton
maintainerPATCH 127/146 mm/damon: move damon_rand() definition into damon.h
In reply to Andrew MortonFrom: Xin Hao <[email protected]>
Subject: mm/damon: move damon_rand() definition into damon.h
damon_rand() is called in three files:damon/core.c, damon/ paddr.c, damon/vaddr.c, i think there is no need to redefine this twice, So move it to damon.h will be a good choice.
---
include/linux/damon.h | 4 ++++ mm/damon/core.c | 4 ---- mm/damon/prmtv-common.h | 4 ---- 3 files changed, 4 insertions(+), 8 deletions(-)
include/linux/damon.h renamed from include/linux/damon.h~mm-damon-move-damon_rand-definition-into-damonh +4 -0
@@ -11,12 +11,16 @@#include <linux/mutex.h> #include <linux/time64.h> #include <linux/types.h> +#include <linux/random.h> /* Minimal region size. Every damon_region is aligned by this. */ #define DAMON_MIN_REGION PAGE_SIZE/* Max priority score for DAMON-based operation schemes */ #define DAMOS_MAX_SCORE (99) +/* Get a random number in [l, r) */ +#define damon_rand (l, r) (l + prandom_u32_max(r - l)) +/** * struct damon_addr_range - Represents an address region of [@start, @end). * @start: Start address of the region (inclusive).
mm/damon/core.c renamed from mm/damon/core.c~mm-damon-move-damon_rand-definition-into-damonh +0 -4
@@ -11,7 +11,6 @@#include <linux/delay.h> #include <linux/kthread.h> #include <linux/mm.h> -#include <linux/random.h> #include <linux/slab.h> #include <linux/string.h>
@@ -23,9 +22,6 @@#define DAMON_MIN_REGION 1#endif -/* Get a random number in [l, r) */ -#define damon_rand (l, r) (l + prandom_u32_max(r - l)) -static DEFINE_MUTEX (damon_lock);static int nr_running_ctxs;
mm/damon/prmtv-common.h renamed from mm/damon/prmtv-common.h~mm-damon-move-damon_rand-definition-into-damonh +0 -4
@@ -6,10 +6,6 @@*/ #include <linux/damon.h> -#include <linux/random.h> - -/* Get a random number in [l, r) */ -#define damon_rand (l, r) (l + prandom_u32_max(r - l))struct page *damon_get_page (unsigned long pfn);
_
Andrew Morton
maintainerPATCH 128/146 mm/damon: modify damon_rand() macro to static inline function
In reply to Andrew MortonFrom: Xin Hao <[email protected]>
Subject: mm/damon: modify damon_rand() macro to static inline function
damon_rand() cannot be implemented as a macro.
Example:
damon_rand(a++, b);
The value of 'a' will be incremented twice, This is obviously unreasonable, So there fix it.
---
include/linux/damon.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
include/linux/damon.h renamed from include/linux/damon.h~mm-damon-modify-damon_rand-macro-to-static-inline-function +4 -1
@@ -19,7 +19,10 @@#define DAMOS_MAX_SCORE (99)/* Get a random number in [l, r) */ -#define damon_rand (l, r) (l + prandom_u32_max(r - l)) +static inline unsigned long damon_rand (unsigned long l,unsigned long r) +{ +return l +prandom_u32_max (r - l); +}/** * struct damon_addr_range - Represents an address region of [@start, @end).
_
Andrew Morton
maintainerPATCH 129/146 mm/damon: convert macro functions to static inline functions
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: mm/damon: convert macro functions to static inline functions
Patch series "mm/damon: Misc cleanups".
This patchset contains miscellaneous cleanups for DAMON's macro functions and documentation.
This patch (of 6):
This commit converts macro functions in DAMON to static inline functions, for better type checking, code documentation, etc[1].
[1] https://lore.kernel.org/linux-mm/[email protected]/
---
include/linux/damon.h | 18 ++++++++++++------ mm/damon/core.c | 5 ++++- mm/damon/vaddr.c | 6 ++++-- 3 files changed, 20 insertions(+), 9 deletions(-)
include/linux/damon.h renamed from include/linux/damon.h~mm-damon-convert-macro-functions-to-static-inline-functions +12 -6
@@ -399,14 +399,20 @@ struct damon_ctx {struct list_head schemes; }; -#define damon_next_region (r) \ - (container_of(r->list.next, struct damon_region, list)) +static inline struct damon_region *damon_next_region (struct damon_region *r) +{ + return container_of(r->list.next, struct damon_region, list); +} -#define damon_prev_region (r) \ - (container_of(r->list.prev, struct damon_region, list)) +static inline struct damon_region *damon_prev_region (struct damon_region *r) +{ + return container_of(r->list.prev, struct damon_region, list); +} -#define damon_last_region (t) \ - (list_last_entry(&t->regions_list, struct damon_region, list)) +static inline struct damon_region *damon_last_region (struct damon_target *t) +{ + return list_last_entry(&t->regions_list, struct damon_region, list); +}#define damon_for_each_region (r, t) \ list_for_each_entry(r, &t->regions_list, list)
mm/damon/core.c renamed from mm/damon/core.c~mm-damon-convert-macro-functions-to-static-inline-functions +4 -1
@@ -729,7 +729,10 @@ static void kdamond_apply_schemes(struct } } -#define sz_damon_region (r) (r->ar.end - r->ar.start) +static inline unsigned long sz_damon_region (struct damon_region *r) +{ +return r->ar.end - r->ar.start; +}/* * Merge two adjacent regions into one region
mm/damon/vaddr.c renamed from mm/damon/vaddr.c~mm-damon-convert-macro-functions-to-static-inline-functions +4 -2
@@ -26,8 +26,10 @@* 't->id' should be the pointer to the relevant 'struct pid' having reference * count. Caller must put the returned task, unless it is NULL. */ -#define damon_get_task_struct (t) \ - (get_pid_task((struct pid *)t->id, PIDTYPE_PID)) +static inline struct task_struct *damon_get_task_struct (struct damon_target *t) +{ +return get_pid_task ((struct pid *)t->id,PIDTYPE_PID ); +}/* * Get the mm_struct of the given target
_
Andrew Morton
maintainerPATCH 130/146 Docs/admin-guide/mm/damon/usage: update for scheme quotas and watermarks
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: Docs/admin-guide/mm/damon/usage: update for scheme quotas and watermarks
DAMOS features including time/space quota limits and watermarks are not described in the DAMON debugfs interface document. This commit updates the document for the features.
---
Documentation/admin-guide/mm/damon/usage.rst | 123 +++++++++++++---- 1 file changed, 98 insertions(+), 25 deletions(-)
Documentation/admin-guide/mm/damon/usage.rst renamed from Documentation/admin-guide/mm/damon/usage.rst~docs-admin-guide-mm-damon-usage-update-for-scheme-quotas-and-watermarks +98 -25
@@ -131,24 +131,38 @@ Schemes For usual DAMON-based data access aware memory management optimizations, users would simply want the system to apply a memory management action to a memory -region of a specific size having a specific access frequency for a specific -time. DAMON receives such formalized operation schemes from the user and -applies those to the target processes. It also counts the total number and -size of regions that each scheme is applied. This statistics can be used for -online analysis or tuning of the schemes. +region of a specific access pattern. DAMON receives such formalized operation +schemes from the user and applies those to the target processes. Users can get and set the schemes by reading from and writing to ``schemes`` debugfs file. Reading the file also shows the statistics of each scheme. To -the file, each of the schemes should be represented in each line in below form: +the file, each of the schemes should be represented in each line in below +form:: - min-size max-size min-acc max-acc min-age max-age action + <target access pattern> <action> <quota> <watermarks> -Note that the ranges are closed interval. Bytes for the size of regions -(``min-size`` and ``max-size``), number of monitored accesses per aggregate -interval for access frequency (``min-acc`` and ``max-acc``), number of -aggregate intervals for the age of regions (``min-age`` and ``max-age``), and a -predefined integer for memory management actions should be used. The supported -numbers and their meanings are as below. +You can disable schemes by simply writing an empty string to the file. + +Target Access Pattern +~~~~~~~~~~~~~~~~~~~~~ + +The ``<target access pattern>`` is constructed with three ranges in below +form:: + + min-size max-size min-acc max-acc min-age max-age + +Specifically, bytes for the size of regions (``min-size`` and ``max-size``), +number of monitored accesses per aggregate interval for access frequency +(``min-acc`` and ``max-acc``), number of aggregate intervals for the age of +regions (``min-age`` and ``max-age``) are specified. Note that the ranges are +closed interval. + +Action +~~~~~~ + +The ``<action>`` is a predefined integer for memory management actions, which +DAMON will apply to the regions having the target access pattern. The +supported numbers and their meanings are as below. - 0: Call ``madvise()`` for the region with ``MADV_WILLNEED`` - 1: Call ``madvise()`` for the region with ``MADV_COLD``
@@ -157,20 +171,79 @@ numbers and their meanings are as below. - 4: Call ``madvise()`` for the region with ``MADV_NOHUGEPAGE`` - 5: Do nothing but count the statistics -You can disable schemes by simply writing an empty string to the file. For -example, below commands applies a scheme saying "If a memory region of size in -[4KiB, 8KiB] is showing accesses per aggregate interval in [0, 5] for aggregate -interval in [10, 20], page out the region", check the entered scheme again, and -finally remove the scheme. :: +Quota +~~~~~ - # cd <debugfs>/damon - # echo "4096 8192 0 5 10 20 2" > schemes - # cat schemes - 4096 8192 0 5 10 20 2 0 0 - # echo > schemes +Optimal ``target access pattern`` for each ``action`` is workload dependent, so +not easy to find. Worse yet, setting a scheme of some action too aggressive +can cause severe overhead. To avoid such overhead, users can limit time and +size quota for the scheme via the ``<quota>`` in below form:: + + <ms> <sz> <reset interval> <priority weights> + +This makes DAMON to try to use only up to ``<ms>`` milliseconds for applying +the action to memory regions of the ``target access pattern`` within the +``<reset interval>`` milliseconds, and to apply the action to only up to +``<sz>`` bytes of memory regions within the ``<reset interval>``. Setting both +``<ms>`` and ``<sz>`` zero disables the quota limits. + +When the quota limit is expected to be exceeded, DAMON prioritizes found memory +regions of the ``target access pattern`` based on their size, access frequency, +and age. For personalized prioritization, users can set the weights for the +three properties in ``<priority weights>`` in below form:: + + <size weight> <access frequency weight> <age weight> + +Watermarks +~~~~~~~~~~ -The last two integers in the 4th line of above example is the total number and -the total size of the regions that the scheme is applied. +Some schemes would need to run based on current value of the system's specific +metrics like free memory ratio. For such cases, users can specify watermarks +for the condition.:: + + <metric> <check interval> <high mark> <middle mark> <low mark> + +``<metric>`` is a predefined integer for the metric to be checked. The +supported numbers and their meanings are as below. + + - 0: Ignore the watermarks + - 1: System's free memory rate (per thousand) + +The value of the metric is checked every ``<check interval>`` microseconds. + +If the value is higher than ``<high mark>`` or lower than ``<low mark>``, the +scheme is deactivated. If the value is lower than ``<mid mark>``, the scheme +is activated. + +Statistics +~~~~~~~~~~ + +It also counts the total number and bytes of regions that each scheme is +applied. This statistics can be used for online analysis or tuning of the +schemes. + +The statistics can be shown by reading the ``schemes`` file. Reading the file +will show each scheme you entered in each line, and the two numbers for the +statistics will be added at the end of each line. + +Example +~~~~~~~ + +Below commands applies a scheme saying "If a memory region of size in [4KiB, +8KiB] is showing accesses per aggregate interval in [0, 5] for aggregate +interval in [10, 20], page out the region. For the paging out, use only up to +10ms per second, and also don't page out more than 1GiB per second. Under the +limitation, page out memory regions having longer age first. Also, check the +free memory rate of the system every 5 seconds, start the monitoring and paging +out when the free memory rate becomes lower than 50%, but stop it if the free +memory rate becomes larger than 60%, or lower than 30%".:: + + # cd <debugfs>/damon + # scheme="4096 8192 0 5 10 20 2" # target access pattern and action + # scheme+=" 10 $((1024*1024*1024)) 1000" # quotas + # scheme+=" 0 0 100" # prioritization weights + # scheme+=" 1 5000000 600 500 300" # watermarks + # echo "$scheme" > schemes Turning On/Off
_
Andrew Morton
maintainerPATCH 131/146 Docs/admin-guide/mm/damon/usage: remove redundant information
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: Docs/admin-guide/mm/damon/usage: remove redundant information
DAMON usage document mentions DAMON user space tool and programming interface twice. This commit integrates those and remove unnecessary part.
---
Documentation/admin-guide/mm/damon/usage.rst | 42 ++++++++--------- 1 file changed, 21 insertions(+), 21 deletions(-)
Documentation/admin-guide/mm/damon/usage.rst renamed from Documentation/admin-guide/mm/damon/usage.rst~docs-admin-guide-mm-damon-usage-remove-redundant-information +21 -21
@@ -7,30 +7,30 @@ Detailed Usages DAMON provides below three interfaces for different users. - *DAMON user space tool.* - This is for privileged people such as system administrators who want a - just-working human-friendly interface. Using this, users can use the DAMON’s - major features in a human-friendly way. It may not be highly tuned for - special cases, though. It supports both virtual and physical address spaces - monitoring. + `This <https://github.com/awslabs/damo>`_ is for privileged people such as + system administrators who want a just-working human-friendly interface. + Using this, users can use the DAMON’s major features in a human-friendly way. + It may not be highly tuned for special cases, though. It supports both + virtual and physical address spaces monitoring. For more detail, please + refer to its `usage document + <https://github.com/awslabs/damo/blob/next/USAGE.md>`_. - *debugfs interface.* - This is for privileged user space programmers who want more optimized use of - DAMON. Using this, users can use DAMON’s major features by reading - from and writing to special debugfs files. Therefore, you can write and use - your personalized DAMON debugfs wrapper programs that reads/writes the - debugfs files instead of you. The DAMON user space tool is also a reference - implementation of such programs. It supports both virtual and physical - address spaces monitoring. + :ref:`This <debugfs_interface>` is for privileged user space programmers who + want more optimized use of DAMON. Using this, users can use DAMON’s major + features by reading from and writing to special debugfs files. Therefore, + you can write and use your personalized DAMON debugfs wrapper programs that + reads/writes the debugfs files instead of you. The `DAMON user space tool + <https://github.com/awslabs/damo>`_ is one example of such programs. It + supports both virtual and physical address spaces monitoring. - *Kernel Space Programming Interface.* - This is for kernel space programmers. Using this, users can utilize every - feature of DAMON most flexibly and efficiently by writing kernel space - DAMON application programs for you. You can even extend DAMON for various - address spaces. + :doc:`This </vm/damon/api>` is for kernel space programmers. Using this, + users can utilize every feature of DAMON most flexibly and efficiently by + writing kernel space DAMON application programs for you. You can even extend + DAMON for various address spaces. For detail, please refer to the interface + :doc:`document </vm/damon/api>`. -Nevertheless, you could write your own user space tool using the debugfs -interface. A reference implementation is available at -https://github.com/awslabs/damo. If you are a kernel programmer, you could -refer to :doc:`/vm/damon/api` for the kernel space programming interface. For -the reason, this document describes only the debugfs interface + +.. _debugfs_interface: debugfs Interface =================
_
Andrew Morton
maintainerPATCH 132/146 Docs/admin-guide/mm/damon/usage: mention tracepoint at the beginning
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: Docs/admin-guide/mm/damon/usage: mention tracepoint at the beginning
To get detailed monitoring results from the user space, users need to use the damon_aggregated tracepoint. This commit adds a brief mention of it at the beginning of the usage document.
---
Documentation/admin-guide/mm/damon/usage.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
Documentation/admin-guide/mm/damon/usage.rst renamed from Documentation/admin-guide/mm/damon/usage.rst~docs-admin-guide-mm-damon-usage-mention-tracepoint-at-the-beginning +8 -1
@@ -21,7 +21,10 @@ DAMON provides below three interfaces fo you can write and use your personalized DAMON debugfs wrapper programs that reads/writes the debugfs files instead of you. The `DAMON user space tool <https://github.com/awslabs/damo>`_ is one example of such programs. It - supports both virtual and physical address spaces monitoring. + supports both virtual and physical address spaces monitoring. Note that this + interface provides only simple :ref:`statistics <damos_stats>` for the + monitoring results. For detailed monitoring results, DAMON provides a + :ref:`tracepoint <tracepoint>`. - *Kernel Space Programming Interface.* :doc:`This </vm/damon/api>` is for kernel space programmers. Using this, users can utilize every feature of DAMON most flexibly and efficiently by
@@ -215,6 +218,8 @@ If the value is higher than ``<high mark scheme is deactivated. If the value is lower than ``<mid mark>``, the scheme is activated. +.. _damos_stats: + Statistics ~~~~~~~~~~
@@ -268,6 +273,8 @@ the monitoring is turned on. If you wri an error code such as ``-EBUSY`` will be returned. +.. _tracepoint: + Tracepoint for Monitoring Results =================================
_
Andrew Morton
maintainerPATCH 133/146 Docs/admin-guide/mm/damon/usage: update for kdamond_pid and (mk|rm)_contexts
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: Docs/admin-guide/mm/damon/usage: update for kdamond_pid and (mk|rm)_contexts
The DAMON debugfs usage document is missing descriptions for 'kdamond_pid', 'mk_contexts', and 'rm_contexts' debugfs files. This commit adds those.
---
Documentation/admin-guide/mm/damon/usage.rst | 52 ++++++++++++++++- 1 file changed, 49 insertions(+), 3 deletions(-)
Documentation/admin-guide/mm/damon/usage.rst renamed from Documentation/admin-guide/mm/damon/usage.rst~docs-admin-guide-mm-damon-usage-update-for-kdamond_pid-and-mkrm_contexts +49 -3
@@ -38,9 +38,9 @@ DAMON provides below three interfaces fo debugfs Interface ================= -DAMON exports five files, ``attrs``, ``target_ids``, ``init_regions``, -``schemes`` and ``monitor_on`` under its debugfs directory, -``<debugfs>/damon/``. +DAMON exports eight files, ``attrs``, ``target_ids``, ``init_regions``, +``schemes``, ``monitor_on``, ``kdamond_pid``, ``mk_contexts`` and +``rm_contexts`` under its debugfs directory, ``<debugfs>/damon/``. Attributes
@@ -273,6 +273,52 @@ the monitoring is turned on. If you wri an error code such as ``-EBUSY`` will be returned. +Monitoring Thread PID +--------------------- + +DAMON does requested monitoring with a kernel thread called ``kdamond``. You +can get the pid of the thread by reading the ``kdamond_pid`` file. When the +monitoring is turned off, reading the file returns ``none``. :: + + # cd <debugfs>/damon + # cat monitor_on + off + # cat kdamond_pid + none + # echo on > monitor_on + # cat kdamond_pid + 18594 + + +Using Multiple Monitoring Threads +--------------------------------- + +One ``kdamond`` thread is created for each monitoring context. You can create +and remove monitoring contexts for multiple ``kdamond`` required use case using +the ``mk_contexts`` and ``rm_contexts`` files. + +Writing the name of the new context to the ``mk_contexts`` file creates a +directory of the name on the DAMON debugfs directory. The directory will have +DAMON debugfs files for the context. :: + + # cd <debugfs>/damon + # ls foo + # ls: cannot access 'foo': No such file or directory + # echo foo > mk_contexts + # ls foo + # attrs init_regions kdamond_pid schemes target_ids + +If the context is not needed anymore, you can remove it and the corresponding +directory by putting the name of the context to the ``rm_contexts`` file. :: + + # echo foo > rm_contexts + # ls foo + # ls: cannot access 'foo': No such file or directory + +Note that ``mk_contexts``, ``rm_contexts``, and ``monitor_on`` files are in the +root directory only. + + .. _tracepoint: Tracepoint for Monitoring Results
_
Andrew Morton
maintainerPATCH 134/146 mm/damon: remove a mistakenly added comment for a future feature
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: mm/damon: remove a mistakenly added comment for a future feature
Due to a mistake in patches reordering, a comment for a future feature called 'arbitrary monitoring target support'[1], which is still under development, has added. Because it only introduces confusion and we don't have a plan to post the patches soon, this commit removes the mistakenly added part.
[1] https://lore.kernel.org/linux-mm/[email protected]/
---
include/linux/damon.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/damon.h renamed from include/linux/damon.h~mm-damon-remove-a-mistakenly-added-comment-for-a-future-feature +1 -1
@@ -281,7 +281,7 @@ struct damon_ctx;* as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided * DAMON-based operation scheme is found. It should apply the scheme's action -* to the region. This is not used for &DAMON_ARBITRARY_TARGET case. +* to the region. * @target_valid should check whether the target is still valid for the * monitoring. * @cleanup is called from @kdamond just before its termination.
_
Andrew Morton
maintainerPATCH 135/146 mm/damon/schemes: account scheme actions that successfully applied
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: mm/damon/schemes: account scheme actions that successfully applied
Patch series "mm/damon/schemes: Extend stats for better online analysis and tuning".
To help online access pattern analysis and tuning of DAMON-based Operation Schemes (DAMOS), DAMOS provides simple statistics for each scheme. Introduction of DAMOS time/space quota further made the tuning easier by making the risk management easier. However, that also made understanding of the working schemes a little bit more difficult.
For an example, progress of a given scheme can now be throttled by not only the aggressiveness of the target access pattern, but also the time/space quotas. So, when a scheme is showing unexpectedly slow progress, it's difficult to know by what the progress of the scheme is throttled, with currently provided statistics.
This patchset extends the statistics to contain some metrics that can be helpful for such online schemes analysis and tuning (patches 1-2), exports those to users (patches 3 and 5), and add documents (patches 4 and 6).
This patch (of 6):
DAMON-based operation schemes (DAMOS) stats provide only the number and the amount of regions that the action of the scheme has tried to be applied. Because the action could be failed for some reasons, the currently provided information is sometimes not useful or convenient enough for schemes profiling and tuning. To improve this situation, this commit extends the DAMOS stats to provide the number and the amount of regions that the action has successfully applied.
---
include/linux/damon.h | 28 +++++++++++++++++++++------- mm/damon/core.c | 13 ++++++++----- mm/damon/dbgfs.c | 2 +- mm/damon/paddr.c | 13 +++++++------ mm/damon/vaddr.c | 30 ++++++++++++++++-------------- 5 files changed, 53 insertions(+), 33 deletions(-)
include/linux/damon.h renamed from include/linux/damon.h~mm-damon-schemes-account-scheme-actions-that-successfully-applied +21 -7
@@ -193,6 +193,20 @@ struct damos_watermarks { };/** +* struct damos_stat - Statistics on a given scheme. +* @nr_tried: Total number of regions that the scheme is tried to be applied. +* @sz_tried: Total size of regions that the scheme is tried to be applied. +* @nr_applied: Total number of regions that the scheme is applied. +* @sz_applied: Total size of regions that the scheme is applied. +*/ +struct damos_stat { +unsigned long nr_tried; +unsigned long sz_tried; +unsigned long nr_applied; +unsigned long sz_applied; +}; + +/** * struct damos - Represents a Data Access Monitoring-based Operation Scheme. * @min_sz_region: Minimum size of target regions. * @max_sz_region: Maximum size of target regions.
@@ -203,8 +217,7 @@ struct damos_watermarks {* @action: &damo_action to be applied to the target regions. * @quota: Control the aggressiveness of this scheme. * @wmarks: Watermarks for automated (in)activation of this scheme. -* @stat_count: Total number of regions that this scheme is applied. -* @stat_sz: Total size of regions that this scheme is applied. +* @stat: Statistics of this scheme. * @list: List head for siblings. * * For each aggregation interval, DAMON finds regions which fit in the
@@ -235,8 +248,7 @@ struct damos {enum damos_action action;struct damos_quota quota;struct damos_watermarks wmarks; -unsigned long stat_count; -unsigned long stat_sz; +struct damos_stat stat;struct list_head list; };
@@ -281,7 +293,8 @@ struct damon_ctx;* as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided * DAMON-based operation scheme is found. It should apply the scheme's action -* to the region. +* to the region and return bytes of the region that the action is successfully +* applied. * @target_valid should check whether the target is still valid for the * monitoring. * @cleanup is called from @kdamond just before its termination.
@@ -295,8 +308,9 @@ struct damon_primitive {int (*get_scheme_score)(struct damon_ctx *context,struct damon_target *t,struct damon_region *r,struct damos *scheme); -int (*apply_scheme)(struct damon_ctx *context,struct damon_target *t, -struct damon_region *r,struct damos *scheme); +unsigned long (*apply_scheme)(struct damon_ctx *context, +struct damon_target *t,struct damon_region *r, +struct damos *scheme);bool (*target_valid)(void *target);void (*cleanup)(struct damon_ctx *context); };
mm/damon/core.c renamed from mm/damon/core.c~mm-damon-schemes-account-scheme-actions-that-successfully-applied +8 -5
@@ -102,8 +102,7 @@ struct damos *damon_new_scheme( scheme->min_age_region = min_age_region; scheme->max_age_region = max_age_region; scheme->action = action; - scheme->stat_count =0 ; - scheme->stat_sz =0 ; + scheme->stat = (struct damos_stat ){};INIT_LIST_HEAD (&scheme->list); scheme->quota.ms = quota->ms;
@@ -574,6 +573,7 @@ static void damon_do_apply_schemes(strucstruct damos_quota *quota = &s->quota;unsigned long sz = r->ar.end - r->ar.start;struct timespec64 begin, end; +unsigned long sz_applied =0 ;if (!s->wmarks.activated)continue ;
@@ -627,7 +627,7 @@ static void damon_do_apply_schemes(strucdamon_split_region_at (c, t, r, sz); }ktime_get_coarse_ts64 (&begin); - c->primitive.apply_scheme (c, t, r, s); + sz_applied = c->primitive.apply_scheme (c, t, r, s);ktime_get_coarse_ts64 (&end); quota->total_charged_ns +=timespec64_to_ns (&end) -timespec64_to_ns (&begin);
@@ -641,8 +641,11 @@ static void damon_do_apply_schemes(struc r->age =0 ; update_stat: - s->stat_count++; - s->stat_sz += sz; + s->stat.nr_tried++; + s->stat.sz_tried += sz; +if (sz_applied) + s->stat.nr_applied++; + s->stat.sz_applied += sz_applied; } }
mm/damon/dbgfs.c renamed from mm/damon/dbgfs.c~mm-damon-schemes-account-scheme-actions-that-successfully-applied +1 -1
@@ -117,7 +117,7 @@ static ssize_t sprint_schemes(struct dam s->quota.weight_age, s->wmarks.metric, s->wmarks.interval, s->wmarks.high, s->wmarks.mid, s->wmarks.low, - s->stat_count, s->stat_sz); + s->stat.nr_tried, s->stat.sz_tried);if (!rc)return -ENOMEM ;
mm/damon/paddr.c renamed from mm/damon/paddr.c~mm-damon-schemes-account-scheme-actions-that-successfully-applied +7 -6
@@ -213,14 +213,15 @@ bool damon_pa_target_valid(void *t)return true; } -static int damon_pa_apply_scheme (struct damon_ctx *ctx,struct damon_target *t, -struct damon_region *r,struct damos *scheme) +static unsigned long damon_pa_apply_scheme (struct damon_ctx *ctx, +struct damon_target *t,struct damon_region *r, +struct damos *scheme) { -unsigned long addr; +unsigned long addr, applied;LIST_HEAD (page_list);if (scheme->action !=DAMOS_PAGEOUT ) -return -EINVAL ; +return 0 ;for (addr = r->ar.start; addr < r->ar.end; addr +=PAGE_SIZE ) {struct page *page =damon_get_page (PHYS_PFN (addr));
@@ -241,9 +242,9 @@ static int damon_pa_apply_scheme(structput_page (page); } } -reclaim_pages (&page_list); + applied =reclaim_pages (&page_list);cond_resched (); -return 0 ; +return applied *PAGE_SIZE ; } static int damon_pa_scheme_score(struct damon_ctx *context,
mm/damon/vaddr.c renamed from mm/damon/vaddr.c~mm-damon-schemes-account-scheme-actions-that-successfully-applied +16 -14
@@ -572,32 +572,34 @@ bool damon_va_target_valid(void *target) }#ifndef CONFIG_ADVISE_SYSCALLS -static int damos_madvise (struct damon_target *target,struct damon_region *r, -int behavior) +static unsigned long damos_madvise (struct damon_target *target, +struct damon_region *r,int behavior) { -return -EINVAL ; +return 0 ; }#else -static int damos_madvise (struct damon_target *target,struct damon_region *r, -int behavior) +static unsigned long damos_madvise (struct damon_target *target, +struct damon_region *r,int behavior) {struct mm_struct *mm; -int ret = -ENOMEM ; +unsigned long start =PAGE_ALIGN (r->ar.start); +unsigned long len =PAGE_ALIGN (r->ar.end - r->ar.start); +unsigned long applied; mm =damon_get_mm (target);if (!mm) - goto out; +return 0 ; - ret =do_madvise (mm,PAGE_ALIGN (r->ar.start), -PAGE_ALIGN (r->ar.end - r->ar.start), behavior); + applied =do_madvise (mm, start, len, behavior) ?0 : len;mmput (mm); -out: -return ret; + +return applied; }#endif /* CONFIG_ADVISE_SYSCALLS */ -static int damon_va_apply_scheme (struct damon_ctx *ctx,struct damon_target *t, -struct damon_region *r,struct damos *scheme) +static unsigned long damon_va_apply_scheme (struct damon_ctx *ctx, +struct damon_target *t,struct damon_region *r, +struct damos *scheme) {int madv_action;
@@ -620,7 +622,7 @@ static int damon_va_apply_scheme(structcase DAMOS_STAT :return 0 ;default : -return -EINVAL ; +return 0 ; }return damos_madvise (t, r, madv_action);
_
Andrew Morton
maintainerPATCH 136/146 mm/damon/schemes: account how many times quota limit has exceeded
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: mm/damon/schemes: account how many times quota limit has exceeded
If the time/space quotas of a given DAMON-based operation scheme is too small, the scheme could show unexpectedly slow progress. However, there is no good way to notice the case in runtime. This commit extends the DAMOS stat to provide how many times the quota limits exceeded so that the users can easily notice the case and tune the scheme.
---
include/linux/damon.h | 2 ++ mm/damon/core.c | 2 ++ 2 files changed, 4 insertions(+)
include/linux/damon.h renamed from include/linux/damon.h~mm-damon-schemes-account-how-many-times-quota-limit-has-exceeded +2 -0
@@ -198,12 +198,14 @@ struct damos_watermarks {* @sz_tried: Total size of regions that the scheme is tried to be applied. * @nr_applied: Total number of regions that the scheme is applied. * @sz_applied: Total size of regions that the scheme is applied. +* @qt_exceeds: Total number of times the quota of the scheme has exceeded. */ struct damos_stat {unsigned long nr_tried;unsigned long sz_tried;unsigned long nr_applied;unsigned long sz_applied; +unsigned long qt_exceeds; };/**
mm/damon/core.c renamed from mm/damon/core.c~mm-damon-schemes-account-how-many-times-quota-limit-has-exceeded +2 -0
@@ -693,6 +693,8 @@ static void kdamond_apply_schemes(structif (time_after_eq (jiffies, quota->charged_from +msecs_to_jiffies ( quota->reset_interval))) { +if (quota->esz && quota->charged_sz >= quota->esz) + s->stat.qt_exceeds++; quota->total_charged_sz += quota->charged_sz; quota->charged_from = jiffies; quota->charged_sz =0 ;
_
Andrew Morton
maintainerPATCH 137/146 mm/damon/reclaim: provide reclamation statistics
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: mm/damon/reclaim: provide reclamation statistics
This commit implements new DAMON_RECLAIM parameters for statistics reporting. Those can be used for understanding how DAMON_RECLAIM is working, and for tuning the other parameters.
---
mm/damon/reclaim.c | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+)
mm/damon/reclaim.c renamed from mm/damon/reclaim.c~mm-damon-reclaim-provide-reclamation-statistics +46 -0
@@ -185,6 +185,36 @@ module_param(monitor_region_end, ulong,static int kdamond_pid__read_mostly =-1 ;module_param (kdamond_pid, int,0400 ); +/* +* Number of memory regions that tried to be reclaimed. +*/ +static unsigned long nr_reclaim_tried_regions__read_mostly ; +module_param (nr_reclaim_tried_regions, ulong,0400 ); + +/* +* Total bytes of memory regions that tried to be reclaimed. +*/ +static unsigned long bytes_reclaim_tried_regions__read_mostly ; +module_param (bytes_reclaim_tried_regions, ulong,0400 ); + +/* +* Number of memory regions that successfully be reclaimed. +*/ +static unsigned long nr_reclaimed_regions__read_mostly ; +module_param (nr_reclaimed_regions, ulong,0400 ); + +/* +* Total bytes of memory regions that successfully be reclaimed. +*/ +static unsigned long bytes_reclaimed_regions__read_mostly ; +module_param (bytes_reclaimed_regions, ulong,0400 ); + +/* +* Number of times that the time/space quota limits have exceeded +*/ +static unsigned long nr_quota_exceeds__read_mostly ; +module_param (nr_quota_exceeds, ulong,0400 ); +static struct damon_ctx *ctx;static struct damon_target *target;
@@ -333,6 +363,21 @@ static void damon_reclaim_timer_fn(struc } static DECLARE_DELAYED_WORK(damon_reclaim_timer, damon_reclaim_timer_fn); +static int damon_reclaim_after_aggregation (struct damon_ctx *c) +{ +struct damos *s; + +/* update the stats parameter */ +damon_for_each_scheme (s, c) { + nr_reclaim_tried_regions = s->stat.nr_tried; + bytes_reclaim_tried_regions = s->stat.sz_tried; + nr_reclaimed_regions = s->stat.nr_applied; + bytes_reclaimed_regions = s->stat.sz_applied; + nr_quota_exceeds = s->stat.qt_exceeds; + } +return 0 ; +} +static int __init damon_reclaim_init (void ) { ctx =damon_new_ctx ();
@@ -340,6 +385,7 @@ static int __init damon_reclaim_init(voireturn -ENOMEM ;damon_pa_set_primitives (ctx); + ctx->callback.after_aggregation = damon_reclaim_after_aggregation;/* 4242 means nothing but fun */ target =damon_new_target (4242 );
_
Andrew Morton
maintainerPATCH 138/146 Docs/admin-guide/mm/damon/reclaim: document statistics parameters
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: Docs/admin-guide/mm/damon/reclaim: document statistics parameters
This commit adds descriptions for the DAMON_RECLAIM statistics parameters.
---
Documentation/admin-guide/mm/damon/reclaim.rst | 25 +++++++++++++++ 1 file changed, 25 insertions(+)
Documentation/admin-guide/mm/damon/reclaim.rst renamed from Documentation/admin-guide/mm/damon/reclaim.rst~docs-admin-guide-mm-damon-reclaim-document-statistics-parameters +25 -0
@@ -208,6 +208,31 @@ PID of the DAMON thread. If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. Else, -1. +nr_reclaim_tried_regions +------------------------ + +Number of memory regions that tried to be reclaimed by DAMON_RECLAIM. + +bytes_reclaim_tried_regions +--------------------------- + +Total bytes of memory regions that tried to be reclaimed by DAMON_RECLAIM. + +nr_reclaimed_regions +-------------------- + +Number of memory regions that successfully be reclaimed by DAMON_RECLAIM. + +bytes_reclaimed_regions +----------------------- + +Total bytes of memory regions that successfully be reclaimed by DAMON_RECLAIM. + +nr_quota_exceeds +---------------- + +Number of times that the time/space quota limits have exceeded. + Example =======
_
Andrew Morton
maintainerPATCH 139/146 mm/damon/dbgfs: support all DAMOS stats
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: mm/damon/dbgfs: support all DAMOS stats
Currently, DAMON debugfs interface is not supporting DAMON-based Operation Schemes (DAMOS) stats for schemes successfully applied regions and time/space quota limit exceeds. This commit adds the support.
---
mm/damon/dbgfs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
mm/damon/dbgfs.c renamed from mm/damon/dbgfs.c~mm-damon-dbgfs-support-all-damos-stats +4 -2
@@ -105,7 +105,7 @@ static ssize_t sprint_schemes(struct damdamon_for_each_scheme (s, c) { rc = scnprintf(&buf[written], len - written, - "%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %d %lu %lu %lu %lu %lu %lu\n", + "%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %d %lu %lu %lu %lu %lu %lu %lu %lu %lu\n", s->min_sz_region, s->max_sz_region, s->min_nr_accesses, s->max_nr_accesses, s->min_age_region, s->max_age_region,
@@ -117,7 +117,9 @@ static ssize_t sprint_schemes(struct dam s->quota.weight_age, s->wmarks.metric, s->wmarks.interval, s->wmarks.high, s->wmarks.mid, s->wmarks.low, - s->stat.nr_tried, s->stat.sz_tried); + s->stat.nr_tried, s->stat.sz_tried, + s->stat.nr_applied, s->stat.sz_applied, + s->stat.qt_exceeds);if (!rc)return -ENOMEM ;
_
Andrew Morton
maintainerPATCH 140/146 Docs/admin-guide/mm/damon/usage: update for schemes statistics
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: Docs/admin-guide/mm/damon/usage: update for schemes statistics
This commit updates DAMON debugfs interface for statistics of schemes successfully applied regions and time/space quota limit exceeds counts.
---
Documentation/admin-guide/mm/damon/usage.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
Documentation/admin-guide/mm/damon/usage.rst renamed from Documentation/admin-guide/mm/damon/usage.rst~docs-admin-guide-mm-damon-usage-update-for-schemes-statistics +5 -4
@@ -223,12 +223,13 @@ is activated. Statistics ~~~~~~~~~~ -It also counts the total number and bytes of regions that each scheme is -applied. This statistics can be used for online analysis or tuning of the -schemes. +It also counts the total number and bytes of regions that each scheme is tried +to be applied, the two numbers for the regions that each scheme is successfully +applied, and the total number of the quota limit exceeds. This statistics can +be used for online analysis or tuning of the schemes. The statistics can be shown by reading the ``schemes`` file. Reading the file -will show each scheme you entered in each line, and the two numbers for the +will show each scheme you entered in each line, and the five numbers for the statistics will be added at the end of each line. Example
_
Andrew Morton
maintainerPATCH 141/146 mm/damon: add access checking for hugetlb pages
In reply to Andrew MortonFrom: Baolin Wang <[email protected]>
Subject: mm/damon: add access checking for hugetlb pages
The process's VMAs can be mapped by hugetlb page, but now the DAMON did not implement the access checking for hugetlb pte, so we can not get the actual access count like below if a process VMAs were mapped by hugetlb.
damon_aggregated: target_id=18446614368406014464 nr_regions=12 4194304-5476352: 0 545damon_aggregated: target_id=18446614368406014464 nr_regions=12 140662370467840-140662372970496: 0 545damon_aggregated: target_id=18446614368406014464 nr_regions=12 140662372970496-140662375460864: 0 545damon_aggregated: target_id=18446614368406014464 nr_regions=12 140662375460864-140662377951232: 0 545damon_aggregated: target_id=18446614368406014464 nr_regions=12 140662377951232-140662380449792: 0 545damon_aggregated: target_id=18446614368406014464 nr_regions=12 140662380449792-140662382944256: 0 545
......
Thus this patch adds hugetlb access checking support, with this patch we can see below VMA mapped by hugetlb access count.
damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296486649856-140296489914368: 1 3damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296489914368-140296492978176: 1 3damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296492978176-140296495439872: 1 3damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296495439872-140296498311168: 1 3damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296498311168-140296501198848: 1 3damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296501198848-140296504320000: 1 3damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296504320000-140296507568128: 1 2
......
---
mm/damon/vaddr.c | 96 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+)
mm/damon/vaddr.c renamed from mm/damon/vaddr.c~mm-damon-add-access-checking-for-hugetlb-pages +96 -0
@@ -388,8 +388,65 @@ out:return 0 ; } +#ifdef CONFIG_HUGETLB_PAGE +static void damon_hugetlb_mkold (pte_t *pte,struct mm_struct *mm, +struct vm_area_struct *vma,unsigned long addr) +{ +bool referenced = false; +pte_t entry =huge_ptep_get (pte); +struct page *page =pte_page (entry); + +if (!page) +return ; + +get_page (page); + +if (pte_young (entry)) { + referenced = true; + entry =pte_mkold (entry); +huge_ptep_set_access_flags (vma, addr, pte, entry, + vma->vm_flags &VM_WRITE ); + } + +#ifdef CONFIG_MMU_NOTIFIER +if (mmu_notifier_clear_young (mm, addr, + addr +huge_page_size (hstate_vma (vma)))) + referenced = true; +#endif /* CONFIG_MMU_NOTIFIER */ + +if (referenced) +set_page_young (page); + +set_page_idle (page); +put_page (page); +} + +static int damon_mkold_hugetlb_entry (pte_t *pte,unsigned long hmask, +unsigned long addr,unsigned long end, +struct mm_walk *walk) +{ +struct hstate *h =hstate_vma (walk->vma); +spinlock_t *ptl; +pte_t entry; + + ptl =huge_pte_lock (h, walk->mm, pte); + entry =huge_ptep_get (pte); +if (!pte_present (entry)) + goto out; + +damon_hugetlb_mkold (pte, walk->mm, walk->vma, addr); + +out: +spin_unlock (ptl); +return 0 ; +} +#else +#define damon_mkold_hugetlb_entry NULL +#endif /* CONFIG_HUGETLB_PAGE */ +static const struct mm_walk_ops damon_mkold_ops = { .pmd_entry = damon_mkold_pmd_entry, + .hugetlb_entry = damon_mkold_hugetlb_entry, };static void damon_va_mkold (struct mm_struct *mm,unsigned long addr)
@@ -484,8 +541,47 @@ out:return 0 ; } +#ifdef CONFIG_HUGETLB_PAGE +static int damon_young_hugetlb_entry (pte_t *pte,unsigned long hmask, +unsigned long addr,unsigned long end, +struct mm_walk *walk) +{ +struct damon_young_walk_private *priv = walk->private; +struct hstate *h =hstate_vma (walk->vma); +struct page *page; +spinlock_t *ptl; +pte_t entry; + + ptl =huge_pte_lock (h, walk->mm, pte); + entry =huge_ptep_get (pte); +if (!pte_present (entry)) + goto out; + + page =pte_page (entry); +if (!page) + goto out; + +get_page (page); + +if (pte_young (entry) || !page_is_idle (page) || +mmu_notifier_test_young (walk->mm, addr)) { + *priv->page_sz =huge_page_size (h); + priv->young = true; + } + +put_page (page); + +out: +spin_unlock (ptl); +return 0 ; +} +#else +#define damon_young_hugetlb_entry NULL +#endif /* CONFIG_HUGETLB_PAGE */ +static const struct mm_walk_ops damon_young_ops = { .pmd_entry = damon_young_pmd_entry, + .hugetlb_entry = damon_young_hugetlb_entry, }; static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
_
Andrew Morton
maintainerPATCH 142/146 mm/damon: move the implementation of damon_insert_region to damon.h
In reply to Andrew MortonFrom: Guoqing Jiang <[email protected]>
Subject: mm/damon: move the implementation of damon_insert_region to damon.h
Usually, inline function is declared static since it should sit between storage and type. And implement it in a header file if used by multiple files.
And this change also fixes compile issue when backport damon to 5.10.
mm/damon/vaddr.c: In function `damon_va_evenly_split_region':./include/linux/damon.h:425:13: error: inlining failed in call to always_inline' damon_insert_region': function body not available 425 | inline void damon_insert_region(struct damon_region *r,
| ^~~~~~~~~~~~~~~~~~~mm/damon/vaddr.c:86:3: note: called from here
86 | damon_insert_region(n, r, next, t);
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
---
include/linux/damon.h | 13 +++++++++++-- mm/damon/core.c | 11 ----------- 2 files changed, 11 insertions(+), 13 deletions(-)
include/linux/damon.h renamed from include/linux/damon.h~mm-damon-move-the-implementation-of-damon_insert_region-to-damonh +11 -2
@@ -451,9 +451,18 @@ static inline struct damon_region *damon#ifdef CONFIG_DAMON struct damon_region *damon_new_region (unsigned long start,unsigned long end); -inline void damon_insert_region (struct damon_region *r, + +/* +* Add a region between two other regions +*/ +static inline void damon_insert_region (struct damon_region *r,struct damon_region *prev,struct damon_region *next, -struct damon_target *t); +struct damon_target *t) +{ +__list_add (&r->list, &prev->list, &next->list); + t->nr_regions++; +} +void damon_add_region (struct damon_region *r,struct damon_target *t);void damon_destroy_region (struct damon_region *r,struct damon_target *t);
mm/damon/core.c renamed from mm/damon/core.c~mm-damon-move-the-implementation-of-damon_insert_region-to-damonh +0 -11
@@ -49,17 +49,6 @@ struct damon_region *damon_new_region(unreturn region; } -/* -* Add a region between two other regions -*/ -inline void damon_insert_region (struct damon_region *r, -struct damon_region *prev,struct damon_region *next, -struct damon_target *t) -{ -__list_add (&r->list, &prev->list, &next->list); - t->nr_regions++; -} -void damon_add_region (struct damon_region *r,struct damon_target *t) {list_add_tail (&r->list, &t->regions_list);
_
Andrew Morton
maintainerPATCH 143/146 mm/damon/dbgfs: remove an unnecessary variable
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: mm/damon/dbgfs: remove an unnecessary variable
Patch series "mm/damon: Hide unnecessary information disclosures".
DAMON is exposing some unnecessary information including kernel pointer in kernel log and tracepoint. This patchset hides such information. The first patch is only for a trivial cleanup, though.
This patch (of 4):
This commit removes a unnecessarily used variable in dbgfs_target_ids_write().
---
mm/damon/dbgfs.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
mm/damon/dbgfs.c renamed from mm/damon/dbgfs.c~mm-damon-dbgfs-remove-a-unnecessary-variable +2 -3
@@ -364,7 +364,7 @@ static ssize_t dbgfs_target_ids_write(ststruct damon_ctx *ctx = file->private_data;struct damon_target *t, *next_t;bool id_is_pid = true; -char *kbuf, *nrs; +char *kbuf;unsigned long *targets;ssize_t nr_targets;ssize_t ret;
@@ -374,14 +374,13 @@ static ssize_t dbgfs_target_ids_write(stif (IS_ERR (kbuf))return PTR_ERR (kbuf); - nrs = kbuf;if (!strncmp (kbuf,"paddr\n" , count)) { id_is_pid = false;/* target id is meaningless here, but we set it just for fun */ scnprintf (kbuf, count,"42 " ); } - targets =str_to_target_ids (nrs, count, &nr_targets); + targets =str_to_target_ids (kbuf, count, &nr_targets);if (!targets) { ret = -ENOMEM ; goto out;
_
Andrew Morton
maintainerPATCH 144/146 mm/damon/vaddr: use pr_debug() for damon_va_three_regions() failure logging
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: mm/damon/vaddr: use pr_debug() for damon_va_three_regions() failure logging
Failure of 'damon_va_three_regions()' is logged using 'pr_err()'. But, the function can fail in legal situations. To avoid making users be surprised and to keep the kernel clean, this commit makes the log to be printed using 'pr_debug()'.
---
mm/damon/vaddr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/damon/vaddr.c renamed from mm/damon/vaddr.c~mm-damon-vaddr-use-pr_debug-for-damon_va_three_regions-failure-logging +1 -1
@@ -238,7 +238,7 @@ static void __damon_va_init_regions(struint i;if (damon_va_three_regions (t, regions)) { -pr_err ("Failed to get three regions of target %lu\n" , t->id); +pr_debug ("Failed to get three regions of target %lu\n" , t->id);return ; }
_
Andrew Morton
maintainerPATCH 145/146 mm/damon/vaddr: hide kernel pointer from damon_va_three_regions() failure log
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: mm/damon/vaddr: hide kernel pointer from damon_va_three_regions() failure log
The failure log message for 'damon_va_three_regions()' prints the target id, which is a 'struct pid' pointer in the case. To avoid exposing the kernel pointer via the log, this commit makes the log to use the index of the target in the context's targets list instead.
---
mm/damon/vaddr.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)
mm/damon/vaddr.c renamed from mm/damon/vaddr.c~mm-damon-vaddr-hide-kernel-pointer-from-damon_va_three_regions-failure-log +8 -2
@@ -232,13 +232,19 @@ static int damon_va_three_regions(structstatic void __damon_va_init_regions (struct damon_ctx *ctx,struct damon_target *t) { +struct damon_target *ti;struct damon_region *r;struct damon_addr_range regions[3 ];unsigned long sz =0 , nr_pieces; -int i; +int i, tidx =0 ;if (damon_va_three_regions (t, regions)) { -pr_debug ("Failed to get three regions of target %lu\n" , t->id); +damon_for_each_target (ti, ctx) { +if (ti == t) +break ; + tidx++; + } +pr_debug ("Failed to get three regions of %dth target\n" , tidx);return ; }
_
Andrew Morton
maintainerPATCH 146/146 mm/damon: hide kernel pointer from tracepoint event
In reply to Andrew MortonFrom: SeongJae Park <[email protected]>
Subject: mm/damon: hide kernel pointer from tracepoint event
DAMON's virtual address spaces monitoring primitive uses 'struct pid *' of the target process as its monitoring target id. The kernel address is exposed as-is to the user space via the DAMON tracepoint, 'damon_aggregated'.
Though primarily only privileged users are allowed to access that, it would be better to avoid unnecessarily exposing kernel pointers so. Because the trace result is only required to be able to distinguish each target, we aren't need to use the pointer as-is.
This commit makes the tracepoint to use the index of the target in the context's targets list as its id in the tracepoint, to hide the kernel space address.
---
include/trace/events/damon.h | 8 ++++---- mm/damon/core.c | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-)
include/trace/events/damon.h renamed from include/trace/events/damon.h~mm-damon-hide-kernel-pointer-from-tracepoint-event +4 -4
@@ -11,10 +11,10 @@ TRACE_EVENT(damon_aggregated, - TP_PROTO(struct damon_target *t, struct damon_region *r, - unsigned int nr_regions), + TP_PROTO(struct damon_target *t, unsigned int target_id, + struct damon_region *r, unsigned int nr_regions), - TP_ARGS(t, r, nr_regions), + TP_ARGS(t, target_id, r, nr_regions), TP_STRUCT__entry( __field(unsigned long, target_id)
@@ -26,7 +26,7 @@ TRACE_EVENT(damon_aggregated, ), TP_fast_assign( - __entry->target_id = t->id; + __entry->target_id = target_id; __entry->nr_regions = nr_regions; __entry->start = r->ar.start; __entry->end = r->ar.end;
mm/damon/core.c renamed from mm/damon/core.c~mm-damon-hide-kernel-pointer-from-tracepoint-event +3 -1
@@ -514,15 +514,17 @@ static bool kdamond_aggregate_interval_pstatic void kdamond_reset_aggregated (struct damon_ctx *c) {struct damon_target *t; +unsigned int ti =0 ;/* target's index */ damon_for_each_target (t, c) {struct damon_region *r;damon_for_each_region (r, t) { -trace_damon_aggregated (t, r,damon_nr_regions (t)); +trace_damon_aggregated (t, ti, r,damon_nr_regions (t)); r->last_nr_accesses = r->nr_accesses; r->nr_accesses =0 ; } + ti++; } }
_