idle memory tracking

Knowing the portion of memory that is not used by a certain application or memory cgroup (idle memory) can be useful for partitioning the system efficiently, e.g. by setting memory cgroup limits appropriately. Currently, the only means to estimate the amount of idle memory provided by the kernel is /proc/PID/{clear_refs,smaps}: the user can clear the access bit for all pages mapped to a particular process by writing 1 to clear_refs, wait for some time, and then count smaps:Referenced. However, this method has two serious shortcomings:

- it does not count unmapped file pages
- it affects the reclaimer logic

To overcome these drawbacks, this patch introduces two new page flags, Idle and Young, and a new proc file, /proc/kpageidle. A page's Idle flag can only be set from userspace by writing 1 to /proc/kpageidle at the offset corresponding to the page, and it is cleared whenever the page is accessed either through page tables (it is cleared in page_referenced() in this case) or using the read(2) system call (mark_page_accessed()). Thus by setting the Idle flag for pages of a particular workload, which can be found e.g. by reading /proc/PID/pagemap, waiting for some time to let the workload access its working set, and then reading the kpageidle file, one can estimate the amount of pages that are not used by the workload.

The Young page flag is used to avoid interference with the memory reclaimer. A page's Young flag is set whenever the Access bit of a page table entry pointing to the page is cleared by writing to kpageidle. If page_referenced() is called on a Young page, it will add 1 to its return value, therefore concealing the fact that the Access bit was cleared.

Note, since there is no room for extra page flags on 32 bit, this feature uses extended page flags when compiled on 32 bit.

Signed-off-by Vladimir Davydov <[email protected]>

---
 Documentation/vm/pagemap.txt |   10 ++-
 fs/proc/page.c               |  154 ++++++++++++++++++++++++++++++++++++++++++
 fs/proc/task_mmu.c           |    4 +-
 include/linux/mm.h           |   88 ++++++++++++++++++++++++
 include/linux/page-flags.h   |    9 +++
 include/linux/page_ext.h     |    4 ++
 mm/Kconfig                   |   12 ++++
 mm/debug.c                   |    4 ++
 mm/page_ext.c                |    3 +
 mm/rmap.c                    |    7 ++
 mm/swap.c                    |    2 +
 11 files changed, 295 insertions(+), 2 deletions(-)

Documentation/vm/pagemap.txt +1 -1

index a9b7afc8fbc6..ac6fd32a9296 100644
@@ -5,7 +5,7 @@ pagemap is a new (as of 2.6.25) set of interfaces in the kernel that allow
 userspace programs to examine the page tables and related information by
 reading files in /proc.
 
-There are four components to pagemap:
+There are five components to pagemap:
 
  * /proc/pid/pagemap.  This file lets a userspace process find out which
    physical frame each virtual page is mapped to.  It contains one 64-bit

Documentation/vm/pagemap.txt +6 -0

+   considered idle if it has not been accessed since it was marked idle. To
+   mark a page idle one should write 1 to this file at the offset corresponding
+   to the page. Only user memory pages can be marked idle, for other page types
+   input is silently ignored. Writing to this file beyond max PFN results in
+   the ENXIO error. Only available when CONFIG_IDLE_PAGE_TRACKING is set.
+
 Short descriptions to the page flags:
 
  0. LOCKED

fs/proc/page.c +3 -0

index 70d23245dd43..cfc55ba7fee6 100644
@@ -275,6 +275,156 @@ static const struct file_operations proc_kpagecgroup_operations = {
 };
 #endif /* CONFIG_MEMCG */
 
+#ifdef CONFIG_IDLE_PAGE_TRACKING
+static struct page *kpageidle_get_page(unsigned long pfn)
+{

fs/proc/page.c +4 -0

+		put_page(page);
+		return NULL;
+	}
+	return page;

fs/proc/page.c +123 -0

+		 * return > 0 (see page_referenced_one)
+		 */
+		set_page_young(page);
+}
+
+static ssize_t kpageidle_read(struct file *file, char __user *buf,
+			      size_t count, loff_t *ppos)
+{
+	u64 __user *out = (u64 __user *)buf;
+	struct page *page;
+	unsigned long src = *ppos;
+	unsigned long pfn;
+	ssize_t ret = 0;
+	u64 val;
+
+	pfn = src / KPMSIZE;
+	count = min_t(unsigned long, count, (max_pfn * KPMSIZE) - src);
+	if (src & KPMMASK || count & KPMMASK)
+		return -EINVAL;
+
+	while (count > 0) {
+		val = 0;
+		page = kpageidle_get_page(pfn);
+		if (page) {
+			if (page_is_idle(page)) {
+				/*
+				 * The page might have been referenced via a
+				 * pte, in which case it is not idle. Clear
+				 * refs and recheck.
+				 */
+				kpageidle_clear_refs(page);
+				if (page_is_idle(page))
+					val = 1;
+			}
+			put_page(page);
+		}
+
+		if (put_user(val, out)) {
+			ret = -EFAULT;
+			break;
+		}
+
+		pfn++;
+		out++;
+		count -= KPMSIZE;
+	}
+
+	*ppos += (char __user *)out - buf;
+	if (!ret)
+		ret = (char __user *)out - buf;
+	return ret;
+}
+
+static ssize_t kpageidle_write(struct file *file, const char __user *buf,
+			       size_t count, loff_t *ppos)
+{
+	const u64 __user *in = (u64 __user *)buf;
+	struct page *page;
+	unsigned long src = *ppos;
+	unsigned long pfn;
+	ssize_t ret = 0;
+	u64 val;
+
+	pfn = src / KPMSIZE;
+	if (src & KPMMASK || count & KPMMASK)
+		return -EINVAL;
+
+	while (count > 0) {
+		if (pfn >= max_pfn) {
+			if ((char __user *)in == buf)
+				ret = -ENXIO;
+			break;
+		}
+
+		if (get_user(val, in)) {
+			ret = -EFAULT;
+			break;
+		}
+
+		if (val == 1) {
+			page = kpageidle_get_page(pfn);
+			if (page) {
+				kpageidle_clear_refs(page);
+				set_page_idle(page);
+				put_page(page);
+			}
+		} else if (val) {
+			ret = -EINVAL;
+			break;
+		}
+
+		pfn++;
+		in++;
+		count -= KPMSIZE;
+	}
+
+	*ppos += (char __user *)in - buf;
+	if (!ret)
+		ret = (char __user *)in - buf;
+	return ret;
+}
+
+static const struct file_operations proc_kpageidle_operations = {
+	.llseek = mem_lseek,
+	.read = kpageidle_read,
+	.write = kpageidle_write,
+};
+
+#ifndef CONFIG_64BIT
+static bool need_page_idle(void)
+{
+	return true;
+}
+struct page_ext_operations page_idle_ops = {
+	.need = need_page_idle,
+};
+#endif
+#endif /* CONFIG_IDLE_PAGE_TRACKING */
+
 static int __init proc_page_init(void)
 {
 	proc_create("kpagecount", S_IRUSR, NULL, &proc_kpagecount_operations);

@@ -282,6 +432,10 @@ static int __init proc_page_init(void)
 #ifdef CONFIG_MEMCG
 	proc_create("kpagecgroup", S_IRUSR, NULL, &proc_kpagecgroup_operations);
 #endif
+#ifdef CONFIG_IDLE_PAGE_TRACKING
+	proc_create("kpageidle", S_IRUSR | S_IWUSR, NULL,
+		    &proc_kpageidle_operations);
+#endif
 	return 0;
 }
 fs_initcall(proc_page_init);

fs/proc/task_mmu.c +3 -1

index 6dee68d013ff..ab04846f7dd5 100644
@@ -458,7 +458,7 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page,
 
 	mss->resident += size;
 	/* Accumulate the size in pages that have been accessed. */
-	if (young || PageReferenced(page))
+	if (young || page_is_young(page) || PageReferenced(page))
 		mss->referenced += size;
 	mapcount = page_mapcount(page);
 	if (mapcount >= 2) {

@@ -808,6 +808,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
 
 		/* Clear accessed and referenced bits. */
 		pmdp_test_and_clear_young(vma, addr, pmd);
+		clear_page_young(page);
 		ClearPageReferenced(page);
 out:
 		spin_unlock(ptl);

@@ -835,6 +836,7 @@ out:
 
 		/* Clear accessed and referenced bits. */
 		ptep_test_and_clear_young(vma, addr, pte);
+		clear_page_young(page);
 		ClearPageReferenced(page);
 	}
 	pte_unmap_unlock(pte - 1, ptl);

include/linux/mm.h +88 -0

index 0755b9fd03a7..794d29aa2317 100644
@@ -2200,5 +2200,93 @@ void __init setup_nr_node_ids(void);
 static inline void setup_nr_node_ids(void) {}
 #endif
 
+#ifdef CONFIG_IDLE_PAGE_TRACKING
+#ifdef CONFIG_64BIT
+static inline bool page_is_young(struct page *page)
+{
+	return PageYoung(page);
+}
+
+static inline void set_page_young(struct page *page)
+{
+	SetPageYoung(page);
+}
+
+static inline void clear_page_young(struct page *page)
+{
+	ClearPageYoung(page);
+}
+
+static inline bool page_is_idle(struct page *page)
+{
+	return PageIdle(page);
+}
+
+static inline void set_page_idle(struct page *page)
+{
+	SetPageIdle(page);
+}
+
+static inline void clear_page_idle(struct page *page)
+{
+	ClearPageIdle(page);
+}
+#else /* !CONFIG_64BIT */
+/*
+ * If there is not enough space to store Idle and Young bits in page flags, use
+ * page ext flags instead.
+ */
+extern struct page_ext_operations page_idle_ops;
+
+static inline bool page_is_young(struct page *page)
+{
+	return test_bit(PAGE_EXT_YOUNG, &lookup_page_ext(page)->flags);
+}
+
+static inline void set_page_young(struct page *page)
+{
+	set_bit(PAGE_EXT_YOUNG, &lookup_page_ext(page)->flags);
+}
+
+static inline void clear_page_young(struct page *page)
+{
+	clear_bit(PAGE_EXT_YOUNG, &lookup_page_ext(page)->flags);
+}
+
+static inline bool page_is_idle(struct page *page)
+{
+	return test_bit(PAGE_EXT_IDLE, &lookup_page_ext(page)->flags);
+}
+
+static inline void set_page_idle(struct page *page)
+{
+	set_bit(PAGE_EXT_IDLE, &lookup_page_ext(page)->flags);
+}
+
+static inline void clear_page_idle(struct page *page)
+{
+	clear_bit(PAGE_EXT_IDLE, &lookup_page_ext(page)->flags);
+}
+#endif /* CONFIG_64BIT */
+#else /* !CONFIG_IDLE_PAGE_TRACKING */
+static inline bool page_is_young(struct page *page)
+{
+	return false;
+}
+
+static inline void clear_page_young(struct page *page)
+{
+}
+
+static inline bool page_is_idle(struct page *page)
+{
+	return false;
+}
+
+static inline void clear_page_idle(struct page *page)
+{
+}
+#endif /* CONFIG_IDLE_PAGE_TRACKING */
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_H */

include/linux/page-flags.h +9 -0

index f34e040b34e9..5e7c4f50a644 100644
@@ -109,6 +109,10 @@ enum pageflags {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 	PG_compound_lock,
 #endif
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
+	PG_young,
+	PG_idle,
+#endif
 	__NR_PAGEFLAGS,
 
 	/* Filesystems */

@@ -289,6 +293,11 @@ PAGEFLAG_FALSE(HWPoison)
 #define __PG_HWPOISON 0
 #endif
 
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
+PAGEFLAG(Young, young)
+PAGEFLAG(Idle, idle)
+#endif
+
 /*
  * On an anonymous page mapped into a user virtual memory area,
  * page->mapping points to its anon_vma, not to a struct address_space;

include/linux/page_ext.h +4 -0

index c42981cd99aa..17f118a82854 100644
@@ -26,6 +26,10 @@ enum page_ext_flags {
 	PAGE_EXT_DEBUG_POISON,		/* Page is poisoned */
 	PAGE_EXT_DEBUG_GUARD,
 	PAGE_EXT_OWNER,
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT)
+	PAGE_EXT_YOUNG,
+	PAGE_EXT_IDLE,
+#endif
 };
 
 /*

mm/Kconfig +12 -0

index 390214da4546..3600eace4774 100644
@@ -635,3 +635,15 @@ config MAX_STACK_SIZE_MB
 	  changed to a smaller value in which case that is used.
 
 	  A sane initial value is 80 MB.
+
+config IDLE_PAGE_TRACKING
+	bool "Enable idle page tracking"
+	select PROC_PAGE_MONITOR
+	select PAGE_EXTENSION if !64BIT
+	help
+	  This feature allows to estimate the amount of user pages that have
+	  not been touched during a given period of time. This information can
+	  be useful to tune memory cgroup limits and/or for job placement
+	  within a compute cluster.
+
+	  See Documentation/vm/pagemap.txt for more details.

mm/debug.c +4 -0

index 3eb3ac2fcee7..bb66f9ccec03 100644
@@ -48,6 +48,10 @@ static const struct trace_print_flags pageflag_names[] = {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 	{1UL << PG_compound_lock,	"compound_lock"	},
 #endif
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
+	{1UL << PG_young,		"young"		},
+	{1UL << PG_idle,		"idle"		},
+#endif
 };
 
 static void dump_flags(unsigned long flags,

mm/page_ext.c +3 -0

index d86fd2f5353f..e4b3af054bf2 100644
@@ -59,6 +59,9 @@ static struct page_ext_operations *page_ext_ops[] = {
 #ifdef CONFIG_PAGE_OWNER
 	&page_owner_ops,
 #endif
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT)
+	&page_idle_ops,
+#endif
 };
 
 static unsigned long total_usage;

mm/rmap.c +0 -0

index 24dd3f9fee27..12e73b758d9e 100644

mm/rmap.c +0 -0

 	}
 
 	pra->mapcount--;

mm/swap.c +2 -0

index a7251a8ed532..6bf6f293a9ea 100644
@@ -623,6 +623,8 @@ void mark_page_accessed(struct page *page)
 	} else if (!PageReferenced(page)) {
 		SetPageReferenced(page);
 	}
+	if (page_is_idle(page))
+		clear_page_idle(page);
 }
 EXPORT_SYMBOL(mark_page_accessed);
 

Signature · 7 lines

1.7.10.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Quoted 8 lines · Vladimir Davydov

fs/proc/page.c +0 -0

index 70d23245dd43..cfc55ba7fee6 100644
@@ -275,6 +275,156 @@ static const struct file_operations proc_kpagecgroup_operations = {
 };
 #endif /* CONFIG_MEMCG */
 

fs/proc/page.c static const struct file_operations proc_kpagecgroup_operations = { · line 290

Quoted 13 lines

fs/proc/page.c +13 -0

+#ifdef CONFIG_IDLE_PAGE_TRACKING
+static struct page *kpageidle_get_page(unsigned long pfn)
+{
+	struct page *page;
+
+	if (!pfn_valid(pfn))
+		return NULL;
+	page = pfn_to_page(pfn);
+	/*
+	 * We are only interested in user memory pages, i.e. pages that are
+	 * allocated and on an LRU list.
+	 */
+	if (!page || page_count(page) == 0 || !PageLRU(page))

Why do you check (page_count == 0) even if we check it with get_page_unless_zero below?

fs/proc/page.c static const struct file_operations proc_kpagecgroup_operations = { · line 294

Quoted 5 lines

fs/proc/page.c +4 -0

+		return NULL;
+	if (!get_page_unless_zero(page))
+		return NULL;
+	if (unlikely(!PageLRU(page))) {

What lock protect the check PageLRU?
If it is racing ClearPageLRU, what happens?

Quoted 5 lines

fs/proc/page.c +4 -0

+		put_page(page);
+		return NULL;
+	}
+	return page;

fs/proc/page.c static const struct file_operations proc_kpagecgroup_operations = { · line 308

Quoted 10 lines

fs/proc/page.c +10 -0

+}
+
+static void kpageidle_clear_refs(struct page *page)
+{
+	unsigned long dummy;
+
+	if (page_referenced(page, 0, NULL, &dummy))
+		/*
+		 * This page was referenced. To avoid interference with the
+		 * reclaimer, mark it young so that the next call will also

next what call?

It just works with mapped page so kpageidle_clear_pte_refs as function name is more clear.

One more, kpageidle_clear_refs removes PG_idle via page_referenced which is important feature for the function. Please document it so we could understand why we need double check for PG_idle after calling kpageidle_clear_refs for pte access bit.

Quoted 11 lines

mm/rmap.c +3 -0

index 24dd3f9fee27..12e73b758d9e 100644
@@ -784,6 +784,13 @@ static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
 	if (referenced) {
 		pra->referenced++;
 		pra->vm_flags |= vma->vm_flags;
+		if (page_is_idle(page))
+			clear_page_idle(page);
+	}

mm/rmap.c static int page_referenced_one(struct page *page, struct vm_area_struct *vma, · line 793

Quoted 4 lines

mm/rmap.c +4 -0

+
+	if (page_is_young(page)) {
+		clear_page_young(page);
+		pra->referenced++;

If a page was page_is_young and referenced recenlty, pra->referenced is increased doubly and it changes current behavior for file-backed page promotion. Look at page_check_references.

Quoted 9 lines · Minchan Kim

Quoted 8 lines · Vladimir Davydov

fs/proc/page.c +0 -0

index 70d23245dd43..cfc55ba7fee6 100644
@@ -275,6 +275,156 @@ static const struct file_operations proc_kpagecgroup_operations = {
 };
 #endif /* CONFIG_MEMCG */
 

Quoted 16 lines

fs/proc/page.c static const struct file_operations proc_kpagecgroup_operations = { · line 290

Quoted 13 lines

fs/proc/page.c +13 -0

+#ifdef CONFIG_IDLE_PAGE_TRACKING
+static struct page *kpageidle_get_page(unsigned long pfn)
+{
+	struct page *page;
+
+	if (!pfn_valid(pfn))
+		return NULL;
+	page = pfn_to_page(pfn);
+	/*
+	 * We are only interested in user memory pages, i.e. pages that are
+	 * allocated and on an LRU list.
+	 */
+	if (!page || page_count(page) == 0 || !PageLRU(page))

Why do you check (page_count == 0) even if we check it with get_page_unless_zero below?

I intended to avoid overhead of cmpxchg in case page_count is 0, but diving deeper into get_page_unless_zero, I see that it already handles such a scenario, so this check is useless. I'll remove it.

Quoted 8 lines

fs/proc/page.c static const struct file_operations proc_kpagecgroup_operations = { · line 294

Quoted 5 lines

fs/proc/page.c +4 -0

+		return NULL;
+	if (!get_page_unless_zero(page))
+		return NULL;
+	if (unlikely(!PageLRU(page))) {

What lock protect the check PageLRU?
If it is racing ClearPageLRU, what happens?

If we hold a reference to a page and see that it's on an LRU list, it will surely remain a user memory page at least until we release the reference to it, so it must be safe to play with idle/young flags. If we

race with isolate_lru_page, or any similar function temporarily clearing PG_lru, we will silently skip the page w/o touching its idle/young flags. We could consider isolated pages too, but that would increase the cost of this function.

If you find this explanation OK, I'll add it to the comment to this function.

Quoted 18 lines

Quoted 5 lines

fs/proc/page.c +4 -0

+		put_page(page);
+		return NULL;
+	}
+	return page;

fs/proc/page.c static const struct file_operations proc_kpagecgroup_operations = { · line 308

Quoted 10 lines

fs/proc/page.c +10 -0

+}
+
+static void kpageidle_clear_refs(struct page *page)
+{
+	unsigned long dummy;
+
+	if (page_referenced(page, 0, NULL, &dummy))
+		/*
+		 * This page was referenced. To avoid interference with the
+		 * reclaimer, mark it young so that the next call will also

next what call?

Quoted 7 lines

It just works with mapped page so kpageidle_clear_pte_refs as function name is more clear.

One more, kpageidle_clear_refs removes PG_idle via page_referenced which is important feature for the function. Please document it so we could understand why we need double check for PG_idle after calling kpageidle_clear_refs for pte access bit.

Sounds reasonable, will do.

Quoted 11 lines

Quoted 11 lines

mm/rmap.c +3 -0

index 24dd3f9fee27..12e73b758d9e 100644
@@ -784,6 +784,13 @@ static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
 	if (referenced) {
 		pra->referenced++;
 		pra->vm_flags |= vma->vm_flags;
+		if (page_is_idle(page))
+			clear_page_idle(page);
+	}

Quoted 8 lines

mm/rmap.c static int page_referenced_one(struct page *page, struct vm_area_struct *vma,

Quoted 4 lines

mm/rmap.c +4 -0

+
+	if (page_is_young(page)) {
+		clear_page_young(page);
+		pra->referenced++;

If a page was page_is_young and referenced recenlty, pra->referenced is increased doubly and it changes current behavior for file-backed page promotion. Look at page_check_references.

Yeah, you're quite right, I missed that. Something like this should get rid of this extra reference:

mm/rmap.c +8 -0

index 24dd3f9fee27..eca7416f55d7 100644
@@ -781,6 +781,14 @@ static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
 		pte_unmap_unlock(pte, ptl);
 	}
 
+	if (referenced && page_is_idle(page))
+		clear_page_idle(page);
+
+	if (page_is_young(page)) {
+		clear_page_young(page);
+		referenced++;
+	}
+
 	if (referenced) {
 		pra->referenced++;
 		pra->vm_flags |= vma->vm_flags;

Thanks,
Vladimir

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Quoted 23 lines · Vladimir Davydov

Quoted 13 lines · Minchan Kim

Quoted 12 lines · Vladimir Davydov

(quoted diff) +12 -0

+#ifdef CONFIG_IDLE_PAGE_TRACKING
+static struct page *kpageidle_get_page(unsigned long pfn)
+{
+	struct page *page;
+
+	if (!pfn_valid(pfn))
+		return NULL;
+	page = pfn_to_page(pfn);
+	/*
+	 * We are only interested in user memory pages, i.e. pages that are
+	 * allocated and on an LRU list.
+	 */

Quoted 8 lines

Review

Quoted 5 lines

(quoted diff) +5 -0

+	if (!page || page_count(page) == 0 || !PageLRU(page))
+		return NULL;
+	if (!get_page_unless_zero(page))
+		return NULL;
+	if (unlikely(!PageLRU(page))) {

What lock protect the check PageLRU?
If it is racing ClearPageLRU, what happens?

Quoted 3 lines

If we hold a reference to a page and see that it's on an LRU list, it will surely remain a user memory page at least until we release the reference to it, so it must be safe to play with idle/young flags. If we

The problem is that you pass the page in rmap reverse logic(ie, page_referenced) once you judge it's LRU page so if it is false-positive, what happens? A question is SetPageLRU, PageLRU, ClearPageLRU keeps memory ordering? IOW, all of fields from struct page rmap can acccess should be set up completely before LRU checking. Otherwise, something will be broken.

Quoted 28 lines · Minchan Kim

Quoted 23 lines · Vladimir Davydov

Quoted 13 lines · Minchan Kim

Quoted 12 lines · Vladimir Davydov

(quoted diff) +12 -0

+#ifdef CONFIG_IDLE_PAGE_TRACKING
+static struct page *kpageidle_get_page(unsigned long pfn)
+{
+	struct page *page;
+
+	if (!pfn_valid(pfn))
+		return NULL;
+	page = pfn_to_page(pfn);
+	/*
+	 * We are only interested in user memory pages, i.e. pages that are
+	 * allocated and on an LRU list.
+	 */

Quoted 8 lines

Review

Quoted 5 lines

(quoted diff) +5 -0

+	if (!page || page_count(page) == 0 || !PageLRU(page))
+		return NULL;
+	if (!get_page_unless_zero(page))
+		return NULL;
+	if (unlikely(!PageLRU(page))) {

What lock protect the check PageLRU?
If it is racing ClearPageLRU, what happens?

Quoted 3 lines

If we hold a reference to a page and see that it's on an LRU list, it will surely remain a user memory page at least until we release the reference to it, so it must be safe to play with idle/young flags. If we

Quoted 5 lines

The problem is that you pass the page in rmap reverse logic(ie, page_referenced) once you judge it's LRU page so if it is false-positive, what happens? A question is SetPageLRU, PageLRU, ClearPageLRU keeps memory ordering? IOW, all of fields from struct page rmap can acccess should be set up completely before LRU checking. Otherwise, something will be broken.

So, basically you are concerned about the case when we encounter a freshly allocated page, which has PG_lru bit set and it's going to become anonymous, but it is still in the process of rmap initialization, i.e. its ->mapping or ->mapcount may still be uninitialized, right?

AFAICS, page_referenced should handle such pages fine. Look, it only needs ->index, ->mapping, and ->mapcount.

If ->mapping is unset, than it is NULL and rmap_walk_anon_lock -> page_lock_anon_vma_read will return NULL so that rmap_walk will be a no-op.

If ->index is not initialized, than at worst we will go to anon_vma_interval_tree_foreach over a wrong interval, in which case we will see that the page is actually not mapped in page_referenced_one -> page_check_address and again do nothing.

If ->mapcount is not initialized it is -1, and page_lock_anon_vma_read will return NULL, just as it does in case ->mapping = NULL.

For file pages, we always take PG_locked before checking ->mapping, so it must be valid.

Thanks,
Vladimir

Quoted 52 lines · Vladimir Davydov

Quoted 28 lines · Minchan Kim

Quoted 23 lines · Vladimir Davydov

Quoted 13 lines · Minchan Kim

Quoted 12 lines · Vladimir Davydov

(quoted diff) +12 -0

+#ifdef CONFIG_IDLE_PAGE_TRACKING
+static struct page *kpageidle_get_page(unsigned long pfn)
+{
+	struct page *page;
+
+	if (!pfn_valid(pfn))
+		return NULL;
+	page = pfn_to_page(pfn);
+	/*
+	 * We are only interested in user memory pages, i.e. pages that are
+	 * allocated and on an LRU list.
+	 */

Quoted 8 lines

Review

Quoted 5 lines

(quoted diff) +5 -0

+	if (!page || page_count(page) == 0 || !PageLRU(page))
+		return NULL;
+	if (!get_page_unless_zero(page))
+		return NULL;
+	if (unlikely(!PageLRU(page))) {

What lock protect the check PageLRU?
If it is racing ClearPageLRU, what happens?

Quoted 3 lines

If we hold a reference to a page and see that it's on an LRU list, it will surely remain a user memory page at least until we release the reference to it, so it must be safe to play with idle/young flags. If we

Quoted 5 lines

The problem is that you pass the page in rmap reverse logic(ie, page_referenced) once you judge it's LRU page so if it is false-positive, what happens? A question is SetPageLRU, PageLRU, ClearPageLRU keeps memory ordering? IOW, all of fields from struct page rmap can acccess should be set up completely before LRU checking. Otherwise, something will be broken.

So, basically you are concerned about the case when we encounter a freshly allocated page, which has PG_lru bit set and it's going to become anonymous, but it is still in the process of rmap initialization, i.e. its ->mapping or ->mapcount may still be uninitialized, right?

AFAICS, page_referenced should handle such pages fine. Look, it only needs ->index, ->mapping, and ->mapcount.

If ->mapping is unset, than it is NULL and rmap_walk_anon_lock -> page_lock_anon_vma_read will return NULL so that rmap_walk will be a no-op.

If ->index is not initialized, than at worst we will go to anon_vma_interval_tree_foreach over a wrong interval, in which case we will see that the page is actually not mapped in page_referenced_one -> page_check_address and again do nothing.

Quoted 8 lines

If ->mapcount is not initialized it is -1, and page_lock_anon_vma_read will return NULL, just as it does in case ->mapping = NULL.

For file pages, we always take PG_locked before checking ->mapping, so it must be valid.

Thanks,
Vladimir

do_anonymous_page
page_add_new_anon_rmap
atomic_set(&page->_mapcount, 0);
__page_set_anon_rmap

   anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
   page->mapping = (struct address_space *) anon_vma;
   page->index = linear_page_index(vma, address);

lru_cache_add
__pagevec_lru_add_fn
SetPageLRU(page);

During the procedure, there is no lock to prevent race. Then, at least, we need a write memory barrier to guarantee other fields set up before SetPageLRU. (Of course, PageLRU should have read-memory barrier to work well) But I can't find any barrier, either.

IOW, any fields you said could be out of order store without any lock or memory barrier. You might argue atomic op is a barrier on x86 but it doesn't guarantee other arches work like that so we need explict momory barrier or lock.

Let's have a theoretical example.

CPU 0 CPU 1

do_anonymous_page
__page_set_anon_rmap
/* out of order happened so SetPageLRU is done ahead */ SetPageLRU(page)
/* Compilr changed store operation like below */

page->mapping = (struct address_space *) anon_vma;
/* Big stall happens */
/* idletacking judged it as LRU page so pass the page
in page_reference */
page_refernced
page_rmapping return true because page->mapping has some vaule but not complete so it calls rmap_walk_file.
it's okay to pass non-completed anon page in rmap_walk_file?

  page->mapping = (struct address_space *)
        ((void *)page_mapping + PAGE_MAPPING_ANON);

It's too theoretical so it might be hard to happen in real practice. My point is there is nothing to prevent explict race. Even if there is no problem with other lock, it's fragile. Do I miss something?

I think general way to handle PageLRU are ahead isolation or zone->lru_lock.

Quoted 86 lines · Minchan Kim

Quoted 52 lines · Vladimir Davydov

Quoted 28 lines · Minchan Kim

Quoted 23 lines · Vladimir Davydov

Quoted 13 lines · Minchan Kim

Quoted 12 lines · Vladimir Davydov

(quoted diff) +12 -0

+#ifdef CONFIG_IDLE_PAGE_TRACKING
+static struct page *kpageidle_get_page(unsigned long pfn)
+{
+	struct page *page;
+
+	if (!pfn_valid(pfn))
+		return NULL;
+	page = pfn_to_page(pfn);
+	/*
+	 * We are only interested in user memory pages, i.e. pages that are
+	 * allocated and on an LRU list.
+	 */

Quoted 8 lines

Review

Quoted 5 lines

(quoted diff) +5 -0

+	if (!page || page_count(page) == 0 || !PageLRU(page))
+		return NULL;
+	if (!get_page_unless_zero(page))
+		return NULL;
+	if (unlikely(!PageLRU(page))) {

What lock protect the check PageLRU?
If it is racing ClearPageLRU, what happens?

Quoted 3 lines

If we hold a reference to a page and see that it's on an LRU list, it will surely remain a user memory page at least until we release the reference to it, so it must be safe to play with idle/young flags. If we

Quoted 5 lines

The problem is that you pass the page in rmap reverse logic(ie, page_referenced) once you judge it's LRU page so if it is false-positive, what happens? A question is SetPageLRU, PageLRU, ClearPageLRU keeps memory ordering? IOW, all of fields from struct page rmap can acccess should be set up completely before LRU checking. Otherwise, something will be broken.

So, basically you are concerned about the case when we encounter a freshly allocated page, which has PG_lru bit set and it's going to become anonymous, but it is still in the process of rmap initialization, i.e. its ->mapping or ->mapcount may still be uninitialized, right?

AFAICS, page_referenced should handle such pages fine. Look, it only needs ->index, ->mapping, and ->mapcount.

If ->mapping is unset, than it is NULL and rmap_walk_anon_lock -> page_lock_anon_vma_read will return NULL so that rmap_walk will be a no-op.

If ->index is not initialized, than at worst we will go to anon_vma_interval_tree_foreach over a wrong interval, in which case we will see that the page is actually not mapped in page_referenced_one -> page_check_address and again do nothing.

Quoted 8 lines

If ->mapcount is not initialized it is -1, and page_lock_anon_vma_read will return NULL, just as it does in case ->mapping = NULL.

For file pages, we always take PG_locked before checking ->mapping, so it must be valid.

Thanks,
Vladimir

do_anonymous_page
page_add_new_anon_rmap
atomic_set(&page->_mapcount, 0);
__page_set_anon_rmap

   anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
   page->mapping = (struct address_space *) anon_vma;
   page->index = linear_page_index(vma, address);

lru_cache_add
__pagevec_lru_add_fn
SetPageLRU(page);

During the procedure, there is no lock to prevent race. Then, at least, we need a write memory barrier to guarantee other fields set up before SetPageLRU. (Of course, PageLRU should have read-memory barrier to work well) But I can't find any barrier, either.

IOW, any fields you said could be out of order store without any lock or memory barrier. You might argue atomic op is a barrier on x86 but it doesn't guarantee other arches work like that so we need explict momory barrier or lock.

Let's have a theoretical example.

Quoted 7 lines

CPU 0 CPU 1

do_anonymous_page
__page_set_anon_rmap
/* out of order happened so SetPageLRU is done ahead */ SetPageLRU(page)
/* Compilr changed store operation like below */

But it couldn't. Quoting Documentation/atomic_ops.txt:

Properly aligned pointers, longs, ints, and chars (and unsigned equivalents) may be atomically loaded from and stored to in the same sense as described for atomic_read() and atomic_set().

__page_set_anon_rmap sets page->mapping using the following expression:

(quoted diff) +0 -0

 	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
 	page->mapping = (struct address_space *) anon_vma;

and it can't be split, i.e. if one concurrently reads page->mapping he/she will see either NULL or (anon_vma+PAGE_MAPPING_ANON), and there can't be any intermediate result in page->mapping, such as anon_vma or PAGE_MAPPING_ANON, because one doesn't expect

	atomic_set(&p, a + b);

to behave like

	atomic_set(&p, a);
	atomic_set(&p, atomic_read(&p) + b);

Quoted 10 lines

page->mapping = (struct address_space *) anon_vma;
/* Big stall happens */
/* idletacking judged it as LRU page so pass the page
in page_reference */
page_refernced
page_rmapping return true because page->mapping has some vaule but not complete so it calls rmap_walk_file.
it's okay to pass non-completed anon page in rmap_walk_file?

Quoted 2 lines

  page->mapping = (struct address_space *)
        ((void *)page_mapping + PAGE_MAPPING_ANON);

Thanks,
Vladimir

Quoted 19 lines

page->mapping = (struct address_space *) anon_vma;
/* Big stall happens */
/* idletacking judged it as LRU page so pass the page
in page_reference */
page_refernced
page_rmapping return true because page->mapping has some vaule but not complete so it calls rmap_walk_file.
it's okay to pass non-completed anon page in rmap_walk_file?

  page->mapping = (struct address_space *)
        ((void *)page_mapping + PAGE_MAPPING_ANON);

It's too theoretical so it might be hard to happen in real practice. My point is there is nothing to prevent explict race. Even if there is no problem with other lock, it's fragile. Do I miss something?

I think general way to handle PageLRU are ahead isolation or zone->lru_lock.

So, I guess once below compiler optimization happens in __page_set_anon_rmap, it could be corrupt in page_refernced.

__page_set_anon_rmap:

        page->mapping = (struct address_space *) anon_vma;
        page->mapping = (struct address_space *)((void *)page_mapping + PAGE_MAPPING_ANON);

Because page_referenced checks it with PageAnon which has no memory barrier. So if above compiler optimization happens, page_referenced can pass the anon page in rmap_walk_file, not ramp_walk_anon. It's my theory. :)

Quoted 10 lines · Minchan Kim

So, I guess once below compiler optimization happens in __page_set_anon_rmap, it could be corrupt in page_refernced.

__page_set_anon_rmap:

        page->mapping = (struct address_space *) anon_vma;
        page->mapping = (struct address_space *)((void *)page_mapping + PAGE_MAPPING_ANON);

Because page_referenced checks it with PageAnon which has no memory barrier. So if above compiler optimization happens, page_referenced can pass the anon page in rmap_walk_file, not ramp_walk_anon. It's my theory. :)

FWIW

If such splits were possible, we would have bugs all over the kernel IMO. An example is do_wp_page() vs shrink_active_list(). In do_wp_page() we can call page_move_anon_rmap(), which sets page->mapping in exactly the same fashion as above-mentioned __page_set_anon_rmap():

	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
	page->mapping = (struct address_space *) anon_vma;

The page in question may be on an LRU list, because nowhere in do_wp_page() we remove it from the list, neither do we take any LRU related locks. The page is locked, that's true, but shrink_active_list() calls page_referenced() on an unlocked page, so according to your logic they can race with the latter receiving a page with page->mapping equal to anon_vma w/o PAGE_MAPPING_ANON bit set:

CPU0 CPU1
---- ----
do_wp_page shrink_active_list
lock_page page_referenced

				  PageAnon->yes, so skip trylock_page
 page_move_anon_rmap
  page->mapping = anon_vma
				  rmap_walk
				   PageAnon->no
				   rmap_walk_file
				    BUG

page->mapping = page->mapping+PAGE_MAPPING_ANON

However, this does not happen.

Thanks,
Vladimir

Quoted 42 lines · Vladimir Davydov

Quoted 10 lines · Minchan Kim

So, I guess once below compiler optimization happens in __page_set_anon_rmap, it could be corrupt in page_refernced.

__page_set_anon_rmap:

        page->mapping = (struct address_space *) anon_vma;
        page->mapping = (struct address_space *)((void *)page_mapping + PAGE_MAPPING_ANON);

Because page_referenced checks it with PageAnon which has no memory barrier. So if above compiler optimization happens, page_referenced can pass the anon page in rmap_walk_file, not ramp_walk_anon. It's my theory. :)

FWIW

If such splits were possible, we would have bugs all over the kernel IMO. An example is do_wp_page() vs shrink_active_list(). In do_wp_page() we can call page_move_anon_rmap(), which sets page->mapping in exactly the same fashion as above-mentioned __page_set_anon_rmap():

	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
	page->mapping = (struct address_space *) anon_vma;

The page in question may be on an LRU list, because nowhere in do_wp_page() we remove it from the list, neither do we take any LRU related locks. The page is locked, that's true, but shrink_active_list() calls page_referenced() on an unlocked page, so according to your logic they can race with the latter receiving a page with page->mapping equal to anon_vma w/o PAGE_MAPPING_ANON bit set:

CPU0 CPU1
---- ----
do_wp_page shrink_active_list
lock_page page_referenced

				  PageAnon->yes, so skip trylock_page
 page_move_anon_rmap
  page->mapping = anon_vma
				  rmap_walk
				   PageAnon->no
				   rmap_walk_file
				    BUG

page->mapping = page->mapping+PAGE_MAPPING_ANON

Quoted 1 line

However, this does not happen.

Good spot.

However, it doesn't mean it's right so you are okay to rely on it. Normally, store tearing is not common and such race would be hard to hit but I want to call it as BUG.

Rik wrote the code and commented out.

"Protected against the rmap code by the page lock"

But unfortunately, page_referenced in shrink_active_list doesn't hold a page lock so isn't it a bug? Rik?

Please, read store tearing section in Documentation/memory-barrier.txt. If you get confused due to aligned memory, please read this link.

        https://lkml.org/lkml/2014/7/16/262

Other quote from Paul in https://lkml.org/lkml/2015/5/1/229
"
..
If the thing read/written does fit into a machine word and if the location read/written is properly aligned, I would be quite surprised if either READ_ONCE() or WRITE_ONCE() resulted in any sort of tearing.
"

I parsed it as that "even store tearing can happen machine word at alinged address and that's why WRITE_ONCE is there to prevent it"

If you want to claim GCC doesn't do it, please read below links

        https://lkml.org/lkml/2015/4/16/527
        http://yarchive.net/comp/linux/ACCESS_ONCE.html

Quote from Linus
"
The thing is, you can't _prove_ that the compiler won't do it, especially if you end up changing the code later (without thinking about the fact that you're loading things without locking).

So the rule is: if you access unlocked values, you use ACCESS_ONCE(). You don't say "but it can't matter". Because you simply don't know. "

Yeb, I might be paranoid but my point is it might work now on most of arch but it seem to be buggy/fragile/subtle because we couldn't prove all arch/compiler don't make any trouble. So, intead of adding more logics based on fragile, please use right lock model. If lock becomes big trouble by overhead, let's fix it(for instance, use WRITE_ONCE for update-side and READ_ONCE for read-side) if I don't miss something.

Quoted 6 lines · Minchan Kim

Yeb, I might be paranoid but my point is it might work now on most of arch but it seem to be buggy/fragile/subtle because we couldn't prove all arch/compiler don't make any trouble. So, intead of adding more logics based on fragile, please use right lock model. If lock becomes big trouble by overhead, let's fix it(for instance, use WRITE_ONCE for update-side and READ_ONCE for read-side) if I don't miss something.

IMO, locking would be an overkill. READ_ONCE is OK, because it has no performance implications, but I would prefer to be convinced that it is 100% necessary before adding it just in case.

Knowing the portion of memory that is not used by a certain application or memory cgroup (idle memory) can be useful for partitioning the system efficiently, e.g. by setting memory cgroup limits appropriately. Currently, the only means to estimate the amount of idle memory provided by the kernel is /proc/PID/{clear_refs,smaps}: the user can clear the access bit for all pages mapped to a particular process by writing 1 to clear_refs, wait for some time, and then count smaps:Referenced. However, this method has two serious shortcomings:

- it does not count unmapped file pages
- it affects the reclaimer logic

To overcome these drawbacks, this patch introduces two new page flags, Idle and Young, and a new proc file, /proc/kpageidle. A page's Idle flag can only be set from userspace by writing 1 to /proc/kpageidle at the offset corresponding to the page, and it is cleared whenever the page is accessed either through page tables (it is cleared in page_referenced() in this case) or using the read(2) system call (mark_page_accessed()). Thus by setting the Idle flag for pages of a particular workload, which can be found e.g. by reading /proc/PID/pagemap, waiting for some time to let the workload access its working set, and then reading the kpageidle file, one can estimate the amount of pages that are not used by the workload.

The Young page flag is used to avoid interference with the memory reclaimer. A page's Young flag is set whenever the Access bit of a page table entry pointing to the page is cleared by writing to kpageidle. If page_referenced() is called on a Young page, it will add 1 to its return value, therefore concealing the fact that the Access bit was cleared.

Note, since there is no room for extra page flags on 32 bit, this feature uses extended page flags when compiled on 32 bit.

Signed-off-by Vladimir Davydov <[email protected]>

---
 Documentation/vm/pagemap.txt |   10 ++-
 fs/proc/page.c               |  154 ++++++++++++++++++++++++++++++++++++++++++
 fs/proc/task_mmu.c           |    4 +-
 include/linux/mm.h           |   88 ++++++++++++++++++++++++
 include/linux/page-flags.h   |    9 +++
 include/linux/page_ext.h     |    4 ++
 mm/Kconfig                   |   12 ++++
 mm/debug.c                   |    4 ++
 mm/page_ext.c                |    3 +
 mm/rmap.c                    |    7 ++
 mm/swap.c                    |    2 +
 11 files changed, 295 insertions(+), 2 deletions(-)

Documentation/vm/pagemap.txt +1 -1

index a9b7afc8fbc6..ac6fd32a9296 100644
@@ -5,7 +5,7 @@ pagemap is a new (as of 2.6.25) set of interfaces in the kernel that allow
 userspace programs to examine the page tables and related information by
 reading files in /proc.
 
-There are four components to pagemap:
+There are five components to pagemap:
 
  * /proc/pid/pagemap.  This file lets a userspace process find out which
    physical frame each virtual page is mapped to.  It contains one 64-bit

@@ -69,6 +69,14 @@ There are four components to pagemap:
    memory cgroup each page is charged to, indexed by PFN. Only available when

Documentation/vm/pagemap.txt There are four components to pagemap: · line 79

Quoted 12 lines · Vladimir Davydov

Documentation/vm/pagemap.txt +8 -0

@@ -69,6 +69,14 @@ There are four components to pagemap:
    memory cgroup each page is charged to, indexed by PFN. Only available when
    CONFIG_MEMCG is set.
 
+ * /proc/kpageidle.  For each page this file contains a 64-bit number, which
+   equals 1 if the page is idle or 0 otherwise, indexed by PFN. A page is
+   considered idle if it has not been accessed since it was marked idle. To
+   mark a page idle one should write 1 to this file at the offset corresponding
+   to the page. Only user memory pages can be marked idle, for other page types
+   input is silently ignored. Writing to this file beyond max PFN results in
+   the ENXIO error. Only available when CONFIG_IDLE_PAGE_TRACKING is set.
+

How about using kpageflags for reading part?

I mean PG_idle is one of the page flags and we already have a feature to parse of each PFN flag so we could reuse existing feature for reading idleness.

Quoted 16 lines · Minchan Kim

Documentation/vm/pagemap.txt There are four components to pagemap: · line 79

Quoted 12 lines · Vladimir Davydov

Documentation/vm/pagemap.txt +8 -0

@@ -69,6 +69,14 @@ There are four components to pagemap:
    memory cgroup each page is charged to, indexed by PFN. Only available when
    CONFIG_MEMCG is set.
 
+ * /proc/kpageidle.  For each page this file contains a 64-bit number, which
+   equals 1 if the page is idle or 0 otherwise, indexed by PFN. A page is
+   considered idle if it has not been accessed since it was marked idle. To
+   mark a page idle one should write 1 to this file at the offset corresponding
+   to the page. Only user memory pages can be marked idle, for other page types
+   input is silently ignored. Writing to this file beyond max PFN results in
+   the ENXIO error. Only available when CONFIG_IDLE_PAGE_TRACKING is set.
+

How about using kpageflags for reading part?

Quoted 3 lines

I mean PG_idle is one of the page flags and we already have a feature to parse of each PFN flag so we could reuse existing feature for reading idleness.

Reading PG_idle implies clearing all pte references to make sure the page was not referenced via a pte. This means that exporting it via /proc/kpageflags would increase the cost of reading this file, even for users that don't care about PG_idle. I'm not sure all users of /proc/kpageflags will be fine with it.

Thanks,
Vladimir

Hi,

This patch set introduces a new user API for tracking user memory pages that have not been used for a given period of time. The purpose of this is to provide the userspace with the means of tracking a workload's working set, i.e. the set of pages that are actively used by the workload. Knowing the working set size can be useful for partitioning the system more efficiently, e.g. by tuning memory cgroup limits appropriately, or for job placement within a compute cluster.

---- USE CASES ----

The unified cgroup hierarchy has memory.low and memory.high knobs, which are defined as the low and high boundaries for the workload working set size. However, the working set size of a workload may be unknown or change in time. With this patch set, one can periodically estimate the amount of memory unused by each cgroup and tune their memory.low and memory.high parameters accordingly, therefore optimizing the overall memory utilization.

Another use case is balancing workloads within a compute cluster. Knowing how much memory is not really used by a workload unit may help take a more optimal decision when considering migrating the unit to another node within the cluster.

---- USER API ----

The user API consists of two new proc files:

* /proc/kpageidle. For each page this file contains a 64-bit number, which equals 1 if the page is idle or 0 otherwise, indexed by PFN. A page is

Quoted 2 lines · Vladimir Davydov

* /proc/kpageidle. For each page this file contains a 64-bit number, which equals 1 if the page is idle or 0 otherwise, indexed by PFN. A page is

Why do we need 64bit per page to indicate just idle or not?

Hi,

This patch set introduces a new user API for tracking user memory pages that have not been used for a given period of time. The purpose of this is to provide the userspace with the means of tracking a workload's working set, i.e. the set of pages that are actively used by the workload. Knowing the working set size can be useful for partitioning the system more efficiently, e.g. by tuning memory cgroup limits appropriately, or for job placement within a compute cluster.

---- USE CASES ----

The unified cgroup hierarchy has memory.low and memory.high knobs, which are defined as the low and high boundaries for the workload working set size. However, the working set size of a workload may be unknown or change in time. With this patch set, one can periodically estimate the amount of memory unused by each cgroup and tune their memory.low and memory.high parameters accordingly, therefore optimizing the overall memory utilization.

Another use case is balancing workloads within a compute cluster. Knowing how much memory is not really used by a workload unit may help take a more optimal decision when considering migrating the unit to another node within the cluster.