idle memory tracking

This function returns the inode number of the closest online ancestor of the memory cgroup a page is charged to. It is required for exporting information about which page is charged to which cgroup to userspace, which will be introduced by a following patch.

...

mm/memcontrol.c +0 -0

@@ -441,6 +441,29 @@ struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
 	return &memcg->css;
 }

Quoted 14 lines · On Sun, 19 Jul 2015 15:31:10 +0300 Vladimir Davydov <[email protected]>…

This function returns the inode number of the closest online ancestor of the memory cgroup a page is charged to. It is required for exporting information about which page is charged to which cgroup to userspace, which will be introduced by a following patch.

...

mm/memcontrol.c +0 -0

@@ -441,6 +441,29 @@ struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
 	return &memcg->css;
 }
 

mm/memcontrol.c struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page) · line 452

Quoted 9 lines

mm/memcontrol.c +9 -0

+/**
+ * page_cgroup_ino - return inode number of the memcg a page is charged to
+ * @page: the page
+ *
+ * Look up the closest online ancestor of the memory cgroup @page is charged to
+ * and return its inode number or 0 if @page is not charged to any cgroup. It
+ * is safe to call this function without holding a reference to @page.
+ */
+unsigned long page_cgroup_ino(struct page *page)

Shouldn't it return an ino_t?

Quoted 12 lines

mm/memcontrol.c +11 -0

+{
+	struct mem_cgroup *memcg;
+	unsigned long ino = 0;
+
+	rcu_read_lock();
+	memcg = READ_ONCE(page->mem_cgroup);
+	while (memcg && !(memcg->css.flags & CSS_ONLINE))
+		memcg = parent_mem_cgroup(memcg);
+	if (memcg)
+		ino = cgroup_ino(memcg->css.cgroup);
+	rcu_read_unlock();

mm/memcontrol.c struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page) · line 465

Quoted 2 lines

mm/memcontrol.c +2 -0

+	return ino;
+}

The function is racy, isn't it? There's nothing to prevent this inode from getting torn down and potentially reallocated one nanosecond after page_cgroup_ino() returns? If so, it is only safely usable by things which don't care (such as procfs interfaces) and this should be documented in some fashion.

Hwpoison allows to filter pages by memory cgroup ino. Currently, it calls try_get_mem_cgroup_from_page to obtain the cgroup from a page and then its ino using cgroup_ino, but now we have an apter method for that, page_cgroup_ino, so use it instead.

mm/hwpoison-inject.c +0 -3

@@ -45,12 +45,9 @@ static int hwpoison_inject(void *data, u64 val)
 	/*
 	 * do a racy check with elevated page count, to make sure PG_hwpoison
 	 * will only be set for the targeted owner (or on a free page).
-	 * We temporarily take page lock for try_get_mem_cgroup_from_page().
 	 * memory_failure() will redo the check reliably inside page lock.
 	 */
-	lock_page(hpage);
 	err = hwpoison_filter(hpage);
-	unlock_page(hpage);
 	if (err)
 		goto put_out;
 

Quoted 4 lines · On Sun, 19 Jul 2015 15:31:11 +0300 Vladimir Davydov…

Hwpoison allows to filter pages by memory cgroup ino. Currently, it calls try_get_mem_cgroup_from_page to obtain the cgroup from a page and then its ino using cgroup_ino, but now we have an apter method for that, page_cgroup_ino, so use it instead.

I assume "an apter" was supposed to be "a helper"?

Quoted 20 lines

mm/hwpoison-inject.c +0 -3

@@ -45,12 +45,9 @@ static int hwpoison_inject(void *data, u64 val)
 	/*
 	 * do a racy check with elevated page count, to make sure PG_hwpoison
 	 * will only be set for the targeted owner (or on a free page).
-	 * We temporarily take page lock for try_get_mem_cgroup_from_page().
 	 * memory_failure() will redo the check reliably inside page lock.
 	 */
-	lock_page(hpage);
 	err = hwpoison_filter(hpage);
-	unlock_page(hpage);
 	if (err)
 		goto put_out;
 

@@ -126,7 +123,7 @@ static int pfn_inject_init(void)
 	if (!dentry)
 		goto fail;
 

mm/hwpoison-inject.c static int pfn_inject_init(void) · line 129

Quoted 5 lines

mm/hwpoison-inject.c +1 -1

-#ifdef CONFIG_MEMCG_SWAP
+#ifdef CONFIG_MEMCG
 	dentry = debugfs_create_u64("corrupt-filter-memcg", 0600,
 				    hwpoison_dir, &hwpoison_filter_memcg);
 	if (!dentry)

Confused. We're changing the conditions under which this debugfs file is created. Is this a typo or some unchangelogged thing or what?

/proc/kpagecgroup contains a 64-bit inode number of the memory cgroup each page is charged to, indexed by PFN. Having this information is useful for estimating a cgroup working set size.

The file is present if CONFIG_PROC_PAGE_MONITOR && CONFIG_MEMCG.

...

fs/proc/page.c +5 -0

@@ -225,10 +226,62 @@ static const struct file_operations proc_kpageflags_operations = {
 	.read = kpageflags_read,
 };
 
+#ifdef CONFIG_MEMCG
+static ssize_t kpagecgroup_read(struct file *file, char __user *buf,
+				size_t count, loff_t *ppos)
+{
+	u64 __user *out = (u64 __user *)buf;

fs/proc/page.c +4 -0

+			break;
+		}
+
+		pfn++;

Quoted 22 lines · On Sun, 19 Jul 2015 15:31:13 +0300 Vladimir Davydov…

/proc/kpagecgroup contains a 64-bit inode number of the memory cgroup each page is charged to, indexed by PFN. Having this information is useful for estimating a cgroup working set size.

The file is present if CONFIG_PROC_PAGE_MONITOR && CONFIG_MEMCG.

...

fs/proc/page.c +10 -0

@@ -225,10 +226,62 @@ static const struct file_operations proc_kpageflags_operations = {
 	.read = kpageflags_read,
 };
 
+#ifdef CONFIG_MEMCG
+static ssize_t kpagecgroup_read(struct file *file, char __user *buf,
+				size_t count, loff_t *ppos)
+{
+	u64 __user *out = (u64 __user *)buf;
+	struct page *ppage;
+	unsigned long src = *ppos;
+	unsigned long pfn;
+	ssize_t ret = 0;
+	u64 ino;

fs/proc/page.c static const struct file_operations proc_kpageflags_operations = { · line 243

Quoted 5 lines

fs/proc/page.c +5 -0

+
+	pfn = src / KPMSIZE;
+	count = min_t(unsigned long, count, (max_pfn * KPMSIZE) - src);
+	if (src & KPMMASK || count & KPMMASK)
+		return -EINVAL;

The user-facing documentation should explain that reads must be performed in multiple-of-8 sizes.

Quoted 11 lines

fs/proc/page.c +10 -0

+	while (count > 0) {
+		if (pfn_valid(pfn))
+			ppage = pfn_to_page(pfn);
+		else
+			ppage = NULL;
+
+		if (ppage)
+			ino = page_cgroup_ino(ppage);
+		else
+			ino = 0;

fs/proc/page.c static const struct file_operations proc_kpageflags_operations = { · line 257

Quoted 3 lines

fs/proc/page.c +3 -0

+
+		if (put_user(ino, out)) {
+			ret = -EFAULT;

Here we do the usual procfs violation of read() behaviour. read() normally only returns an error if it read nothing. This code will transfer a megabyte then return -EFAULT so userspace doesn't know that it got that megabyte.

That's easy to fix, but procfs files do this all over the place anyway :(

Quoted 14 lines

fs/proc/page.c +14 -0

+			break;
+		}
+
+		pfn++;
+		out++;
+		count -= KPMSIZE;
+	}
+
+	*ppos += (char __user *)out - buf;
+	if (!ret)
+		ret = (char __user *)out - buf;
+	return ret;
+}
+

In the scope of the idle memory tracking feature, which is introduced by the following patch, we need to clear the referenced/accessed bit not only in primary, but also in secondary ptes. The latter is required in order to estimate wss of KVM VMs. At the same time we want to avoid flushing tlb, because it is quite expensive and it won't really affect the final result.

Currently, there is no function for clearing pte young bit that would meet our requirements, so this patch introduces one. To achieve that we have to add a new mmu-notifier callback, clear_young, since there is no method for testing-and-clearing a secondary pte w/o flushing tlb. The new method is not mandatory and currently only implemented by KVM.

Signed-off-by Vladimir Davydov <[email protected]>

Reviewed-by Andres Lagar-Cavilla <[email protected]>

Acked-by Paolo Bonzini <[email protected]>

---
 include/linux/mmu_notifier.h | 44
++++++++++++++++++++++++++++++++++++++++++++
 mm/mmu_notifier.c            | 17 +++++++++++++++++
 virt/kvm/kvm_main.c          | 18 ++++++++++++++++++
 3 files changed, 79 insertions(+)

include/linux/mmu_notifier.h +1 -0

index 61cd67f4d788..a5b17137c683 100644
@@ -66,6 +66,16 @@ struct mmu_notifier_ops {
                                 unsigned long end);
 
        /*
+        * clear_young is a lightweight version of clear_flush_young. Like

the
+ * latter, it is supposed to test-and-clear the young/accessed
bitflag
+ * in the secondary pte, but it may omit flushing the secondary
tlb.

include/linux/mmu_notifier.h +10 -0

+        */
+       int (*clear_young)(struct mmu_notifier *mn,
+                          struct mm_struct *mm,
+                          unsigned long start,
+                          unsigned long end);
+
+       /*
         * test_young is called to check the young/accessed bitflag in
         * the secondary pte. This is used to know if the page is
         * frequently used without actually clearing the flag or tearing

@@ -203,6 +213,9 @@ extern void __mmu_notifier_release(struct mm_struct
 *mm);
 extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
                                          unsigned long start,
                                          unsigned long end);
+extern int __mmu_notifier_clear_young(struct mm_struct *mm,
+                                     unsigned long start,
+                                     unsigned long end);
 extern int __mmu_notifier_test_young(struct mm_struct *mm,
                                     unsigned long address);

extern void __mmu_notifier_change_pte(struct mm_struct *mm,

include/linux/mmu_notifier.h +9 -0

@@ -231,6 +244,15 @@ static inline int
 mmu_notifier_clear_flush_young(struct mm_struct *mm,
        return 0;
 }
 
+static inline int mmu_notifier_clear_young(struct mm_struct *mm,
+                                          unsigned long start,
+                                          unsigned long end)
+{
+       if (mm_has_notifiers(mm))
+               return __mmu_notifier_clear_young(mm, start, end);
+       return 0;
+}
+
 static inline int mmu_notifier_test_young(struct mm_struct *mm,
                                          unsigned long address)

{

include/linux/mmu_notifier.h +22 -0

@@ -311,6 +333,28 @@ static inline void mmu_notifier_mm_destroy(struct
 mm_struct *mm)
        __young;                                                        \
 })
 
+#define ptep_clear_young_notify(__vma, __address, __ptep)              \
+({                                                                     \
+       int __young;                                                    \
+       struct vm_area_struct *___vma = __vma;                          \
+       unsigned long ___address = __address;                           \
+       __young = ptep_test_and_clear_young(___vma, ___address, __ptep);\
+       __young |= mmu_notifier_clear_young(___vma->vm_mm, ___address,  \
+                                           ___address + PAGE_SIZE);    \
+       __young;                                                        \
+})
+
+#define pmdp_clear_young_notify(__vma, __address, __pmdp)              \
+({                                                                     \
+       int __young;                                                    \
+       struct vm_area_struct *___vma = __vma;                          \
+       unsigned long ___address = __address;                           \
+       __young = pmdp_test_and_clear_young(___vma, ___address, __pmdp);\
+       __young |= mmu_notifier_clear_young(___vma->vm_mm, ___address,  \
+                                           ___address + PMD_SIZE);     \
+       __young;                                                        \
+})
+
 #define        ptep_clear_flush_notify(__vma, __address, __ptep)
     \

 ({                                                                     \
        unsigned long ___addr = __address & PAGE_MASK;                  \

mm/mmu_notifier.c +17 -0

index 3b9b3d0741b2..5fbdd367bbed 100644
@@ -123,6 +123,23 @@ int __mmu_notifier_clear_flush_young(struct mm_struct
 *mm,
        return young;
 }
 
+int __mmu_notifier_clear_young(struct mm_struct *mm,
+                              unsigned long start,
+                              unsigned long end)
+{
+       struct mmu_notifier *mn;
+       int young = 0, id;
+
+       id = srcu_read_lock(&srcu);
+       hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
+               if (mn->ops->clear_young)
+                       young |= mn->ops->clear_young(mn, mm, start, end);
+       }
+       srcu_read_unlock(&srcu, id);
+
+       return young;
+}
+
 int __mmu_notifier_test_young(struct mm_struct *mm,
                              unsigned long address)

{

virt/kvm/kvm_main.c +0 -0

index 8b8a44453670..ff4173ce6924 100644
@@ -387,6 +387,23 @@ static int kvm_mmu_notifier_clear_flush_young(struct
 mmu_notifier *mn,
        return young;
 }

virt/kvm/kvm_main.c +9 -0

+       idx = srcu_read_lock(&kvm->srcu);
+       spin_lock(&kvm->mmu_lock);
+       young = kvm_age_hva(kvm, start, end);
+       spin_unlock(&kvm->mmu_lock);
+       srcu_read_unlock(&kvm->srcu, idx);
+
+       return young;
+}
+
 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,

struct mm_struct *mm, unsigned long address)

virt/kvm/kvm_main.c +1 -0

@@ -419,6 +436,7 @@ static const struct mmu_notifier_ops
 kvm_mmu_notifier_ops = {
        .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
        .invalidate_range_end   = kvm_mmu_notifier_invalidate_range_end,
        .clear_flush_young      = kvm_mmu_notifier_clear_flush_young,
+       .clear_young            = kvm_mmu_notifier_clear_young,
        .test_young             = kvm_mmu_notifier_test_young,
        .change_pte             = kvm_mmu_notifier_change_pte,

.release = kvm_mmu_notifier_release,
--
2.1.4

On Sun, Jul 19, 2015 at 5:31 AM, Vladimir Davydov <[email protected]>

Quoted 5 lines

(quoted diff) +5 -0

+static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
+                                       struct mm_struct *mm,
+                                       unsigned long start,
+                                       unsigned long end)
+{

Review

Quoted 4 lines

(quoted diff) +3 -0

+       struct kvm *kvm = mmu_notifier_to_kvm(mn);
+       int young, idx;
+

If you need to cut out another version please add comments as to the two issues raised:
- This doesn't proactively flush TLBs -- not obvious if it should.
- This adversely affects performance in Pre_haswell Intel EPT.

Quoted 1 line · Andres Lagar-Cavilla

On Sun, Jul 19, 2015 at 5:31 AM, Vladimir Davydov <[email protected]>

[...]

Quoted 5 lines

Quoted 5 lines

(quoted diff) +5 -0

+static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
+                                       struct mm_struct *mm,
+                                       unsigned long start,
+                                       unsigned long end)
+{

Quoted 8 lines

Review

Quoted 4 lines

(quoted diff) +3 -0

+       struct kvm *kvm = mmu_notifier_to_kvm(mn);
+       int young, idx;
+

If you need to cut out another version please add comments as to the two issues raised:
- This doesn't proactively flush TLBs -- not obvious if it should.
- This adversely affects performance in Pre_haswell Intel EPT.

Oops, I stopped reading your e-mail in reply to the previous version of this patch as soon as I saw the Reviewed-by tag, so I missed your request for the comment, sorry about that.

Here it goes (incremental):
---

virt/kvm/kvm_main.c +6 -0

index ff4173ce6924..e69a5cb99571 100644
@@ -397,6 +397,19 @@ static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
 
 	idx = srcu_read_lock(&kvm->srcu);
 	spin_lock(&kvm->mmu_lock);
+	/*
+	 * Even though we do not flush TLB, this will still adversely
+	 * affect performance on pre-Haswell Intel EPT, where there is
+	 * no EPT Access Bit to clear so that we have to tear down EPT
+	 * tables instead. If we find this unacceptable, we can always
+	 * add a parameter to kvm_age_hva so that it effectively doesn't

Knowing the portion of memory that is not used by a certain application or memory cgroup (idle memory) can be useful for partitioning the system efficiently, e.g. by setting memory cgroup limits appropriately. Currently, the only means to estimate the amount of idle memory provided by the kernel is /proc/PID/{clear_refs,smaps}: the user can clear the access bit for all pages mapped to a particular process by writing 1 to clear_refs, wait for some time, and then count smaps:Referenced. However, this method has two serious shortcomings:

- it does not count unmapped file pages
- it affects the reclaimer logic

To overcome these drawbacks, this patch introduces two new page flags, Idle and Young, and a new proc file, /proc/kpageidle. A page's Idle flag can only be set from userspace by setting bit in /proc/kpageidle at the offset corresponding to the page, and it is cleared whenever the page is accessed either through page tables (it is cleared in page_referenced() in this case) or using the read(2) system call (mark_page_accessed()). Thus by setting the Idle flag for pages of a particular workload, which can be found e.g. by reading /proc/PID/pagemap, waiting for some time to let the workload access its working set, and then reading the kpageidle file, one can estimate the amount of pages that are not used by the workload.

The Young page flag is used to avoid interference with the memory reclaimer. A page's Young flag is set whenever the Access bit of a page table entry pointing to the page is cleared by writing to kpageidle. If page_referenced() is called on a Young page, it will add 1 to its return value, therefore concealing the fact that the Access bit was cleared.

Note, since there is no room for extra page flags on 32 bit, this feature uses extended page flags when compiled on 32 bit.

...

...

fs/proc/page.c +10 -0

+		return;
+
+	rmap_walk(page, &rwc);
+
+	if (need_lock)
+		unlock_page(page);
+}
+
+static ssize_t kpageidle_read(struct file *file, char __user *buf,
+			      size_t count, loff_t *ppos)

fs/proc/page.c +40 -0

+			}
+			put_page(page);
+		}
+		if (bit == KPMBITS - 1) {
+			if (put_user(idle_bitmap, out)) {
+				ret = -EFAULT;
+				break;
+			}
+			idle_bitmap = 0;
+			out++;
+		}
+	}
+
+	*ppos += (char __user *)out - buf;
+	if (!ret)
+		ret = (char __user *)out - buf;
+	return ret;
+}
+
+static ssize_t kpageidle_write(struct file *file, const char __user *buf,
+			       size_t count, loff_t *ppos)
+{
+	const u64 __user *in = (const u64 __user *)buf;
+	struct page *page;
+	unsigned long pfn, end_pfn;
+	ssize_t ret = 0;
+	u64 idle_bitmap = 0;
+	int bit;
+
+	if (*ppos & KPMMASK || count & KPMMASK)
+		return -EINVAL;
+
+	pfn = *ppos * BITS_PER_BYTE;
+	if (pfn >= max_pfn)
+		return -ENXIO;
+
+	end_pfn = pfn + count * BITS_PER_BYTE;
+	if (end_pfn > max_pfn)
+		end_pfn = ALIGN(max_pfn, KPMBITS);
+

fs/proc/page.c +7 -0

+			page = kpageidle_get_page(pfn);
+			if (page) {
+				kpageidle_clear_pte_refs(page);
+				set_page_idle(page);
+				put_page(page);
+			}
+		}

Quoted 38 lines · On Sun, 19 Jul 2015 15:31:15 +0300 Vladimir Davydov…

Knowing the portion of memory that is not used by a certain application or memory cgroup (idle memory) can be useful for partitioning the system efficiently, e.g. by setting memory cgroup limits appropriately. Currently, the only means to estimate the amount of idle memory provided by the kernel is /proc/PID/{clear_refs,smaps}: the user can clear the access bit for all pages mapped to a particular process by writing 1 to clear_refs, wait for some time, and then count smaps:Referenced. However, this method has two serious shortcomings:

- it does not count unmapped file pages
- it affects the reclaimer logic

To overcome these drawbacks, this patch introduces two new page flags, Idle and Young, and a new proc file, /proc/kpageidle. A page's Idle flag can only be set from userspace by setting bit in /proc/kpageidle at the offset corresponding to the page, and it is cleared whenever the page is accessed either through page tables (it is cleared in page_referenced() in this case) or using the read(2) system call (mark_page_accessed()). Thus by setting the Idle flag for pages of a particular workload, which can be found e.g. by reading /proc/PID/pagemap, waiting for some time to let the workload access its working set, and then reading the kpageidle file, one can estimate the amount of pages that are not used by the workload.

The Young page flag is used to avoid interference with the memory reclaimer. A page's Young flag is set whenever the Access bit of a page table entry pointing to the page is cleared by writing to kpageidle. If page_referenced() is called on a Young page, it will add 1 to its return value, therefore concealing the fact that the Access bit was cleared.

Note, since there is no room for extra page flags on 32 bit, this feature uses extended page flags when compiled on 32 bit.

...

...

Review

Quoted 6 lines

(quoted diff) +6 -0

+static void kpageidle_clear_pte_refs(struct page *page)
+{
+	struct rmap_walk_control rwc = {
+		.rmap_one = kpageidle_clear_pte_refs_one,
+		.anon_lock = page_lock_anon_vma_read,
+	};

I think this can be static const, since `arg' is unused? That would save some cycles and stack.

Review

Quoted 9 lines

(quoted diff) +8 -0

+	bool need_lock;
+
+	if (!page_mapped(page) ||
+	    !page_rmapping(page))
+		return;
+
+	need_lock = !PageAnon(page) || PageKsm(page);
+	if (need_lock && !trylock_page(page))

Oh. So the feature is a bit unreliable.

I'm not immediately seeing anything which would prevent us from using plain old lock_page() here. What's going on?

Quoted 11 lines

(quoted diff) +10 -0

+		return;
+
+	rmap_walk(page, &rwc);
+
+	if (need_lock)
+		unlock_page(page);
+}
+
+static ssize_t kpageidle_read(struct file *file, char __user *buf,
+			      size_t count, loff_t *ppos)

Review

Quoted 10 lines

(quoted diff) +10 -0

+{
+	u64 __user *out = (u64 __user *)buf;
+	struct page *page;
+	unsigned long pfn, end_pfn;
+	ssize_t ret = 0;
+	u64 idle_bitmap = 0;
+	int bit;
+
+	if (*ppos & KPMMASK || count & KPMMASK)
+		return -EINVAL;

Interface requires 8-byte aligned offset and size.

Review

Quoted 7 lines

(quoted diff) +7 -0

+	pfn = *ppos * BITS_PER_BYTE;
+	if (pfn >= max_pfn)
+		return 0;
+
+	end_pfn = pfn + count * BITS_PER_BYTE;
+	if (end_pfn > max_pfn)
+		end_pfn = ALIGN(max_pfn, KPMBITS);

So we lose up to 63 pages. Presumably max_pfn is well enough aligned for this to not matter, dunno.

Quoted 4 lines

(quoted diff) +3 -0

+	for (; pfn < end_pfn; pfn++) {
+		bit = pfn % KPMBITS;
+		page = kpageidle_get_page(pfn);

Review

Quoted 10 lines

(quoted diff) +10 -0

+		if (page) {
+			if (page_is_idle(page)) {
+				/*
+				 * The page might have been referenced via a
+				 * pte, in which case it is not idle. Clear
+				 * refs and recheck.
+				 */
+				kpageidle_clear_pte_refs(page);
+				if (page_is_idle(page))
+					idle_bitmap |= 1ULL << bit;

I don't understand what's going on here. More details, please?

Quoted 41 lines

(quoted diff) +40 -0

+			}
+			put_page(page);
+		}
+		if (bit == KPMBITS - 1) {
+			if (put_user(idle_bitmap, out)) {
+				ret = -EFAULT;
+				break;
+			}
+			idle_bitmap = 0;
+			out++;
+		}
+	}
+
+	*ppos += (char __user *)out - buf;
+	if (!ret)
+		ret = (char __user *)out - buf;
+	return ret;
+}
+
+static ssize_t kpageidle_write(struct file *file, const char __user *buf,
+			       size_t count, loff_t *ppos)
+{
+	const u64 __user *in = (const u64 __user *)buf;
+	struct page *page;
+	unsigned long pfn, end_pfn;
+	ssize_t ret = 0;
+	u64 idle_bitmap = 0;
+	int bit;
+
+	if (*ppos & KPMMASK || count & KPMMASK)
+		return -EINVAL;
+
+	pfn = *ppos * BITS_PER_BYTE;
+	if (pfn >= max_pfn)
+		return -ENXIO;
+
+	end_pfn = pfn + count * BITS_PER_BYTE;
+	if (end_pfn > max_pfn)
+		end_pfn = ALIGN(max_pfn, KPMBITS);
+

Review

Quoted 10 lines

(quoted diff) +10 -0

+	for (; pfn < end_pfn; pfn++) {
+		bit = pfn % KPMBITS;
+		if (bit == 0) {
+			if (get_user(idle_bitmap, in)) {
+				ret = -EFAULT;
+				break;
+			}
+			in++;
+		}
+		if (idle_bitmap >> bit & 1) {

Hate it when I have to go look up a C precedence table. This is

		if ((idle_bitmap >> bit) & 1) {

Knowing the portion of memory that is not used by a certain application or memory cgroup (idle memory) can be useful for partitioning the system efficiently, e.g. by setting memory cgroup limits appropriately.

Currently, the only means to estimate the amount of idle memory provided by the kernel is /proc/PID/{clear_refs,smaps}: the user can clear the access bit for all pages mapped to a particular process by writing 1 to clear_refs, wait for some time, and then count smaps:Referenced. However, this method has two serious shortcomings:

- it does not count unmapped file pages
- it affects the reclaimer logic

To overcome these drawbacks, this patch introduces two new page flags, Idle and Young, and a new proc file, /proc/kpageidle. A page's Idle flag can only be set from userspace by setting bit in /proc/kpageidle at the offset corresponding to the page, and it is cleared whenever the page is accessed either through page tables (it is cleared in page_referenced() in this case) or using the read(2) system call (mark_page_accessed()). Thus by setting the Idle flag for pages of a particular workload, which can be found e.g. by reading /proc/PID/pagemap, waiting for some time to let the workload access its working set, and then reading the kpageidle file, one can estimate the amount of pages that are not used by the workload.

The Young page flag is used to avoid interference with the memory reclaimer. A page's Young flag is set whenever the Access bit of a page table entry pointing to the page is cleared by writing to kpageidle. If page_referenced() is called on a Young page, it will add 1 to its return value, therefore concealing the fact that the Access bit was cleared.

Note, since there is no room for extra page flags on 32 bit, this feature uses extended page flags when compiled on 32 bit.

Signed-off-by Vladimir Davydov <[email protected]>

---
 Documentation/vm/pagemap.txt |  12 ++-
 fs/proc/page.c               | 218 +++++++++++++++++++++++++++++++++++++++++++
 fs/proc/task_mmu.c           |   4 +-
 include/linux/mm.h           |  98 +++++++++++++++++++
 include/linux/page-flags.h   |  11 +++
 include/linux/page_ext.h     |   4 +
 mm/Kconfig                   |  12 +++
 mm/debug.c                   |   4 +
 mm/huge_memory.c             |  11 ++-
 mm/migrate.c                 |   5 +
 mm/page_ext.c                |   3 +
 mm/rmap.c                    |   5 +
 mm/swap.c                    |   2 +
 13 files changed, 385 insertions(+), 4 deletions(-)

Documentation/vm/pagemap.txt +11 -1

index 3a37ed184258..34fe828c3007 100644
@@ -5,7 +5,7 @@ pagemap is a new (as of 2.6.25) set of interfaces in the kernel that allow
 userspace programs to examine the page tables and related information by
 reading files in /proc.
 
-There are four components to pagemap:
+There are five components to pagemap:
 
  * /proc/pid/pagemap.  This file lets a userspace process find out which
    physical frame each virtual page is mapped to.  It contains one 64-bit

@@ -70,6 +70,16 @@ There are four components to pagemap:
    memory cgroup each page is charged to, indexed by PFN. Only available when
    CONFIG_MEMCG is set.
 
+ * /proc/kpageidle.  This file implements a bitmap where each bit corresponds
+   to a page, indexed by PFN. When the bit is set, the corresponding page is
+   idle. A page is considered idle if it has not been accessed since it was
+   marked idle. To mark a page idle one should set the bit corresponding to the
+   page by writing to the file. A value written to the file is OR-ed with the
+   current bitmap value. Only user memory pages can be marked idle, for other
+   page types input is silently ignored. Writing to this file beyond max PFN
+   results in the ENXIO error. Only available when CONFIG_IDLE_PAGE_TRACKING is
+   set.
+
 Short descriptions to the page flags:
 
  0. LOCKED

fs/proc/page.c +218 -0

index 70d23245dd43..273537885ab4 100644
@@ -5,6 +5,8 @@
 #include <linux/ksm.h>
 #include <linux/mm.h>
 #include <linux/mmzone.h>
+#include <linux/rmap.h>
+#include <linux/mmu_notifier.h>
 #include <linux/huge_mm.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>

@@ -16,6 +18,7 @@
 
 #define KPMSIZE sizeof(u64)
 #define KPMMASK (KPMSIZE - 1)
+#define KPMBITS (KPMSIZE * BITS_PER_BYTE)
 
 /* /proc/kpagecount - an array exposing page counts
  *

@@ -275,6 +278,217 @@ static const struct file_operations proc_kpagecgroup_operations = {
 };
 #endif /* CONFIG_MEMCG */
 
+#ifdef CONFIG_IDLE_PAGE_TRACKING
+/*
+ * Idle page tracking only considers user memory pages, for other types of
+ * pages the idle flag is always unset and an attempt to set it is silently
+ * ignored.
+ *
+ * We treat a page as a user memory page if it is on an LRU list, because it is
+ * always safe to pass such a page to rmap_walk(), which is essential for idle
+ * page tracking. With such an indicator of user pages we can skip isolated
+ * pages, but since there are not usually many of them, it will hardly affect
+ * the overall result.
+ *
+ * This function tries to get a user memory page by pfn as described above.
+ */
+static struct page *kpageidle_get_page(unsigned long pfn)
+{
+       struct page *page;
+       struct zone *zone;
+
+       if (!pfn_valid(pfn))
+               return NULL;
+
+       page = pfn_to_page(pfn);
+       if (!page || !PageLRU(page) ||
+           !get_page_unless_zero(page))
+               return NULL;
+
+       zone = page_zone(page);
+       spin_lock_irq(&zone->lru_lock);
+       if (unlikely(!PageLRU(page))) {
+               put_page(page);
+               page = NULL;
+       }
+       spin_unlock_irq(&zone->lru_lock);
+       return page;
+}
+
+static int kpageidle_clear_pte_refs_one(struct page *page,
+                                       struct vm_area_struct *vma,
+                                       unsigned long addr, void *arg)
+{
+       struct mm_struct *mm = vma->vm_mm;
+       spinlock_t *ptl;
+       pmd_t *pmd;
+       pte_t *pte;
+       bool referenced = false;
+
+       if (unlikely(PageTransHuge(page))) {
+               pmd = page_check_address_pmd(page, mm, addr,
+                                            PAGE_CHECK_ADDRESS_PMD_FLAG, &ptl);
+               if (pmd) {
+                       referenced = pmdp_clear_young_notify(vma, addr, pmd);
+                       spin_unlock(ptl);
+               }
+       } else {
+               pte = page_check_address(page, mm, addr, &ptl, 0);
+               if (pte) {
+                       referenced = ptep_clear_young_notify(vma, addr, pte);
+                       pte_unmap_unlock(pte, ptl);
+               }
+       }
+       if (referenced) {
+               clear_page_idle(page);
+               /*
+                * We cleared the referenced bit in a mapping to this page. To
+                * avoid interference with page reclaim, mark it young so that
+                * page_referenced() will return > 0.
+                */
+               set_page_young(page);
+       }
+       return SWAP_AGAIN;
+}
+
+static void kpageidle_clear_pte_refs(struct page *page)
+{
+       struct rmap_walk_control rwc = {
+               .rmap_one = kpageidle_clear_pte_refs_one,
+               .anon_lock = page_lock_anon_vma_read,
+       };
+       bool need_lock;
+
+       if (!page_mapped(page) ||
+           !page_rmapping(page))
+               return;
+
+       need_lock = !PageAnon(page) || PageKsm(page);
+       if (need_lock && !trylock_page(page))
+               return;
+
+       rmap_walk(page, &rwc);
+
+       if (need_lock)
+               unlock_page(page);
+}
+
+static ssize_t kpageidle_read(struct file *file, char __user *buf,
+                             size_t count, loff_t *ppos)
+{
+       u64 __user *out = (u64 __user *)buf;
+       struct page *page;
+       unsigned long pfn, end_pfn;
+       ssize_t ret = 0;
+       u64 idle_bitmap = 0;
+       int bit;
+
+       if (*ppos & KPMMASK || count & KPMMASK)
+               return -EINVAL;
+
+       pfn = *ppos * BITS_PER_BYTE;
+       if (pfn >= max_pfn)
+               return 0;
+
+       end_pfn = pfn + count * BITS_PER_BYTE;
+       if (end_pfn > max_pfn)
+               end_pfn = ALIGN(max_pfn, KPMBITS);
+
+       for (; pfn < end_pfn; pfn++) {
+               bit = pfn % KPMBITS;
+               page = kpageidle_get_page(pfn);
+               if (page) {
+                       if (page_is_idle(page)) {
+                               /*
+                                * The page might have been referenced via a
+                                * pte, in which case it is not idle. Clear
+                                * refs and recheck.
+                                */
+                               kpageidle_clear_pte_refs(page);
+                               if (page_is_idle(page))
+                                       idle_bitmap |= 1ULL << bit;
+                       }
+                       put_page(page);
+               }
+               if (bit == KPMBITS - 1) {
+                       if (put_user(idle_bitmap, out)) {
+                               ret = -EFAULT;
+                               break;
+                       }
+                       idle_bitmap = 0;
+                       out++;
+               }
+       }
+
+       *ppos += (char __user *)out - buf;
+       if (!ret)
+               ret = (char __user *)out - buf;
+       return ret;
+}
+
+static ssize_t kpageidle_write(struct file *file, const char __user *buf,
+                              size_t count, loff_t *ppos)
+{
+       const u64 __user *in = (const u64 __user *)buf;
+       struct page *page;
+       unsigned long pfn, end_pfn;
+       ssize_t ret = 0;
+       u64 idle_bitmap = 0;
+       int bit;
+
+       if (*ppos & KPMMASK || count & KPMMASK)
+               return -EINVAL;
+
+       pfn = *ppos * BITS_PER_BYTE;
+       if (pfn >= max_pfn)
+               return -ENXIO;
+
+       end_pfn = pfn + count * BITS_PER_BYTE;
+       if (end_pfn > max_pfn)
+               end_pfn = ALIGN(max_pfn, KPMBITS);
+
+       for (; pfn < end_pfn; pfn++) {
+               bit = pfn % KPMBITS;
+               if (bit == 0) {
+                       if (get_user(idle_bitmap, in)) {
+                               ret = -EFAULT;
+                               break;
+                       }
+                       in++;
+               }
+               if (idle_bitmap >> bit & 1) {
+                       page = kpageidle_get_page(pfn);
+                       if (page) {
+                               kpageidle_clear_pte_refs(page);
+                               set_page_idle(page);
+                               put_page(page);
+                       }
+               }
+       }
+
+       *ppos += (const char __user *)in - buf;
+       if (!ret)
+               ret = (const char __user *)in - buf;
+       return ret;
+}
+
+static const struct file_operations proc_kpageidle_operations = {
+       .llseek = mem_lseek,
+       .read = kpageidle_read,
+       .write = kpageidle_write,
+};
+
+#ifndef CONFIG_64BIT
+static bool need_page_idle(void)
+{
+       return true;
+}
+struct page_ext_operations page_idle_ops = {
+       .need = need_page_idle,
+};
+#endif
+#endif /* CONFIG_IDLE_PAGE_TRACKING */
+
 static int __init proc_page_init(void)
 {
        proc_create("kpagecount", S_IRUSR, NULL, &proc_kpagecount_operations);

@@ -282,6 +496,10 @@ static int __init proc_page_init(void)
 #ifdef CONFIG_MEMCG
        proc_create("kpagecgroup", S_IRUSR, NULL, &proc_kpagecgroup_operations);
 #endif
+#ifdef CONFIG_IDLE_PAGE_TRACKING
+       proc_create("kpageidle", S_IRUSR | S_IWUSR, NULL,
+                   &proc_kpageidle_operations);
+#endif
        return 0;
 }
 fs_initcall(proc_page_init);

fs/proc/task_mmu.c +3 -1

index 860bb0f30f14..7c9a17414106 100644
@@ -459,7 +459,7 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page,
 
        mss->resident += size;
        /* Accumulate the size in pages that have been accessed. */
-       if (young || PageReferenced(page))
+       if (young || page_is_young(page) || PageReferenced(page))
                mss->referenced += size;
        mapcount = page_mapcount(page);
        if (mapcount >= 2) {

@@ -808,6 +808,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
 
                /* Clear accessed and referenced bits. */
                pmdp_test_and_clear_young(vma, addr, pmd);
+               test_and_clear_page_young(page);
                ClearPageReferenced(page);
 out:
                spin_unlock(ptl);

@@ -835,6 +836,7 @@ out:
 
                /* Clear accessed and referenced bits. */
                ptep_test_and_clear_young(vma, addr, pte);
+               test_and_clear_page_young(page);
                ClearPageReferenced(page);
        }
        pte_unmap_unlock(pte - 1, ptl);

include/linux/mm.h +98 -0

index c3a2b37365f6..0e62be7d5138 100644
@@ -2202,5 +2202,103 @@ void __init setup_nr_node_ids(void);
 static inline void setup_nr_node_ids(void) {}
 #endif
 
+#ifdef CONFIG_IDLE_PAGE_TRACKING
+#ifdef CONFIG_64BIT
+static inline bool page_is_young(struct page *page)
+{
+       return PageYoung(page);
+}
+
+static inline void set_page_young(struct page *page)
+{
+       SetPageYoung(page);
+}
+
+static inline bool test_and_clear_page_young(struct page *page)
+{
+       return TestClearPageYoung(page);
+}
+
+static inline bool page_is_idle(struct page *page)
+{
+       return PageIdle(page);
+}
+
+static inline void set_page_idle(struct page *page)
+{
+       SetPageIdle(page);
+}
+
+static inline void clear_page_idle(struct page *page)
+{
+       ClearPageIdle(page);
+}
+#else /* !CONFIG_64BIT */
+/*
+ * If there is not enough space to store Idle and Young bits in page flags, use
+ * page ext flags instead.
+ */
+extern struct page_ext_operations page_idle_ops;
+
+static inline bool page_is_young(struct page *page)
+{
+       return test_bit(PAGE_EXT_YOUNG, &lookup_page_ext(page)->flags);
+}
+
+static inline void set_page_young(struct page *page)
+{
+       set_bit(PAGE_EXT_YOUNG, &lookup_page_ext(page)->flags);
+}
+
+static inline bool test_and_clear_page_young(struct page *page)
+{
+       return test_and_clear_bit(PAGE_EXT_YOUNG,
+                                 &lookup_page_ext(page)->flags);
+}
+
+static inline bool page_is_idle(struct page *page)
+{
+       return test_bit(PAGE_EXT_IDLE, &lookup_page_ext(page)->flags);
+}
+
+static inline void set_page_idle(struct page *page)
+{
+       set_bit(PAGE_EXT_IDLE, &lookup_page_ext(page)->flags);
+}
+
+static inline void clear_page_idle(struct page *page)
+{
+       clear_bit(PAGE_EXT_IDLE, &lookup_page_ext(page)->flags);
+}
+#endif /* CONFIG_64BIT */
+#else /* !CONFIG_IDLE_PAGE_TRACKING */
+static inline bool page_is_young(struct page *page)
+{
+       return false;
+}
+
+static inline void set_page_young(struct page *page)
+{
+}
+
+static inline bool test_and_clear_page_young(struct page *page)
+{
+       return false;
+}
+
+static inline bool page_is_idle(struct page *page)
+{
+       return false;
+}
+
+static inline void set_page_idle(struct page *page)
+{
+}
+
+static inline void clear_page_idle(struct page *page)
+{
+}
+#endif /* CONFIG_IDLE_PAGE_TRACKING */
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_H */

include/linux/page-flags.h +11 -0

index 91b7f9b2b774..478f2241f284 100644
@@ -109,6 +109,10 @@ enum pageflags {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
        PG_compound_lock,
 #endif
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
+       PG_young,
+       PG_idle,
+#endif
        __NR_PAGEFLAGS,
 
        /* Filesystems */

@@ -363,6 +367,13 @@ PAGEFLAG_FALSE(HWPoison)
 #define __PG_HWPOISON 0
 #endif
 
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
+TESTPAGEFLAG(Young, young, PF_ANY)
+SETPAGEFLAG(Young, young, PF_ANY)
+TESTCLEARFLAG(Young, young, PF_ANY)
+PAGEFLAG(Idle, idle, PF_ANY)
+#endif
+
 /*
  * On an anonymous page mapped into a user virtual memory area,
  * page->mapping points to its anon_vma, not to a struct address_space;

include/linux/page_ext.h +4 -0

index c42981cd99aa..17f118a82854 100644
@@ -26,6 +26,10 @@ enum page_ext_flags {
        PAGE_EXT_DEBUG_POISON,          /* Page is poisoned */
        PAGE_EXT_DEBUG_GUARD,
        PAGE_EXT_OWNER,
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT)
+       PAGE_EXT_YOUNG,
+       PAGE_EXT_IDLE,
+#endif
 };
 
 /*

mm/Kconfig +12 -0

index e79de2bd12cd..db817e2c2ec8 100644
@@ -654,3 +654,15 @@ config DEFERRED_STRUCT_PAGE_INIT
          when kswapd starts. This has a potential performance impact on
          processes running early in the lifetime of the systemm until kswapd
          finishes the initialisation.
+
+config IDLE_PAGE_TRACKING
+       bool "Enable idle page tracking"
+       select PROC_PAGE_MONITOR
+       select PAGE_EXTENSION if !64BIT
+       help
+         This feature allows to estimate the amount of user pages that have
+         not been touched during a given period of time. This information can
+         be useful to tune memory cgroup limits and/or for job placement
+         within a compute cluster.
+
+         See Documentation/vm/pagemap.txt for more details.

mm/debug.c +4 -0

index 76089ddf99ea..6c1b3ea61bfd 100644
@@ -48,6 +48,10 @@ static const struct trace_print_flags pageflag_names[] = {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
        {1UL << PG_compound_lock,       "compound_lock" },
 #endif
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
+       {1UL << PG_young,               "young"         },
+       {1UL << PG_idle,                "idle"          },
+#endif
 };
 
 static void dump_flags(unsigned long flags,

mm/huge_memory.c +9 -2

index 8f9a334a6c66..5ab46adca104 100644
@@ -1806,6 +1806,11 @@ static void __split_huge_page_refcount(struct page *page,
                /* clear PageTail before overwriting first_page */
                smp_wmb();
 
+               if (page_is_young(page))
+                       set_page_young(page_tail);
+               if (page_is_idle(page))
+                       set_page_idle(page_tail);
+
                /*
                 * __split_huge_page_splitting() already set the
                 * splitting bit in all pmd that could map this

@@ -2311,7 +2316,8 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
                VM_BUG_ON_PAGE(PageLRU(page), page);
 
                /* If there is no mapped pte young don't collapse the page */
-               if (pte_young(pteval) || PageReferenced(page) ||
+               if (pte_young(pteval) ||
+                   page_is_young(page) || PageReferenced(page) ||
                    mmu_notifier_test_young(vma->vm_mm, address))
                        referenced = true;
        }

@@ -2738,7 +2744,8 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
                 */
                if (page_count(page) != 1 + !!PageSwapCache(page))
                        goto out_unmap;
-               if (pte_young(pteval) || PageReferenced(page) ||
+               if (pte_young(pteval) ||
+                   page_is_young(page) || PageReferenced(page) ||
                    mmu_notifier_test_young(vma->vm_mm, address))
                        referenced = true;
        }

mm/migrate.c +5 -0

index d3529d620a5b..d86cec005aa6 100644
@@ -524,6 +524,11 @@ void migrate_page_copy(struct page *newpage, struct page *page)
                        __set_page_dirty_nobuffers(newpage);
        }
 
+       if (page_is_young(page))
+               set_page_young(newpage);
+       if (page_is_idle(page))
+               set_page_idle(newpage);
+
        /*
         * Copy NUMA information to the new page, to prevent over-eager
         * future migrations of this same page.

mm/page_ext.c +3 -0

index d86fd2f5353f..e4b3af054bf2 100644
@@ -59,6 +59,9 @@ static struct page_ext_operations *page_ext_ops[] = {
 #ifdef CONFIG_PAGE_OWNER
        &page_owner_ops,
 #endif
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT)
+       &page_idle_ops,
+#endif
 };
 
 static unsigned long total_usage;

mm/rmap.c +5 -0

index 30812e9042ae..9e411aa03176 100644
@@ -900,6 +900,11 @@ static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
                pte_unmap_unlock(pte, ptl);
        }
 
+       if (referenced)
+               clear_page_idle(page);
+       if (test_and_clear_page_young(page))
+               referenced++;
+
        if (referenced) {
                pra->referenced++;
                pra->vm_flags |= vma->vm_flags;

mm/swap.c +2 -1

index d398860badd1..04b6ce51bcf0 100644
@@ -623,6 +623,8 @@ void mark_page_accessed(struct page *page)
        } else if (!PageReferenced(page)) {
                SetPageReferenced(page);
        }
+       if (page_is_idle(page))
+               clear_page_idle(page);
 }
 EXPORT_SYMBOL(mark_page_accessed);
 
--

2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

fs/proc/page.c:341:4: error: implicit declaration of function 'pmdp_clear_young_notify' [-Werror=implicit-function-declaration] fs/proc/page.c:347:4: error: implicit declaration of function 'ptep_clear_young_notify' [-Werror=implicit-function-declaration]
cc1: some warnings being treated as errors
make[3]: *** [fs/proc/page.o] Error 1
make[2]: *** [fs/proc] Error 2

As noted by Minchan, a benefit of reading idle flag from /proc/kpageflags is that one can easily filter dirty and/or unevictable pages while estimating the size of unused memory.

Note that idle flag read from /proc/kpageflags may be stale in case the page was accessed via a PTE, because it would be too costly to iterate over all page mappings on each /proc/kpageflags read to provide an up-to-date value. To make sure the flag is up-to-date one has to read /proc/kpageidle first.

Quoted 4 lines · On Sun, 19 Jul 2015 15:31:16 +0300 Vladimir Davydov…

As noted by Minchan, a benefit of reading idle flag from /proc/kpageflags is that one can easily filter dirty and/or unevictable pages while estimating the size of unused memory.

Quoted 5 lines

Note that idle flag read from /proc/kpageflags may be stale in case the page was accessed via a PTE, because it would be too costly to iterate over all page mappings on each /proc/kpageflags read to provide an up-to-date value. To make sure the flag is up-to-date one has to read /proc/kpageidle first.

Is there any value in teaching the regular old page scanner to update these flags? If it's doing an rmap scan anyway...

Quoted 12 lines · Andrew Morton

Quoted 9 lines · On Sun, 19 Jul 2015 15:31:16 +0300 Vladimir Davydov…

As noted by Minchan, a benefit of reading idle flag from /proc/kpageflags is that one can easily filter dirty and/or unevictable pages while estimating the size of unused memory.

Note that idle flag read from /proc/kpageflags may be stale in case the page was accessed via a PTE, because it would be too costly to iterate over all page mappings on each /proc/kpageflags read to provide an up-to-date value. To make sure the flag is up-to-date one has to read /proc/kpageidle first.

Quoted 2 lines

Is there any value in teaching the regular old page scanner to update these flags? If it's doing an rmap scan anyway...

I don't understand what you mean by "regular old page scanner". Could you please elaborate?

Quoted 2 lines · On Wed, 22 Jul 2015 19:25:28 +0300 Vladimir Davydov < [email protected]>…

Quoted 1 line · Andrew Morton

On Sun, 19 Jul 2015 15:31:16 +0300 Vladimir Davydov <

Quoted 3 lines · [email protected]> wrote:

Quoted 3 lines

Quoted 3 lines

As noted by Minchan, a benefit of reading idle flag from /proc/kpageflags is that one can easily filter dirty and/or

unevictable

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

pages while estimating the size of unused memory.

Note that idle flag read from /proc/kpageflags may be stale in case

the

Quoted 1 line

Quoted 1 line

Quoted 1 line

page was accessed via a PTE, because it would be too costly to

iterate

Quoted 9 lines

Quoted 6 lines

Quoted 3 lines

over all page mappings on each /proc/kpageflags read to provide an up-to-date value. To make sure the flag is up-to-date one has to read /proc/kpageidle first.

Is there any value in teaching the regular old page scanner to update these flags? If it's doing an rmap scan anyway...

I don't understand what you mean by "regular old page scanner". Could you please elaborate?

Whenever kswapd or direct reclaim perform an rmap scan, take that as an opportunity to also update PageIdle().

---- PERFORMANCE EVALUATION ----

SPECjvm2008 (https://www.spec.org/jvm2008/) was used to evaluate the performance impact introduced by this patch set. Three runs were carried out:

- base: kernel without the patch
- patched: patched kernel, the feature is not used
- patched-active: patched kernel, 1 minute-period daemon is used for tracking idle memory

For tracking idle memory, idlememstat utility was used:
https://github.com/locker/idlememstat

testcase base patched patched-active

compiler       537.40 ( 0.00)%   532.26 (-0.96)%   538.31 ( 0.17)%
compress       305.47 ( 0.00)%   301.08 (-1.44)%   300.71 (-1.56)%
crypto         284.32 ( 0.00)%   282.21 (-0.74)%   284.87 ( 0.19)%
derby          411.05 ( 0.00)%   413.44 ( 0.58)%   412.07 ( 0.25)%
mpegaudio      189.96 ( 0.00)%   190.87 ( 0.48)%   189.42 (-0.28)%
scimark.large   46.85 ( 0.00)%    46.41 (-0.94)%    47.83 ( 2.09)%
scimark.small  412.91 ( 0.00)%   415.41 ( 0.61)%   421.17 ( 2.00)%
serial         204.23 ( 0.00)%   213.46 ( 4.52)%   203.17 (-0.52)%
startup         36.76 ( 0.00)%    35.49 (-3.45)%    35.64 (-3.05)%
sunflow        115.34 ( 0.00)%   115.08 (-0.23)%   117.37 ( 1.76)%
xml            620.55 ( 0.00)%   619.95 (-0.10)%   620.39 (-0.03)%

composite 211.50 ( 0.00)% 211.15 (-0.17)% 211.67 ( 0.08)%

time idlememstat:

17.20user 65.16system 2:15:23elapsed 1%CPU (0avgtext+0avgdata 8476maxresident)k 448inputs+40outputs (1major+36052minor)pagefaults 0swaps

Hi,

This patch set introduces a new user API for tracking user memory pages that have not been used for a given period of time. The purpose of this is to provide the userspace with the means of tracking a workload's working set, i.e. the set of pages that are actively used by the workload. Knowing the working set size can be useful for partitioning the system more efficiently, e.g. by tuning memory cgroup limits appropriately, or for job placement within a compute cluster.

It is based on top of v4.2-rc2-mmotm-2015-07-15-16-46
It applies without conflicts to v4.2-rc2-mmotm-2015-07-17-16-04 as well

---- USE CASES ----

The unified cgroup hierarchy has memory.low and memory.high knobs, which are defined as the low and high boundaries for the workload working set size. However, the working set size of a workload may be unknown or change in time. With this patch set, one can periodically estimate the amount of memory unused by each cgroup and tune their memory.low and memory.high parameters accordingly, therefore optimizing the overall memory utilization.

Another use case is balancing workloads within a compute cluster. Knowing how much memory is not really used by a workload unit may help take a more optimal decision when considering migrating the unit to another node within the cluster.

Also, as noted by Minchan, this would be useful for per-process reclaim
(https://lwn.net/Articles/545668/). With idle tracking, we could reclaim idle
pages only by smart user memory manager.

---- USER API ----

The user API consists of two new proc files:

* /proc/kpageidle. This file implements a bitmap where each bit
corresponds
to a page, indexed by PFN. When the bit is set, the corresponding page
is
idle. A page is considered idle if it has not been accessed since it was marked idle. To mark a page idle one should set the bit corresponding
to the
page by writing to the file. A value written to the file is OR-ed with
the
current bitmap value. Only user memory pages can be marked idle, for
other
page types input is silently ignored. Writing to this file beyond max
PFN
results in the ENXIO error. Only available when
CONFIG_IDLE_PAGE_TRACKING is
set.

This file can be used to estimate the amount of pages that are not used by a particular workload as follows:

1. mark all pages of interest idle by setting corresponding bits in the /proc/kpageidle bitmap
2. wait until the workload accesses its working set
3. read /proc/kpageidle and count the number of bits set

* /proc/kpagecgroup. This file contains a 64-bit inode number of the memory cgroup each page is charged to, indexed by PFN. Only available
when
CONFIG_MEMCG is set.

This file can be used to find all pages (including unmapped file pages) accounted to a particular cgroup. Using /proc/kpageidle, one can then estimate the cgroup working set size.

For an example of using these files for estimating the amount of unused memory pages per each memory cgroup, please see the script attached below.

---- REASONING ----

The reason to introduce the new user API instead of using /proc/PID/{clear_refs,smaps} is that the latter has two serious drawbacks:

- it does not count unmapped file pages
- it affects the reclaimer logic

The new API attempts to overcome them both. For more details on how it is achieved, please see the comment to patch 6.

---- CHANGE LOG ----

Changes in v9:

- add cond_resched to /proc/kpage* read/write loop (Andres)
- rebase on top of v4.2-rc2-mmotm-2015-07-15-16-46

Changes in v8:

- clear referenced/accessed bit in secondary ptes while accessing /proc/kpageidle; this is required to estimate wss of KVM VMs (Andres)
- check the young flag when collapsing a huge page
- copy idle/young flags on page migration

Changes in v7:

This iteration addresses Andres's comments to v6:

- do not reuse page_referenced for clearing idle flag, introduce a separate function instead; this way we won't issue expensive tlb flushes on /proc/kpageidle read/write
- propagate young/idle flags from head to tail pages on thp split
- skip compound tail pages while reading/writing /proc/kpageidle
- cleanup page_referenced_one

Changes in v6:

- Split the patch introducing page_cgroup_ino helper to ease review.
- Rebase on top of v4.1-rc7-mmotm-2015-06-09-16-55

Changes in v5:

- Fix possible race between kpageidle_clear_pte_refs() and __page_set_anon_rmap() by checking that a page is on an LRU list under zone->lru_lock (Minchan).
- Export idle flag via /proc/kpageflags (Minchan).
- Rebase on top of 4.1-rc3.

Changes in v4:

This iteration primarily addresses Minchan's comments to v3:

- Implement /proc/kpageidle as a bitmap instead of using u64 per each
page,
because there does not seem to be any future uses for the other 63 bits.
- Do not double-increase pra->referenced in page_referenced_one() if the
page
was young and referenced recently.
- Remove the pointless (page_count == 0) check from kpageidle_get_page().
- Rename kpageidle_clear_refs() to kpageidle_clear_pte_refs().
- Improve comments to kpageidle-related functions.
- Rebase on top of 4.1-rc2.

Note it does not address Minchan's concern of possible __page_set_anon_rmap vs
page_referenced race (see https://lkml.org/lkml/2015/5/3/220) since it is still
unclear if this race can really happen (see
https://lkml.org/lkml/2015/5/4/160)

Changes in v3:

- Enable CONFIG_IDLE_PAGE_TRACKING for 32 bit. Since this feature requires two extra page flags and there is no space for them on 32 bit, page ext is used (thanks to Minchan Kim).
- Minor code cleanups and comments improved.
- Rebase on top of 4.1-rc1.

Changes in v2:

- The main difference from v1 is the API change. In v1 the user can only set the idle flag for all pages at once, and for clearing the Idle flag on pages accessed via page tables /proc/PID/clear_refs should be used.
The main drawback of the v1 approach, as noted by Minchan, is that on big machines setting the idle flag for each pages can result in CPU bursts, which would be especially frustrating if the user only wanted to estimate the amount of idle pages for a particular process or VMA. With the new API a more fine-grained approach is possible: one can read a process's /proc/PID/pagemap and set/check the Idle flag only for those pages of the process's address space he or she is interested in.
Another good point about the v2 API is that it is possible to limit /proc/kpage* scanning rate when the user wants to estimate the total number of idle pages, which is unachievable with the v1 approach.
- Make /proc/kpagecgroup return the ino of the closest online ancestor in case the cgroup a page is charged to is offline.
- Fix /proc/PID/clear_refs not clearing Young page flag.
- Rebase on top of v4.0-rc6-mmotm-2015-04-01-14-54

v8: https://lkml.org/lkml/2015/7/15/587
v7: https://lkml.org/lkml/2015/7/11/119
v6: https://lkml.org/lkml/2015/6/12/301
v5: https://lkml.org/lkml/2015/5/12/449
v4: https://lkml.org/lkml/2015/5/7/580
v3: https://lkml.org/lkml/2015/4/28/224
v2: https://lkml.org/lkml/2015/4/7/260
v1: https://lkml.org/lkml/2015/3/18/794

---- PATCH SET STRUCTURE ----

The patch set is organized as follows:

- patch 1 adds page_cgroup_ino() helper for the sake of /proc/kpagecgroup and patches 2-3 do related cleanup
- patch 4 adds /proc/kpagecgroup, which reports cgroup ino each page is charged to
- patch 5 introduces a new mmu notifier callback, clear_young, which is a lightweight version of clear_flush_young; it is used in patch 6
- patch 6 implements the idle page tracking feature, including the userspace API, /proc/kpageidle
- patch 7 exports idle flag via /proc/kpageflags

---- SIMILAR WORKS ----

Originally, the patch for tracking idle memory was proposed back in 2011 by Michel Lespinasse (see http://lwn.net/Articles/459269/). The main difference between Michel's patch and this one is that Michel implemented a kernel space daemon for estimating idle memory size per cgroup while this patch only provides the userspace with the minimal API for doing the job, leaving the rest up to the userspace. However, they both share the same idea of Idle/Young page flags to avoid affecting the reclaimer logic.

---- PERFORMANCE EVALUATION ----

SPECjvm2008 (https://www.spec.org/jvm2008/) was used to evaluate the performance impact introduced by this patch set. Three runs were carried out:

- base: kernel without the patch
- patched: patched kernel, the feature is not used
- patched-active: patched kernel, 1 minute-period daemon is used for tracking idle memory

For tracking idle memory, idlememstat utility was used:
https://github.com/locker/idlememstat

testcase base patched patched-active

compiler       537.40 ( 0.00)%   532.26 (-0.96)%   538.31 ( 0.17)%
compress       305.47 ( 0.00)%   301.08 (-1.44)%   300.71 (-1.56)%
crypto         284.32 ( 0.00)%   282.21 (-0.74)%   284.87 ( 0.19)%
derby          411.05 ( 0.00)%   413.44 ( 0.58)%   412.07 ( 0.25)%
mpegaudio      189.96 ( 0.00)%   190.87 ( 0.48)%   189.42 (-0.28)%
scimark.large   46.85 ( 0.00)%    46.41 (-0.94)%    47.83 ( 2.09)%
scimark.small  412.91 ( 0.00)%   415.41 ( 0.61)%   421.17 ( 2.00)%
serial         204.23 ( 0.00)%   213.46 ( 4.52)%   203.17 (-0.52)%
startup         36.76 ( 0.00)%    35.49 (-3.45)%    35.64 (-3.05)%
sunflow        115.34 ( 0.00)%   115.08 (-0.23)%   117.37 ( 1.76)%
xml            620.55 ( 0.00)%   619.95 (-0.10)%   620.39 (-0.03)%

composite 211.50 ( 0.00)% 211.15 (-0.17)% 211.67 ( 0.08)%

time idlememstat:

17.20user 65.16system 2:15:23elapsed 1%CPU (0avgtext+0avgdata 8476maxresident)k
448inputs+40outputs (1major+36052minor)pagefaults 0swaps

---- SCRIPT FOR COUNTING IDLE PAGES PER CGROUP ---- #! /usr/bin/python
#

import os
import stat
import errno
import struct

CGROUP_MOUNT = "/sys/fs/cgroup/memory"
BUFSIZE = 8 * 1024 # must be multiple of 8

def get_hugepage_size():

    with open("/proc/meminfo", "r") as f:
        for s in f:
            k, v = s.split(":")
            if k == "Hugepagesize":
                return int(v.split()[0]) * 1024

PAGE_SIZE = os.sysconf("SC_PAGE_SIZE")
HUGEPAGE_SIZE = get_hugepage_size()

def set_idle():

    f = open("/proc/kpageidle", "wb", BUFSIZE)
    while True:
        try:
            f.write(struct.pack("Q", pow(2, 64) - 1))
        except IOError as err:
            if err.errno == errno.ENXIO:
                break
            raise
    f.close()

def count_idle():

    f_flags = open("/proc/kpageflags", "rb", BUFSIZE)
    f_cgroup = open("/proc/kpagecgroup", "rb", BUFSIZE)

    with open("/proc/kpageidle", "rb", BUFSIZE) as f:
        while f.read(BUFSIZE): pass  # update idle flag

    idlememsz = {}
    while True:
        s1, s2 = f_flags.read(8), f_cgroup.read(8)
        if not s1 or not s2:
            break

        flags, = struct.unpack('Q', s1)
        cgino, = struct.unpack('Q', s2)

        unevictable = (flags >> 18) & 1
        huge = (flags >> 22) & 1
        idle = (flags >> 25) & 1

        if idle and not unevictable:
            idlememsz[cgino] = idlememsz.get(cgino, 0) + \
                (HUGEPAGE_SIZE if huge else PAGE_SIZE)

    f_flags.close()
    f_cgroup.close()
    return idlememsz

if __name__ == "__main__":
print "Setting the idle flag for each page..." set_idle()

raw_input("Wait until the workload accesses its working set, "
"then press Enter")

print "Counting idle pages..."
idlememsz = count_idle()

for dir, subdirs, files in os.walk(CGROUP_MOUNT):
ino = os.stat(dir)[stat.ST_INO]
print dir + ": " + str(idlememsz.get(ino, 0) / 1024) + " kB"
---- END SCRIPT ----

Comments are more than welcome.

Thanks,

Vladimir Davydov (8):
memcg: add page_cgroup_ino helper
hwpoison: use page_cgroup_ino for filtering by memcg
memcg: zap try_get_mem_cgroup_from_page
proc: add kpagecgroup file
mmu-notifier: add clear_young callback
proc: add kpageidle file
proc: export idle flag via kpageflags
proc: add cond_resched to /proc/kpage* read/write loop

 Documentation/vm/pagemap.txt           |  22 ++-
 fs/proc/page.c                         | 282
+++++++++++++++++++++++++++++++++
 fs/proc/task_mmu.c                     |   4 +-
 include/linux/memcontrol.h             |  10 +-
 include/linux/mm.h                     |  98 ++++++++++++
 include/linux/mmu_notifier.h           |  44 +++++
 include/linux/page-flags.h             |  11 ++
 include/linux/page_ext.h               |   4 +
 include/uapi/linux/kernel-page-flags.h |   1 +
 mm/Kconfig                             |  12 ++
 mm/debug.c                             |   4 +
 mm/huge_memory.c                       |  11 +-
 mm/hwpoison-inject.c                   |   5 +-
 mm/memcontrol.c                        |  71 ++++-----
 mm/memory-failure.c                    |  16 +-
 mm/migrate.c                           |   5 +
 mm/mmu_notifier.c                      |  17 ++
 mm/page_ext.c                          |   3 +
 mm/rmap.c                              |   5 +
 mm/swap.c                              |   2 +
 virt/kvm/kvm_main.c                    |  18 +++
 21 files changed, 579 insertions(+), 66 deletions(-)

--
2.1.4

Hi,

This patch set introduces a new user API for tracking user memory pages that have not been used for a given period of time. The purpose of this is to provide the userspace with the means of tracking a workload's working set, i.e. the set of pages that are actively used by the workload. Knowing the working set size can be useful for partitioning the system more efficiently, e.g. by tuning memory cgroup limits appropriately, or for job placement within a compute cluster.

It is based on top of v4.2-rc2-mmotm-2015-07-15-16-46
It applies without conflicts to v4.2-rc2-mmotm-2015-07-17-16-04 as well

---- USE CASES ----

The unified cgroup hierarchy has memory.low and memory.high knobs, which are defined as the low and high boundaries for the workload working set size. However, the working set size of a workload may be unknown or change in time. With this patch set, one can periodically estimate the amount of memory unused by each cgroup and tune their memory.low and memory.high parameters accordingly, therefore optimizing the overall memory utilization.

Another use case is balancing workloads within a compute cluster. Knowing how much memory is not really used by a workload unit may help take a more optimal decision when considering migrating the unit to another node within the cluster.

Also, as noted by Minchan, this would be useful for per-process reclaim
(https://lwn.net/Articles/545668/). With idle tracking, we could reclaim idle pages only by smart user memory manager.

---- USER API ----

The user API consists of two new proc files:

* /proc/kpageidle. This file implements a bitmap where each bit corresponds to a page, indexed by PFN.

When the bit is set, the corresponding page is
idle. A page is considered idle if it has not been accessed since it was marked idle.

To mark a page idle one should set the bit corresponding to the
page by writing to the file. A value written to the file is OR-ed with the current bitmap value. Only user memory pages can be marked idle, for other page types input is silently ignored. Writing to this file beyond max PFN results in the ENXIO error. Only available when CONFIG_IDLE_PAGE_TRACKING is set.

This file can be used to estimate the amount of pages that are not used by a particular workload as follows:

1. mark all pages of interest idle by setting corresponding bits in the /proc/kpageidle bitmap
2. wait until the workload accesses its working set
3. read /proc/kpageidle and count the number of bits set

* /proc/kpagecgroup. This file contains a 64-bit inode number of the memory cgroup each page is charged to, indexed by PFN.

Only available when CONFIG_MEMCG is set.

This file can be used to find all pages (including unmapped file pages) accounted to a particular cgroup. Using /proc/kpageidle, one can then estimate the cgroup working set size.

For an example of using these files for estimating the amount of unused memory pages per each memory cgroup, please see the script attached below.

---- PERFORMANCE EVALUATION ----

Documentation/vm/pagemap.txt | 22 ++-

Quoted 34 lines · On Sun, 19 Jul 2015 15:31:09 +0300 Vladimir Davydov <[email protected]>…

Hi,

This patch set introduces a new user API for tracking user memory pages that have not been used for a given period of time. The purpose of this is to provide the userspace with the means of tracking a workload's working set, i.e. the set of pages that are actively used by the workload. Knowing the working set size can be useful for partitioning the system more efficiently, e.g. by tuning memory cgroup limits appropriately, or for job placement within a compute cluster.

It is based on top of v4.2-rc2-mmotm-2015-07-15-16-46
It applies without conflicts to v4.2-rc2-mmotm-2015-07-17-16-04 as well

---- USE CASES ----

The unified cgroup hierarchy has memory.low and memory.high knobs, which are defined as the low and high boundaries for the workload working set size. However, the working set size of a workload may be unknown or change in time. With this patch set, one can periodically estimate the amount of memory unused by each cgroup and tune their memory.low and memory.high parameters accordingly, therefore optimizing the overall memory utilization.

Another use case is balancing workloads within a compute cluster. Knowing how much memory is not really used by a workload unit may help take a more optimal decision when considering migrating the unit to another node within the cluster.

Also, as noted by Minchan, this would be useful for per-process reclaim
(https://lwn.net/Articles/545668/). With idle tracking, we could reclaim idle pages only by smart user memory manager.

---- USER API ----

Quoted 4 lines

The user API consists of two new proc files:

* /proc/kpageidle. This file implements a bitmap where each bit corresponds to a page, indexed by PFN.

What are the bit mappings? If I read the first byte of /proc/kpageidle I get PFN #0 in bit zero of that byte? And the second byte of /proc/kpageidle contains PFN #8 in its LSB, etc?

Maybe this is covered in the documentation file.

Quoted 3 lines

When the bit is set, the corresponding page is
idle. A page is considered idle if it has not been accessed since it was marked idle.

Perhaps we can spell out in some detail what "accessed" means? I see you've hooked into mark_page_accessed(), so a read from disk is an access. What about a write to disk? And what about a page being accessed from some random device (could hook into get_user_pages()?) Is getting written to swap an access? When a dirty pagecache page is written out by kswapd or direct reclaim?

This also should be in the permanent documentation.

Quoted 8 lines

To mark a page idle one should set the bit corresponding to the
page by writing to the file. A value written to the file is OR-ed with the current bitmap value. Only user memory pages can be marked idle, for other page types input is silently ignored. Writing to this file beyond max PFN results in the ENXIO error. Only available when CONFIG_IDLE_PAGE_TRACKING is set.

Quoted 7 lines

This file can be used to estimate the amount of pages that are not used by a particular workload as follows:

1. mark all pages of interest idle by setting corresponding bits in the /proc/kpageidle bitmap
2. wait until the workload accesses its working set
3. read /proc/kpageidle and count the number of bits set

Security implications. This interface could be used to learn about a sensitive application by poking data at it and then observing its memory access patterns. Perhaps this is why the proc files are root-only (whcih I assume is sufficient).

Some words here about the security side of things and the reasoning behind the chosen permissions would be good to have.

Quoted 2 lines

* /proc/kpagecgroup. This file contains a 64-bit inode number of the memory cgroup each page is charged to, indexed by PFN.

Actually "closest online ancestor". This also should be in the interface documentation.

Quoted 2 lines

Only available when CONFIG_MEMCG is set.

CONFIG_MEMCG and CONFIG_IDLE_PAGE_TRACKING I assume?

Quoted 6 lines

This file can be used to find all pages (including unmapped file pages) accounted to a particular cgroup. Using /proc/kpageidle, one can then estimate the cgroup working set size.

Quoted 3 lines

For an example of using these files for estimating the amount of unused memory pages per each memory cgroup, please see the script attached below.

Why were these put in /proc anyway? Rather than under /sys/fs/cgroup somewhere? Presumably because /proc/kpageidle is useful in non-memcg setups.

Quoted 2 lines

---- PERFORMANCE EVALUATION ----

"^___" means "end of changelog". Perhaps that should have been "^---\n" - unclear.

Quoted 2 lines

Documentation/vm/pagemap.txt | 22 ++-

I think we'll need quite a lot more than this to fully describe the interface?

Quoted 6 lines · Andrew Morton

Quoted 1 line · On Sun, 19 Jul 2015 15:31:09 +0300 Vladimir Davydov…

Documentation/vm/pagemap.txt | 22 ++-

I think we'll need quite a lot more than this to fully describe the interface?

Agree, the documentation sucks :-( Will try to forge something more thorough.

Quoted 7 lines · On Sun, 19 Jul 2015 15:31:09 +0300 Vladimir Davydov <[email protected]>…

To mark a page idle one should set the bit corresponding to the
page by writing to the file. A value written to the file is OR-ed with the current bitmap value. Only user memory pages can be marked idle, for other page types input is silently ignored. Writing to this file beyond max PFN results in the ENXIO error. Only available when CONFIG_IDLE_PAGE_TRACKING is set.

Quoted 7 lines

This file can be used to estimate the amount of pages that are not used by a particular workload as follows:

1. mark all pages of interest idle by setting corresponding bits in the /proc/kpageidle bitmap
2. wait until the workload accesses its working set
3. read /proc/kpageidle and count the number of bits set

Security implications. This interface could be used to learn about a sensitive application by poking data at it and then observing its memory access patterns. Perhaps this is why the proc files are root-only (whcih I assume is sufficient). Some words here about the security side of things and the reasoning behind the chosen permissions would be good to have.

Quoted 2 lines

* /proc/kpagecgroup. This file contains a 64-bit inode number of the memory cgroup each page is charged to, indexed by PFN.

Actually "closest online ancestor". This also should be in the interface documentation.

Quoted 1 line

Only available when CONFIG_MEMCG is set.

CONFIG_MEMCG and CONFIG_IDLE_PAGE_TRACKING I assume?

Quoted 5 lines

This file can be used to find all pages (including unmapped file pages) accounted to a particular cgroup. Using /proc/kpageidle, one can then estimate the cgroup working set size.

Quoted 3 lines

For an example of using these files for estimating the amount of unused memory pages per each memory cgroup, please see the script attached below.

Why were these put in /proc anyway? Rather than under /sys/fs/cgroup somewhere? Presumably because /proc/kpageidle is useful in non-memcg setups.

Quoted 3 lines

Why were these put in /proc anyway? Rather than under /sys/fs/cgroup somewhere? Presumably because /proc/kpageidle is useful in non-memcg setups.

Do we need a /proc/vm/ for holding these kinds of things? We're collecting a lot there. Or invent some way for this to be sensible in /sys?

---- USER API ----

The user API consists of two new proc files:

On Sun 19-07-15 15:31:09, Vladimir Davydov wrote:
[...]

Quoted 3 lines

---- USER API ----

The user API consists of two new proc files:

I was thinking about this for a while. I dislike the interface. It is quite awkward to use - e.g. you have to read the full memory to check a single memcg idleness. This might turn out being a problem especially on large machines.

It also provides a very low level information (per-pfn idleness) which is inherently racy. Does anybody really require this level of detail?

I would assume that most users are interested only in a single number which tells the idleness of the system/memcg.

Well, you have mentioned a per-process reclaim but I am quite skeptical about this.

I guess the primary reason to rely on the pfn rather than the LRU walk, which would be more targeted (especially for memcg cases), is that we cannot hold lru lock for the whole LRU walk and we cannot continue walking after the lock is dropped. Maybe we can try to address that instead? I do not think this is easy to achieve but have you considered that as an option?

Quoted 6 lines

I guess the primary reason to rely on the pfn rather than the LRU walk, which would be more targeted (especially for memcg cases), is that we cannot hold lru lock for the whole LRU walk and we cannot continue walking after the lock is dropped. Maybe we can try to address that instead? I do not think this is easy to achieve but have you considered that as an option?

Yes, I have, and I've come to a conclusion it's not doable, because LRU lists can be constantly rotating at an arbitrary rate. If you have an idea in mind how this could be done, please share.

Speaking of LRU-vs-PFN walk, iterating over PFNs has its own advantages:
- You can distribute a walk in time to avoid CPU bursts.
- You are free to parallelize the scanner as you wish to decrease the scan time.

Quoted 11 lines · Vladimir Davydov

Quoted 6 lines

I guess the primary reason to rely on the pfn rather than the LRU walk, which would be more targeted (especially for memcg cases), is that we cannot hold lru lock for the whole LRU walk and we cannot continue walking after the lock is dropped. Maybe we can try to address that instead? I do not think this is easy to achieve but have you considered that as an option?

Yes, I have, and I've come to a conclusion it's not doable, because LRU lists can be constantly rotating at an arbitrary rate. If you have an idea in mind how this could be done, please share.

Quoted 4 lines

Speaking of LRU-vs-PFN walk, iterating over PFNs has its own advantages:
- You can distribute a walk in time to avoid CPU bursts.
- You are free to parallelize the scanner as you wish to decrease the scan time.

There is a third way: one could go through every MM in the system and scan their page tables. Doing things that way turns out to be generally faster than scanning by physical address, because you don't have to go through RMAP for every page. But, you end up needing to take the mmap_sem lock of every MM (in turn) while scanning them, and that degrades quickly under memory load, which is exactly when you most need this feature. So, scan by address is still what we use here.

My only concern about the interface is that it exposes the fact that the scan is done by address - if the interface only showed per-memcg totals, it would make it possible to change the implementation underneath if we somehow figure out how to work around the mmap_sem issue in the future. I don't think that is necessarily a blocker but this is something to keep in mind IMO.

Quoted 7 lines · Vladimir Davydov

Quoted 6 lines

I guess the primary reason to rely on the pfn rather than the LRU walk, which would be more targeted (especially for memcg cases), is that we cannot hold lru lock for the whole LRU walk and we cannot continue walking after the lock is dropped. Maybe we can try to address that instead? I do not think this is easy to achieve but have you considered that as an option?

Quoted 8 lines

Yes, I have, and I've come to a conclusion it's not doable, because LRU lists can be constantly rotating at an arbitrary rate. If you have an idea in mind how this could be done, please share.

Speaking of LRU-vs-PFN walk, iterating over PFNs has its own advantages:
- You can distribute a walk in time to avoid CPU bursts.
- You are free to parallelize the scanner as you wish to decrease the scan time.

There is a third way: one could go through every MM in the system and scan their page tables. Doing things that way turns out to be generally faster than scanning by physical address, because you don't have to go through RMAP for every page. But, you end up needing to take the mmap_sem lock of every MM (in turn) while scanning them, and that degrades quickly under memory load, which is exactly when you most need this feature. So, scan by address is still what we use here.

Page table scan approach has the inherent problem - it ignores unmapped page cache. If a workload does a lot of read/write or map-access-unmap operations, we won't be able to even roughly estimate its wss.

Quoted 3 lines · Vladimir Davydov

Page table scan approach has the inherent problem - it ignores unmapped page cache. If a workload does a lot of read/write or map-access-unmap operations, we won't be able to even roughly estimate its wss.

You can catch that in mark_page_accessed on those paths, though.

Quoted 1 line · Michel Lespinasse

On Wed, Jul 29, 2015 at 7:45 AM, Vladimir Davydov <

[email protected]>

Quoted 1 line

wrote:

Quoted 5 lines

Quoted 3 lines

Page table scan approach has the inherent problem - it ignores unmapped page cache. If a workload does a lot of read/write or map-access-unmap operations, we won't be able to even roughly estimate its wss.

You can catch that in mark_page_accessed on those paths, though.

Actually, the problem here is how to find an unmapped page cache page *to mark it idle*, not to mark it accessed.

Quoted 18 lines · Michel Lespinasse

Quoted 7 lines · Vladimir Davydov

Quoted 6 lines

I guess the primary reason to rely on the pfn rather than the LRU walk, which would be more targeted (especially for memcg cases), is that we cannot hold lru lock for the whole LRU walk and we cannot continue walking after the lock is dropped. Maybe we can try to address that instead? I do not think this is easy to achieve but have you considered that as an option?

Quoted 8 lines

Yes, I have, and I've come to a conclusion it's not doable, because LRU lists can be constantly rotating at an arbitrary rate. If you have an idea in mind how this could be done, please share.

Speaking of LRU-vs-PFN walk, iterating over PFNs has its own advantages:
- You can distribute a walk in time to avoid CPU bursts.
- You are free to parallelize the scanner as you wish to decrease the scan time.

Quoted 7 lines

There is a third way: one could go through every MM in the system and scan their page tables. Doing things that way turns out to be generally faster than scanning by physical address, because you don't have to go through RMAP for every page. But, you end up needing to take the mmap_sem lock of every MM (in turn) while scanning them, and that degrades quickly under memory load, which is exactly when you most need this feature. So, scan by address is still what we use here.

Page table scan approach has the inherent problem - it ignores unmapped page cache. If a workload does a lot of read/write or map-access-unmap operations, we won't be able to even roughly estimate its wss.

Quoted 27 lines · Vladimir Davydov

Quoted 25 lines · Michel Lespinasse

Quoted 7 lines · Vladimir Davydov

Quoted 6 lines

I guess the primary reason to rely on the pfn rather than the LRU walk, which would be more targeted (especially for memcg cases), is that we cannot hold lru lock for the whole LRU walk and we cannot continue walking after the lock is dropped. Maybe we can try to address that instead? I do not think this is easy to achieve but have you considered that as an option?

Quoted 8 lines

Yes, I have, and I've come to a conclusion it's not doable, because LRU lists can be constantly rotating at an arbitrary rate. If you have an idea in mind how this could be done, please share.

Speaking of LRU-vs-PFN walk, iterating over PFNs has its own advantages:
- You can distribute a walk in time to avoid CPU bursts.
- You are free to parallelize the scanner as you wish to decrease the scan time.

There is a third way: one could go through every MM in the system and scan their page tables. Doing things that way turns out to be generally faster than scanning by physical address, because you don't have to go through RMAP for every page. But, you end up needing to take the mmap_sem lock of every MM (in turn) while scanning them, and that degrades quickly under memory load, which is exactly when you most need this feature. So, scan by address is still what we use here.

Quoted 3 lines

Page table scan approach has the inherent problem - it ignores unmapped page cache. If a workload does a lot of read/write or map-access-unmap operations, we won't be able to even roughly estimate its wss.

That page cache is trivially reclaimable if it is clean. If it needs writeback then it is non-idle only until the next writeback. So why does it matter for the estimation?

Quoted 1 line · Michal Hocko

On Wed 29-07-15 17:45:39, Vladimir Davydov wrote:

Quoted 7 lines

Quoted 3 lines

Page table scan approach has the inherent problem - it ignores unmapped page cache. If a workload does a lot of read/write or map-access-unmap operations, we won't be able to even roughly estimate its wss.

That page cache is trivially reclaimable if it is clean. If it needs writeback then it is non-idle only until the next writeback. So why does it matter for the estimation?

Because it might be a part of a workload's working set, in which case evicting it will make the workload lag.

Quoted 10 lines · Michal Hocko

On Sun 19-07-15 15:31:09, Vladimir Davydov wrote:
[...]

Quoted 3 lines

---- USER API ----

The user API consists of two new proc files:

I was thinking about this for a while. I dislike the interface. It is quite awkward to use - e.g. you have to read the full memory to check a single memcg idleness. This might turn out being a problem especially on large machines.

Yes, with this API estimating the wss of a single memory cgroup will cost almost as much as doing this for the whole system.

Come to think of it, does anyone really need to estimate idleness of one particular cgroup?

If we are doing this for finding an optimal memcg
limits configuration or while considering a load move within a cluster (which I think are the primary use cases for the feature), we must do it system-wide to see the whole picture.

Quoted 2 lines

It also provides a very low level information (per-pfn idleness) which is inherently racy. Does anybody really require this level of detail?

Well, one might want to do it per-process, obtaining PFNs from /proc/pid/pagemap.

Quoted 2 lines

I would assume that most users are interested only in a single number which tells the idleness of the system/memcg.

Yes, that's what I need it for - estimating containers' wss for setting their limits accordingly.

Quoted 2 lines

Well, you have mentioned a per-process reclaim but I am quite skeptical about this.

This is what Minchan mentioned initially. Personally, I'm not going to use it per-process, but I wouldn't rule out this use case either.

Quoted 6 lines

I guess the primary reason to rely on the pfn rather than the LRU walk, which would be more targeted (especially for memcg cases), is that we cannot hold lru lock for the whole LRU walk and we cannot continue walking after the lock is dropped. Maybe we can try to address that instead? I do not think this is easy to achieve but have you considered that as an option?

Yes, I have, and I've come to a conclusion it's not doable, because LRU lists can be constantly rotating at an arbitrary rate. If you have an idea in mind how this could be done, please share.

Speaking of LRU-vs-PFN walk, iterating over PFNs has its own advantages:
- You can distribute a walk in time to avoid CPU bursts.

- You are free to parallelize the scanner as you wish to decrease the scan time.

Quoted 12 lines · Vladimir Davydov

Quoted 10 lines · Michal Hocko

On Sun 19-07-15 15:31:09, Vladimir Davydov wrote:
[...]

Quoted 3 lines

---- USER API ----

The user API consists of two new proc files:

I was thinking about this for a while. I dislike the interface. It is quite awkward to use - e.g. you have to read the full memory to check a single memcg idleness. This might turn out being a problem especially on large machines.

Quoted 5 lines

Yes, with this API estimating the wss of a single memory cgroup will cost almost as much as doing this for the whole system.

Come to think of it, does anyone really need to estimate idleness of one particular cgroup?

It is certainly interesting for setting the low limit.

Quoted 6 lines

If we are doing this for finding an optimal memcg
limits configuration or while considering a load move within a cluster (which I think are the primary use cases for the feature), we must do it system-wide to see the whole picture.

Quoted 5 lines

Quoted 2 lines

It also provides a very low level information (per-pfn idleness) which is inherently racy. Does anybody really require this level of detail?

Well, one might want to do it per-process, obtaining PFNs from /proc/pid/pagemap.

Sure once the interface is exported you can do whatever ;) But my question is whether any real usecase _requires_ it.

Quoted 6 lines

Quoted 3 lines

I would assume that most users are interested only in a single number which tells the idleness of the system/memcg.

Yes, that's what I need it for - estimating containers' wss for setting their limits accordingly.

So why don't we export the single per memcg and global knobs then? This would have few advantages. First of all it would be much easier to use, you wouldn't have to export memcg ids and finally the implementation could be changed without any user visible changes (e.g. lru vs. pfn walks), potential caching and who knows what. In other words. Michel had a single number interface AFAIR, what was the primary reason to move away from that API?

Quoted 6 lines

Quoted 3 lines

Well, you have mentioned a per-process reclaim but I am quite skeptical about this.

This is what Minchan mentioned initially. Personally, I'm not going to use it per-process, but I wouldn't rule out this use case either.

Considering how many times we have been bitten by too broad interfaces I would rather be conservative.

Quoted 8 lines

Quoted 7 lines

I guess the primary reason to rely on the pfn rather than the LRU walk, which would be more targeted (especially for memcg cases), is that we cannot hold lru lock for the whole LRU walk and we cannot continue walking after the lock is dropped. Maybe we can try to address that instead? I do not think this is easy to achieve but have you considered that as an option?

Quoted 3 lines

Yes, I have, and I've come to a conclusion it's not doable, because LRU lists can be constantly rotating at an arbitrary rate. If you have an idea in mind how this could be done, please share.

Yes this is really tricky with the current LRU implementation. I was playing with some ideas (do some checkpoints on the way) but none of them was really working out on a busy systems. But the LRU implementation might change in the future.

I didn't mean this as a hard requirement it just sounds that the current implementation restrictions shape the user visible API which is a good sign to think twice about it.

Quoted 3 lines

Speaking of LRU-vs-PFN walk, iterating over PFNs has its own advantages:
- You can distribute a walk in time to avoid CPU bursts.

This would make the information even more volatile. I am not sure how helpful it would be in the end.

Quoted 3 lines

- You are free to parallelize the scanner as you wish to decrease the scan time.

This is true but you could argue similar with per-node/lru threads if this was implemented in the kernel and really needed. I am not sure it would be really needed though. I would expect this would be a low priority thing.

Quoted 16 lines · Michal Hocko

Quoted 15 lines · Vladimir Davydov

Quoted 10 lines · Michal Hocko

On Sun 19-07-15 15:31:09, Vladimir Davydov wrote:
[...]

Quoted 3 lines

---- USER API ----

The user API consists of two new proc files:

I was thinking about this for a while. I dislike the interface. It is quite awkward to use - e.g. you have to read the full memory to check a single memcg idleness. This might turn out being a problem especially on large machines.

Yes, with this API estimating the wss of a single memory cgroup will cost almost as much as doing this for the whole system.

Quoted 4 lines

Quoted 2 lines

Come to think of it, does anyone really need to estimate idleness of one particular cgroup?

It is certainly interesting for setting the low limit.

Yes, but IMO there is no point in setting the low limit for one particular cgroup w/o considering what's going on with the rest of the system.

Quoted 6 lines

Quoted 5 lines

Quoted 2 lines

I would assume that most users are interested only in a single number which tells the idleness of the system/memcg.

Yes, that's what I need it for - estimating containers' wss for setting their limits accordingly.

Quoted 7 lines

So why don't we export the single per memcg and global knobs then? This would have few advantages. First of all it would be much easier to use, you wouldn't have to export memcg ids and finally the implementation could be changed without any user visible changes (e.g. lru vs. pfn walks), potential caching and who knows what. In other words. Michel had a single number interface AFAIR, what was the primary reason to move away from that API?

Because there is too much to be taken care of in the kernel with such an approach and chances are high that it won't satisfy everyone. What should the scan period be equal too?

Knob. How many kthreads do we want?
Knob. I want to keep history for last N intervals (this was a part of Michel's implementation), what should N be equal to? Knob.

I want to be
able to choose between an instant scan and a scan distributed in time. Knob. I want to see stats for anon/locked/file/dirty memory separately,

Quoted 4 lines

Yes this is really tricky with the current LRU implementation. I was playing with some ideas (do some checkpoints on the way) but none of them was really working out on a busy systems. But the LRU implementation might change in the future.

It might. Then we could come up with a new /proc or /sys file which would do the same as /proc/kpageidle, but on per LRU^w whatever-it-is basis, and give people a choice which one to use.

Quoted 22 lines · Vladimir Davydov

Quoted 20 lines · Michal Hocko

Quoted 12 lines · Vladimir Davydov

Quoted 10 lines · Michal Hocko

On Sun 19-07-15 15:31:09, Vladimir Davydov wrote:
[...]

Quoted 3 lines

---- USER API ----

The user API consists of two new proc files:

I was thinking about this for a while. I dislike the interface. It is quite awkward to use - e.g. you have to read the full memory to check a single memcg idleness. This might turn out being a problem especially on large machines.

Quoted 5 lines

Yes, with this API estimating the wss of a single memory cgroup will cost almost as much as doing this for the whole system.

Come to think of it, does anyone really need to estimate idleness of one particular cgroup?

It is certainly interesting for setting the low limit.

Quoted 3 lines

Yes, but IMO there is no point in setting the low limit for one particular cgroup w/o considering what's going on with the rest of the system.

If you use the low limit for isolating an important load then you do not have to care about the others that much. All you care about is to set the reasonable protection level and let others to compete for the rest.

[...]

Quoted 14 lines

Quoted 13 lines

Quoted 5 lines

Quoted 2 lines

I would assume that most users are interested only in a single number which tells the idleness of the system/memcg.

Yes, that's what I need it for - estimating containers' wss for setting their limits accordingly.

So why don't we export the single per memcg and global knobs then? This would have few advantages. First of all it would be much easier to use, you wouldn't have to export memcg ids and finally the implementation could be changed without any user visible changes (e.g. lru vs. pfn walks), potential caching and who knows what. In other words. Michel had a single number interface AFAIR, what was the primary reason to move away from that API?

Quoted 3 lines

Because there is too much to be taken care of in the kernel with such an approach and chances are high that it won't satisfy everyone. What should the scan period be equal too?

No, just gather the data on the read request and let the userspace to decide when/how often etc. If we are clever enough we can cache the numbers and prevent from the walk. Write to the file and do the mark_idle stuff.

Quoted 4 lines

Knob. How many kthreads do we want?
Knob. I want to keep history for last N intervals (this was a part of Michel's implementation), what should N be equal to? Knob.

This all relates to the kernel thread implementation which I wasn't suggesting. I was referring to Michel's work which might induce that. I was merely referring to a single number output. Sorry about the confusion.

Quoted 4 lines

I want to be
able to choose between an instant scan and a scan distributed in time. Knob. I want to see stats for anon/locked/file/dirty memory separately,

Why is this useful for the memcg limits setting or the wss estimation? I can imagine that a further drop down numbers might be interesting from the debugging POV but I fail to see what kind of decisions from userspace you would do based on them.

[...]

Quoted 5 lines

Quoted 4 lines

Yes this is really tricky with the current LRU implementation. I was playing with some ideas (do some checkpoints on the way) but none of them was really working out on a busy systems. But the LRU implementation might change in the future.

Quoted 3 lines

It might. Then we could come up with a new /proc or /sys file which would do the same as /proc/kpageidle, but on per LRU^w whatever-it-is basis, and give people a choice which one to use.

This just leads to proc files count explosion we are seeing already... Proc ended up in dump ground for different things which didn't fit elsewhere and I am not very much happy about it to be honest.

/proc/kpageidle should probably live somewhere in /sys/kernel/mm, but I added it where similar files are located (kpagecount, kpageflags) to keep things consistent.

Quoted 3 lines · On Wed, 29 Jul 2015 19:29:08 +0300 Vladimir Davydov <[email protected]>…

/proc/kpageidle should probably live somewhere in /sys/kernel/mm, but I added it where similar files are located (kpagecount, kpageflags) to keep things consistent.

I think these files should be moved elsewhere. Consistency is good, but not when we're being consistent with a bad thing.

So let's place these in /sys/kernel/mm and then start being consistent with that?

Quoted 11 lines · Andrew Morton

Quoted 3 lines · On Wed, 29 Jul 2015 19:29:08 +0300 Vladimir Davydov…

/proc/kpageidle should probably live somewhere in /sys/kernel/mm, but I added it where similar files are located (kpagecount, kpageflags) to keep things consistent.

I think these files should be moved elsewhere. Consistency is good, but not when we're being consistent with a bad thing.

So let's place these in /sys/kernel/mm and then start being consistent with that?

I really don't think we should separate kpagecgroup from kpagecount and kpageflags, because they look very similar (each of them is read-only, contains an array of u64 values referenced by PFN). Scattering these files between different filesystems would look ugly IMO.

However, kpageidle is somewhat different (it's read-write, contains a bitmap) so I think it's worth moving it to /sys/kernel/mm. We have to move the code from fs/proc to mm/something then to remove dependency from PROC_FS, which would be unnecessary. Let me give it a try.