mm: add last level page table numa info to /proc/pid/numa_pgtable

In many data center servers, the shared memory architectures is Non-Uniform Memory Access (NUMA), remote numa node data access often brings a high latency problem, but what we are easy to ignore is that the page table remote numa access, It can also leads to a performance degradation.

So there add a new interface in /proc, This will help developers to get more info about performance issues if they are caused by cross-NUMA.

Reported-by kernel test robot <[email protected]>

Signed-off-by Xin Hao <[email protected]>

---
 fs/proc/base.c     |  2 ++
 fs/proc/internal.h |  1 +
 fs/proc/task_mmu.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 90 insertions(+)

fs/proc/base.c +2 -0

index 8dfa36a99c74..487e82dd3275 100644
@@ -3224,6 +3224,7 @@ static const struct pid_entry tgid_base_stuff[] = {
 	REG("maps",       S_IRUGO, proc_pid_maps_operations),
 #ifdef CONFIG_NUMA
 	REG("numa_maps",  S_IRUGO, proc_pid_numa_maps_operations),
+	REG("numa_pgtable", S_IRUGO, proc_pid_numa_pgtable_operations),
 #endif
 	REG("mem",        S_IRUSR|S_IWUSR, proc_mem_operations),
 	LNK("cwd",        proc_cwd_link),

@@ -3571,6 +3572,7 @@ static const struct pid_entry tid_base_stuff[] = {
 #endif
 #ifdef CONFIG_NUMA
 	REG("numa_maps", S_IRUGO, proc_pid_numa_maps_operations),
+	REG("numa_pgtable", S_IRUGO, proc_pid_numa_pgtable_operations),
 #endif
 	REG("mem",       S_IRUSR|S_IWUSR, proc_mem_operations),
 	LNK("cwd",       proc_cwd_link),

fs/proc/internal.h +1 -0

index 06a80f78433d..e7ed9ef097b6 100644
@@ -296,6 +296,7 @@ struct mm_struct *proc_mem_open(struct inode *inode, unsigned int mode);
 
 extern const struct file_operations proc_pid_maps_operations;
 extern const struct file_operations proc_pid_numa_maps_operations;
+extern const struct file_operations proc_pid_numa_pgtable_operations;
 extern const struct file_operations proc_pid_smaps_operations;
 extern const struct file_operations proc_pid_smaps_rollup_operations;
 extern const struct file_operations proc_clear_refs_operations;

fs/proc/task_mmu.c +20 -0

index 2d04e3470d4c..77b7a49757f5 100644
@@ -1999,4 +1999,91 @@ const struct file_operations proc_pid_numa_maps_operations = {
 	.release	= proc_map_release,
 };
 
+struct pgtable_numa_private {
+	struct proc_maps_private proc_maps;
+	unsigned long node[MAX_NUMNODES];
+};
+
+static int gather_pgtable_numa_stats(pmd_t *pmd, unsigned long addr,
+				     unsigned long end, struct mm_walk *walk)
+{
+	struct pgtable_numa_private *priv = walk->private;
+	struct page *page;
+	int nid;
+
+	if (pmd_huge(*pmd)) {
+		page = virt_to_page(pmd);
+	} else {
+		page = pmd_page(*pmd);
+	}
+
+	nid = page_to_nid(page);
+	priv->node[nid]++;

fs/proc/task_mmu.c +57 -1

+ */
+static int show_numa_pgtable(struct seq_file *m, void *v)
+{
+	struct pgtable_numa_private *numa_priv = m->private;
+	struct vm_area_struct *vma = v;
+	struct mm_struct *mm = vma->vm_mm;
+	struct file *file = vma->vm_file;
+	int nid;
+
+	if (!mm)
+		return 0;
+
+	memset(numa_priv->node, 0, sizeof(numa_priv->node));
+
+	seq_printf(m, "%08lx ", vma->vm_start);
+
+	if (file) {
+		seq_puts(m, " file=");
+		seq_file_path(m, file, "\n\t= ");
+	} else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
+		seq_puts(m, " heap");
+	} else if (is_stack(vma)) {
+		seq_puts(m, " stack");
+	}
+
+	/* mmap_lock is held by m_start */
+	walk_page_vma(vma, &show_numa_pgtable_ops, numa_priv);
+
+	for_each_node_state(nid, N_MEMORY) {
+		if (numa_priv->node[nid])
+			seq_printf(m, " N%d=%lu", nid, numa_priv->node[nid]);
+	}
+	seq_putc(m, '\n');
+
+	return 0;
+}
+
+static const struct seq_operations proc_pid_numa_pgtable_op = {
+	.start  = m_start,
+	.next   = m_next,
+	.stop   = m_stop,
+	.show   = show_numa_pgtable,
+};
+
+static int pid_numa_pgtable_open(struct inode *inode, struct file *file)
+{
+	return proc_maps_open(inode, file, &proc_pid_numa_pgtable_op,
+			sizeof(struct pgtable_numa_private));
+}
+
+const struct file_operations proc_pid_numa_pgtable_operations = {
+	.open		= pid_numa_pgtable_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= proc_map_release,
+};
+
 #endif /* CONFIG_NUMA */
--

2.31.0

Quoted 5 lines · Xin Hao

In many data center servers, the shared memory architectures is Non-Uniform Memory Access (NUMA), remote numa node data access often brings a high latency problem, but what we are easy to ignore is that the page table remote numa access, It can also leads to a performance degradation.

Let me try rewriting:

"
Many data center servers employ Non-Uniform Memory Access (NUMA) architectures. Remote numa memory access results in high latency. While memory placement is one issue, sub-optimal page table placement can also result in surprise performance degradation.
"

Quoted 2 lines

So there add a new interface in /proc, This will help developers to get more info about performance issues if they are caused by cross-NUMA.

Why do we only care about "last level page table", why not about the others?

IMHO, we could emit something like "0, 1, 3, 0" instead for a given user space address, showing the NUMA node the page table belongs to from highest to lowest page table level.

Quoted 2 lines

Reported-by kernel test robot <[email protected]>

The kernel test robot reported that we need "/proc/pid/numa_pgtable" ?! :)

Just drop that unless it's a follow-up fix.

Quoted 68 lines

Signed-off-by Xin Hao <[email protected]>

---
  fs/proc/base.c     |  2 ++
  fs/proc/internal.h |  1 +
  fs/proc/task_mmu.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++
  3 files changed, 90 insertions(+)

fs/proc/base.c +2 -0

index 8dfa36a99c74..487e82dd3275 100644
@@ -3224,6 +3224,7 @@ static const struct pid_entry tgid_base_stuff[] = {
  	REG("maps",       S_IRUGO, proc_pid_maps_operations),
  #ifdef CONFIG_NUMA
  	REG("numa_maps",  S_IRUGO, proc_pid_numa_maps_operations),
+	REG("numa_pgtable", S_IRUGO, proc_pid_numa_pgtable_operations),
  #endif
  	REG("mem",        S_IRUSR|S_IWUSR, proc_mem_operations),
  	LNK("cwd",        proc_cwd_link),

@@ -3571,6 +3572,7 @@ static const struct pid_entry tid_base_stuff[] = {
  #endif
  #ifdef CONFIG_NUMA
  	REG("numa_maps", S_IRUGO, proc_pid_numa_maps_operations),
+	REG("numa_pgtable", S_IRUGO, proc_pid_numa_pgtable_operations),
  #endif
  	REG("mem",       S_IRUSR|S_IWUSR, proc_mem_operations),
  	LNK("cwd",       proc_cwd_link),

fs/proc/internal.h +1 -0

index 06a80f78433d..e7ed9ef097b6 100644
@@ -296,6 +296,7 @@ struct mm_struct *proc_mem_open(struct inode *inode, unsigned int mode);
 
  extern const struct file_operations proc_pid_maps_operations;
  extern const struct file_operations proc_pid_numa_maps_operations;
+extern const struct file_operations proc_pid_numa_pgtable_operations;
  extern const struct file_operations proc_pid_smaps_operations;
  extern const struct file_operations proc_pid_smaps_rollup_operations;
  extern const struct file_operations proc_clear_refs_operations;

fs/proc/task_mmu.c +20 -0

index 2d04e3470d4c..77b7a49757f5 100644
@@ -1999,4 +1999,91 @@ const struct file_operations proc_pid_numa_maps_operations = {
  	.release	= proc_map_release,
  };
 
+struct pgtable_numa_private {
+	struct proc_maps_private proc_maps;
+	unsigned long node[MAX_NUMNODES];
+};
+
+static int gather_pgtable_numa_stats(pmd_t *pmd, unsigned long addr,
+				     unsigned long end, struct mm_walk *walk)
+{
+	struct pgtable_numa_private *priv = walk->private;
+	struct page *page;
+	int nid;
+
+	if (pmd_huge(*pmd)) {
+		page = virt_to_page(pmd);
+	} else {
+		page = pmd_page(*pmd);
+	}
+
+	nid = page_to_nid(page);
+	priv->node[nid]++;

fs/proc/task_mmu.c const struct file_operations proc_pid_numa_maps_operations = { · line 2031

Quoted 10 lines

fs/proc/task_mmu.c +10 -0

+
+	return 0;
+}
+
+static const struct mm_walk_ops show_numa_pgtable_ops = {
+	.pmd_entry = gather_pgtable_numa_stats,
+};
+
+/*
+ * Display the page talbe allocated per node via /proc.

s/talbe/table/

but the comment somehow doesn't make sense. We don't display a page table.

Quoted 61 lines

fs/proc/task_mmu.c +57 -1

+ */
+static int show_numa_pgtable(struct seq_file *m, void *v)
+{
+	struct pgtable_numa_private *numa_priv = m->private;
+	struct vm_area_struct *vma = v;
+	struct mm_struct *mm = vma->vm_mm;
+	struct file *file = vma->vm_file;
+	int nid;
+
+	if (!mm)
+		return 0;
+
+	memset(numa_priv->node, 0, sizeof(numa_priv->node));
+
+	seq_printf(m, "%08lx ", vma->vm_start);
+
+	if (file) {
+		seq_puts(m, " file=");
+		seq_file_path(m, file, "\n\t= ");
+	} else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
+		seq_puts(m, " heap");
+	} else if (is_stack(vma)) {
+		seq_puts(m, " stack");
+	}
+
+	/* mmap_lock is held by m_start */
+	walk_page_vma(vma, &show_numa_pgtable_ops, numa_priv);
+
+	for_each_node_state(nid, N_MEMORY) {
+		if (numa_priv->node[nid])
+			seq_printf(m, " N%d=%lu", nid, numa_priv->node[nid]);
+	}
+	seq_putc(m, '\n');
+
+	return 0;
+}
+
+static const struct seq_operations proc_pid_numa_pgtable_op = {
+	.start  = m_start,
+	.next   = m_next,
+	.stop   = m_stop,
+	.show   = show_numa_pgtable,
+};
+
+static int pid_numa_pgtable_open(struct inode *inode, struct file *file)
+{
+	return proc_maps_open(inode, file, &proc_pid_numa_pgtable_op,
+			sizeof(struct pgtable_numa_private));
+}
+
+const struct file_operations proc_pid_numa_pgtable_operations = {
+	.open		= pid_numa_pgtable_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= proc_map_release,
+};
+
  #endif /* CONFIG_NUMA */
--

2.31.0

Quoted 6 lines · David Hildenbrand

Quoted 5 lines · Xin Hao

In many data center servers, the shared memory architectures is Non-Uniform Memory Access (NUMA), remote numa node data access often brings a high latency problem, but what we are easy to ignore is that the page table remote numa access, It can also leads to a performance degradation.

Quoted 8 lines

Let me try rewriting:

"
Many data center servers employ Non-Uniform Memory Access (NUMA) architectures. Remote numa memory access results in high latency. While memory placement is one issue, sub-optimal page table placement can also result in surprise performance degradation.
"

Thanks,  it reads more clearly.

Quoted 3 lines

Quoted 2 lines

So there add a new interface in /proc, This will help developers to get more info about performance issues if they are caused by cross-NUMA.

Quoted 5 lines

Why do we only care about "last level page table", why not about the others?

IMHO, we could emit something like "0, 1, 3, 0" instead for a given user space address, showing the NUMA node the page table belongs to from highest to lowest page table level.

I have planned to implement the PTE page table in this version first, and then support other page tables in the next patch later.

Quoted 21 lines · haoxin

Quoted 6 lines · David Hildenbrand

Quoted 5 lines · Xin Hao

In many data center servers, the shared memory architectures is Non-Uniform Memory Access (NUMA), remote numa node data access often brings a high latency problem, but what we are easy to ignore is that the page table remote numa access, It can also leads to a performance degradation.

Quoted 8 lines

Let me try rewriting:

"
Many data center servers employ Non-Uniform Memory Access (NUMA) architectures. Remote numa memory access results in high latency. While memory placement is one issue, sub-optimal page table placement can also result in surprise performance degradation.
"

Thanks,  it reads more clearly.

Quoted 4 lines

Quoted 2 lines

So there add a new interface in /proc, This will help developers to get more info about performance issues if they are caused by cross-NUMA.

Why do we only care about "last level page table", why not about the others?

Quoted 5 lines

Quoted 3 lines

IMHO, we could emit something like "0, 1, 3, 0" instead for a given user space address, showing the NUMA node the page table belongs to from highest to lowest page table level.

I have planned to implement the PTE page table in this version first, and then support other page tables in the next patch later.

If there are plans, let's do it all at once, to get a good and single interface to expose that information.