new procfs memory analysis feature

PATCH David Singleton 4 participants 9 messages Dec 7, 2006 – Dec 11, 2006

Subsystems (4)

Memory management Documentation Proc filesystem Memory mapping

lore

Share & tools

lore

David Singleton

author

2006-12-07 22:09 UTC #0 lore raw

Andrew,

this implements a feature for memory analysis tools to go along with
smaps.
It shows reference counts for individual pages instead of aggregate totals for a given VMA. It helps memory analysis tools determine how well pages are being shared, or not, in a shared libraries, etc.

The per page information is presented in /proc/<pid>/pagemaps.

Signed-off-by David SIngleton <[email protected]>

Signed-off-by Joe Green <[email protected]>

Documentation/filesystems/proc.txt |    3 -
fs/proc/base.c                     |   15 +++++
fs/proc/internal.h                 |    5 -
fs/proc/task_mmu.c                 |  110 +++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 5 deletions(-)

Documentation/filesystems/proc.txt +2 -1

Index: linux-2.6.18/Documentation/filesystems/proc.txt
===================================================================
@@ -128,12 +128,13 @@ Table 1-1: Process specific entries in /
 fd      Directory, which contains all file descriptors
 maps   Memory maps to executables and library files           (2.4)
 mem     Memory held by this process
+ pagemaps Based on maps, presents page ref counts for each mapped file
 root   Link to the root directory of this process
+ smaps  Extension based on maps, presenting the rss size for each mapped file
 stat    Process status
 statm   Process memory status information
 status  Process status in human readable form
 wchan   If CONFIG_KALLSYMS is set, a pre-decoded wchan
- smaps  Extension based on maps, presenting the rss size for each mapped file

..............................................................................

For example, to get the status information of a process, all you have to do is

fs/proc/base.c +5 -0

Index: linux-2.6.18/fs/proc/base.c
===================================================================
@@ -182,6 +182,11 @@ enum pid_directory_inos {
       PROC_TID_OOM_SCORE,
       PROC_TID_OOM_ADJUST,
 
+#ifdef CONFIG_MMU
+       PROC_TGID_PAGEMAPS,
+       PROC_TID_PAGEMAPS,
+#endif
+
       /* Add new entries before this */
       PROC_TID_FD_DIR = 0x8000,       /* 0x8000-0xffff */

};
@@ -240,6 +245,9 @@ static struct pid_entry tgid_base_stuff[ #ifdef CONFIG_AUDITSYSCALL
E(PROC_TGID_LOGINUID, "loginuid", S_IFREG|S_IWUSR|S_IRUGO),
#endif
+#ifdef CONFIG_MMU
+ E(PROC_TGID_PAGEMAPS, "pagemaps", S_IFREG|S_IRUGO),
+#endif

       {0,0,NULL,0}
};

static struct pid_entry tid_base_stuff[] = {
@@ -282,6 +290,9 @@ static struct pid_entry tid_base_stuff[] #ifdef CONFIG_AUDITSYSCALL
E(PROC_TID_LOGINUID, "loginuid", S_IFREG|S_IWUSR|S_IRUGO),
#endif
+#ifdef CONFIG_MMU
+ E(PROC_TID_PAGEMAPS, "pagemaps", S_IFREG|S_IRUGO),
+#endif

       {0,0,NULL,0}
};

@@ -1769,6 +1780,10 @@ static struct dentry *proc_pident_lookup

               case PROC_TGID_SMAPS:
                       inode->i_fop = &proc_smaps_operations;
                       break;

+ case PROC_TID_PAGEMAPS:
+ case PROC_TGID_PAGEMAPS:
+ inode->i_fop = &proc_pagemaps_operations;
+ break;
#endif
case PROC_TID_MOUNTSTATS:
case PROC_TGID_MOUNTSTATS:

fs/proc/internal.h +0 -0

Index: linux-2.6.18/fs/proc/internal.h
===================================================================
@@ -40,10 +40,7 @@ extern int proc_pid_statm(struct task_st

extern struct file_operations proc_maps_operations; extern struct file_operations proc_numa_maps_operations; extern struct file_operations proc_smaps_operations; -
-extern struct file_operations proc_maps_operations; -extern struct file_operations proc_numa_maps_operations; -extern struct file_operations proc_smaps_operations; +extern struct file_operations proc_pagemaps_operations;

void free_proc_entry(struct proc_dir_entry *de);

fs/proc/task_mmu.c +0 -0

Index: linux-2.6.18/fs/proc/task_mmu.c
===================================================================
@@ -436,6 +436,116 @@ static int do_maps_open(struct inode *in
       return ret;

}

+static void pagemaps_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
+ unsigned long addr, unsigned long end,
+ struct seq_file *m)
+{

+       pte_t *pte, ptent;
+       spinlock_t *ptl;
+       struct page *page;
+       int mapcount = 0;

+
+ pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
+ do {
+ ptent = *pte;
+ if (pte_present(ptent)) {
+ page = vm_normal_page(vma, addr, ptent);
+ if (page) {
+ if (pte_dirty(ptent))
+ mapcount = -page_mapcount(page);
+ else
+ mapcount = page_mapcount(page);
+ } else {
+ mapcount = 1;
+ }
+ }
+ seq_printf(m, " %d", mapcount);
+
+ } while (pte++, addr += PAGE_SIZE, addr != end);
+ seq_putc(m, '\n');
+
+ pte_unmap_unlock(pte - 1, ptl);
+ cond_resched();
+
+}
+
+static inline void pagemaps_pmd_range(struct vm_area_struct *vma, pud_t *pud,
+ unsigned long addr, unsigned long end,
+ struct seq_file *m)
+{
+ pmd_t *pmd;
+ unsigned long next;
+
+ pmd = pmd_offset(pud, addr);
+ do {
+ next = pmd_addr_end(addr, end);
+ if (pmd_none_or_clear_bad(pmd))
+ continue;
+ pagemaps_pte_range(vma, pmd, addr, next, m);
+ } while (pmd++, addr = next, addr != end);
+}
+
+static inline void pagemaps_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
+ unsigned long addr, unsigned long end,
+ struct seq_file *m)
+{
+ pud_t *pud;
+ unsigned long next;
+
+ pud = pud_offset(pgd, addr);
+ do {
+ next = pud_addr_end(addr, end);
+ if (pud_none_or_clear_bad(pud))
+ continue;
+ pagemaps_pmd_range(vma, pud, addr, next, m);
+ } while (pud++, addr = next, addr != end);
+}
+
+static inline void pagemaps_pgd_range(struct vm_area_struct *vma,
+ unsigned long addr, unsigned long end,
+ struct seq_file *m)
+{
+ pgd_t *pgd;
+ unsigned long next;
+
+ pgd = pgd_offset(vma->vm_mm, addr);
+ do {
+ next = pgd_addr_end(addr, end);
+ if (pgd_none_or_clear_bad(pgd))
+ continue;
+ pagemaps_pud_range(vma, pgd, addr, next, m);
+ } while (pgd++, addr = next, addr != end);
+}
+
+static int show_pagemap(struct seq_file *m, void *v)
+{
+ struct vm_area_struct *vma = v;
+
+ show_map_internal(m, v, NULL);
+ if (vma->vm_mm && !is_vm_hugetlb_page(vma))
+ pagemaps_pgd_range(vma, vma->vm_start, vma->vm_end, m);
+ return 0;
+}
+
+static struct seq_operations proc_pid_pagemaps_op = {

+       .start  = m_start,
+       .next   = m_next,
+       .stop   = m_stop,
+       .show   = show_pagemap

+};
+
+static int pagemaps_open(struct inode *inode, struct file *file) +{
+ return do_maps_open(inode, file, &proc_pid_pagemaps_op);
+}
+
+struct file_operations proc_pagemaps_operations = {

+       .open           = pagemaps_open,
+       .read           = seq_read,
+       .llseek         = seq_lseek,
+       .release        = seq_release_private,
+};
+
static int maps_open(struct inode *inode, struct file *file)
{
       return do_maps_open(inode, file, &proc_pid_maps_op);

Andrew Morton

2006-12-07 22:36 UTC #1 lore raw

Quoted 11 lines · On Thu, 07 Dec 2006 14:09:40 -0800 David Singleton <[email protected]>…

Andrew,
this implements a feature for memory analysis tools to go along with
smaps.
It shows reference counts for individual pages instead of aggregate totals for a given VMA.
It helps memory analysis tools determine how well pages are being shared, or not,
in a shared libraries, etc.

Quoted 2 lines

The per page information is presented in /proc/<pid>/pagemaps.

I think the concept is not a bad one, frankly - this requirement arises frequently. What bugs me is that it only displays the mapcount and dirtiness. Perhaps there are other things which people want to know. I'm not sure what they would be though.

I wonder if it would be insane to display the info via a filesystem:

	cat /mnt/pagemaps/$(pidof crond)/pgd0/pmd1/pte45

Probably it would.

Quoted 1 line

Index: linux-2.6.18/Documentation/filesystems/proc.txt

Against 2.6.18? I didn't know you could still buy copies of that ;)

This patch's changelog should include sample output.

Your email client wordwraps patches, and it replaces tabs with spaces.

Quoted 18 lines

...

(quoted diff) +16 -0

+static void pagemaps_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pte_t *pte, ptent;
+       spinlock_t *ptl;
+       struct page *page;
+       int mapcount = 0;
+
+       pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
+       do {
+               ptent = *pte;
+               if (pte_present(ptent)) {
+                       page = vm_normal_page(vma, addr, ptent);
+                       if (page) {
+                               if (pte_dirty(ptent))

Review

Quoted 10 lines

(quoted diff) +10 -0

+                                       mapcount = -page_mapcount(page);
+                               else
+                                       mapcount = page_mapcount(page);
+                       } else {
+                               mapcount = 1;
+                       }
+               }
+               seq_printf(m, " %d", mapcount);
+
+       } while (pte++, addr += PAGE_SIZE, addr != end);

Well that's cute. As long as both seq_file and pte-pages are of size PAGE_SIZE, and as long as pte's are more than three bytes, this will not overflow the seq_file output buffer.

hm. Unless the pages are all dirty and the mapcounts are all 10000. I think it will overflow then?

Quoted 40 lines

(quoted diff) +2 -0

+
+static inline void pagemaps_pmd_range(struct vm_area_struct *vma, pud_t

*pud,

(quoted diff) +16 -0

+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pmd_t *pmd;
+       unsigned long next;
+
+       pmd = pmd_offset(pud, addr);
+       do {
+               next = pmd_addr_end(addr, end);
+               if (pmd_none_or_clear_bad(pmd))
+                       continue;
+               pagemaps_pte_range(vma, pmd, addr, next, m);
+       } while (pmd++, addr = next, addr != end);
+}
+
+static inline void pagemaps_pud_range(struct vm_area_struct *vma, pgd_t

*pgd,

(quoted diff) +20 -0

+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pud_t *pud;
+       unsigned long next;
+
+       pud = pud_offset(pgd, addr);
+       do {
+               next = pud_addr_end(addr, end);
+               if (pud_none_or_clear_bad(pud))
+                       continue;
+               pagemaps_pmd_range(vma, pud, addr, next, m);
+       } while (pud++, addr = next, addr != end);
+}
+
+static inline void pagemaps_pgd_range(struct vm_area_struct *vma,
+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pgd_t *pgd;

Review

Quoted 10 lines

(quoted diff) +10 -0

+       unsigned long next;
+
+       pgd = pgd_offset(vma->vm_mm, addr);
+       do {
+               next = pgd_addr_end(addr, end);
+               if (pgd_none_or_clear_bad(pgd))
+                       continue;
+               pagemaps_pud_range(vma, pgd, addr, next, m);
+       } while (pgd++, addr = next, addr != end);
+}

I think that's our eighth open-coded pagetable walker. Apparently they are all slightly different. Perhaps we shouild do something about that one day.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

david singleton

author

2006-12-08 00:30 UTC #2 lore raw

Quoted 25 lines · Andrew Morton

Quoted 12 lines · On Thu, 07 Dec 2006 14:09:40 -0800 David Singleton <[email protected]>…
Andrew,
this implements a feature for memory analysis tools to go along
with
smaps.
It shows reference counts for individual pages instead of aggregate totals for a given VMA.
It helps memory analysis tools determine how well pages are being shared, or not,
in a shared libraries, etc.
Quoted 2 lines
The per page information is presented in /proc/<pid>/pagemaps.
I think the concept is not a bad one, frankly - this requirement arises frequently. What bugs me is that it only displays the mapcount and dirtiness. Perhaps there are other things which people want to know. I'm not sure what they would be though.
I wonder if it would be insane to display the info via a filesystem:

Quoted 7 lines

	cat /mnt/pagemaps/$(pidof crond)/pgd0/pmd1/pte45
Probably it would.
Quoted 1 line
Index: linux-2.6.18/Documentation/filesystems/proc.txt
Against 2.6.18? I didn't know you could still buy copies of that ;)

whoops, I have an old copy. let me make a patch against 2.6.19.

Quoted 2 lines

This patch's changelog should include sample output.

okay.

Quoted 2 lines

Your email client wordwraps patches, and it replaces tabs with spaces.

Is an attachment okay? gziped tarfile? a new mailer?

David

Quoted 88 lines

Quoted 25 lines

...

+static void pagemaps_pte_range(struct vm_area_struct *vma, pmd_t *pmd,

(quoted diff) +18 -1

+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pte_t *pte, ptent;
+       spinlock_t *ptl;
+       struct page *page;
+       int mapcount = 0;
+
+       pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
+       do {
+               ptent = *pte;
+               if (pte_present(ptent)) {
+                       page = vm_normal_page(vma, addr, ptent);
+                       if (page) {
+                               if (pte_dirty(ptent))
+                                       mapcount = 
-page_mapcount(page);
+                               else
+                                       mapcount =

page_mapcount(page);

Review

Quoted 7 lines

(quoted diff) +7 -0

+                       } else {
+                               mapcount = 1;
+                       }
+               }
+               seq_printf(m, " %d", mapcount);
+
+       } while (pte++, addr += PAGE_SIZE, addr != end);

Well that's cute. As long as both seq_file and pte-pages are of size PAGE_SIZE, and as long as pte's are more than three bytes, this will not overflow the seq_file output buffer.

hm. Unless the pages are all dirty and the mapcounts are all 10000. I think it will overflow then?

Quoted 48 lines

(quoted diff) +2 -0

+
+static inline void pagemaps_pmd_range(struct vm_area_struct *vma,

pud_t
*pud,

(quoted diff) +16 -0

+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pmd_t *pmd;
+       unsigned long next;
+
+       pmd = pmd_offset(pud, addr);
+       do {
+               next = pmd_addr_end(addr, end);
+               if (pmd_none_or_clear_bad(pmd))
+                       continue;
+               pagemaps_pte_range(vma, pmd, addr, next, m);
+       } while (pmd++, addr = next, addr != end);
+}
+
+static inline void pagemaps_pud_range(struct vm_area_struct *vma,

pgd_t
*pgd,

(quoted diff) +26 -0

+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pud_t *pud;
+       unsigned long next;
+
+       pud = pud_offset(pgd, addr);
+       do {
+               next = pud_addr_end(addr, end);
+               if (pud_none_or_clear_bad(pud))
+                       continue;
+               pagemaps_pmd_range(vma, pud, addr, next, m);
+       } while (pud++, addr = next, addr != end);
+}
+
+static inline void pagemaps_pgd_range(struct vm_area_struct *vma,
+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pgd_t *pgd;
+       unsigned long next;
+
+       pgd = pgd_offset(vma->vm_mm, addr);
+       do {
+               next = pgd_addr_end(addr, end);
+               if (pgd_none_or_clear_bad(pgd))

Quoted 10 lines

Review
Quoted 4 lines
(quoted diff) +4 -0
+                       continue;
+               pagemaps_pud_range(vma, pgd, addr, next, m);
+       } while (pgd++, addr = next, addr != end);
+}
I think that's our eighth open-coded pagetable walker. Apparently they are all slightly different. Perhaps we shouild do something about that one day.

david singleton

author

2006-12-08 01:07 UTC #3 lore raw

In reply to Andrew Morton

Attached is the 2.6.19 patch.

Signed-off-by David Singleton <[email protected]>

Signed-off-by Joe Green <[email protected]>

Add variation of /proc/PID/smaps called /proc/PID/pagemaps. Shows reference counts for individual pages instead of aggregate totals. Allows more detailed memory usage information for memory analysis tools. An example of the output shows the shared text VMA for ld.so and the share depths of the pages in the VMA.

a7f4b000-a7f65000 r-xp 00000000 00:0d 19185826 /lib/ld-2.5.90.so
11 11 11 11 11 11 11 11 11 13 13 13 13 13 13 13 8 8 8 13 13 13 13 13 13 13

 Documentation/filesystems/proc.txt |    3 -
 fs/proc/base.c                     |    2 
 fs/proc/internal.h                 |    5 -
 fs/proc/task_mmu.c                 |  110 +++++++++++++++++++++++++++++++++++++
 4 files changed, 115 insertions(+), 5 deletions(-)

Documentation/filesystems/proc.txt +2 -1

Index: linux-2.6.19/Documentation/filesystems/proc.txt
===================================================================
@@ -130,12 +130,13 @@ Table 1-1: Process specific entries in /
  fd      Directory, which contains all file descriptors 
  maps	 Memory maps to executables and library files		(2.4)
  mem     Memory held by this process                    
+ pagemaps Based on maps, presents page ref counts for each mapped file
  root	 Link to the root directory of this process
+ smaps	 Extension based on maps, presenting the rss size for each mapped file
  stat    Process status                                 
  statm   Process memory status information              
  status  Process status in human readable form          
  wchan   If CONFIG_KALLSYMS is set, a pre-decoded wchan
- smaps	 Extension based on maps, presenting the rss size for each mapped file
 ..............................................................................
 
 For example, to get the status information of a process, all you have to do is

fs/proc/base.c +2 -0

Index: linux-2.6.19/fs/proc/base.c
===================================================================
@@ -1773,6 +1773,7 @@ static struct pid_entry tgid_base_stuff[
 	REG("mountstats", S_IRUSR, mountstats),
 #ifdef CONFIG_MMU
 	REG("smaps",      S_IRUGO, smaps),
+	REG("pagemaps",   S_IRUGO, pagemaps),
 #endif
 #ifdef CONFIG_SECURITY
 	DIR("attr",       S_IRUGO|S_IXUGO, attr_dir),

@@ -2047,6 +2048,7 @@ static struct pid_entry tid_base_stuff[]
 	REG("mounts",    S_IRUGO, mounts),
 #ifdef CONFIG_MMU
 	REG("smaps",     S_IRUGO, smaps),
+	REG("pagemaps",  S_IRUGO, pagemaps),
 #endif
 #ifdef CONFIG_SECURITY
 	DIR("attr",      S_IRUGO|S_IXUGO, attr_dir),

fs/proc/internal.h +1 -4

Index: linux-2.6.19/fs/proc/internal.h
===================================================================
@@ -41,10 +41,7 @@ extern int proc_pid_statm(struct task_st
 extern struct file_operations proc_maps_operations;
 extern struct file_operations proc_numa_maps_operations;
 extern struct file_operations proc_smaps_operations;
-
-extern struct file_operations proc_maps_operations;
-extern struct file_operations proc_numa_maps_operations;
-extern struct file_operations proc_smaps_operations;
+extern struct file_operations proc_pagemaps_operations;
 
 
 void free_proc_entry(struct proc_dir_entry *de);

fs/proc/task_mmu.c +110 -0

116 lines not on this page · raw message

Quoted 134 lines · Andrew Morton

Quoted 12 lines · On Thu, 07 Dec 2006 14:09:40 -0800 David Singleton <[email protected]>…
Andrew,
this implements a feature for memory analysis tools to go along
with
smaps.
It shows reference counts for individual pages instead of aggregate totals for a given VMA.
It helps memory analysis tools determine how well pages are being shared, or not,
in a shared libraries, etc.
Quoted 2 lines
The per page information is presented in /proc/<pid>/pagemaps.
I think the concept is not a bad one, frankly - this requirement arises frequently. What bugs me is that it only displays the mapcount and dirtiness. Perhaps there are other things which people want to know. I'm not sure what they would be though.
I wonder if it would be insane to display the info via a filesystem:
	cat /mnt/pagemaps/$(pidof crond)/pgd0/pmd1/pte45
Probably it would.
Quoted 1 line
Index: linux-2.6.18/Documentation/filesystems/proc.txt
Against 2.6.18? I didn't know you could still buy copies of that ;)
This patch's changelog should include sample output.
Your email client wordwraps patches, and it replaces tabs with spaces.
Quoted 24 lines
...
+static void pagemaps_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
(quoted diff) +18 -1
+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pte_t *pte, ptent;
+       spinlock_t *ptl;
+       struct page *page;
+       int mapcount = 0;
+
+       pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
+       do {
+               ptent = *pte;
+               if (pte_present(ptent)) {
+                       page = vm_normal_page(vma, addr, ptent);
+                       if (page) {
+                               if (pte_dirty(ptent))
+                                       mapcount = 
-page_mapcount(page);
+                               else
+                                       mapcount = 
page_mapcount(page);
Review
Quoted 7 lines
(quoted diff) +7 -0
+                       } else {
+                               mapcount = 1;
+                       }
+               }
+               seq_printf(m, " %d", mapcount);
+
+       } while (pte++, addr += PAGE_SIZE, addr != end);
Well that's cute. As long as both seq_file and pte-pages are of size PAGE_SIZE, and as long as pte's are more than three bytes, this will not overflow the seq_file output buffer.
hm. Unless the pages are all dirty and the mapcounts are all 10000. I think it will overflow then?
Quoted 42 lines
(quoted diff) +2 -0
+
+static inline void pagemaps_pmd_range(struct vm_area_struct *vma, 
pud_t
*pud,
(quoted diff) +16 -0
+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pmd_t *pmd;
+       unsigned long next;
+
+       pmd = pmd_offset(pud, addr);
+       do {
+               next = pmd_addr_end(addr, end);
+               if (pmd_none_or_clear_bad(pmd))
+                       continue;
+               pagemaps_pte_range(vma, pmd, addr, next, m);
+       } while (pmd++, addr = next, addr != end);
+}
+
+static inline void pagemaps_pud_range(struct vm_area_struct *vma, 
pgd_t
*pgd,
(quoted diff) +20 -0
+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pud_t *pud;
+       unsigned long next;
+
+       pud = pud_offset(pgd, addr);
+       do {
+               next = pud_addr_end(addr, end);
+               if (pud_none_or_clear_bad(pud))
+                       continue;
+               pagemaps_pmd_range(vma, pud, addr, next, m);
+       } while (pud++, addr = next, addr != end);
+}
+
+static inline void pagemaps_pgd_range(struct vm_area_struct *vma,
+                               unsigned long addr, unsigned long end,
+                               struct seq_file *m)
+{
+       pgd_t *pgd;
Review
Quoted 10 lines
(quoted diff) +10 -0
+       unsigned long next;
+
+       pgd = pgd_offset(vma->vm_mm, addr);
+       do {
+               next = pgd_addr_end(addr, end);
+               if (pgd_none_or_clear_bad(pgd))
+                       continue;
+               pagemaps_pud_range(vma, pgd, addr, next, m);
+       } while (pgd++, addr = next, addr != end);
+}
I think that's our eighth open-coded pagetable walker. Apparently they are all slightly different. Perhaps we shouild do something about that one day.

Andrew Morton

2006-12-08 01:46 UTC #4 lore raw

Quoted 1 line · On Thu, 7 Dec 2006 17:07:22 -0800 david singleton <[email protected]> wrote:

Attached is the 2.6.19 patch.

It still has the overflow bug.

david singleton

author

2006-12-08 01:53 UTC #5 lore raw

Quoted 29 lines · Andrew Morton

Quoted 1 line · On Thu, 7 Dec 2006 17:07:22 -0800 david singleton <[email protected]> wrote:
Attached is the 2.6.19 patch.
It still has the overflow bug.
Quoted 11 lines
fs/proc/task_mmu.c +9 -1
+       do {
+               ptent = *pte;
+               if (pte_present(ptent)) {
+                       page = vm_normal_page(vma, addr, ptent);
+                       if (page) {
+                               if (pte_dirty(ptent))
+                                       mapcount = 
-page_mapcount(page);
+                               else
+                                       mapcount = 
page_mapcount(page);
fs/proc/task_mmu.c static int do_maps_open(struct inode *in · line 457
Quoted 7 lines
fs/proc/task_mmu.c +7 -0
+                       } else {
+                               mapcount = 1;
+                       }
+               }
+               seq_printf(m, " %d", mapcount);
+
+       } while (pte++, addr += PAGE_SIZE, addr != end);
Well that's cute. As long as both seq_file and pte-pages are of size PAGE_SIZE, and as long as pte's are more than three bytes, this will not overflow the seq_file output buffer.

Quoted 3 lines

hm. Unless the pages are all dirty and the mapcounts are all 10000. I think it will overflow then?

I guess that could happen? Any suggestions?

Paul Cameron Davies

2006-12-08 06:21 UTC #6 lore raw

In reply to Andrew Morton

Quoted 3 lines · Andrew Morton

I think that's our eighth open-coded pagetable walker. Apparently they are all slightly different. Perhaps we shouild do something about that one day.

At UNSW we have abstracted the page table into its own layer, and are running an alternate page table (a GPT), under a clean page table interface (PTI).

The PTI gathers all the open coded iterators togethers into one place, which would be a good precursor to providing generic iterators for non performance critical iterations.

We are completing the updating/enhancements to this PTI for the latest kernel, to be released just prior to LCA. This PTI is benchmarking well. We also plan to release the experimental guarded page table (GPT) running under this PTI.

Paul Davies
Gelato@UNSW
~

Jeremy Fitzhardinge

2006-12-08 21:46 UTC #7 lore raw

Quoted 8 lines · Paul Cameron Davies

The PTI gathers all the open coded iterators togethers into one place, which would be a good precursor to providing generic iterators for non performance critical iterations.
We are completing the updating/enhancements to this PTI for the latest kernel, to be released just prior to LCA. This PTI is benchmarking well. We also plan to release the experimental guarded page table (GPT) running under this PTI.

I looked at implementing linear pagetable mappings for x86 as a way of getting rid of CONFIG_HIGHPTE, and to make pagetable manipulations generally more efficient. I gave up on it after a while because all the existing pagetable accessors are not suitable for a linear pagetable, and I didn't want to have to introduce a pile of new pagetable interfaces. Would the PTI interface be helpful for this?

Thanks,
J

Paul Cameron Davies

2006-12-11 02:19 UTC #8 lore raw

Quoted 6 lines · Jeremy Fitzhardinge

I looked at implementing linear pagetable mappings for x86 as a way of getting rid of CONFIG_HIGHPTE, and to make pagetable manipulations generally more efficient. I gave up on it after a while because all the existing pagetable accessors are not suitable for a linear pagetable, and I didn't want to have to introduce a pile of new pagetable interfaces. Would the PTI interface be helpful for this?

Yes. The PTI is a useful vehicle for experimentation with page tables.
The PTI has two components. The first component provides for architectural
and implementation independent page table access. The second component
provides for architecture dependendent access, but I have only done this for IA64. However, abstracting out the page table implementation for the arch dependent stuff on x86 would enable experimentation with implementing linear page table mappings for x86, while leaving
the current implementation in place as an alternative page table.

Cheers

Paul Davies