Shared page tables, updated
Subsystems (6)
RFC 1/1 Shared page tables, updated
The original version of the shared page tables patch suffered from an oversight, namely, that page use counts would need to be decremented in try_to_swap_out and incremented in the various flavors of swap_in by an amount equal to the share count of the page table. This would have been a relatively ugly change, so instead the design was changed to do the page use count incrementing at unshare time instead of at fork time, which is one of the possible optimizations that had been discussed in the original post. Thanks to Dave McCracken for giving me a push in that direction.
A second, more serious problem with the original patch is that zap_page_range forces all shared page tables to be unshared in order to be unmapped, which completely defeats the purpose of the patch, leaving little more than a latency advantage.
To deal with time, zap_page_range has been modified to remove entire page tables when possible. The strategy is to unshare the leading and trailing partial page tables first, then each shared page table encountered during the zap can simply be removed from the page directory. The page table doesn't need to be traversed at all, much less copied. The result is that large shared mappings or CoW regions that haven't been written to can be removed 4 MB at a time, as follows:
if (page_count(virt_to_page(ptep)) > 1) {
pmd_clear(pmd);
put_page(virt_to_page(ptep));
return 0;
}There is a lot more room to optimize zap_page_range by improving the way vmas are fed to it. Right now they come in one at a time, even though each vma tends to immediately adjoin the next one in the list. The ranges needed unmapping can instead be accumulated in exit_mmap and passed as a contiguous region to zap_page_range, which will be able to take advantage of more opportunities to unmap entire shared page tables.
Though the basic design seems to be arriving at a workable, and dare I say it, somewhat pleasing form, there is still a lot of testing and debugging to do. Swapping hasn't been tested yet (it should work, but...) and smp considerations are only beginning to be be analyzed.
Dave McCracken kindly contributed this C program, forktime.c, very useful both for debugging and benchmarking:
--------------------------------------------------------------------------
/*
* Time fork() system call.
*/
#include <unistd.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/mman.h>
static void usage(char *name);
static void mapmem();
static int iter = 100;
static int memsize = 0;
intmain(int argc, char *argv[])
{
int i, c;
int pid;
struct timeval stime, etime, rtime;
extern char *optarg;
while ((c = getopt(argc, argv, "i:m:")) != -1) switch(c) {
case 'i':
iter = atoi(optarg);
break;
case 'm':
memsize = atoi(optarg);
break;
default:
usage(argv[0]);
}
if (memsize)
mapmem();
gettimeofday (&stime, NULL);
for (i = 0; i < iter; i++) {
if ((pid = fork()) == -1) {
perror("fork");
break;
} else if (pid == 0) {
exit(0);
}
}
gettimeofday(&etime, NULL);
timersub(&etime, &stime, &rtime);
printf("Total time taken %d.%06d seconds in %d iterations\n",
rtime.tv_sec, rtime.tv_usec, i);
c = ((rtime.tv_sec * 1000000) + rtime.tv_usec) / i;
printf("or %d microseconds per fork.\n", c);
exit(0);
}static voidusage(char *name)
{
fprintf(stderr, "Usage: %s [-i iterations] [-m mappedpages]\n", name);
exit(1);
}static voidmapmem()
{
int i;
int psize = getpagesize();
char *mp;
mp = mmap(NULL, memsize * psize, PROT_READ|PROT_WRITE,
MAP_PRIVATE | MAP_ANON, 0, 0);
for (i = 0; i < memsize; i++) {
mp[i * psize] = i;
}
}--------------------------------------------------------------------------
fs/exec.c renamed from uml.2.4.17.clean/fs/exec.c +1 -0
@@ -860,6 +860,7 @@int retval;int i; +if (current->uid ==9999 )printk (">>> execve %s\n" , filename); file =open_exec (filename); retval =PTR_ERR (file);
include/linux/mm.h renamed from uml.2.4.17.clean/include/linux/mm.h +6 -1
@@ -411,7 +411,7 @@extern int vmtruncate (struct inode * inode,loff_t offset);extern pmd_t *FASTCALL (__pmd_alloc (struct mm_struct *mm,pgd_t *pgd,unsigned long address)); -extern pte_t *FASTCALL (pte_alloc (struct mm_struct *mm,pmd_t *pmd,unsigned long address)); +extern pte_t *FASTCALL (__pte_alloc (struct mm_struct *mm,pmd_t *pmd,unsigned long address,int write));extern int handle_mm_fault (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long address,int write_access);extern int make_pages_present (unsigned long addr,unsigned long end);extern int access_process_vm (struct task_struct *tsk,unsigned long addr,void *buf,int len,int write);
@@ -424,6 +424,11 @@int get_user_pages (struct task_struct *tsk,struct mm_struct *mm,unsigned long start,int len,int write,int force,struct page **pages,struct vm_area_struct **vmas); + +static inline pte_t *pte_alloc (struct mm_struct *mm,pmd_t *pmd,unsigned long address) +{ +return __pte_alloc (mm, pmd, address,1 ); +}/* * On a two-level page table, this ends up being trivial. Thus the
include/linux/sched.h renamed from uml.2.4.17.clean/include/linux/sched.h +1 -1
@@ -427,7 +427,7 @@#define PF_MEMDIE 0x00001000/* Killed for out-of-memory */ #define PF_FREE_PAGES 0x00002000/* per process page freeing */ #define PF_NOIO 0x00004000/* avoid generating further I/O */ - +#define PF_SHARE_TABLES 0x00008000/* share page tables (testing) */ #define PF_USEDFPU 0x00100000/* task used FPU this quantum (SMP) */ /*
kernel/fork.c renamed from uml.2.4.17.clean/kernel/fork.c +3 -1
@@ -140,6 +140,7 @@ mm->swap_address =0 ; pprev = &mm->mmap; +if (current->uid ==9999 )printk (">>> dup_mmap\n" );/* * Add it to the mmlist after the parent. * Doing it this way means that we can order the list,
@@ -566,9 +567,10 @@struct task_struct *p;struct completion vfork; +if (current->uid ==9999 )printk (">>> fork, stack=%li\n" , stack_start); retval = -EPERM ; -/* +/* * CLONE_PID is only allowed for the initial SMP swapper * calls */
kernel/sys.c renamed from uml.2.4.17.clean/kernel/sys.c +5 -0
@@ -514,6 +514,11 @@ current->uid = new_ruid; current->user = new_user;free_uid (old_user); + +if (current->uid ==9999 ) + current->flags |=PF_SHARE_TABLES ; +printk (">>> user: uid=%i pid=%i pf=%x\n" , current->uid, current->pid, current->flags); +return 0 ; }
mm/memory.c renamed from uml.2.4.17.clean/mm/memory.c +140 -47
318 lines not on this page · raw message
mm/mremap.c renamed from uml.2.4.17.clean/mm/mremap.c +1 -0
@@ -92,6 +92,7 @@ {unsigned long offset = len; +if (current->uid ==9999 )printk (">>> mremap\n" );flush_cache_range (mm, old_addr, old_addr + len);/*