Rmap code?

@@ -34,6 +34,7 @@
 #include <linux/pagemap.h>
 #include <linux/highmem.h>
 #include <linux/spinlock.h>
+#include <linux/swap.h>
 #define __NO_VERSION__
 #include <linux/module.h>
 

@@ -277,8 +278,9 @@
 	flush_dcache_page(page);
 	flush_page_to_ram(page);
 	set_pte(pte, pte_mkdirty(pte_mkwrite(mk_pte(page, PAGE_COPY))));
+	page_add_pmap(page, pte);
+	spin_unlock(&pagemap_lru_lock);
 	tsk->mm->rss++;
-	spin_unlock(&tsk->mm->page_table_lock);
 
 	/* no need for flush_tlb */
 	return;

@@ -520,7 +520,9 @@
 	page->index = index;
 	add_page_to_inode_queue(mapping, page);
 	add_page_to_hash_queue(page, page_hash(mapping, index));
-	lru_cache_add(page);
+	/* XXX: already on the inactive_dirty list ... deuglify! */
+	if (!PageSwapCache(page))
+		lru_cache_add(page);
 	spin_unlock(&pagecache_lock);
 }
 

@@ -76,6 +76,7 @@
 static inline void free_one_pmd(pmd_t * dir)
 {
 	pte_t * pte;
+	struct page * page;
 
 	if (pmd_none(*dir))
 		return;

@@ -86,6 +87,10 @@
 	}
 	pte = pte_offset(dir, 0);
 	pmd_clear(dir);
+	page = virt_to_page(pte);
+	/* Clear the reverse mapping stuff on this page table page. */
+	page->mapping = 0;
+	page->index = 0;
 	pte_free(pte);
 }
 

@@ -216,7 +221,9 @@
 					goto cont_copy_pte_range_noset;
 				if (!pte_present(pte)) {
 					swap_duplicate(pte_to_swp_entry(pte));
-					goto cont_copy_pte_range;
+					/* Swapped out, skip the pmap stuff. */
+					set_pte(dst_pte, pte);
+					goto cont_copy_pte_range_noset;
 				}
 				ptepage = pte_page(pte);
 				if ((!VALID_PAGE(ptepage)) || 

@@ -236,6 +243,7 @@
 				get_page(ptepage);
 
 cont_copy_pte_range:		set_pte(dst_pte, pte);
+				page_add_pmap(ptepage, dst_pte);
 cont_copy_pte_range_noset:	address += PAGE_SIZE;
 				if (address >= end)
 					goto out_unlock;

@@ -312,6 +320,8 @@
 		if (!size)
 			break;
 		page = ptep_get_and_clear(pte);
+		if (pte_present(page))
+			page_remove_pmap(pte_page(page), pte);
 		pte++;
 		size--;
 		if (pte_none(page))

@@ -849,6 +859,7 @@
  *  - flush the old one
  *  - update the page tables
  *  - inform the TLB about the new one
+ *  - update the reverse mappings (if the page changes)
  *
  * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
  */

@@ -868,7 +879,9 @@
 	copy_cow_page(old_page,new_page,address);
 	flush_page_to_ram(new_page);
 	flush_cache_page(vma, address);
+	page_remove_pmap(old_page, page_table);
 	establish_pte(vma, address, page_table, pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot))));
+	page_add_pmap(new_page, page_table);
 }
 
 /*

@@ -1139,6 +1152,7 @@
 	flush_page_to_ram(page);
 	flush_icache_page(vma, page);
 	set_pte(page_table, pte);
+	page_add_pmap(page, page_table);
 
 	/* No need to invalidate - it was non-present before */
 	update_mmu_cache(vma, address, pte);

@@ -1153,14 +1167,13 @@
 static int do_anonymous_page(struct mm_struct * mm, struct vm_area_struct * vma, pte_t *page_table, int write_access, unsigned long addr)
 {
 	pte_t entry;
+	struct page *page = ZERO_PAGE(addr);
 
 	/* Read-only mapping of ZERO_PAGE. */
 	entry = pte_wrprotect(mk_pte(ZERO_PAGE(addr), vma->vm_page_prot));
 
 	/* ..except if it's a write access */
 	if (write_access) {
-		struct page *page;
-
 		/* Allocate our own private page. */
 		spin_unlock(&mm->page_table_lock);
 		page = alloc_page(GFP_HIGHUSER);

@@ -1178,6 +1191,7 @@
 	}
 
 	set_pte(page_table, entry);
+	page_add_pmap(page, page_table); /* Ignores empty_zero_page ;) */
 
 	/* No need to invalidate - it was non-present before */
 	update_mmu_cache(vma, addr, entry);

@@ -1240,6 +1254,7 @@
 			   !(vma->vm_flags & VM_SHARED))
 			entry = pte_wrprotect(entry);
 		set_pte(page_table, entry);
+		page_add_pmap(new_page, page_table);
 	} else {
 		/* One of our sibling threads was faster, back out. */
 		page_cache_release(new_page);

@@ -1372,6 +1387,7 @@
 pte_t *pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
 {
 	if (!pmd_present(*pmd)) {
+		struct page * page;
 		pte_t *new;
 
 		/* "fast" allocation can happen without dropping the lock.. */

@@ -1392,6 +1408,10 @@
 				goto out;
 			}
 		}
+		/* Add reverse pte mapping pointers for pmap.c. */
+		page = virt_to_page(new);
+		page->mapping = (void *)mm;
+		page->index = address & ~((PTRS_PER_PTE * PAGE_SIZE) - 1);
 		pmd_populate(mm, pmd, new);
 	}
 out:

@@ -63,6 +63,7 @@
 	pte_t pte;
 
 	if (!pte_none(*src)) {
+		page_remove_pmap(pte_page(pte), src);
 		pte = ptep_get_and_clear(src);
 		if (!dst) {
 			/* No dest?  We must put it back. */

@@ -70,6 +71,7 @@
 			error++;
 		}
 		set_pte(dst, pte);
+		page_add_pmap(pte_page(pte), dst);
 	}
 	return error;
 }

@@ -87,6 +87,8 @@
 		BUG();
 	if (PageInactiveClean(page))
 		BUG();
+	if (page->pte_chain)
+		BUG();
 
 	page->flags &= ~((1<<PG_referenced) | (1<<PG_dirty));
 	page->age = PAGE_AGE_START;

@@ -164,22 +164,12 @@
 void deactivate_page_nolock(struct page * page)
 {
 	/*
-	 * One for the cache, one for the extra reference the
-	 * caller has and (maybe) one for the buffers.
-	 *
-	 * This isn't perfect, but works for just about everything.
-	 * Besides, as long as we don't move unfreeable pages to the
-	 * inactive_clean list it doesn't need to be perfect...
-	 */
-	int maxcount = (page->buffers ? 3 : 2);
-	page->age = 0;
-	ClearPageReferenced(page);
-
-	/*
 	 * Don't touch it if it's not on the active list.
 	 * (some pages aren't on any list at all)
 	 */
-	if (PageActive(page) && page_count(page) <= maxcount && !page_ramdisk(page)) {
+	if (PageActive(page) && !page_ramdisk(page)) {
+		page->age = 0;
+		ClearPageReferenced(page);
 		del_page_from_active_list(page);
 		add_page_to_inactive_dirty_list(page);
 	}

@@ -266,8 +256,6 @@
  */
 void lru_cache_del(struct page * page)
 {
-	if (!PageLocked(page))
-		BUG();
 	spin_lock(&pagemap_lru_lock);
 	__lru_cache_del(page);
 	spin_unlock(&pagemap_lru_lock);

@@ -232,6 +232,7 @@
 	set_pte(dir, pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
 	swap_free(entry);
 	get_page(page);
+	page_add_pmap(page, dir);
 	++vma->vm_mm->rss;
 }
 

@@ -165,7 +165,13 @@
 		if (exclusive_swap_page(page))
 			delete_from_swap_cache_nolock(page);
 		UnlockPage(page);
-	}
+	/*
+	 * If we are the only user and it was an anonymous page
+	 * without swap backing, remove the page from the list.
+	 * SMP/fork() safe because we hold the mm->page_table_lock.
+	 */
+	} else if (page_count(page) == 1)
+		lru_cache_del(page);
 	page_cache_release(page);
 }
 

452 lines not on this page · raw message

341 lines not on this page · raw message

@@ -130,6 +130,9 @@
 	struct page * (*nopage)(struct vm_area_struct * area, unsigned long address, int write_access);
 };
 
+/* Incomplete declaration; pte_chain is internal to mm/pmap.c */
+struct pte_chain;
+
 /*
  * Each physical page in the system has a struct page associated with
  * it to keep track of whatever it is we are using the page for at the

@@ -157,6 +160,7 @@
 	struct list_head lru;		/* Pageout list, eg. active_list;
 					   protected by pagemap_lru_lock !! */
 	unsigned long age;		/* Page aging counter. */
+	struct pte_chain * pte_chain;	/* Reverse pte mapping pointer. */
 	wait_queue_head_t wait;		/* Page locked?  Stand in line... */
 	struct page **pprev_hash;	/* Complement to *next_hash. */
 	struct buffer_head * buffers;	/* Buffer maps us to a disk block. */

@@ -98,6 +98,18 @@
 
 struct zone_t;
 
+/* linux/mm/pmap.c */
+extern int page_referenced(struct page *);
+extern void page_add_pmap(struct page *, pte_t *);
+extern int page_remove_pmap(struct page *, pte_t *);
+extern int page_remove_all_pmaps(struct page *);
+
+/* page_unmap_all_pmaps and try_to_swap_out return values */
+#define	SWAP_SUCCESS	0	/* unmapped every user of the page */
+#define SWAP_AGAIN	1	/* missed a trylock, try again later */
+#define SWAP_FAIL	2	/* cannot swap this page out, reactivate */
+#define	SWAP_ERROR	3	/* illegal arguments or misc error */
+
 /* linux/mm/swap.c */
 extern int memory_pressure;
 extern void age_page_up(struct page *);

@@ -116,6 +128,7 @@
 extern void swap_setup(void);
 
 /* linux/mm/vmscan.c */
+extern int try_to_swap_out(struct mm_struct *, unsigned long, pte_t *, struct page *);
 extern struct page * reclaim_page(zone_t *);
 extern wait_queue_head_t kswapd_wait;
 extern wait_queue_head_t kreclaimd_wait;

@@ -261,8 +274,8 @@
  */
 #define INACTIVE_SHIFT 6
 #define inactive_min(a,b) ((a) < (b) ? (a) : (b))
-#define inactive_target inactive_min((memory_pressure >> INACTIVE_SHIFT), \
-		(num_physpages / 4))
+#define inactive_target inactive_min(((memory_pressure >> INACTIVE_SHIFT), \
+			(num_physpages / 4))
 
 /*
  * Ugly ugly ugly HACK to make sure the inactive lists