Containers: Pagecache accounting and control subsystem (v4)

@@ -24,6 +24,8 @@ struct rss_container {
 	 * the counter to account for RSS
 	 */
 	struct res_counter res;
+	/* the counter to account for pagecache pages */
+ 	struct res_counter pagecache_res;
 	/*
 	 * the lists of pages within the container.
 	 * actually these lists store the page containers (see below), not

@@ -347,6 +349,7 @@ static int rss_create(struct container_s
 		return -ENOMEM;
 
 	res_counter_init(&rss->res);
+	res_counter_init(&rss->pagecache_res);
 	INIT_LIST_HEAD(&rss->inactive_list);
 	INIT_LIST_HEAD(&rss->active_list);
 	rss_container_attach(rss, cont);

@@ -359,6 +362,21 @@ static void rss_destroy(struct container
 	kfree(rss_from_cont(cont));
 }
 
+static ssize_t pagecache_read(struct container *cont, struct cftype *cft,
+		struct file *file, char __user *userbuf,
+		size_t nbytes, loff_t *ppos)
+{
+	return res_counter_read(&rss_from_cont(cont)->pagecache_res,
+			cft->private, userbuf, nbytes, ppos);
+}
+
+static ssize_t pagecache_write(struct container *cont, struct cftype *cft,
+		struct file *file, const char __user *userbuf,
+		size_t nbytes, loff_t *ppos)
+{
+	return res_counter_write(&rss_from_cont(cont)->pagecache_res,
+			cft->private, userbuf, nbytes, ppos);
+}
 
 static ssize_t rss_read(struct container *cont, struct cftype *cft,
 		struct file *file, char __user *userbuf,

@@ -418,6 +436,25 @@ static struct cftype rss_reclaimed = {
 	.read = rss_read_reclaimed,
 };
 
+static struct cftype pagecache_usage = {
+	.name = "pagecache_usage",
+	.private = RES_USAGE,
+	.read = pagecache_read,
+};
+
+static struct cftype pagecache_limit = {
+	.name = "pagecache_limit",
+	.private = RES_LIMIT,
+	.read = pagecache_read,
+	.write = pagecache_write,
+};
+
+static struct cftype pagecache_failcnt = {
+	.name = "pagecache_failcnt",
+	.private = RES_FAILCNT,
+	.read = pagecache_read,
+};
+
 static int rss_populate(struct container_subsys *ss,
 		struct container *cont)
 {

@@ -431,6 +468,12 @@ static int rss_populate(struct container
 		return rc;
 	if ((rc = container_add_file(cont, &rss_reclaimed)) < 0)
 		return rc;
+	if ((rc = container_add_file(cont, &pagecache_usage)) < 0)
+		return rc;
+	if ((rc = container_add_file(cont, &pagecache_failcnt)) < 0)
+		return rc;
+	if ((rc = container_add_file(cont, &pagecache_limit)) < 0)
+		return rc;
 
 	return 0;
 }

@@ -12,6 +12,14 @@
 struct page_container;
 struct rss_container;
 
+/* Flags for container_page_xxx calls */
+#define PAGE_TYPE_RSS		0x00000001
+#define PAGE_TYPE_PAGECACHE	0x00000002
+#define PAGE_TYPE_SWAPCACHE	0x00000004
+
+#define PAGE_SUBTYPE_ANON	0x00010000
+#define PAGE_SUBTYPE_FILE	0x00020000
+
 #ifdef CONFIG_RSS_CONTAINER
 /*
  * This is how the RSS accounting works.

@@ -68,11 +76,12 @@ struct rss_container;
  *
  */
 
-int container_rss_prepare(struct page *, struct vm_area_struct *vma,
-		struct page_container **);
-void container_rss_add(struct page_container *);
-void container_rss_del(struct page_container *);
-void container_rss_release(struct page_container *);
+int container_page_prepare(struct page *page, struct mm_struct *mm,
+		struct page_container **ppc, unsigned int flags);
+
+void container_page_add(struct page_container *, unsigned int flags);
+void container_page_del(struct page_container *, unsigned int flags);
+void container_page_release(struct page_container *, unsigned int flags);
 
 void container_out_of_memory(struct rss_container *);
 

@@ -92,15 +101,15 @@ static inline int container_rss_prepare(
 	return 0;
 }
 
-static inline void container_rss_add(struct page_container *pc)
+static inline void container_page_add(struct page_container *pc, unsigned int flags)
 {
 }
 
-static inline void container_rss_del(struct page_container *pc)
+static inline void container_page_del(struct page_container *pc, unsigned int flags)
 {
 }
 
-static inline void container_rss_release(struct page_container *pc)
+static inline void container_page_release(struct page_container *pc, unsigned int flags)
 {
 }
 

223 lines not on this page · raw message

@@ -321,7 +321,7 @@ void install_arg_page(struct vm_area_str
 	if (unlikely(anon_vma_prepare(vma)))
 		goto out;
 
-	if (container_rss_prepare(page, vma, &pcont))
+	if (container_page_prepare(page, vma->vm_mm, &pcont, PAGE_TYPE_RSS))
 		goto out;
 
 	flush_dcache_page(page);

@@ -343,7 +343,7 @@ void install_arg_page(struct vm_area_str
 	return;
 
 out_release:
-	container_rss_release(pcont);
+	container_page_release(pcont, PAGE_TYPE_RSS);
 out:
 	__free_page(page);
 	force_sig(SIGKILL, current);

@@ -30,6 +30,7 @@
 #include <linux/security.h>
 #include <linux/syscalls.h>
 #include <linux/cpuset.h>
+#include <linux/rss_container.h>
 #include "filemap.h"
 #include "internal.h"
 

@@ -117,6 +118,9 @@ void __remove_from_page_cache(struct pag
 	struct address_space *mapping = page->mapping;
 
 	radix_tree_delete(&mapping->page_tree, page->index);
+	/* Uncharge before the mapping is gone */
+	if (page_container(page))
+		container_page_del(page_container(page), PAGE_TYPE_PAGECACHE);
 	page->mapping = NULL;
 	mapping->nrpages--;
 	__dec_zone_page_state(page, NR_FILE_PAGES);

@@ -440,6 +444,8 @@ int add_to_page_cache(struct page *page,
 		pgoff_t offset, gfp_t gfp_mask)
 {
 	int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
+	struct page_container *pc;
+	struct mm_struct *mm;
 
 	if (error == 0) {
 		write_lock_irq(&mapping->tree_lock);

@@ -453,6 +459,18 @@ int add_to_page_cache(struct page *page,
 			__inc_zone_page_state(page, NR_FILE_PAGES);
 		}
 		write_unlock_irq(&mapping->tree_lock);
+		/* Unlock before charge, because we may reclaim this inline */
+		if(!error) {
+			if (current->mm)
+				mm = current->mm;
+			else
+				mm = &init_mm;
+			if (!container_page_prepare(page, mm, &pc, PAGE_TYPE_PAGECACHE))
+				container_page_add(pc, PAGE_TYPE_PAGECACHE);
+			else
+				BUG();
+		}
+
 		radix_tree_preload_end();
 	}
 	return error;

@@ -1755,7 +1755,7 @@ gotten:
 		cow_user_page(new_page, old_page, address, vma);
 	}
 
-	if (container_rss_prepare(new_page, vma, &pcont))
+	if (container_page_prepare(new_page, vma->vm_mm, &pcont, PAGE_TYPE_RSS))
 		goto oom;
 
 	/*

@@ -1791,7 +1791,7 @@ gotten:
 		new_page = old_page;
 		ret |= VM_FAULT_WRITE;
 	} else
-		container_rss_release(pcont);
+		container_page_release(pcont, PAGE_TYPE_RSS);
 
 	if (new_page)
 		page_cache_release(new_page);

@@ -2217,7 +2217,7 @@ static int do_swap_page(struct mm_struct
 		count_vm_event(PGMAJFAULT);
 	}
 
-	if (container_rss_prepare(page, vma, &pcont)) {
+	if (container_page_prepare(page, vma->vm_mm, &pcont, PAGE_TYPE_RSS)) {
 		ret = VM_FAULT_OOM;
 		goto out;
 	}

@@ -2235,7 +2235,7 @@ static int do_swap_page(struct mm_struct
 
 	if (unlikely(!PageUptodate(page))) {
 		ret = VM_FAULT_SIGBUS;
-		container_rss_release(pcont);
+		container_page_release(pcont, PAGE_TYPE_RSS);
 		goto out_nomap;
 	}
 

@@ -2271,7 +2271,7 @@ unlock:
 out:
 	return ret;
 out_nomap:
-	container_rss_release(pcont);
+	container_page_release(pcont, PAGE_TYPE_RSS);
 	pte_unmap_unlock(page_table, ptl);
 	unlock_page(page);
 	page_cache_release(page);

@@ -2302,7 +2302,7 @@ static int do_anonymous_page(struct mm_s
 		if (!page)
 			goto oom;
 
-		if (container_rss_prepare(page, vma, &pcont))
+		if (container_page_prepare(page, vma->vm_mm, &pcont, PAGE_TYPE_RSS))
 			goto oom_release;
 
 		entry = mk_pte(page, vma->vm_page_prot);

@@ -2338,7 +2338,7 @@ unlock:
 	pte_unmap_unlock(page_table, ptl);
 	return VM_FAULT_MINOR;
 release_container:
-	container_rss_release(pcont);
+	container_page_release(pcont, PAGE_TYPE_RSS);
 release:
 	page_cache_release(page);
 	goto unlock;

@@ -2442,7 +2442,7 @@ static int __do_fault(struct mm_struct *
 
 	}
 
-	if (container_rss_prepare(page, vma, &pcont)) {
+	if (container_page_prepare(page, vma->vm_mm, &pcont, PAGE_TYPE_RSS)) {
 		fdata.type = VM_FAULT_OOM;
 		goto out;
 	}

@@ -2483,7 +2483,7 @@ static int __do_fault(struct mm_struct *
 		update_mmu_cache(vma, address, entry);
 		lazy_mmu_prot_update(entry);
 	} else {
-		container_rss_release(pcont);
+		container_page_release(pcont, PAGE_TYPE_RSS);
 		if (anon)
 			page_cache_release(page);
 		else

@@ -159,7 +159,7 @@ static void remove_migration_pte(struct
  		return;
  	}
 
-	if (container_rss_prepare(new, vma, &pcont)) {
+	if (container_page_prepare(new, vma->vm_mm, &pcont, PAGE_TYPE_RSS)) {
 		pte_unmap(ptep);
 		return;
 	}

@@ -194,7 +194,7 @@ static void remove_migration_pte(struct
 
 out:
 	pte_unmap_unlock(ptep, ptl);
-	container_rss_release(pcont);
+	container_page_release(pcont, PAGE_TYPE_RSS);
 }
 
 /*

@@ -578,14 +578,14 @@ void page_add_anon_rmap(struct page *pag
 	if (atomic_inc_and_test(&page->_mapcount)) {
 		__page_set_anon_rmap(page, vma, address);
 		/* 0 -> 1 state change */
-		container_rss_add(pcont);
+		container_page_add(pcont, PAGE_TYPE_RSS|PAGE_SUBTYPE_ANON);
 	} else {
 		__page_check_anon_rmap(page, vma, address);
 		/*
 		 * we raced with another touch or just mapped the page
 		 * for the N-th time
 		 */
-		container_rss_release(pcont);
+		container_page_release(pcont, PAGE_TYPE_RSS|PAGE_SUBTYPE_ANON);
 	}
 }
 

@@ -606,7 +606,7 @@ void page_add_new_anon_rmap(struct page
 {
 	BUG_ON(address < vma->vm_start || address >= vma->vm_end);
 	atomic_set(&page->_mapcount, 0); /* elevate count by 1 (starts at -1) */
-	container_rss_add(pcont);
+	container_page_add(pcont, PAGE_TYPE_RSS|PAGE_SUBTYPE_ANON);
 	__page_set_anon_rmap(page, vma, address);
 }
 

@@ -628,10 +628,10 @@ void page_add_file_rmap(struct page *pag
 			 * are not added to the container as they do not imply
 			 * RSS consumption  --xemul
 			 */
-			container_rss_add(pcont);
+			container_page_add(pcont, PAGE_TYPE_RSS|PAGE_SUBTYPE_FILE);
 		__inc_zone_page_state(page, NR_FILE_MAPPED);
 	} else if (pcont)
-		container_rss_release(pcont);
+		container_page_release(pcont, PAGE_TYPE_RSS|PAGE_SUBTYPE_FILE);
 }
 
 #ifdef CONFIG_DEBUG_VM

@@ -689,7 +689,7 @@ void page_remove_rmap(struct page *page,
 		}
 
 		if (pcont)
-			container_rss_del(pcont);
+			container_page_del(pcont, PAGE_TYPE_RSS);
 		/*
 		 * It would be tidy to reset the PageAnon mapping here,
 		 * but that might overwrite a racing page_add_anon_rmap

@@ -17,6 +17,7 @@
 #include <linux/backing-dev.h>
 #include <linux/pagevec.h>
 #include <linux/migrate.h>
+#include <linux/rss_container.h>
 
 #include <asm/pgtable.h>
 

@@ -74,6 +75,8 @@ static int __add_to_swap_cache(struct pa
 			       gfp_t gfp_mask)
 {
 	int error;
+	struct page_container *pc;
+	struct mm_struct *mm;
 
 	BUG_ON(PageSwapCache(page));
 	BUG_ON(PagePrivate(page));

@@ -92,6 +95,17 @@ static int __add_to_swap_cache(struct pa
 		}
 		write_unlock_irq(&swapper_space.tree_lock);
 		radix_tree_preload_end();
+		/* Unlock before charge, because we may reclaim this inline */
+		if(!error) {
+			if (current->mm)
+				mm = current->mm;
+			else
+				mm = &init_mm;
+			if (!container_page_prepare(page, mm, &pc, PAGE_TYPE_PAGECACHE))
+				container_page_add(pc, PAGE_TYPE_PAGECACHE);
+			else
+				BUG();
+		}
 	}
 	return error;
 }

@@ -130,6 +144,8 @@ void __delete_from_swap_cache(struct pag
 	BUG_ON(PagePrivate(page));
 
 	radix_tree_delete(&swapper_space.page_tree, page_private(page));
+	if (page_container(page))
+		container_page_del(page_container(page), PAGE_TYPE_PAGECACHE);
 	set_page_private(page, 0);
 	ClearPageSwapCache(page);
 	total_swapcache_pages--;

@@ -535,7 +535,7 @@ static int unuse_pte_range(struct vm_are
 	int found = 0;
 	struct page_container *pcont;
 
-	if (container_rss_prepare(page, vma, &pcont))
+	if (container_page_prepare(page, vma->vm_mm, &pcont, PAGE_TYPE_RSS))
 		return 0;
 
 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);

@@ -552,7 +552,7 @@ static int unuse_pte_range(struct vm_are
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 	pte_unmap_unlock(pte - 1, ptl);
 	if (!found)
-		container_rss_release(pcont);
+		container_page_release(pcont, PAGE_TYPE_RSS);
 	return found;
 }
 

@@ -274,9 +274,6 @@ void container_rss_move_lists(struct pag
 	struct rss_container *rss;
 	struct page_container *pc;
 
-	if (!page_mapped(pg))
-		return;
-
 	pc = page_container(pg);
 	if (pc == NULL)
 		return;