Skip to contentSkip to conversation

nommu: add anonymous page memcg accounting

PATCH RESEND Superseded Steven J. Magnani 4 participants 9 messages Oct 1, 2010 – Oct 4, 2010

Subsystems (2)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH RESEND 1/1 nommu: add anonymous page memcg accounting

Add the necessary calls to track VM anonymous page usage.

---

mm/nommu.c +29 -5
diff -uprN a/mm/nommu.c b/mm/nommu.c
@@ -524,8 +524,10 @@ static void delete_nommu_region(struct v
 /*
  * free a contiguous series of pages
  */
-static void free_page_series(unsigned long from, unsigned long to)
+static void free_page_series(unsigned long from, unsigned long to,
+			     const struct file *file)
 {
+	mem_cgroup_uncharge_start();
 	for (; from < to; from += PAGE_SIZE) {
 		struct page *page = virt_to_page(from);
 
@@ -534,8 +536,12 @@ static void free_page_series(unsigned lo
 		if (page_count(page) != 1)
 			kdebug("free page %p: refcount not one: %d",
 			       page, page_count(page));
+		if (!file)
+			mem_cgroup_uncharge_page(page);
+
 		put_page(page);
 	}
+	mem_cgroup_uncharge_end();
 }
 
 /*
@@ -563,7 +569,8 @@ static void __put_nommu_region(struct vm
 		 * from ramfs/tmpfs mustn't be released here */
 		if (region->vm_flags & VM_MAPPED_COPY) {
 			kdebug("free series");
-			free_page_series(region->vm_start, region->vm_top);
+			free_page_series(region->vm_start, region->vm_top,
+					 region->vm_file);
 		}
 		kmem_cache_free(vm_region_jar, region);
 	} else {
@@ -1117,9 +1124,26 @@ static int do_mmap_private(struct vm_are
 		set_page_refcounted(&pages[point]);
 
 	base = page_address(pages);
-	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
+
 	region->vm_start = (unsigned long) base;
 	region->vm_end   = region->vm_start + rlen;
+
+	if (!vma->vm_file) {
+		for (point = 0; point < total; point++) {
+			int charge_failed =
+				mem_cgroup_newpage_charge(&pages[point],
+							  current->mm,
+							  GFP_KERNEL);
+			if (charge_failed) {
+				free_page_series(region->vm_start,
+						 region->vm_end, NULL);
+				region->vm_start = region->vm_end = 0;
+				goto enomem;
+			}
+		}
+	}
+
+	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
 	region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
 
 	vma->vm_start = region->vm_start;
@@ -1150,7 +1174,7 @@ static int do_mmap_private(struct vm_are
 	return 0;
 
 error_free:
-	free_page_series(region->vm_start, region->vm_end);
+	free_page_series(region->vm_start, region->vm_end, vma->vm_file);
 	region->vm_start = vma->vm_start = 0;
 	region->vm_end   = vma->vm_end = 0;
 	region->vm_top   = 0;
@@ -1555,7 +1579,7 @@ static int shrink_vma(struct mm_struct *
 	add_nommu_region(region);
 	up_write(&nommu_region_sem);
 
-	free_page_series(from, to);
+	free_page_series(from, to, vma->vm_file);
 	return 0;
 }
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#1 lore raw
Quoted 1 line · Steven J. Magnani

Add the necessary calls to track VM anonymous page usage.

Do we really need to do memcg accounting in NOMMU mode? Might it be better to just apply the attached patch instead?

David
---

init/Kconfig +1 -1
index 2de5b1c..aecff10 100644
@@ -555,7 +555,7 @@ config RESOURCE_COUNTERS
 
 config CGROUP_MEM_RES_CTLR
 	bool "Memory Resource Controller for Control Groups"
-	depends on CGROUPS && RESOURCE_COUNTERS
+	depends on CGROUPS && RESOURCE_COUNTERS && MMU
 	select MM_OWNER
 	help
 	  Provides a memory resource controller that manages both anonymous

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#2 lore raw
Quoted 11 lines · David Howells

Do we really need to do memcg accounting in NOMMU mode? Might it be better to just apply the attached patch instead?

David
---

init/Kconfig +0 -0
index 2de5b1c..aecff10 100644

init/Kconfig config RESOURCE_COUNTERS · line 561

Quoted 8 lines
init/Kconfig +1 -1
@@ -555,7 +555,7 @@ config RESOURCE_COUNTERS
 
 config CGROUP_MEM_RES_CTLR
 	bool "Memory Resource Controller for Control Groups"
-	depends on CGROUPS && RESOURCE_COUNTERS
+	depends on CGROUPS && RESOURCE_COUNTERS && MMU
 	select MM_OWNER
 	help
 	  Provides a memory resource controller that manages both anonymous

If anything I think nommu is one of the better applications of memcg. Since nommu typically == embedded, being able to put potential memory pigs in a sandbox so they can't destabilize the system is a Good Thing. That was my motivation for doing this in the first place and it works quite well.

If it would be better to make nommu memcg contingent on some new Kconfig option, we can do that.

Steve

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#3 lore raw
Quoted 4 lines · Steve Magnani

If anything I think nommu is one of the better applications of memcg. Since nommu typically embedded, being able to put potential memory pigs in a sandbox so they can't destabilize the system is a Good Thing. That was my motivation for doing this in the first place and it works quite well.

I suspect it's not useful for a few reasons:

(1) You don't normally run many applications on a NOMMU system. Typically, you'll run just one, probably threaded app, I think.

(2) In general, you won't be able to cull processes to make space. If the OOM killer runs your application has a bug in it.

(3) memcg has a huge overhead. 20 bytes per page! On a 4K page 32-bit system, that's nearly 5% of your RAM, assuming I understand the CGROUP_MEM_RES_CTLR config help text correctly.

(4) There's no swapping, no page faults, no migration and little shareable memory. Being able to allocate large blocks of contiguous memory is much more important and much more of a bottleneck than this. The 5% of RAM lost makes that just that little bit harder.

If it's memory sandboxing you require, ulimit might be sufficient for NOMMU mode.

However, I suppose there's little harm in letting the patch in. I would guess the additions all optimise away if memcg isn't enabled.

A question for you: why does struct page_cgroup need a page pointer? If an array of page_cgroup structs is allocated per array of page structs, then you should be able to use the array index to map between them.

David

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#4 lore raw
Quoted 11 lines · David Howells
Quoted 4 lines · Steve Magnani

If anything I think nommu is one of the better applications of memcg. Since nommu typically embedded, being able to put potential memory pigs in a sandbox so they can't destabilize the system is a Good Thing. That was my motivation for doing this in the first place and it works quite well.

I suspect it's not useful for a few reasons:

(1) You don't normally run many applications on a NOMMU system. Typically, you'll run just one, probably threaded app, I think.

Not always.

Quoted 3 lines

(2) In general, you won't be able to cull processes to make space. If the OOM killer runs your application has a bug in it.

Not always. Every now and then applications have to deal with user-supplied input of some sort.

In our case it's a user-formatted disk drive that can have some arbitrarily-sized FAT32 partition on which we are required to run dosfsck. Now, dosfsck is the epitome of a memory pig; its memory requirements scale with partition size, number of dentries, and any damage encountered - none of which can be predicted. There is a set of partitions we are able to check with no problem, but no guarantee the user won't present us with one that would bring down the whole system, were the OOM killer to get involved. Putting just dosfsck in its own sandbox ensures this can't happen. See also my response to #4 below.

Quoted 4 lines

(3) memcg has a huge overhead. 20 bytes per page! On a 4K page 32-bit system, that's nearly 5% of your RAM, assuming I understand the CGROUP_MEM_RES_CTLR config help text correctly.

When you use 16K pages, 20 bytes/page isn't so huge :)

Quoted 8 lines

(4) There's no swapping, no page faults, no migration and little shareable memory. Being able to allocate large blocks of contiguous memory is much more important and much more of a bottleneck than this. The 5% of RAM lost makes that just that little bit harder.

If it's memory sandboxing you require, ulimit might be sufficient for NOMMU mode.

dosfsck is written to handle memory allocation failures properly (bailing out) but I have not been able to get this code to execute when the system runs out of memory - the OOM killer gets invoked and that's all she wrote. Will a ulimit violation return control back to the process, or terminate it in some graceful manner?

Quoted 7 lines

However, I suppose there's little harm in letting the patch in. I would guess the additions all optimise away if memcg isn't enabled.

A question for you: why does struct page_cgroup need a page pointer? If an array of page_cgroup structs is allocated per array of page structs, then you should be able to use the array index to map between them.

Kame is probably better able to answer this.

Steve

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#5 lore raw
Quoted 55 lines · Steven J. Magnani
Quoted 11 lines · David Howells
Quoted 4 lines · Steve Magnani

If anything I think nommu is one of the better applications of memcg. Since nommu typically embedded, being able to put potential memory pigs in a sandbox so they can't destabilize the system is a Good Thing. That was my motivation for doing this in the first place and it works quite well.

I suspect it's not useful for a few reasons:

(1) You don't normally run many applications on a NOMMU system. Typically, you'll run just one, probably threaded app, I think.

Not always.

Quoted 3 lines

(2) In general, you won't be able to cull processes to make space. If the OOM killer runs your application has a bug in it.

Not always. Every now and then applications have to deal with user-supplied input of some sort.

In our case it's a user-formatted disk drive that can have some arbitrarily-sized FAT32 partition on which we are required to run dosfsck. Now, dosfsck is the epitome of a memory pig; its memory requirements scale with partition size, number of dentries, and any damage encountered - none of which can be predicted. There is a set of partitions we are able to check with no problem, but no guarantee the user won't present us with one that would bring down the whole system, were the OOM killer to get involved. Putting just dosfsck in its own sandbox ensures this can't happen. See also my response to #4 below.

Quoted 4 lines

(3) memcg has a huge overhead. 20 bytes per page! On a 4K page 32-bit system, that's nearly 5% of your RAM, assuming I understand the CGROUP_MEM_RES_CTLR config help text correctly.

When you use 16K pages, 20 bytes/page isn't so huge :)

Quoted 8 lines

(4) There's no swapping, no page faults, no migration and little shareable memory. Being able to allocate large blocks of contiguous memory is much more important and much more of a bottleneck than this. The 5% of RAM lost makes that just that little bit harder.

If it's memory sandboxing you require, ulimit might be sufficient for NOMMU mode.

dosfsck is written to handle memory allocation failures properly (bailing out) but I have not been able to get this code to execute when the system runs out of memory - the OOM killer gets invoked and that's all she wrote. Will a ulimit violation return control back to the process, or terminate it in some graceful manner?

Quoted 1 line
Quoted 9 lines
Quoted 6 lines

However, I suppose there's little harm in letting the patch in. I would guess the additions all optimise away if memcg isn't enabled.

A question for you: why does struct page_cgroup need a page pointer? If an array of page_cgroup structs is allocated per array of page structs, then you should be able to use the array index to map between them.

Kame is probably better able to answer this.

To answer David's question: We have no notion of pfn in page_cgroup, how do we do the indexing? BTW, we are moving to cgroup ids that will take just 16 bits instead of 64 on a 64 bit system.

Signature · 8 lines
	Three Cheers,
	Balbir

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#6 lore raw
In reply to Steven J. Magnani
Quoted 7 lines · On Fri, 01 Oct 2010 11:41:07 -0500 "Steven J. Magnani" <[email protected]>…
Quoted 6 lines

However, I suppose there's little harm in letting the patch in. I would guess the additions all optimise away if memcg isn't enabled.

A question for you: why does struct page_cgroup need a page pointer? If an array of page_cgroup structs is allocated per array of page structs, then you should be able to use the array index to map between them.

No reason. It was not array in the 1st implemenation and ->page still remains. At 2nd implementation, I didn't know embeded people has any interests on memcg. And I wasn't sure how
page_cgroup_to_page() : pfn_to_page(pagec_cgroup_to_pfn(pc))
will be widely used.

Now, we know page_cgroup->page is not used in very critical path _if_ node-id and zone-id can be directly got from page_cgroup.

I'm now preparing a patch to remove struct page* pointer. I'm wondering whether it's ok that some architecuture cannot drop struct page pointer. If SPARSEMEM is used on 32bit arch, I'm not sure whether # of bits isn't enough. I may have to add overhead to get nid, zid in critical path.
(for example, s390/32bit, x86-32/HIGHMEM, ARM/HIGHMEM?)

Current out priority is supporting dirty_ratio rather than memory usage diet. Please wait. Removing page_cgroup->page patch will add something a bit complex.

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#7 lore raw
In reply to Steve Magnani
Quoted 13 lines · Steve Magnani
Quoted 12 lines · David Howells

Do we really need to do memcg accounting in NOMMU mode? Might it be better to just apply the attached patch instead?

David
---

init/Kconfig +0 -0
index 2de5b1c..aecff10 100644
@@ -555,7 +555,7 @@ config RESOURCE_COUNTERS
 
Quoted 12 lines

init/Kconfig config RESOURCE_COUNTERS · line 561

Quoted 7 lines
init/Kconfig +1 -1
 config CGROUP_MEM_RES_CTLR
 	bool "Memory Resource Controller for Control Groups"
-	depends on CGROUPS && RESOURCE_COUNTERS
+	depends on CGROUPS && RESOURCE_COUNTERS && MMU
 	select MM_OWNER
 	help
 	  Provides a memory resource controller that manages both anonymous

If anything I think nommu is one of the better applications of memcg. Since nommu typically == embedded, being able to put potential memory pigs in a sandbox so they can't destabilize the system is a Good Thing. That was my motivation for doing this in the first place and it works quite well.

Good to know, but I want to point out that I never explictly tested it for NOMMU when I created memcg. I thought like the rest that not having reclaim capability would limit memcg usage in the NOMMU world.

Signature · 8 lines
	Three Cheers,
	Balbir

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#8 lore raw
In reply to Steven J. Magnani
Quoted 2 lines · Steven J. Magnani

Add the necessary calls to track VM anonymous page usage.

Is there a motivation to have these changes for nommu as well?

Quoted 18 lines

---
diff -uprN a/mm/nommu.c b/mm/nommu.c

mm/nommu.c +3 -1
@@ -524,8 +524,10 @@ static void delete_nommu_region(struct v
 /*
  * free a contiguous series of pages
  */
-static void free_page_series(unsigned long from, unsigned long to)
+static void free_page_series(unsigned long from, unsigned long to,
+			     const struct file *file)
 {
+	mem_cgroup_uncharge_start();
 	for (; from < to; from += PAGE_SIZE) {
 		struct page *page = virt_to_page(from);
 

mm/nommu.c static void free_page_series(unsigned lo · line 541

Quoted 6 lines
mm/nommu.c +3 -0
@@ -534,8 +536,12 @@ static void free_page_series(unsigned lo
 		if (page_count(page) != 1)
 			kdebug("free page %p: refcount not one: %d",
 			       page, page_count(page));
+		if (!file)
+			mem_cgroup_uncharge_page(page);
+

I don't understand the if !file bits, do you want to selectively enable memcg for !file for nommu?

Quoted 55 lines
mm/nommu.c +22 -3
 		put_page(page);
 	}
+	mem_cgroup_uncharge_end();
 }
 
 /*
@@ -563,7 +569,8 @@ static void __put_nommu_region(struct vm
 		 * from ramfs/tmpfs mustn't be released here */
 		if (region->vm_flags & VM_MAPPED_COPY) {
 			kdebug("free series");
-			free_page_series(region->vm_start, region->vm_top);
+			free_page_series(region->vm_start, region->vm_top,
+					 region->vm_file);
 		}
 		kmem_cache_free(vm_region_jar, region);
 	} else {
@@ -1117,9 +1124,26 @@ static int do_mmap_private(struct vm_are
 		set_page_refcounted(&pages[point]);
 
 	base = page_address(pages);
-	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
+
 	region->vm_start = (unsigned long) base;
 	region->vm_end   = region->vm_start + rlen;
+
+	if (!vma->vm_file) {
+		for (point = 0; point < total; point++) {
+			int charge_failed =
+				mem_cgroup_newpage_charge(&pages[point],
+							  current->mm,
+							  GFP_KERNEL);
+			if (charge_failed) {
+				free_page_series(region->vm_start,
+						 region->vm_end, NULL);
+				region->vm_start = region->vm_end = 0;
+				goto enomem;
+			}
+		}
+	}
+
+	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
 	region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
 
 	vma->vm_start = region->vm_start;
@@ -1150,7 +1174,7 @@ static int do_mmap_private(struct vm_are
 	return 0;
 
 error_free:
-	free_page_series(region->vm_start, region->vm_end);
+	free_page_series(region->vm_start, region->vm_end, vma->vm_file);
 	region->vm_start = vma->vm_start = 0;
 	region->vm_end   = vma->vm_end = 0;
 	region->vm_top   = 0;

mm/nommu.c static int shrink_vma(struct mm_struct * · line 1585

Quoted 8 lines
mm/nommu.c +1 -1
@@ -1555,7 +1579,7 @@ static int shrink_vma(struct mm_struct *
 	add_nommu_region(region);
 	up_write(&nommu_region_sem);
 
-	free_page_series(from, to);
+	free_page_series(from, to, vma->vm_file);
 	return 0;
 }
 

Could you please add more details on your use and update the Documentation/cgroups/memory.txt file as well.

Signature · 8 lines
	Three Cheers,
	Balbir

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>