Skip to contentSkip to conversation

Copy on write for hugetlbfs

RFC In review Adam Litke 3 participants 11 messages Nov 7, 2005 – Nov 8, 2005

Subsystems (3)
Memory management Memory mapping HugeTLB
lore
Share & tools
lore

Adam Litke

author
#0 lore raw

cover Copy on write for hugetlbfs

The following two patches implement copy-on-write for hugetlbfs (thus enabling MAP_PRIVATE mappings). Patch 1/2 (previously posted by David Gibson) contains a couple small fixes to the demand fault handler and makes COW fit in nicely. Patch 2/2 is the cow changes. Comments?

Signature · 8 lines
Adam Litke - (agl at us.ibm.com)
IBM Linux Technology Center

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
author
#1 lore raw

RFC 1/2 Hugetlb fault fixes and reorg

[RFC] Cleanup / small fixes to hugetlb fault handling (Patch originally from David Gibson <[email protected]>) Initial Post: Tue. 25 Oct 2005

Quoted 10 lines · David Gibson

This patch makes some slight tweaks / cleanups to the fault handling path for huge pages in -mm. My main motivation is to make it simpler to fit COW in, but along the way it addresses a few minor problems with the existing code:

- The check against i_size was duplicated: once in find_lock_huge_page() and again in hugetlb_fault() after taking the page_table_lock. We only really need the locked one, so remove the other.

Quoted 5 lines

- find_lock_huge_page() isn't a great name, since it does extra things not analagous to find_lock_page(). Rename it find_or_alloc_huge_page() which is closer to the mark.

Signed-off-by David Gibson <[email protected]>
Acked-by Adam Litke <[email protected]>
---
 hugetlb.c |   77 +++++++++++++++++++++++++++++++++++++-------------------------
 1 files changed, 46 insertions(+), 31 deletions(-)
mm/hugetlb.c +46 -31
diff -upN reference/mm/hugetlb.c current/mm/hugetlb.c
@@ -339,30 +339,24 @@ void unmap_hugepage_range(struct vm_area
 	flush_tlb_range(vma, start, end);
 }
 
-static struct page *find_lock_huge_page(struct address_space *mapping,
-			unsigned long idx)
+static struct page *find_or_alloc_huge_page(struct address_space *mapping,
+					    unsigned long idx)
 {
 	struct page *page;
 	int err;
-	struct inode *inode = mapping->host;
-	unsigned long size;
 
 retry:
 	page = find_lock_page(mapping, idx);
 	if (page)
-		goto out;
-
-	/* Check to make sure the mapping hasn't been truncated */
-	size = i_size_read(inode) >> HPAGE_SHIFT;
-	if (idx >= size)
-		goto out;
+		return page;
 
 	if (hugetlb_get_quota(mapping))
-		goto out;
+		return NULL;
+
 	page = alloc_huge_page();
 	if (!page) {
 		hugetlb_put_quota(mapping);
-		goto out;
+		return NULL;
 	}
 
 	err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);

@@ -373,50 +367,49 @@ retry:
 			goto retry;
 		page = NULL;
 	}
-out:
+
 	return page;
 }
 
-int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
-			unsigned long address, int write_access)
+int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
+		    unsigned long address, pte_t *ptep)
 {
-	int ret = VM_FAULT_SIGBUS;
+	int ret;
 	unsigned long idx;
 	unsigned long size;
-	pte_t *pte;
 	struct page *page;
 	struct address_space *mapping;
 
-	pte = huge_pte_alloc(mm, address);
-	if (!pte)
-		goto out;
-
 	mapping = vma->vm_file->f_mapping;
 	idx = ((address - vma->vm_start) >> HPAGE_SHIFT)
 		+ (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
 
-	/*
-	 * Use page lock to guard against racing truncation
-	 * before we get page_table_lock.
-	 */
-	page = find_lock_huge_page(mapping, idx);
+	/* This returns a locked page, which keeps us safe in the
+	 * event of a race with truncate() */
+	page = find_or_alloc_huge_page(mapping, idx);
 	if (!page)
-		goto out;
+		return VM_FAULT_SIGBUS;
 
 	spin_lock(&mm->page_table_lock);
+
+	ret = VM_FAULT_SIGBUS;
+
 	size = i_size_read(mapping->host) >> HPAGE_SHIFT;
 	if (idx >= size)
 		goto backout;
 
 	ret = VM_FAULT_MINOR;
-	if (!pte_none(*pte))
+
+	if (!pte_none(*ptep))
+		/* oops, someone instantiated this PTE before us */
 		goto backout;
 
 	add_mm_counter(mm, file_rss, HPAGE_SIZE / PAGE_SIZE);
-	set_huge_pte_at(mm, address, pte, make_huge_pte(vma, page));
+	set_huge_pte_at(mm, address, ptep, make_huge_pte(vma, page));
+
 	spin_unlock(&mm->page_table_lock);
 	unlock_page(page);
-out:
+
 	return ret;
 
 backout:

@@ -424,7 +417,29 @@ backout:
 	hugetlb_put_quota(mapping);
 	unlock_page(page);
 	put_page(page);
-	goto out;
+
+	return ret;
+}
+
+int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
+		  unsigned long address, int write_access)
+{
+	pte_t *ptep;
+	pte_t entry;
+
+	ptep = huge_pte_alloc(mm, address);
+	if (! ptep)
+		return VM_FAULT_OOM;
+
+	entry = *ptep;
+
+	if (pte_none(entry))
+		return hugetlb_no_page(mm, vma, address, ptep);
+
+	/* we could get here if another thread instantiated the pte
+	 * before the test above */
+
+	return VM_FAULT_MINOR;
 }
 
 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
Signature · 8 lines
Adam Litke - (agl at us.ibm.com)
IBM Linux Technology Center

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#2 lore raw

Re: [RFC 1/2] Hugetlb fault fixes and reorg

mm/hugetlb.c void unmap_hugepage_range(struct vm_area · line 348

Quoted 11 lines · Adam Litke

(Patch originally from David Gibson <[email protected]>) Initial Post: Tue. 25 Oct 2005

mm/hugetlb.c +2 -4
-static struct page *find_lock_huge_page(struct address_space *mapping,
-			unsigned long idx)
+static struct page *find_or_alloc_huge_page(struct address_space *mapping,
+					    unsigned long idx)
 {
 	struct page *page;
 	int err;
-	struct inode *inode = mapping->host;
-	unsigned long size;

This patch is a combination of function renaming, variable initialization/assignment and return path/etc. oddities, plus some functional changes (did I catch them all?) which apparently took a bit of effort to get to after sifting through the rest of that.

Dump the parallel cleanups or split them into pure cleanup and pure functional patches. I don't mind the cleanups, I mind the mixing.

-- wli

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#3 lore raw

Re: [RFC 1/2] Hugetlb fault fixes and reorg

Quoted 17 lines · William Lee Irwin

mm/hugetlb.c void unmap_hugepage_range(struct vm_area · line 348

Quoted 11 lines · Adam Litke

(Patch originally from David Gibson <[email protected]>) Initial Post: Tue. 25 Oct 2005

mm/hugetlb.c +2 -4
-static struct page *find_lock_huge_page(struct address_space *mapping,
-			unsigned long idx)
+static struct page *find_or_alloc_huge_page(struct address_space *mapping,
+					    unsigned long idx)
 {
 	struct page *page;
 	int err;
-	struct inode *inode = mapping->host;
-	unsigned long size;

This patch is a combination of function renaming, variable initialization/assignment and return path/etc. oddities, plus some functional changes (did I catch them all?) which apparently took a bit of effort to get to after sifting through the rest of that.

Functional changes? There shouldn't be...

Quoted 2 lines

Dump the parallel cleanups or split them into pure cleanup and pure functional patches. I don't mind the cleanups, I mind the mixing.

Signature · 10 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
author
#4 lore raw

RFC 2/2 Hugetlb COW

In reply to Adam Litke

[RFC] COW for hugepages
(Patch originally from David Gibson <[email protected]>)

This patch implements copy-on-write for hugepages, hence allowing MAP_PRIVATE mappings of hugetlbfs.

This is chiefly useful for cases where we want to use hugepages "automatically" - that is to map hugepages without the knowledge of the code in the final application (either via kernel hooks, or with LD_PRELOAD). We can use various heuristics to determine when hugepages might be a good idea, but changing the semantics of anonymous memory from MAP_PRIVATE to MAP_SHARED without the app's knowledge is clearly wrong.

---
 fs/hugetlbfs/inode.c    |    3 -
 include/linux/hugetlb.h |   11 ++++
 mm/hugetlb.c            |  113 ++++++++++++++++++++++++++++++++++++++++--------
 mm/mmap.c               |    2 
 4 files changed, 107 insertions(+), 22 deletions(-)
fs/hugetlbfs/inode.c +0 -3
diff -upN reference/fs/hugetlbfs/inode.c current/fs/hugetlbfs/inode.c
@@ -100,9 +100,6 @@ static int hugetlbfs_file_mmap(struct fi
 	loff_t len, vma_len;
 	int ret;
 
-	if ((vma->vm_flags & (VM_MAYSHARE | VM_WRITE)) == VM_WRITE)
-		return -EINVAL;
-
 	if (vma->vm_pgoff & (HPAGE_SIZE / PAGE_SIZE - 1))
 		return -EINVAL;
include/linux/hugetlb.h +11 -0
diff -upN reference/include/linux/hugetlb.h current/include/linux/hugetlb.h
@@ -65,6 +65,17 @@ pte_t huge_ptep_get_and_clear(struct mm_
 			      pte_t *ptep);
 #endif
 
+#define huge_ptep_set_wrprotect(mm, addr, ptep) ptep_set_wrprotect(mm, addr, ptep)
+static inline void set_huge_ptep_writable(struct vm_area_struct *vma,
+		unsigned long address, pte_t *ptep)
+{
+	pte_t entry;
+
+	entry = pte_mkwrite(pte_mkdirty(*ptep));
+	ptep_set_access_flags(vma, address, ptep, entry, 1);
+	update_mmu_cache(vma, address, entry);
+}
+
 #ifndef ARCH_HAS_HUGETLB_PREFAULT_HOOK
 #define hugetlb_prefault_arch_hook(mm)		do { } while (0)
 #else
mm/hugetlb.c +95 -18

183 lines not on this page · raw message

mm/mmap.c +1 -1
diff -upN reference/mm/mmap.c current/mm/mmap.c
@@ -1077,7 +1077,7 @@ munmap_back:
 		error = file->f_op->mmap(file, vma);
 		if (error)
 			goto unmap_and_free_vma;
-		if ((vma->vm_flags & (VM_SHARED | VM_WRITE | VM_RESERVED))
+		if ((vma->vm_flags & (VM_SHARED | VM_WRITE | VM_RESERVED | VM_HUGETLB))
 						== (VM_WRITE | VM_RESERVED)) {
 			printk(KERN_WARNING "program %s is using MAP_PRIVATE, "
 				"PROT_WRITE mmap of VM_RESERVED memory, which "
Signature · 8 lines
Adam Litke - (agl at us.ibm.com)
IBM Linux Technology Center

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
author
#5 lore raw

Re: [RFC 2/2] Hugetlb COW

Quoted 6 lines · Adam Litke

[RFC] COW for hugepages
(Patch originally from David Gibson <[email protected]>)

This patch implements copy-on-write for hugepages, hence allowing MAP_PRIVATE mappings of hugetlbfs.

Quoted 7 lines

This is chiefly useful for cases where we want to use hugepages "automatically" - that is to map hugepages without the knowledge of the code in the final application (either via kernel hooks, or with LD_PRELOAD). We can use various heuristics to determine when hugepages might be a good idea, but changing the semantics of anonymous memory from MAP_PRIVATE to MAP_SHARED without the app's knowledge is clearly wrong.

I forgot to mention in the original post that this patch is currently broken on ppc64 due to a problem with update_mmu_cache(). The proper fix is understood but backed up behind the powerpc merge activity.

Signature · 8 lines
Adam Litke - (agl at us.ibm.com)
IBM Linux Technology Center

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#6 lore raw

Re: [RFC 2/2] Hugetlb COW

Quoted 15 lines · Adam Litke
Quoted 6 lines · Adam Litke

[RFC] COW for hugepages
(Patch originally from David Gibson <[email protected]>)

This patch implements copy-on-write for hugepages, hence allowing MAP_PRIVATE mappings of hugetlbfs.

Quoted 7 lines

This is chiefly useful for cases where we want to use hugepages "automatically" - that is to map hugepages without the knowledge of the code in the final application (either via kernel hooks, or with LD_PRELOAD). We can use various heuristics to determine when hugepages might be a good idea, but changing the semantics of anonymous memory from MAP_PRIVATE to MAP_SHARED without the app's knowledge is clearly wrong.

Quoted 3 lines

I forgot to mention in the original post that this patch is currently broken on ppc64 due to a problem with update_mmu_cache(). The proper fix is understood but backed up behind the powerpc merge activity.

Now that the merge tree and 64k pages have been pulled into mainline I updated this patch, and sent it off to paulus. You'll fnid it under the subject "ppc64: Make hash_preload() and update_mmu_cache() cope with hugepages".

Signature · 10 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#7 lore raw

Re: [RFC 2/2] Hugetlb COW

In reply to Adam Litke
Quoted 15 lines · Adam Litke
Quoted 6 lines · Adam Litke

[RFC] COW for hugepages
(Patch originally from David Gibson <[email protected]>)

This patch implements copy-on-write for hugepages, hence allowing MAP_PRIVATE mappings of hugetlbfs.

Quoted 7 lines

This is chiefly useful for cases where we want to use hugepages "automatically" - that is to map hugepages without the knowledge of the code in the final application (either via kernel hooks, or with LD_PRELOAD). We can use various heuristics to determine when hugepages might be a good idea, but changing the semantics of anonymous memory from MAP_PRIVATE to MAP_SHARED without the app's knowledge is clearly wrong.

Quoted 3 lines

I forgot to mention in the original post that this patch is currently broken on ppc64 due to a problem with update_mmu_cache(). The proper fix is understood but backed up behind the powerpc merge activity.

Actually, BenH just pointed out that the version of update_mmu_cache() we now have from the 64k patch already half-handled this situation - it wouldn't actually do anything on a hugepage fault, but it wouldn't blow up either. So the COW patch should work as is on ppc64, although the update_mmu_cache change is probably an important optimization (otherwise we'll take two faults on every COW - one to actually do the COW, and one to twiddle the bits in the hash page table).

Signature · 10 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#8 lore raw

Re: [RFC 2/2] Hugetlb COW

In reply to Adam Litke
Quoted 11 lines · Adam Litke

[RFC] COW for hugepages
(Patch originally from David Gibson <[email protected]>) This patch implements copy-on-write for hugepages, hence allowing MAP_PRIVATE mappings of hugetlbfs.
This is chiefly useful for cases where we want to use hugepages "automatically" - that is to map hugepages without the knowledge of the code in the final application (either via kernel hooks, or with LD_PRELOAD). We can use various heuristics to determine when hugepages might be a good idea, but changing the semantics of anonymous memory from MAP_PRIVATE to MAP_SHARED without the app's knowledge is clearly wrong.

I'll go check for architectures where page protections may be encoded differently depending on the size of the translation, or whose code is otherwise unprepared to cope with protection bits.

If you've done such checking already, I'd be much obliged to hear of it (in fact, I'd much prefer you to have done so).

-- wli

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#9 lore raw

Re: [RFC 2/2] Hugetlb COW

Quoted 13 lines · William Lee Irwin
Quoted 11 lines · Adam Litke

[RFC] COW for hugepages
(Patch originally from David Gibson <[email protected]>) This patch implements copy-on-write for hugepages, hence allowing MAP_PRIVATE mappings of hugetlbfs.
This is chiefly useful for cases where we want to use hugepages "automatically" - that is to map hugepages without the knowledge of the code in the final application (either via kernel hooks, or with LD_PRELOAD). We can use various heuristics to determine when hugepages might be a good idea, but changing the semantics of anonymous memory from MAP_PRIVATE to MAP_SHARED without the app's knowledge is clearly wrong.

Quoted 6 lines

I'll go check for architectures where page protections may be encoded differently depending on the size of the translation, or whose code is otherwise unprepared to cope with protection bits.

If you've done such checking already, I'd be much obliged to hear of it (in fact, I'd much prefer you to have done so).

I can't see how the COW catch could be any more broken in this regard than we are already: make_huge_pte() in mm/hugetlb.c already assumes that pte_mkwrite() and pte_wrprotect() will work properly on hugepage PTEs. COW doesn't use anything more.

Signature · 10 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#10 lore raw

Re: [RFC 2/2] Hugetlb COW

In reply to Adam Litke
Quoted 2 lines · Adam Litke

[RFC] COW for hugepages
(Patch originally from David Gibson <[email protected]>)

Just realised there's at least one seriouss bug in the patch - find_or_alloc_huge_page() can return a non-locked page when demand faulting a PRIVATE mapping. That will cause the unlock_page() further down to BUG(). Here's a corrected version.

This patch implements copy-on-write for hugepages, hence allowing MAP_SHARED mappings of hugetlbfs.

This is chiefly useful for cases where we want to use hugepages "automatically" - that is to map hugepages without the knowledge of the code in the final application (either via kernel hooks, or with LD_PRELOAD). We can use various heuristics to determine when hugepages might be a good idea, but changing the semantics of anonymous memory from MAP_PRIVATE to MAP_SHARED without the app's knowledge is clearly wrong.

fs/hugetlbfs/inode.c +0 -3
Index: working-2.6/fs/hugetlbfs/inode.c
===================================================================
@@ -100,9 +100,6 @@
 	loff_t len, vma_len;
 	int ret;
 
-	if ((vma->vm_flags & (VM_MAYSHARE | VM_WRITE)) == VM_WRITE)
-		return -EINVAL;
-
 	if (vma->vm_pgoff & (HPAGE_SIZE / PAGE_SIZE - 1))
 		return -EINVAL;
include/linux/hugetlb.h +11 -0
Index: working-2.6/include/linux/hugetlb.h
===================================================================
@@ -65,6 +65,17 @@
 			      pte_t *ptep);
 #endif
 
+#define huge_ptep_set_wrprotect(mm, addr, ptep) ptep_set_wrprotect(mm, addr, ptep)
+static inline void set_huge_ptep_writable(struct vm_area_struct *vma,
+		unsigned long address, pte_t *ptep)
+{
+	pte_t entry;
+
+	entry = pte_mkwrite(pte_mkdirty(*ptep));
+	ptep_set_access_flags(vma, address, ptep, entry, 1);
+	update_mmu_cache(vma, address, entry);
+}
+
 #ifndef ARCH_HAS_HUGETLB_PREFAULT_HOOK
 #define hugetlb_prefault_arch_hook(mm)		do { } while (0)
 #else
mm/hugetlb.c +100 -18

191 lines not on this page · raw message

mm/mmap.c +1 -1
Index: working-2.6/mm/mmap.c
===================================================================
@@ -1076,7 +1076,7 @@
 		error = file->f_op->mmap(file, vma);
 		if (error)
 			goto unmap_and_free_vma;
-		if ((vma->vm_flags & (VM_SHARED | VM_WRITE | VM_RESERVED))
+		if ((vma->vm_flags & (VM_SHARED | VM_WRITE | VM_RESERVED | VM_HUGETLB))
 						== (VM_WRITE | VM_RESERVED)) {
 			printk(KERN_WARNING "program %s is using MAP_PRIVATE, "
 				"PROT_WRITE mmap of VM_RESERVED memory, which "
Signature · 10 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>