shared page table for hugetlb page - v4
Subsystems (4)
PATCH 1/2 shared page table for hugetlb page - v4
Following up with the work on shared page table done by Dave McCracken. This set of patch target shared page table for hugetlb memory only.
The shared page table is particular useful in the situation of large number of independent processes sharing large shared memory segments. In the normal page case, the amount of memory saved from process' page table is quite significant. For hugetlb, the saving on page table memory is not the primary objective (as hugetlb itself already cuts down page table overhead significantly), instead, the purpose of using shared page table on hugetlb is to allow faster TLB refill and smaller cache pollution upon TLB miss.
With PT sharing, pte entries are shared among hundreds of processes, the cache consumption used by all the page table is smaller and in return, application gets much higher cache hit ratio. One other effect is that cache hit ratio with hardware page walker hitting on pte in cache will be higher and this helps to reduce tlb miss latency. These two effects contribute to higher application performance.
fs/hugetlbfs/inode.c renamed from fs/hugetlbfs/inode.c.orig +1 -1
@@ -293,7 +293,7 @@ hugetlb_vmtruncate_list(struct prio_treeif (h_vm_pgoff >= h_pgoff) v_offset =0 ; -unmap_hugepage_range (vma, +__unmap_hugepage_range (vma, vma->vm_start + v_offset, vma->vm_end); } }
include/linux/hugetlb.h renamed from include/linux/hugetlb.h.orig +0 -0
@@ -17,6 +17,7 @@ int hugetlb_sysctl_handler(struct ctl_taint copy_hugetlb_page_range (struct mm_struct *,struct mm_struct *,struct vm_area_struct *); int follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *, struct page **, struct vm_area_struct **, unsigned long *, int
*, int);
void unmap_hugepage_range(struct vm_area_struct *, unsigned long, unsigned long);
+void __unmap_hugepage_range(struct vm_area_struct *, unsigned long, unsigned long);
int hugetlb_prefault(struct address_space *, struct vm_area_struct *); int hugetlb_report_meminfo(char *);
int hugetlb_report_node_meminfo(int, char *);
@@ -34,6 +35,7 @@ extern int sysctl_hugetlb_shm_group;
pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr); pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr);
+int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep);
struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
int write);
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
mm/hugetlb.c renamed from mm/hugetlb.c.orig +19 -2
@@ -356,8 +356,8 @@ nomem:return -ENOMEM ; } -void unmap_hugepage_range (struct vm_area_struct *vma,unsigned long start, -unsigned long end) +void __unmap_hugepage_range (struct vm_area_struct *vma,unsigned long start, +unsigned long end) {struct mm_struct *mm = vma->vm_mm;unsigned long address;
@@ -381,6 +381,9 @@ void unmap_hugepage_range(struct vm_areaif (!ptep)continue ; +if (huge_pmd_unshare (mm, &address, ptep)) +continue ; + pte =huge_ptep_get_and_clear (mm, address, ptep);if (pte_none (pte))continue ;
@@ -398,6 +401,16 @@ void unmap_hugepage_range(struct vm_area } } +void unmap_hugepage_range (struct vm_area_struct *vma,unsigned long start, +unsigned long end) +{ +if (vma->vm_file) { +spin_lock (&vma->vm_file->f_mapping->i_mmap_lock); +__unmap_hugepage_range (vma, start, end); +spin_unlock (&vma->vm_file->f_mapping->i_mmap_lock); + } +} +static int hugetlb_cow (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long address,pte_t *ptep,pte_t pte) {
@@ -632,11 +645,14 @@ void hugetlb_change_protection(struct vmBUG_ON (address >= end);flush_cache_range (vma, address, end); +spin_lock (&vma->vm_file->f_mapping->i_mmap_lock);spin_lock (&mm->page_table_lock);for (; address < end; address +=HPAGE_SIZE ) { ptep =huge_pte_offset (mm, address);if (!ptep)continue ; +if (huge_pmd_unshare (mm, &address, ptep)) +continue ;if (!pte_none (*ptep)) { pte =huge_ptep_get_and_clear (mm, address, ptep); pte =pte_mkhuge (pte_modify (pte, newprot));
@@ -645,6 +661,7 @@ void hugetlb_change_protection(struct vm } }spin_unlock (&mm->page_table_lock); +spin_unlock (&vma->vm_file->f_mapping->i_mmap_lock);flush_tlb_range (vma, start, end); }
arch/i386/mm/hugetlbpage.c renamed from arch/i386/mm/hugetlbpage.c.orig +111 -1
125 lines not on this page · raw message
arch/ia64/mm/hugetlbpage.c renamed from arch/ia64/mm/hugetlbpage.c.orig +5 -0
@@ -64,6 +64,11 @@ huge_pte_offset (struct mm_struct *mm, ureturn pte; } +int huge_pmd_unshare (struct mm_struct *mm,unsigned long *addr,pte_t *ptep) +{ +return 0 ; +} +#define mk_pte_huge (entry) { pte_val(entry) |= _PAGE_P; }/*
arch/powerpc/mm/hugetlbpage.c renamed from arch/powerpc/mm/hugetlbpage.c.orig +5 -0
@@ -146,6 +146,11 @@ pte_t *huge_pte_alloc(struct mm_struct *return hugepte_offset (hpdp, addr); } +int huge_pmd_unshare (struct mm_struct *mm,unsigned long *addr,pte_t *ptep) +{ +return 0 ; +} +static void free_hugepte_range (struct mmu_gather *tlb,hugepd_t *hpdp) {pte_t *hugepte =hugepd_page (*hpdp);
arch/sh/mm/hugetlbpage.c renamed from arch/sh/mm/hugetlbpage.c.orig +5 -0
@@ -63,6 +63,11 @@ pte_t *huge_pte_offset(struct mm_structreturn pte; } +int huge_pmd_unshare (struct mm_struct *mm,unsigned long *addr,pte_t *ptep) +{ +return 0 ; +} +struct page *follow_huge_addr (struct mm_struct *mm,unsigned long address,int write) {
arch/sh64/mm/hugetlbpage.c renamed from arch/sh64/mm/hugetlbpage.c.orig +5 -0
@@ -53,6 +53,11 @@ pte_t *huge_pte_offset(struct mm_structreturn pte; } +int huge_pmd_unshare (struct mm_struct *mm,unsigned long *addr,pte_t *ptep) +{ +return 0 ; +} +void set_huge_pte_at (struct mm_struct *mm,unsigned long addr,pte_t *ptep,pte_t entry) {
arch/sparc64/mm/hugetlbpage.c renamed from arch/sparc64/mm/hugetlbpage.c.orig +5 -0
@@ -235,6 +235,11 @@ pte_t *huge_pte_offset(struct mm_structreturn pte; } +int huge_pmd_unshare (struct mm_struct *mm,unsigned long *addr,pte_t *ptep) +{ +return 0 ; +} +void set_huge_pte_at (struct mm_struct *mm,unsigned long addr,pte_t *ptep,pte_t entry) {
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>