From: "Jan Beulich" Subject: [PATCH] linux: allocate and free cold pages Date: Fri, 08 Feb 2008 15:20:19 +0000 Message-ID: <47AC8143.76E4.0078.0@novell.com> Mime-Version: 1.0 Content-Type: text/plain; charset=US-ASCII Content-Transfer-Encoding: quoted-printable Return-path: Content-Disposition: inline List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xensource.com Errors-To: xen-devel-bounces@lists.xensource.com To: xen-devel@lists.xensource.com List-Id: xen-devel@lists.xenproject.org To reduce the performance side effects of ballooning, use and return cold pages. To limit the impact scrubbing of these (and other) pages has on the cache, also implement a dedicated scrubbing function on x86 which uses non-temporal stores (when available). As usual, written and tested on 2.6.24 and made apply to the 2.6.18 tree without further testing. Signed-off-by: Jan Beulich Index: head-2008-01-28/arch/i386/lib/Makefile =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- head-2008-01-28.orig/arch/i386/lib/Makefile 2008-01-24 23:58:37.0000000= 00 +0100 +++ head-2008-01-28/arch/i386/lib/Makefile 2008-02-08 10:46:49.0000000= 00 +0100 @@ -7,3 +7,4 @@ include ${srctree}/arch/x86/lib/Makefile bitops.o =20 lib-$(CONFIG_X86_USE_3DNOW) +=3D mmx.o +lib-$(CONFIG_XEN_SCRUB_PAGES) +=3D scrub.o Index: head-2008-01-28/arch/i386/lib/scrub.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ head-2008-01-28/arch/i386/lib/scrub.c 2008-02-08 12:30:51.0000000= 00 +0100 @@ -0,0 +1,21 @@ +#include +#include +#include + +void scrub_pages(void *v, unsigned int count) +{ + if (likely(cpu_has_xmm2)) { + unsigned long n =3D count * (PAGE_SIZE / sizeof(long) / = 4); + + for (; n--; v +=3D sizeof(long) * 4) + asm("movnti %1,(%0)\n\t" + "movnti %1,%c2(%0)\n\t" + "movnti %1,2*%c2(%0)\n\t" + "movnti %1,3*%c2(%0)\n\t" + : : "r" (v), "r" (0L), "i" (sizeof(long)) + : "memory"); + asm volatile("sfence" : : : "memory"); + } else + for (; count--; v +=3D PAGE_SIZE) + clear_page(v); +} Index: head-2008-01-28/arch/i386/mm/hypervisor.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- head-2008-01-28.orig/arch/i386/mm/hypervisor.c 2008-02-08 = 14:29:22.000000000 +0100 +++ head-2008-01-28/arch/i386/mm/hypervisor.c 2008-02-08 11:30:34.0000000= 00 +0100 @@ -281,7 +281,7 @@ int xen_create_contiguous_region( set_xen_guest_handle(exchange.in.extent_start, in_frames); set_xen_guest_handle(exchange.out.extent_start, &out_frame); =20 - scrub_pages(vstart, 1 << order); + scrub_pages((void *)vstart, 1 << order); =20 balloon_lock(flags); =20 @@ -374,7 +374,7 @@ void xen_destroy_contiguous_region(unsig set_xen_guest_handle(exchange.in.extent_start, &in_frame); set_xen_guest_handle(exchange.out.extent_start, out_frames); =20 - scrub_pages(vstart, 1 << order); + scrub_pages((void *)vstart, 1 << order); =20 balloon_lock(flags); =20 Index: head-2008-01-28/arch/x86_64/lib/Makefile =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- head-2008-01-28.orig/arch/x86_64/lib/Makefile 2008-01-24 = 23:58:37.000000000 +0100 +++ head-2008-01-28/arch/x86_64/lib/Makefile 2008-02-08 10:46:49.0000000= 00 +0100 @@ -10,3 +10,4 @@ include ${srctree}/arch/x86/lib/Makefile usercopy.o getuser.o putuser.o \ thunk.o clear_page.o copy_page.o bitstr.o bitops.o lib-y +=3D memcpy.o memmove.o memset.o copy_user.o +lib-$(CONFIG_XEN_SCRUB_PAGES) +=3D scrub.o Index: head-2008-01-28/arch/x86_64/lib/scrub.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ head-2008-01-28/arch/x86_64/lib/scrub.c 2008-02-08 12:30:51.0000000= 00 +0100 @@ -0,0 +1,1 @@ +#include "../../i386/lib/scrub.c" Index: head-2008-01-28/drivers/xen/balloon/balloon.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- head-2008-01-28.orig/drivers/xen/balloon/balloon.c 2008-02-08 = 14:29:22.000000000 +0100 +++ head-2008-01-28/drivers/xen/balloon/balloon.c 2008-02-08 = 14:29:36.000000000 +0100 @@ -105,7 +105,7 @@ static struct timer_list balloon_timer; /* When ballooning out (allocating memory to return to Xen) we don't = really=20 want the kernel to try too hard since that can trigger the oom killer. = */ #define GFP_BALLOON \ - (GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC) + (GFP_HIGHUSER|__GFP_NOWARN|__GFP_NORETRY|__GFP_NOMEMALLOC|__GFP_COL= D) =20 #define PAGE_TO_LIST(p) (&(p)->lru) #define LIST_TO_PAGE(l) list_entry((l), struct page, lru) @@ -175,6 +175,17 @@ static struct page *balloon_next_page(st return LIST_TO_PAGE(next); } =20 +static inline void balloon_free_page(struct page *page) +{ +#ifndef MODULE + if (put_page_testzero(page)) + free_cold_page(page); +#else + /* free_cold_page() is not being exported. */ + __free_page(page); +#endif +} + static void balloon_alarm(unsigned long unused) { schedule_work(&balloon_worker); @@ -292,7 +303,7 @@ static int increase_reservation(unsigned /* Relinquish the page back to the allocator. */ ClearPageReserved(page); init_page_count(page); - __free_page(page); + balloon_free_page(page); } =20 bs.current_pages +=3D nr_pages; @@ -623,7 +634,8 @@ static int dealloc_pte_fn( =20 struct page **alloc_empty_pages_and_pagevec(int nr_pages) { - unsigned long vaddr, flags; + unsigned long flags; + void *v; struct page *page, **pagevec; int i, ret; =20 @@ -632,13 +644,12 @@ struct page **alloc_empty_pages_and_page return NULL; =20 for (i =3D 0; i < nr_pages; i++) { - page =3D pagevec[i] =3D alloc_page(GFP_KERNEL); + page =3D pagevec[i] =3D alloc_page(GFP_KERNEL|__GFP_COLD); if (page =3D=3D NULL) goto err; =20 - vaddr =3D (unsigned long)page_address(page); - - scrub_pages(vaddr, 1); + v =3D page_address(page); + scrub_pages(v, 1); =20 balloon_lock(flags); =20 @@ -656,8 +667,9 @@ struct page **alloc_empty_pages_and_page ret =3D 0; /* success */ } else { #ifdef CONFIG_XEN - ret =3D apply_to_page_range(&init_mm, vaddr, = PAGE_SIZE, - dealloc_pte_fn, NULL); + ret =3D apply_to_page_range(&init_mm, (unsigned = long)v, + PAGE_SIZE, dealloc_pte_fn= , + NULL); #else /* Cannot handle non-auto translate mode. */ ret =3D 1; @@ -668,7 +679,7 @@ struct page **alloc_empty_pages_and_page =20 if (ret !=3D 0) { balloon_unlock(flags); - __free_page(page); + balloon_free_page(page); goto err; } =20 Index: head-2008-01-28/include/asm-i386/mach-xen/asm/hypervisor.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- head-2008-01-28.orig/include/asm-i386/mach-xen/asm/hypervisor.h = 2008-02-08 14:29:22.000000000 +0100 +++ head-2008-01-28/include/asm-i386/mach-xen/asm/hypervisor.h 2008-02-08 = 10:42:13.000000000 +0100 @@ -121,7 +121,7 @@ int xen_limit_pages_to_max_mfn( u64 jiffies_to_st(unsigned long jiffies); =20 #ifdef CONFIG_XEN_SCRUB_PAGES -#define scrub_pages(_p,_n) memset((void *)(_p), 0, (_n) << PAGE_SHIFT) +void scrub_pages(void *, unsigned int); #else #define scrub_pages(_p,_n) ((void)0) #endif