From: "Jan Beulich" Subject: [PATCH 1/5] x86-64: use MFNs for linking together pages on lists Date: Fri, 30 Jan 2009 10:29:22 +0000 Message-ID: <4982E492.76E4.0078.0@novell.com> Mime-Version: 1.0 Content-Type: text/plain; charset=US-ASCII Content-Transfer-Encoding: quoted-printable Return-path: Content-Disposition: inline List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xensource.com Errors-To: xen-devel-bounces@lists.xensource.com To: xen-devel@lists.xensource.com List-Id: xen-devel@lists.xenproject.org Unless more than 16Tb are going to ever be supported in Xen, this will allow reducing the linked list entries in struct page_info from 16 to 8 bytes. This doesn't modify struct shadow_page_info, yet, so in order to meet the constraints of that 'mirror' structure the list entry gets artificially forced to be 16 bytes in size. That workaround will be removed in a subsequent patch. One question related to the special list manipulation methods used in relinquish_memory(): Couldn't relinquish_memory() remove the page to be dealt with from the list and re-add it in all cases where it's not being put on relmem_list (would eliminate the need for page_list_move_tail(), page_list_splice_init(), and perhaps also page_list_is_eol())? Signed-off-by: Jan Beulich --- 2009-01-30.orig/xen/arch/x86/domain.c 2009-01-30 09:53:28.0000000= 00 +0100 +++ 2009-01-30/xen/arch/x86/domain.c 2009-01-30 10:13:53.000000000 = +0100 @@ -141,7 +141,7 @@ void dump_pageframe_info(struct domain * } else { - list_for_each_entry ( page, &d->page_list, list ) + page_list_for_each ( page, &d->page_list ) { printk(" DomPage %p: caf=3D%08lx, taf=3D%" PRtype_info = "\n", _p(page_to_mfn(page)), @@ -154,7 +154,7 @@ void dump_pageframe_info(struct domain * p2m_pod_dump_data(d); } =20 - list_for_each_entry ( page, &d->xenpage_list, list ) + page_list_for_each ( page, &d->xenpage_list ) { printk(" XenPage %p: caf=3D%08lx, taf=3D%" PRtype_info "\n", _p(page_to_mfn(page)), @@ -380,7 +380,7 @@ int arch_domain_create(struct domain *d, INIT_LIST_HEAD(&d->arch.pdev_list); =20 d->arch.relmem =3D RELMEM_not_started; - INIT_LIST_HEAD(&d->arch.relmem_list); + INIT_PAGE_LIST_HEAD(&d->arch.relmem_list); =20 pdpt_order =3D get_order_from_bytes(PDPT_L1_ENTRIES * sizeof(l1_pgentr= y_t)); d->arch.mm_perdomain_pt =3D alloc_xenheap_pages(pdpt_order, 0); @@ -1655,27 +1655,25 @@ int hypercall_xlat_continuation(unsigned #endif =20 static int relinquish_memory( - struct domain *d, struct list_head *list, unsigned long type) + struct domain *d, struct page_list_head *list, unsigned long type) { - struct list_head *ent; - struct page_info *page; + struct page_info *page, *cur; unsigned long x, y; int ret =3D 0; =20 /* Use a recursive lock, as we may enter 'free_domheap_page'. */ spin_lock_recursive(&d->page_alloc_lock); =20 - ent =3D list->next; - while ( ent !=3D list ) + page =3D page_list_first(list); + while ( !page_list_is_eol(page, list) ) { - page =3D list_entry(ent, struct page_info, list); - /* Grab a reference to the page so it won't disappear from under = us. */ if ( unlikely(!get_page(page, d)) ) { /* Couldn't get a reference -- someone is freeing this page. = */ - ent =3D ent->next; - list_move_tail(&page->list, &d->arch.relmem_list); + cur =3D page; + page =3D page_list_next(page, list); + page_list_move_tail(cur, list, &d->arch.relmem_list); continue; } =20 @@ -1747,9 +1745,10 @@ static int relinquish_memory( } =20 /* Follow the list chain and /then/ potentially free the page. */ - ent =3D ent->next; - list_move_tail(&page->list, &d->arch.relmem_list); - put_page(page); + cur =3D page; + page =3D page_list_next(page, list); + page_list_move_tail(cur, list, &d->arch.relmem_list); + put_page(cur); =20 if ( hypercall_preempt_check() ) { @@ -1758,7 +1757,7 @@ static int relinquish_memory( } } =20 - list_splice_init(&d->arch.relmem_list, list); + page_list_splice_init(&d->arch.relmem_list, list); =20 out: spin_unlock_recursive(&d->page_alloc_lock); --- 2009-01-30.orig/xen/arch/x86/domain_build.c 2009-01-28 08:53:49.0000000= 00 +0100 +++ 2009-01-30/xen/arch/x86/domain_build.c 2009-01-30 10:13:53.0000000= 00 +0100 @@ -880,7 +880,7 @@ int __init construct_dom0( } si->first_p2m_pfn =3D pfn; si->nr_p2m_frames =3D d->tot_pages - count; - list_for_each_entry ( page, &d->page_list, list ) + page_list_for_each ( page, &d->page_list ) { mfn =3D page_to_mfn(page); if ( get_gpfn_from_mfn(mfn) >=3D count ) --- 2009-01-30.orig/xen/arch/x86/domctl.c 2009-01-30 08:27:02.0000000= 00 +0100 +++ 2009-01-30/xen/arch/x86/domctl.c 2009-01-30 10:13:53.000000000 = +0100 @@ -240,7 +240,7 @@ long arch_do_domctl( struct domain *d =3D rcu_lock_domain_by_id(domctl->domain); unsigned long max_pfns =3D domctl->u.getmemlist.max_pfns; uint64_t mfn; - struct list_head *list_ent; + struct page_info *page; =20 ret =3D -EINVAL; if ( d !=3D NULL ) @@ -259,19 +259,19 @@ long arch_do_domctl( goto getmemlist_out; } =20 - ret =3D 0; - list_ent =3D d->page_list.next; - for ( i =3D 0; (i < max_pfns) && (list_ent !=3D &d->page_list)= ; i++ ) + ret =3D i =3D 0; + page_list_for_each(page, &d->page_list) { - mfn =3D page_to_mfn(list_entry( - list_ent, struct page_info, list)); + if ( i >=3D max_pfns ) + break; + mfn =3D page_to_mfn(page); if ( copy_to_guest_offset(domctl->u.getmemlist.buffer, i, &mfn, 1) ) { ret =3D -EFAULT; break; } - list_ent =3D mfn_to_page(mfn)->list.next; + ++i; } =20 spin_unlock(&d->page_alloc_lock); --- 2009-01-30.orig/xen/arch/x86/e820.c 2009-01-30 09:53:28.000000000 = +0100 +++ 2009-01-30/xen/arch/x86/e820.c 2009-01-30 10:13:53.000000000 = +0100 @@ -1,10 +1,10 @@ #include #include #include +#include #include #include #include -#include #include =20 /* opt_mem: Limit of physical RAM. Any RAM beyond this point is ignored. = */ --- 2009-01-30.orig/xen/arch/x86/mm.c 2009-01-28 08:53:49.000000000 = +0100 +++ 2009-01-30/xen/arch/x86/mm.c 2009-01-30 10:13:53.000000000 = +0100 @@ -333,7 +333,7 @@ void share_xen_page_with_guest( page->count_info |=3D PGC_allocated | 1; if ( unlikely(d->xenheap_pages++ =3D=3D 0) ) get_knownalive_domain(d); - list_add_tail(&page->list, &d->xenpage_list); + page_list_add_tail(page, &d->xenpage_list); } =20 spin_unlock(&d->page_alloc_lock); @@ -3508,7 +3508,7 @@ int steal_page( /* Unlink from original owner. */ if ( !(memflags & MEMF_no_refcount) ) d->tot_pages--; - list_del(&page->list); + page_list_del(page, &d->page_list); =20 spin_unlock(&d->page_alloc_lock); return 0; --- 2009-01-30.orig/xen/arch/x86/mm/hap/hap.c 2009-01-30 08:27:02.0000000= 00 +0100 +++ 2009-01-30/xen/arch/x86/mm/hap/hap.c 2009-01-30 10:13:53.0000000= 00 +0100 @@ -96,11 +96,10 @@ static struct page_info *hap_alloc(struc =20 ASSERT(hap_locked_by_me(d)); =20 - if ( unlikely(list_empty(&d->arch.paging.hap.freelist)) ) + pg =3D page_list_remove_head(&d->arch.paging.hap.freelist); + if ( unlikely(!pg) ) return NULL; =20 - pg =3D list_entry(d->arch.paging.hap.freelist.next, struct page_info, = list); - list_del(&pg->list); d->arch.paging.hap.free_pages--; =20 p =3D hap_map_domain_page(page_to_mfn(pg)); @@ -118,7 +117,7 @@ static void hap_free(struct domain *d, m ASSERT(hap_locked_by_me(d)); =20 d->arch.paging.hap.free_pages++; - list_add_tail(&pg->list, &d->arch.paging.hap.freelist); + page_list_add_tail(pg, &d->arch.paging.hap.freelist); } =20 static struct page_info *hap_alloc_p2m_page(struct domain *d) @@ -210,15 +209,13 @@ hap_set_allocation(struct domain *d, uns } d->arch.paging.hap.free_pages++; d->arch.paging.hap.total_pages++; - list_add_tail(&pg->list, &d->arch.paging.hap.freelist); + page_list_add_tail(pg, &d->arch.paging.hap.freelist); } else if ( d->arch.paging.hap.total_pages > pages ) { /* Need to return memory to domheap */ - ASSERT(!list_empty(&d->arch.paging.hap.freelist)); - pg =3D list_entry(d->arch.paging.hap.freelist.next, - struct page_info, list); - list_del(&pg->list); + pg =3D page_list_remove_head(&d->arch.paging.hap.freelist); + ASSERT(pg); d->arch.paging.hap.free_pages--; d->arch.paging.hap.total_pages--; pg->count_info =3D 0; @@ -393,7 +390,7 @@ static void hap_destroy_monitor_table(st void hap_domain_init(struct domain *d) { hap_lock_init(d); - INIT_LIST_HEAD(&d->arch.paging.hap.freelist); + INIT_PAGE_LIST_HEAD(&d->arch.paging.hap.freelist); =20 /* This domain will use HAP for log-dirty mode */ paging_log_dirty_init(d, hap_enable_log_dirty, hap_disable_log_dirty, --- 2009-01-30.orig/xen/arch/x86/mm/hap/p2m-ept.c 2009-01-30 = 08:27:02.000000000 +0100 +++ 2009-01-30/xen/arch/x86/mm/hap/p2m-ept.c 2009-01-30 10:13:53.0000000= 00 +0100 @@ -63,7 +63,7 @@ static int ept_set_middle_entry(struct d =20 pg->count_info =3D 1; pg->u.inuse.type_info =3D 1 | PGT_validated; - list_add_tail(&pg->list, &d->arch.p2m->pages); + page_list_add_tail(pg, &d->arch.p2m->pages); =20 ept_entry->emt =3D 0; ept_entry->igmt =3D 0; --- 2009-01-30.orig/xen/arch/x86/mm/p2m.c 2009-01-30 09:53:28.0000000= 00 +0100 +++ 2009-01-30/xen/arch/x86/mm/p2m.c 2009-01-30 10:13:53.000000000 = +0100 @@ -175,7 +175,7 @@ p2m_next_level(struct domain *d, mfn_t * struct page_info *pg =3D d->arch.p2m->alloc_page(d); if ( pg =3D=3D NULL ) return 0; - list_add_tail(&pg->list, &d->arch.p2m->pages); + page_list_add_tail(pg, &d->arch.p2m->pages); pg->u.inuse.type_info =3D type | 1 | PGT_validated; pg->count_info =3D 1; =20 @@ -214,7 +214,7 @@ p2m_next_level(struct domain *d, mfn_t * struct page_info *pg =3D d->arch.p2m->alloc_page(d); if ( pg =3D=3D NULL ) return 0; - list_add_tail(&pg->list, &d->arch.p2m->pages); + page_list_add_tail(pg, &d->arch.p2m->pages); pg->u.inuse.type_info =3D PGT_l1_page_table | 1 | PGT_validated; pg->count_info =3D 1; =20 @@ -300,18 +300,18 @@ p2m_pod_cache_add(struct domain *d, for(i=3D0; i < 1 << order ; i++) { p =3D page + i; - list_del(&p->list); + page_list_del(p, &d->page_list); } =20 /* Then add the first one to the appropriate populate-on-demand list = */ switch(order) { case 9: - list_add_tail(&page->list, &p2md->pod.super); /* lock: page_alloc = */ + page_list_add_tail(page, &p2md->pod.super); /* lock: page_alloc = */ p2md->pod.count +=3D 1 << order; break; case 0: - list_add_tail(&page->list, &p2md->pod.single); /* lock: page_alloc= */ + page_list_add_tail(page, &p2md->pod.single); /* lock: page_alloc = */ p2md->pod.count +=3D 1 ; break; default: @@ -334,54 +334,51 @@ static struct page_info * p2m_pod_cache_ struct page_info *p =3D NULL; int i; =20 - if ( order =3D=3D 9 && list_empty(&p2md->pod.super) ) + if ( order =3D=3D 9 && page_list_empty(&p2md->pod.super) ) { return NULL; } - else if ( order =3D=3D 0 && list_empty(&p2md->pod.single) ) + else if ( order =3D=3D 0 && page_list_empty(&p2md->pod.single) ) { unsigned long mfn; struct page_info *q; =20 - BUG_ON( list_empty(&p2md->pod.super) ); + BUG_ON( page_list_empty(&p2md->pod.super) ); =20 /* Break up a superpage to make single pages. NB count doesn't * need to be adjusted. */ printk("%s: Breaking up superpage.\n", __func__); - p =3D list_entry(p2md->pod.super.next, struct page_info, list); - list_del(&p->list); + p =3D page_list_remove_head(&p2md->pod.super); mfn =3D mfn_x(page_to_mfn(p)); =20 for ( i=3D0; i<(1<<9); i++ ) { q =3D mfn_to_page(_mfn(mfn+i)); - list_add_tail(&q->list, &p2md->pod.single); + page_list_add_tail(q, &p2md->pod.single); } } =20 switch ( order ) { case 9: - BUG_ON( list_empty(&p2md->pod.super) ); - p =3D list_entry(p2md->pod.super.next, struct page_info, = list);=20 + BUG_ON( page_list_empty(&p2md->pod.super) ); + p =3D page_list_remove_head(&p2md->pod.super); p2md->pod.count -=3D 1 << order; /* Lock: page_alloc */ break; case 0: - BUG_ON( list_empty(&p2md->pod.single) ); - p =3D list_entry(p2md->pod.single.next, struct page_info, list); + BUG_ON( page_list_empty(&p2md->pod.single) ); + p =3D page_list_remove_head(&p2md->pod.single); p2md->pod.count -=3D 1; break; default: BUG(); } =20 - list_del(&p->list); - /* Put the pages back on the domain page_list */ for ( i =3D 0 ; i < (1 << order) ; i++ ) { BUG_ON(page_get_owner(p + i) !=3D d); - list_add_tail(&p[i].list, &d->page_list); + page_list_add_tail(p + i, &d->page_list); } =20 return p; @@ -425,7 +422,7 @@ p2m_pod_set_cache_target(struct domain * spin_lock(&d->page_alloc_lock); =20 if ( (p2md->pod.count - pod_target) > (1>>9) - && !list_empty(&p2md->pod.super) ) + && !page_list_empty(&p2md->pod.super) ) order =3D 9; else order =3D 0; @@ -535,38 +532,27 @@ void p2m_pod_empty_cache(struct domain *d) { struct p2m_domain *p2md =3D d->arch.p2m; - struct list_head *q, *p; + struct page_info *page; =20 spin_lock(&d->page_alloc_lock); =20 - list_for_each_safe(p, q, &p2md->pod.super) /* lock: page_alloc */ + while ( (page =3D page_list_remove_head(&p2md->pod.super)) ) { int i; - struct page_info *page; =20 - list_del(p); - =20 - page =3D list_entry(p, struct page_info, list); - for ( i =3D 0 ; i < (1 << 9) ; i++ ) { BUG_ON(page_get_owner(page + i) !=3D d); - list_add_tail(&page[i].list, &d->page_list); + page_list_add_tail(page + i, &d->page_list); } =20 p2md->pod.count -=3D 1<<9; } =20 - list_for_each_safe(p, q, &p2md->pod.single) + while ( (page =3D page_list_remove_head(&p2md->pod.single)) ) { - struct page_info *page; - =20 - list_del(p); - =20 - page =3D list_entry(p, struct page_info, list); - BUG_ON(page_get_owner(page) !=3D d); - list_add_tail(&page->list, &d->page_list); + page_list_add_tail(page, &d->page_list); =20 p2md->pod.count -=3D 1; } @@ -952,7 +938,7 @@ p2m_pod_emergency_sweep_super(struct dom * NB that this is a zero-sum game; we're increasing our cache = size * by increasing our 'debt'. Since we hold the p2m lock, * (entry_count - count) must remain the same. */ - if ( !list_empty(&p2md->pod.super) && i < limit ) + if ( !page_list_empty(&p2md->pod.super) && i < limit ) break; } =20 @@ -1035,12 +1021,12 @@ p2m_pod_demand_populate(struct domain *d } =20 /* If we're low, start a sweep */ - if ( order =3D=3D 9 && list_empty(&p2md->pod.super) ) + if ( order =3D=3D 9 && page_list_empty(&p2md->pod.super) ) p2m_pod_emergency_sweep_super(d); =20 - if ( list_empty(&p2md->pod.single) && + if ( page_list_empty(&p2md->pod.single) && ( ( order =3D=3D 0 ) - || (order =3D=3D 9 && list_empty(&p2md->pod.super) ) ) ) + || (order =3D=3D 9 && page_list_empty(&p2md->pod.super) ) ) ) p2m_pod_emergency_sweep(d); =20 /* Keep track of the highest gfn demand-populated by a guest fault */ @@ -1477,9 +1463,9 @@ int p2m_init(struct domain *d) =20 memset(p2m, 0, sizeof(*p2m)); p2m_lock_init(p2m); - INIT_LIST_HEAD(&p2m->pages); - INIT_LIST_HEAD(&p2m->pod.super); - INIT_LIST_HEAD(&p2m->pod.single); + INIT_PAGE_LIST_HEAD(&p2m->pages); + INIT_PAGE_LIST_HEAD(&p2m->pod.super); + INIT_PAGE_LIST_HEAD(&p2m->pod.single); =20 p2m->set_entry =3D p2m_set_entry; p2m->get_entry =3D p2m_gfn_to_mfn; @@ -1540,7 +1526,6 @@ int p2m_alloc_table(struct domain *d, =20 { mfn_t mfn =3D _mfn(INVALID_MFN); - struct list_head *entry; struct page_info *page, *p2m_top; unsigned int page_count =3D 0; unsigned long gfn =3D -1UL; @@ -1566,7 +1551,7 @@ int p2m_alloc_table(struct domain *d, p2m_unlock(p2m); return -ENOMEM; } - list_add_tail(&p2m_top->list, &p2m->pages); + page_list_add_tail(p2m_top, &p2m->pages); =20 p2m_top->count_info =3D 1; p2m_top->u.inuse.type_info =3D @@ -1587,11 +1572,8 @@ int p2m_alloc_table(struct domain *d, goto error; =20 /* Copy all existing mappings from the page list and m2p */ - for ( entry =3D d->page_list.next; - entry !=3D &d->page_list; - entry =3D entry->next ) + page_list_for_each(page, &d->page_list) { - page =3D list_entry(entry, struct page_info, list); mfn =3D page_to_mfn(page); gfn =3D get_gpfn_from_mfn(mfn_x(mfn)); page_count++; @@ -1621,19 +1603,14 @@ void p2m_teardown(struct domain *d) /* Return all the p2m pages to Xen. * We know we don't have any extra mappings to these pages */ { - struct list_head *entry, *n; struct page_info *pg; struct p2m_domain *p2m =3D d->arch.p2m; =20 p2m_lock(p2m); d->arch.phys_table =3D pagetable_null(); =20 - list_for_each_safe(entry, n, &p2m->pages) - { - pg =3D list_entry(entry, struct page_info, list); - list_del(entry); + while ( (pg =3D page_list_remove_head(&p2m->pages)) ) p2m->free_page(d, pg); - } p2m_unlock(p2m); } =20 --- 2009-01-30.orig/xen/arch/x86/mm/shadow/common.c 2009-01-28 = 08:53:49.000000000 +0100 +++ 2009-01-30/xen/arch/x86/mm/shadow/common.c 2009-01-30 10:13:53.0000000= 00 +0100 @@ -49,7 +49,7 @@ void shadow_domain_init(struct domain *d shadow_lock_init(d); for ( i =3D 0; i <=3D SHADOW_MAX_ORDER; i++ ) INIT_LIST_HEAD(&d->arch.paging.shadow.freelists[i]); - INIT_LIST_HEAD(&d->arch.paging.shadow.p2m_freelist); + INIT_PAGE_LIST_HEAD(&d->arch.paging.shadow.p2m_freelist); INIT_LIST_HEAD(&d->arch.paging.shadow.pinned_shadows); =20 /* Use shadow pagetables for log-dirty support */ @@ -1672,7 +1672,7 @@ sh_alloc_p2m_pages(struct domain *d) */ page_set_owner(&pg[i], d); pg[i].count_info =3D 1; - list_add_tail(&pg[i].list, &d->arch.paging.shadow.p2m_freelist); + page_list_add_tail(&pg[i], &d->arch.paging.shadow.p2m_freelist); } return 1; } @@ -1681,25 +1681,22 @@ sh_alloc_p2m_pages(struct domain *d) static struct page_info * shadow_alloc_p2m_page(struct domain *d) { - struct list_head *entry; struct page_info *pg; mfn_t mfn; void *p; =20 shadow_lock(d); =20 - if ( list_empty(&d->arch.paging.shadow.p2m_freelist) && + if ( page_list_empty(&d->arch.paging.shadow.p2m_freelist) && !sh_alloc_p2m_pages(d) ) { shadow_unlock(d); return NULL; } - entry =3D d->arch.paging.shadow.p2m_freelist.next; - list_del(entry); + pg =3D page_list_remove_head(&d->arch.paging.shadow.p2m_freelist); =20 shadow_unlock(d); =20 - pg =3D list_entry(entry, struct page_info, list); mfn =3D page_to_mfn(pg); p =3D sh_map_domain_page(mfn); clear_page(p); @@ -3156,7 +3153,6 @@ void shadow_teardown(struct domain *d) { struct vcpu *v; mfn_t mfn; - struct list_head *entry, *n; struct page_info *pg; =20 ASSERT(d->is_dying); @@ -3208,12 +3204,8 @@ void shadow_teardown(struct domain *d) } #endif /* (SHADOW_OPTIMIZATIONS & (SHOPT_VIRTUAL_TLB|SHOPT_OUT_OF_SYNC)) = */ =20 - list_for_each_safe(entry, n, &d->arch.paging.shadow.p2m_freelist) - { - list_del(entry); - pg =3D list_entry(entry, struct page_info, list); + while ( (pg =3D page_list_remove_head(&d->arch.paging.shadow.p2m_freel= ist)) ) shadow_free_p2m_page(d, pg); - } =20 if ( d->arch.paging.shadow.total_pages !=3D 0 ) { --- 2009-01-30.orig/xen/arch/x86/numa.c 2009-01-30 08:27:02.000000000 = +0100 +++ 2009-01-30/xen/arch/x86/numa.c 2009-01-30 10:13:53.000000000 = +0100 @@ -312,7 +312,7 @@ static void dump_numa(unsigned char key) for_each_online_node(i) page_num_node[i] =3D 0; =20 - list_for_each_entry(page, &d->page_list, list) + page_list_for_each(page, &d->page_list) { i =3D phys_to_nid(page_to_mfn(page) << PAGE_SHIFT);= page_num_node[i]++; --- 2009-01-30.orig/xen/common/domain.c 2009-01-30 09:53:28.000000000 = +0100 +++ 2009-01-30/xen/common/domain.c 2009-01-30 10:13:53.000000000 = +0100 @@ -233,8 +233,8 @@ struct domain *domain_create( spin_lock_init(&d->page_alloc_lock); spin_lock_init(&d->shutdown_lock); spin_lock_init(&d->hypercall_deadlock_mutex); - INIT_LIST_HEAD(&d->page_list); - INIT_LIST_HEAD(&d->xenpage_list); + INIT_PAGE_LIST_HEAD(&d->page_list); + INIT_PAGE_LIST_HEAD(&d->xenpage_list); =20 if ( domcr_flags & DOMCRF_hvm ) d->is_hvm =3D 1; --- 2009-01-30.orig/xen/common/grant_table.c 2009-01-30 09:53:28.0000000= 00 +0100 +++ 2009-01-30/xen/common/grant_table.c 2009-01-30 10:13:53.000000000 = +0100 @@ -1192,7 +1192,7 @@ gnttab_transfer( /* Okay, add the page to 'e'. */ if ( unlikely(e->tot_pages++ =3D=3D 0) ) get_knownalive_domain(e); - list_add_tail(&page->list, &e->page_list); + page_list_add_tail(page, &e->page_list); page_set_owner(page, e); =20 spin_unlock(&e->page_alloc_lock); --- 2009-01-30.orig/xen/common/memory.c 2009-01-30 08:27:02.000000000 = +0100 +++ 2009-01-30/xen/common/memory.c 2009-01-30 10:13:53.000000000 = +0100 @@ -218,8 +218,8 @@ static void decrease_reservation(struct=20 static long memory_exchange(XEN_GUEST_HANDLE(xen_memory_exchange_t) arg) { struct xen_memory_exchange exch; - LIST_HEAD(in_chunk_list); - LIST_HEAD(out_chunk_list); + PAGE_LIST_HEAD(in_chunk_list); + PAGE_LIST_HEAD(out_chunk_list); unsigned long in_chunk_order, out_chunk_order; xen_pfn_t gpfn, gmfn, mfn; unsigned long i, j, k; @@ -325,7 +325,7 @@ static long memory_exchange(XEN_GUEST_HA goto fail; } =20 - list_add(&page->list, &in_chunk_list); + page_list_add(page, &in_chunk_list); } } =20 @@ -339,7 +339,7 @@ static long memory_exchange(XEN_GUEST_HA goto fail; } =20 - list_add(&page->list, &out_chunk_list); + page_list_add(page, &out_chunk_list); } =20 /* @@ -347,10 +347,8 @@ static long memory_exchange(XEN_GUEST_HA */ =20 /* Destroy final reference to each input page. */ - while ( !list_empty(&in_chunk_list) ) + while ( (page =3D page_list_remove_head(&in_chunk_list)) ) { - page =3D list_entry(in_chunk_list.next, struct page_info, = list); - list_del(&page->list); if ( !test_and_clear_bit(_PGC_allocated, &page->count_info) ) BUG(); mfn =3D page_to_mfn(page); @@ -360,10 +358,8 @@ static long memory_exchange(XEN_GUEST_HA =20 /* Assign each output page to the domain. */ j =3D 0; - while ( !list_empty(&out_chunk_list) ) + while ( (page =3D page_list_remove_head(&out_chunk_list)) ) { - page =3D list_entry(out_chunk_list.next, struct page_info, = list); - list_del(&page->list); if ( assign_pages(d, page, exch.out.extent_order, MEMF_no_refcount) ) BUG(); @@ -399,21 +395,13 @@ static long memory_exchange(XEN_GUEST_HA */ fail: /* Reassign any input pages we managed to steal. */ - while ( !list_empty(&in_chunk_list) ) - { - page =3D list_entry(in_chunk_list.next, struct page_info, list); - list_del(&page->list); + while ( (page =3D page_list_remove_head(&in_chunk_list)) ) if ( assign_pages(d, page, 0, MEMF_no_refcount) ) BUG(); - } =20 /* Free any output pages we managed to allocate. */ - while ( !list_empty(&out_chunk_list) ) - { - page =3D list_entry(out_chunk_list.next, struct page_info, list); - list_del(&page->list); + while ( (page =3D page_list_remove_head(&out_chunk_list)) ) free_domheap_pages(page, exch.out.extent_order); - } =20 exch.nr_exchanged =3D i << in_chunk_order; =20 --- 2009-01-30.orig/xen/common/page_alloc.c 2009-01-30 09:53:28.0000000= 00 +0100 +++ 2009-01-30/xen/common/page_alloc.c 2009-01-30 10:13:53.000000000 = +0100 @@ -71,7 +71,7 @@ integer_param("dma_bits", dma_bitsize); #endif =20 static DEFINE_SPINLOCK(page_scrub_lock); -LIST_HEAD(page_scrub_list); +PAGE_LIST_HEAD(page_scrub_list); static unsigned long scrub_pages; =20 /********************* @@ -264,7 +264,7 @@ unsigned long __init alloc_boot_pages( #define page_to_zone(pg) (is_xen_heap_page(pg) ? MEMZONE_XEN : \ (fls(page_to_mfn(pg)) - 1)) =20 -typedef struct list_head heap_by_zone_and_order_t[NR_ZONES][MAX_ORDER+1]; +typedef struct page_list_head heap_by_zone_and_order_t[NR_ZONES][MAX_ORDER= +1]; static heap_by_zone_and_order_t *_heap[MAX_NUMNODES]; #define heap(node, zone, order) ((*_heap[node])[zone][order]) =20 @@ -298,7 +298,7 @@ static void init_node_heap(int node) =20 for ( i =3D 0; i < NR_ZONES; i++ ) for ( j =3D 0; j <=3D MAX_ORDER; j++ ) - INIT_LIST_HEAD(&(*_heap[node])[i][j]); + INIT_PAGE_LIST_HEAD(&(*_heap[node])[i][j]); } =20 /* Allocate 2^@order contiguous pages. */ @@ -340,7 +340,7 @@ static struct page_info *alloc_heap_page =20 /* Find smallest order which can satisfy the request. */ for ( j =3D order; j <=3D MAX_ORDER; j++ ) - if ( !list_empty(&heap(node, zone, j)) ) + if ( (pg =3D page_list_remove_head(&heap(node, zone, j))) = ) goto found; } while ( zone-- > zone_lo ); /* careful: unsigned zone may wrap = */ =20 @@ -354,14 +354,11 @@ static struct page_info *alloc_heap_page return NULL; =20 found:=20 - pg =3D list_entry(heap(node, zone, j).next, struct page_info, list); - list_del(&pg->list); - /* We may have to halve the chunk a number of times. */ while ( j !=3D order ) { PFN_ORDER(pg) =3D --j; - list_add_tail(&pg->list, &heap(node, zone, j)); + page_list_add_tail(pg, &heap(node, zone, j)); pg +=3D 1 << j; } =20 @@ -452,8 +449,8 @@ static void free_heap_pages( if ( allocated_in_map(page_to_mfn(pg)-mask) || (PFN_ORDER(pg-mask) !=3D order) ) break; - list_del(&(pg-mask)->list); pg -=3D mask; + page_list_del(pg, &heap(node, zone, order)); } else { @@ -461,7 +458,7 @@ static void free_heap_pages( if ( allocated_in_map(page_to_mfn(pg)+mask) || (PFN_ORDER(pg+mask) !=3D order) ) break; - list_del(&(pg+mask)->list); + page_list_del(pg + mask, &heap(node, zone, order)); } =20 order++; @@ -471,7 +468,7 @@ static void free_heap_pages( } =20 PFN_ORDER(pg) =3D order; - list_add_tail(&pg->list, &heap(node, zone, order)); + page_list_add_tail(pg, &heap(node, zone, order)); =20 spin_unlock(&heap_lock); } @@ -786,7 +783,7 @@ int assign_pages( page_set_owner(&pg[i], d); wmb(); /* Domain pointer must be visible before updating refcnt. = */ pg[i].count_info =3D PGC_allocated | 1; - list_add_tail(&pg[i].list, &d->page_list); + page_list_add_tail(&pg[i], &d->page_list); } =20 spin_unlock(&d->page_alloc_lock); @@ -844,7 +841,7 @@ void free_domheap_pages(struct page_info spin_lock_recursive(&d->page_alloc_lock); =20 for ( i =3D 0; i < (1 << order); i++ ) - list_del(&pg[i].list); + page_list_del2(&pg[i], &d->xenpage_list, &d->arch.relmem_list)= ; =20 d->xenheap_pages -=3D 1 << order; drop_dom_ref =3D (d->xenheap_pages =3D=3D 0); @@ -859,7 +856,7 @@ void free_domheap_pages(struct page_info for ( i =3D 0; i < (1 << order); i++ ) { BUG_ON((pg[i].u.inuse.type_info & PGT_count_mask) !=3D 0); - list_del(&pg[i].list); + page_list_del2(&pg[i], &d->page_list, &d->arch.relmem_list); } =20 d->tot_pages -=3D 1 << order; @@ -882,7 +879,7 @@ void free_domheap_pages(struct page_info { page_set_owner(&pg[i], NULL); spin_lock(&page_scrub_lock); - list_add(&pg[i].list, &page_scrub_list); + page_list_add(&pg[i], &page_scrub_list); scrub_pages++; spin_unlock(&page_scrub_lock); } @@ -965,7 +962,7 @@ static DEFINE_PER_CPU(struct timer, page =20 static void page_scrub_softirq(void) { - struct list_head *ent; + PAGE_LIST_HEAD(list); struct page_info *pg; void *p; int i; @@ -983,32 +980,26 @@ static void page_scrub_softirq(void) do { spin_lock(&page_scrub_lock); =20 - if ( unlikely((ent =3D page_scrub_list.next) =3D=3D &page_scrub_li= st) ) - { - spin_unlock(&page_scrub_lock); - goto out; - } - =20 /* Peel up to 16 pages from the list. */ for ( i =3D 0; i < 16; i++ ) { - if ( ent->next =3D=3D &page_scrub_list ) + if ( !(pg =3D page_list_remove_head(&page_scrub_list)) ) break; - ent =3D ent->next; + page_list_add_tail(pg, &list); } =20 - /* Remove peeled pages from the list. */ - ent->next->prev =3D &page_scrub_list; - page_scrub_list.next =3D ent->next; - scrub_pages -=3D (i+1); + if ( unlikely(i =3D=3D 0) ) + { + spin_unlock(&page_scrub_lock); + goto out; + } + + scrub_pages -=3D i; =20 spin_unlock(&page_scrub_lock); =20 - /* Working backwards, scrub each page in turn. */ - while ( ent !=3D &page_scrub_list ) - { - pg =3D list_entry(ent, struct page_info, list); - ent =3D ent->prev; + /* Scrub each page in turn. */ + while ( (pg =3D page_list_remove_head(&list)) ) { p =3D map_domain_page(page_to_mfn(pg)); scrub_page(p); unmap_domain_page(p); --- 2009-01-30.orig/xen/drivers/passthrough/amd/iommu_map.c 2009-01-30 = 08:27:02.000000000 +0100 +++ 2009-01-30/xen/drivers/passthrough/amd/iommu_map.c 2009-01-30 = 10:13:53.000000000 +0100 @@ -552,7 +552,6 @@ int amd_iommu_sync_p2m(struct domain *d) { unsigned long mfn, gfn, flags; u64 iommu_l2e; - struct list_head *entry; struct page_info *page; struct hvm_iommu *hd; int iw =3D IOMMU_IO_WRITE_ENABLED; @@ -568,10 +567,8 @@ int amd_iommu_sync_p2m(struct domain *d) if ( hd->p2m_synchronized ) goto out; =20 - for ( entry =3D d->page_list.next; entry !=3D &d->page_list; - entry =3D entry->next ) + page_list_for_each ( page, &d->page_list ) { - page =3D list_entry(entry, struct page_info, list); mfn =3D page_to_mfn(page); gfn =3D get_gpfn_from_mfn(mfn); =20 --- 2009-01-30.orig/xen/drivers/passthrough/amd/pci_amd_iommu.c 2009-01-30 = 08:27:02.000000000 +0100 +++ 2009-01-30/xen/drivers/passthrough/amd/pci_amd_iommu.c 2009-01-30 = 10:13:53.000000000 +0100 @@ -23,7 +23,6 @@ #include #include #include -#include =20 extern unsigned short ivrs_bdf_entries; extern struct ivrs_mappings *ivrs_mappings; --- 2009-01-30.orig/xen/drivers/passthrough/iommu.c 2009-01-30 = 08:27:02.000000000 +0100 +++ 2009-01-30/xen/drivers/passthrough/iommu.c 2009-01-30 10:13:53.0000000= 00 +0100 @@ -141,7 +141,7 @@ static int iommu_populate_page_table(str =20 spin_lock(&d->page_alloc_lock); =20 - list_for_each_entry ( page, &d->page_list, list ) + page_list_for_each ( page, &d->page_list ) { if ( (page->u.inuse.type_info & PGT_type_mask) =3D=3D PGT_writable= _page ) { --- 2009-01-30.orig/xen/include/asm-x86/domain.h 2009-01-30 = 08:27:02.000000000 +0100 +++ 2009-01-30/xen/include/asm-x86/domain.h 2009-01-30 10:13:53.0000000= 00 +0100 @@ -83,7 +83,7 @@ struct shadow_domain { =20 /* Memory allocation */ struct list_head freelists[SHADOW_MAX_ORDER + 1]; - struct list_head p2m_freelist; + struct page_list_head p2m_freelist; unsigned int total_pages; /* number of pages allocated */ unsigned int free_pages; /* number of pages on freelists */ unsigned int p2m_pages; /* number of pages allocates to p2m = */ @@ -143,7 +143,7 @@ struct hap_domain { int locker; const char *locker_function; =20 - struct list_head freelist; + struct page_list_head freelist; unsigned int total_pages; /* number of pages allocated */ unsigned int free_pages; /* number of pages on freelists */ unsigned int p2m_pages; /* number of pages allocates to p2m = */ @@ -265,7 +265,7 @@ struct arch_domain RELMEM_l2, RELMEM_done, } relmem; - struct list_head relmem_list; + struct page_list_head relmem_list; =20 cpuid_input_t cpuids[MAX_CPUID_INPUT]; } __cacheline_aligned; --- 2009-01-30.orig/xen/include/asm-x86/mm.h 2009-01-30 09:53:28.0000000= 00 +0100 +++ 2009-01-30/xen/include/asm-x86/mm.h 2009-01-30 10:13:53.000000000 = +0100 @@ -12,15 +12,24 @@ * Per-page-frame information. *=20 * Every architecture must ensure the following: - * 1. 'struct page_info' contains a 'struct list_head list'. + * 1. 'struct page_info' contains a 'struct page_list_entry list'. * 2. Provide a PFN_ORDER() macro for accessing the order of a free = page. */ #define PFN_ORDER(_pfn) ((_pfn)->u.free.order) =20 +#ifndef __i386__ +# undef page_list_entry +struct page_list_entry +{ + unsigned int next, prev; + unsigned long _pad_for_sh_; /* until struct shadow_page_info gets = updated */ +}; +#endif + struct page_info { /* Each frame can be threaded onto a doubly-linked list. */ - struct list_head list; + struct page_list_entry list; =20 /* Reference count and various PGC_xxx flags and fields. */ unsigned long count_info; --- 2009-01-30.orig/xen/include/asm-x86/p2m.h 2009-01-30 08:27:02.0000000= 00 +0100 +++ 2009-01-30/xen/include/asm-x86/p2m.h 2009-01-30 10:13:53.0000000= 00 +0100 @@ -110,7 +110,7 @@ struct p2m_domain { const char *locker_function; /* Func that took it */ =20 /* Pages used to construct the p2m */ - struct list_head pages; + struct page_list_head pages; =20 /* Functions to call to get or free pages for the p2m */ struct page_info * (*alloc_page )(struct domain *d); @@ -148,7 +148,7 @@ struct p2m_domain { * protect moving stuff from the PoD cache to the domain page list. */ struct { - struct list_head super, /* List of superpages = */ + struct page_list_head super, /* List of superpages = */ single; /* Non-super lists = */ int count, /* # of pages in cache lists = */ entry_count; /* # of pages in p2m marked pod = */ --- 2009-01-30.orig/xen/include/asm-x86/perfc.h 2009-01-30 08:27:02.0000000= 00 +0100 +++ 2009-01-30/xen/include/asm-x86/perfc.h 2009-01-30 10:13:53.0000000= 00 +0100 @@ -1,6 +1,5 @@ #ifndef __ASM_PERFC_H__ #define __ASM_PERFC_H__ -#include =20 static inline void arch_perfc_printall(void) { --- 2009-01-30.orig/xen/include/xen/mm.h 2009-01-30 09:53:28.0000000= 00 +0100 +++ 2009-01-30/xen/include/xen/mm.h 2009-01-30 10:13:53.000000000 = +0100 @@ -85,22 +85,221 @@ int assign_pages( #define MAX_ORDER 20 /* 2^20 contiguous pages */ #endif =20 +#define page_list_entry list_head + +#include + +#ifndef page_list_entry +struct page_list_head +{ + struct page_info *next, *tail; +}; +/* These must only have instances in struct page_info. */ +# define page_list_entry + +# define PAGE_LIST_HEAD_INIT(name) { NULL, NULL } +# define PAGE_LIST_HEAD(name) \ + struct page_list_head name =3D PAGE_LIST_HEAD_INIT(name) +# define INIT_PAGE_LIST_HEAD(head) ((head)->tail =3D (head)->next =3D = NULL) +# define INIT_PAGE_LIST_ENTRY(ent) ((ent)->prev =3D (ent)->next =3D ~0) + +static inline int +page_list_empty(const struct page_list_head *head) +{ + return !head->next; +} +static inline struct page_info * +page_list_first(const struct page_list_head *head) +{ + return head->next; +} +static inline struct page_info * +page_list_next(const struct page_info *page, + const struct page_list_head *head) +{ + return page !=3D head->tail ? mfn_to_page(page->list.next) : NULL; +} +static inline struct page_info * +page_list_prev(const struct page_info *page, + const struct page_list_head *head) +{ + return page !=3D head->next ? mfn_to_page(page->list.prev) : NULL; +} +static inline int +page_list_is_eol(const struct page_info *page, + const struct page_list_head *head) +{ + return !page; +} +static inline void +page_list_add(struct page_info *page, struct page_list_head *head) +{ + if ( head->next ) + { + page->list.next =3D page_to_mfn(head->next); + head->next->list.prev =3D page_to_mfn(page); + } + else + { + head->tail =3D page; + page->list.next =3D ~0; + } + page->list.prev =3D ~0; + head->next =3D page; +} +static inline void +page_list_add_tail(struct page_info *page, struct page_list_head *head) +{ + page->list.next =3D ~0; + if ( head->next ) + { + page->list.prev =3D page_to_mfn(head->tail); + head->tail->list.next =3D page_to_mfn(page); + } + else + { + page->list.prev =3D ~0; + head->next =3D page; + } + head->tail =3D page; +} +static inline bool_t +__page_list_del_head(struct page_info *page, struct page_list_head *head, + struct page_info *next, struct page_info *prev) +{ + if ( head->next =3D=3D page ) + { + if ( head->tail !=3D page ) + { + next->list.prev =3D ~0; + head->next =3D next; + } + else + head->tail =3D head->next =3D NULL; + return 1; + } + + if ( head->tail =3D=3D page ) + { + prev->list.next =3D ~0; + head->tail =3D prev; + return 1; + } + + return 0; +} +static inline void +page_list_del(struct page_info *page, struct page_list_head *head) +{ + struct page_info *next =3D mfn_to_page(page->list.next); + struct page_info *prev =3D mfn_to_page(page->list.prev); + + if ( !__page_list_del_head(page, head, next, prev) ) + { + next->list.prev =3D page->list.prev; + prev->list.next =3D page->list.next; + } +} +static inline void +page_list_del2(struct page_info *page, struct page_list_head *head1, + struct page_list_head *head2) +{ + struct page_info *next =3D mfn_to_page(page->list.next); + struct page_info *prev =3D mfn_to_page(page->list.prev); + + if ( !__page_list_del_head(page, head1, next, prev) && + !__page_list_del_head(page, head2, next, prev) ) + { + next->list.prev =3D page->list.prev; + prev->list.next =3D page->list.next; + } +} +static inline void +page_list_move_tail(struct page_info *page, struct page_list_head *list, + struct page_list_head *head) +{ + page_list_del(page, list); + page_list_add_tail(page, head); +} +static inline struct page_info * +page_list_remove_head(struct page_list_head *head) +{ + struct page_info *page =3D head->next; + + if ( page ) + page_list_del(page, head); + + return page; +} +static inline void +page_list_splice_init(struct page_list_head *list, struct page_list_head = *head) +{ + if ( !page_list_empty(list) ) + { + if ( head->next ) + head->tail->list.next =3D page_to_mfn(list->next); + else + head->next =3D list->next; + head->tail =3D list->tail; + INIT_PAGE_LIST_HEAD(list); + } +} + +#define page_list_for_each(pos, head) \ + for ( pos =3D (head)->next; pos; pos =3D page_list_next(pos, head) ) +#define page_list_for_each_safe(pos, tmp, head) \ + for ( pos =3D (head)->next; \ + pos ? (tmp =3D page_list_next(pos, head), 1) : 0; \ + pos =3D tmp ) +#define page_list_for_each_safe_reverse(pos, tmp, head) \ + for ( pos =3D (head)->tail; \ + pos ? (tmp =3D page_list_prev(pos, head), 1) : 0; \ + pos =3D tmp ) +#else +# define page_list_head list_head +# define PAGE_LIST_HEAD_INIT LIST_HEAD_INIT +# define PAGE_LIST_HEAD LIST_HEAD +# define INIT_PAGE_LIST_HEAD INIT_LIST_HEAD +# define INIT_PAGE_LIST_ENTRY INIT_LIST_HEAD +# define page_list_empty list_empty +# define page_list_first(hd) list_entry((hd)->next, \ + struct page_info, = list) +# define page_list_next(pg, hd) list_entry((pg)->list.next, \ + struct page_info, = list) +# define page_list_is_eol(pg, hd) (&(pg)->list =3D=3D (hd)) +# define page_list_add(pg, hd) list_add(&(pg)->list, hd) +# define page_list_add_tail(pg, hd) list_add_tail(&(pg)->list, hd) +# define page_list_del(pg, hd) list_del(&(pg)->list) +# define page_list_del2(pg, hd1, hd2) list_del(&(pg)->list) +# define page_list_move_tail(pg, o, n) list_move_tail(&(pg)->list, n) +# define page_list_remove_head(hd) (!page_list_empty(hd) ? \ + ({ \ + struct page_info *__pg =3D page_list_first(hd); \ + list_del(&__pg->list); \ + __pg; \ + }) : NULL) +# define page_list_splice_init list_splice_init +# define page_list_for_each(pos, head) list_for_each_entry(pos, head, = list) +# define page_list_for_each_safe(pos, tmp, head) \ + list_for_each_entry_safe(pos, tmp, head, list) +# define page_list_for_each_safe_reverse(pos, tmp, head) \ + list_for_each_entry_safe_reverse(pos, tmp, head, list) +#endif + /* Automatic page scrubbing for dead domains. */ -extern struct list_head page_scrub_list; -#define page_scrub_schedule_work() \ - do { \ - if ( !list_empty(&page_scrub_list) ) \ - raise_softirq(PAGE_SCRUB_SOFTIRQ); \ +extern struct page_list_head page_scrub_list; +#define page_scrub_schedule_work() \ + do { \ + if ( !page_list_empty(&page_scrub_list) ) \ + raise_softirq(PAGE_SCRUB_SOFTIRQ); \ } while ( 0 ) #define page_scrub_kick() \ do { \ - if ( !list_empty(&page_scrub_list) ) \ + if ( !page_list_empty(&page_scrub_list) ) \ cpumask_raise_softirq(cpu_online_map, PAGE_SCRUB_SOFTIRQ); \ } while ( 0 ) unsigned long avail_scrub_pages(void); =20 -#include - int guest_remove_page(struct domain *d, unsigned long gmfn); =20 /* Returns TRUE if the whole page at @mfn is ordinary RAM. */ --- 2009-01-30.orig/xen/include/xen/sched.h 2009-01-30 08:27:02.0000000= 00 +0100 +++ 2009-01-30/xen/include/xen/sched.h 2009-01-30 10:13:53.000000000 = +0100 @@ -171,8 +171,8 @@ struct domain spinlock_t domain_lock; =20 spinlock_t page_alloc_lock; /* protects all the following = fields */ - struct list_head page_list; /* linked list, of size tot_pages = */ - struct list_head xenpage_list; /* linked list, of size xenheap_page= s */ + struct page_list_head page_list; /* linked list, of size tot_pages = */ + struct page_list_head xenpage_list; /* linked list (size xenheap_pages= ) */ unsigned int tot_pages; /* number of pages currently = possesed */ unsigned int max_pages; /* maximum value for tot_pages = */ unsigned int xenheap_pages; /* # pages allocated from Xen heap = */