x86-64: use MFNs for linking together pages on lists
PATCH 1/5 x86-64: use MFNs for linking together pages on lists
Unless more than 16Tb are going to ever be supported in Xen, this will allow reducing the linked list entries in struct page_info from 16 to 8 bytes.
This doesn't modify struct shadow_page_info, yet, so in order to meet the constraints of that 'mirror' structure the list entry gets artificially forced to be 16 bytes in size. That workaround will be removed in a subsequent patch.
One question related to the special list manipulation methods used in relinquish_memory(): Couldn't relinquish_memory() remove the page to be dealt with from the list and re-add it in all cases where it's not being put on relmem_list (would eliminate the need for page_list_move_tail(), page_list_splice_init(), and perhaps also page_list_is_eol())?
xen/arch/x86/domain.c +15 -16
@@ -141,7 +141,7 @@ void dump_pageframe_info(struct domain * }else { - list_for_each_entry ( page, &d->page_list, list ) + page_list_for_each ( page, &d->page_list ) { printk(" DomPage %p: caf=%08lx, taf=%" PRtype_info "\n", _p(page_to_mfn(page)),
@@ -154,7 +154,7 @@ void dump_pageframe_info(struct domain *p2m_pod_dump_data (d); } - list_for_each_entry ( page, &d->xenpage_list, list ) + page_list_for_each ( page, &d->xenpage_list ) { printk(" XenPage %p: caf=%08lx, taf=%" PRtype_info "\n", _p(page_to_mfn(page)),
@@ -380,7 +380,7 @@ int arch_domain_create(struct domain *d,INIT_LIST_HEAD (&d->arch.pdev_list); d->arch.relmem = RELMEM_not_started; -INIT_LIST_HEAD (&d->arch.relmem_list); +INIT_PAGE_LIST_HEAD (&d->arch.relmem_list); pdpt_order =get_order_from_bytes (PDPT_L1_ENTRIES *sizeof (l1_pgentry_t)); d->arch.mm_perdomain_pt =alloc_xenheap_pages (pdpt_order,0 );
@@ -1655,27 +1655,25 @@ int hypercall_xlat_continuation(unsigned#endif static int relinquish_memory ( -struct domain *d,struct list_head *list,unsigned long type) +struct domain *d,struct page_list_head *list,unsigned long type) { -struct list_head *ent; -struct page_info *page; +struct page_info *page, *cur;unsigned long x, y;int ret =0 ;/* Use a recursive lock, as we may enter 'free_domheap_page'. */ spin_lock_recursive (&d->page_alloc_lock); - ent = list->next; -while ( ent != list ) + page =page_list_first (list); +while ( !page_list_is_eol (page, list) ) { - page = list_entry(ent, struct page_info, list); -/* Grab a reference to the page so it won't disappear from under us. */ if (unlikely (!get_page (page, d)) ) {/* Couldn't get a reference -- someone is freeing this page. */ - ent = ent->next; -list_move_tail (&page->list, &d->arch.relmem_list); + cur = page; + page =page_list_next (page, list); +page_list_move_tail (cur, list, &d->arch.relmem_list);continue ; }
@@ -1747,9 +1745,10 @@ static int relinquish_memory( }/* Follow the list chain and /then/ potentially free the page. */ - ent = ent->next; -list_move_tail (&page->list, &d->arch.relmem_list); -put_page (page); + cur = page; + page =page_list_next (page, list); +page_list_move_tail (cur, list, &d->arch.relmem_list); +put_page (cur);if (hypercall_preempt_check () ) {
@@ -1758,7 +1757,7 @@ static int relinquish_memory( } } -list_splice_init (&d->arch.relmem_list, list); +page_list_splice_init (&d->arch.relmem_list, list); out:spin_unlock_recursive (&d->page_alloc_lock);
xen/arch/x86/domain_build.c +1 -1
@@ -880,7 +880,7 @@ int __init construct_dom0( } si->first_p2m_pfn = pfn; si->nr_p2m_frames = d->tot_pages - count; - list_for_each_entry ( page, &d->page_list, list ) + page_list_for_each ( page, &d->page_list ) { mfn =page_to_mfn (page);if (get_gpfn_from_mfn (mfn) >= count )
xen/arch/x86/domctl.c +7 -7
@@ -240,7 +240,7 @@ long arch_do_domctl(struct domain *d =rcu_lock_domain_by_id (domctl->domain);unsigned long max_pfns = domctl->u.getmemlist.max_pfns;uint64_t mfn; -struct list_head *list_ent; +struct page_info *page; ret = -EINVAL ;if ( d !=NULL )
@@ -259,19 +259,19 @@ long arch_do_domctl( goto getmemlist_out; } - ret =0 ; - list_ent = d->page_list.next; -for ( i =0 ; (i < max_pfns) && (list_ent != &d->page_list); i++ ) + ret = i =0 ; +page_list_for_each (page, &d->page_list) { - mfn =page_to_mfn (list_entry ( - list_ent, struct page_info, list)); +if ( i >= max_pfns ) +break ; + mfn =page_to_mfn (page);if (copy_to_guest_offset (domctl->u.getmemlist.buffer, i, &mfn,1 ) ) { ret = -EFAULT ;break ; } - list_ent =mfn_to_page (mfn)->list.next; + ++i; }spin_unlock (&d->page_alloc_lock);
xen/arch/x86/e820.c +1 -1
@@ -1,10 +1,10 @@#include <xen/config.h> #include <xen/init.h> #include <xen/lib.h> +#include <xen/mm.h> #include <xen/compat.h> #include <xen/dmi.h> #include <asm/e820.h> -#include <asm/mm.h> #include <asm/page.h> /* opt_mem: Limit of physical RAM. Any RAM beyond this point is ignored. */
xen/arch/x86/mm.c +2 -2
@@ -333,7 +333,7 @@ void share_xen_page_with_guest( page->count_info |= PGC_allocated |1 ;if (unlikely (d->xenheap_pages++ ==0 ) )get_knownalive_domain (d); -list_add_tail (&page->list, &d->xenpage_list); +page_list_add_tail (page, &d->xenpage_list); }spin_unlock (&d->page_alloc_lock);
@@ -3508,7 +3508,7 @@ int steal_page(/* Unlink from original owner. */ if ( !(memflags & MEMF_no_refcount) ) d->tot_pages--; -list_del (&page->list); +page_list_del (page, &d->page_list);spin_unlock (&d->page_alloc_lock);return 0 ;
xen/arch/x86/mm/hap/hap.c +7 -10
@@ -96,11 +96,10 @@ static struct page_info *hap_alloc(strucASSERT (hap_locked_by_me (d)); -if (unlikely (list_empty (&d->arch.paging.hap.freelist)) ) + pg =page_list_remove_head (&d->arch.paging.hap.freelist); +if (unlikely (!pg) )return NULL ; - pg = list_entry(d->arch.paging.hap.freelist.next, struct page_info, list); -list_del (&pg->list); d->arch.paging.hap.free_pages--; p =hap_map_domain_page (page_to_mfn (pg));
@@ -118,7 +117,7 @@ static void hap_free(struct domain *d, mASSERT (hap_locked_by_me (d)); d->arch.paging.hap.free_pages++; -list_add_tail (&pg->list, &d->arch.paging.hap.freelist); +page_list_add_tail (pg, &d->arch.paging.hap.freelist); }static struct page_info *hap_alloc_p2m_page (struct domain *d)
@@ -210,15 +209,13 @@ hap_set_allocation(struct domain *d, uns } d->arch.paging.hap.free_pages++; d->arch.paging.hap.total_pages++; -list_add_tail (&pg->list, &d->arch.paging.hap.freelist); +page_list_add_tail (pg, &d->arch.paging.hap.freelist); }else if ( d->arch.paging.hap.total_pages > pages ) {/* Need to return memory to domheap */ -ASSERT (!list_empty (&d->arch.paging.hap.freelist)); - pg =list_entry (d->arch.paging.hap.freelist.next, - struct page_info, list); -list_del (&pg->list); + pg =page_list_remove_head (&d->arch.paging.hap.freelist); +ASSERT (pg); d->arch.paging.hap.free_pages--; d->arch.paging.hap.total_pages--; pg->count_info =0 ;
@@ -393,7 +390,7 @@ static void hap_destroy_monitor_table(stvoid hap_domain_init (struct domain *d) {hap_lock_init (d); -INIT_LIST_HEAD (&d->arch.paging.hap.freelist); +INIT_PAGE_LIST_HEAD (&d->arch.paging.hap.freelist);/* This domain will use HAP for log-dirty mode */ paging_log_dirty_init(d, hap_enable_log_dirty, hap_disable_log_dirty,
xen/arch/x86/mm/hap/p2m-ept.c +1 -1
@@ -63,7 +63,7 @@ static int ept_set_middle_entry(struct d pg->count_info =1 ; pg->u.inuse.type_info =1 | PGT_validated; -list_add_tail (&pg->list, &d->arch.p2m->pages); +page_list_add_tail (pg, &d->arch.p2m->pages); ept_entry->emt =0 ; ept_entry->igmt =0 ;
xen/arch/x86/mm/p2m.c +31 -54
234 lines not on this page · raw message
xen/arch/x86/mm/shadow/common.c +5 -13
@@ -49,7 +49,7 @@ void shadow_domain_init(struct domain *dshadow_lock_init (d);for ( i =0 ; i <=SHADOW_MAX_ORDER ; i++ )INIT_LIST_HEAD (&d->arch.paging.shadow.freelists[i]); -INIT_LIST_HEAD (&d->arch.paging.shadow.p2m_freelist); +INIT_PAGE_LIST_HEAD (&d->arch.paging.shadow.p2m_freelist);INIT_LIST_HEAD (&d->arch.paging.shadow.pinned_shadows);/* Use shadow pagetables for log-dirty support */
@@ -1672,7 +1672,7 @@ sh_alloc_p2m_pages(struct domain *d)*/ page_set_owner (&pg[i], d); pg[i].count_info =1 ; -list_add_tail (&pg[i].list, &d->arch.paging.shadow.p2m_freelist); +page_list_add_tail (&pg[i], &d->arch.paging.shadow.p2m_freelist); }return 1 ; }
@@ -1681,25 +1681,22 @@ sh_alloc_p2m_pages(struct domain *d)static struct page_info *shadow_alloc_p2m_page (struct domain *d) { -struct list_head *entry;struct page_info *pg;mfn_t mfn;void *p;shadow_lock (d); -if (list_empty (&d->arch.paging.shadow.p2m_freelist) && +if (page_list_empty (&d->arch.paging.shadow.p2m_freelist) && !sh_alloc_p2m_pages (d) ) {shadow_unlock (d);return NULL ; } - entry = d->arch.paging.shadow.p2m_freelist.next; -list_del (entry); + pg =page_list_remove_head (&d->arch.paging.shadow.p2m_freelist);shadow_unlock (d); - pg = list_entry(entry, struct page_info, list); mfn =page_to_mfn (pg); p =sh_map_domain_page (mfn);clear_page (p);
@@ -3156,7 +3153,6 @@ void shadow_teardown(struct domain *d) {struct vcpu *v;mfn_t mfn; -struct list_head *entry, *n;struct page_info *pg;ASSERT (d->is_dying);
@@ -3208,12 +3204,8 @@ void shadow_teardown(struct domain *d) }#endif /* (SHADOW_OPTIMIZATIONS & (SHOPT_VIRTUAL_TLB|SHOPT_OUT_OF_SYNC)) */ -list_for_each_safe (entry, n, &d->arch.paging.shadow.p2m_freelist) - { -list_del (entry); - pg = list_entry(entry, struct page_info, list); +while ( (pg =page_list_remove_head (&d->arch.paging.shadow.p2m_freelist)) )shadow_free_p2m_page (d, pg); - }if ( d->arch.paging.shadow.total_pages !=0 ) {
xen/arch/x86/numa.c +1 -1
@@ -312,7 +312,7 @@ static void dump_numa(unsigned char key)for_each_online_node (i) page_num_node[i] =0 ; -list_for_each_entry (page, &d->page_list, list) +page_list_for_each (page, &d->page_list) { i =phys_to_nid (page_to_mfn (page) <<PAGE_SHIFT ); page_num_node[i]++;
xen/common/domain.c +2 -2
@@ -233,8 +233,8 @@ struct domain *domain_create(spin_lock_init (&d->page_alloc_lock);spin_lock_init (&d->shutdown_lock);spin_lock_init (&d->hypercall_deadlock_mutex); -INIT_LIST_HEAD (&d->page_list); -INIT_LIST_HEAD (&d->xenpage_list); +INIT_PAGE_LIST_HEAD (&d->page_list); +INIT_PAGE_LIST_HEAD (&d->xenpage_list);if ( domcr_flags & DOMCRF_hvm ) d->is_hvm =1 ;
xen/common/grant_table.c +1 -1
@@ -1192,7 +1192,7 @@ gnttab_transfer(/* Okay, add the page to 'e'. */ if (unlikely (e->tot_pages++ ==0 ) )get_knownalive_domain (e); -list_add_tail (&page->list, &e->page_list); +page_list_add_tail (page, &e->page_list);page_set_owner (page, e);spin_unlock (&e->page_alloc_lock);
xen/common/memory.c +8 -20
@@ -218,8 +218,8 @@ static void decrease_reservation(structstatic long memory_exchange (XEN_GUEST_HANDLE (xen_memory_exchange_t) arg) {struct xen_memory_exchange exch; -LIST_HEAD (in_chunk_list); -LIST_HEAD (out_chunk_list); +PAGE_LIST_HEAD (in_chunk_list); +PAGE_LIST_HEAD (out_chunk_list);unsigned long in_chunk_order, out_chunk_order;xen_pfn_t gpfn, gmfn, mfn;unsigned long i, j, k;
@@ -325,7 +325,7 @@ static long memory_exchange(XEN_GUEST_HA goto fail; } -list_add (&page->list, &in_chunk_list); +page_list_add (page, &in_chunk_list); } }
@@ -339,7 +339,7 @@ static long memory_exchange(XEN_GUEST_HA goto fail; } -list_add (&page->list, &out_chunk_list); +page_list_add (page, &out_chunk_list); }/*
@@ -347,10 +347,8 @@ static long memory_exchange(XEN_GUEST_HA*/ /* Destroy final reference to each input page. */ -while ( !list_empty (&in_chunk_list) ) +while ( (page =page_list_remove_head (&in_chunk_list)) ) { - page = list_entry(in_chunk_list.next, struct page_info, list); -list_del (&page->list);if ( !test_and_clear_bit (_PGC_allocated, &page->count_info) )BUG (); mfn =page_to_mfn (page);
@@ -360,10 +358,8 @@ static long memory_exchange(XEN_GUEST_HA/* Assign each output page to the domain. */ j =0 ; -while ( !list_empty (&out_chunk_list) ) +while ( (page =page_list_remove_head (&out_chunk_list)) ) { - page = list_entry(out_chunk_list.next, struct page_info, list); -list_del (&page->list);if (assign_pages (d, page, exch.out.extent_order, MEMF_no_refcount) )BUG ();
@@ -399,21 +395,13 @@ static long memory_exchange(XEN_GUEST_HA*/ fail:/* Reassign any input pages we managed to steal. */ -while ( !list_empty (&in_chunk_list) ) - { - page = list_entry(in_chunk_list.next, struct page_info, list); -list_del (&page->list); +while ( (page =page_list_remove_head (&in_chunk_list)) )if (assign_pages (d, page,0 , MEMF_no_refcount) )BUG (); - }/* Free any output pages we managed to allocate. */ -while ( !list_empty (&out_chunk_list) ) - { - page = list_entry(out_chunk_list.next, struct page_info, list); -list_del (&page->list); +while ( (page =page_list_remove_head (&out_chunk_list)) )free_domheap_pages (page, exch.out.extent_order); - } exch.nr_exchanged = i << in_chunk_order;
xen/common/page_alloc.c +24 -33
155 lines not on this page · raw message
xen/drivers/passthrough/amd/iommu_map.c +1 -4
@@ -552,7 +552,6 @@ int amd_iommu_sync_p2m(struct domain *d) {unsigned long mfn, gfn, flags;u64 iommu_l2e; -struct list_head *entry;struct page_info *page;struct hvm_iommu *hd;int iw =IOMMU_IO_WRITE_ENABLED ;
@@ -568,10 +567,8 @@ int amd_iommu_sync_p2m(struct domain *d)if ( hd->p2m_synchronized ) goto out; -for ( entry = d->page_list.next; entry != &d->page_list; - entry = entry->next ) + page_list_for_each ( page, &d->page_list ) { - page = list_entry(entry, struct page_info, list); mfn =page_to_mfn (page); gfn =get_gpfn_from_mfn (mfn);
xen/drivers/passthrough/amd/pci_amd_iommu.c +0 -1
@@ -23,7 +23,6 @@#include <xen/pci_regs.h> #include <asm/amd-iommu.h> #include <asm/hvm/svm/amd-iommu-proto.h> -#include <asm/mm.h> extern unsigned short ivrs_bdf_entries;extern struct ivrs_mappings *ivrs_mappings;
xen/drivers/passthrough/iommu.c +1 -1
@@ -141,7 +141,7 @@ static int iommu_populate_page_table(strspin_lock (&d->page_alloc_lock); - list_for_each_entry ( page, &d->page_list, list ) + page_list_for_each ( page, &d->page_list ) {if ( (page->u.inuse.type_info & PGT_type_mask) == PGT_writable_page ) {
xen/include/asm-x86/domain.h +3 -3
@@ -83,7 +83,7 @@ struct shadow_domain {/* Memory allocation */ struct list_head freelists[SHADOW_MAX_ORDER +1 ]; -struct list_head p2m_freelist; +struct page_list_head p2m_freelist;unsigned int total_pages;/* number of pages allocated */ unsigned int free_pages;/* number of pages on freelists */ unsigned int p2m_pages;/* number of pages allocates to p2m */
@@ -143,7 +143,7 @@ struct hap_domain {int locker;const char *locker_function; -struct list_head freelist; +struct page_list_head freelist;unsigned int total_pages;/* number of pages allocated */ unsigned int free_pages;/* number of pages on freelists */ unsigned int p2m_pages;/* number of pages allocates to p2m */
@@ -265,7 +265,7 @@ struct arch_domain RELMEM_l2, RELMEM_done, } relmem; -struct list_head relmem_list; +struct page_list_head relmem_list;cpuid_input_t cpuids[MAX_CPUID_INPUT ]; } __cacheline_aligned;
xen/include/asm-x86/mm.h +11 -2
@@ -12,15 +12,24 @@* Per-page-frame information. * * Every architecture must ensure the following: -* 1. 'struct page_info' contains a 'struct list_head list'. +* 1. 'struct page_info' contains a 'struct page_list_entry list'. * 2. Provide a PFN_ORDER() macro for accessing the order of a free page. */ #define PFN_ORDER (_pfn) ((_pfn)->u.free.order) +#ifndef __i386__ +# undef page_list_entry +struct page_list_entry +{ +unsigned int next, prev; +unsigned long _pad_for_sh_;/* until struct shadow_page_info gets updated */ +}; +#endif +struct page_info {/* Each frame can be threaded onto a doubly-linked list. */ -struct list_head list; +struct page_list_entry list;/* Reference count and various PGC_xxx flags and fields. */ unsigned long count_info;
xen/include/asm-x86/p2m.h +2 -2
@@ -110,7 +110,7 @@ struct p2m_domain {const char *locker_function;/* Func that took it */ /* Pages used to construct the p2m */ -struct list_head pages; +struct page_list_head pages;/* Functions to call to get or free pages for the p2m */ struct page_info * (*alloc_page )(struct domain *d);
@@ -148,7 +148,7 @@ struct p2m_domain {* protect moving stuff from the PoD cache to the domain page list. */ struct { -struct list_head super,/* List of superpages */ +struct page_list_head super,/* List of superpages */ single;/* Non-super lists */ int count,/* # of pages in cache lists */ entry_count;/* # of pages in p2m marked pod */
xen/include/asm-x86/perfc.h +0 -1
@@ -1,6 +1,5 @@#ifndef __ASM_PERFC_H__#define __ASM_PERFC_H__ -#include <asm/mm.h> static inline void arch_perfc_printall (void ) {
xen/include/xen/mm.h +207 -8
229 lines not on this page · raw message
xen/include/xen/sched.h +2 -2
@@ -171,8 +171,8 @@ struct domainspinlock_t domain_lock;spinlock_t page_alloc_lock;/* protects all the following fields */ -struct list_head page_list;/* linked list, of size tot_pages */ -struct list_head xenpage_list;/* linked list, of size xenheap_pages */ +struct page_list_head page_list;/* linked list, of size tot_pages */ +struct page_list_head xenpage_list;/* linked list (size xenheap_pages) */ unsigned int tot_pages;/* number of pages currently possesed */ unsigned int max_pages;/* maximum value for tot_pages */ unsigned int xenheap_pages;/* # pages allocated from Xen heap */