gnttab: Improve scaleability
v3 cover gnttab: Improve scaleability
This patch series changes the grant table locking to a more fain grained locking protocol. The result is a performance boost measured with blkfront/blkback. Document the locking protocol.
v3:
* Addressed gnttab_swap_grant_ref() comment from Andrew Cooper
v2:
* Add arm part per request from Julien Grall
Christoph Egger (1):
gnttab: Introduce rwlock to protect updates to grant table state
Matt Wilson (1):
gnttab: refactor locking for scalability
docs/misc/grant-tables.txt | 49 ++++++- xen/arch/arm/mm.c | 4 +- xen/arch/x86/mm.c | 4 +- xen/common/grant_table.c | 321 +++++++++++++++++++++++++---------------- xen/include/xen/grant_table.h | 9 +- 5 files changed, 258 insertions(+), 129 deletions(-)
Signature · 1 line
1.7.9.5
PATCH v3 1/2 gnttab: Introduce rwlock to protect updates to grant table state
Split grant table lock into two separate locks. One to protect maptrack state and change the other into a rwlock.
The rwlock is used to prevent readers from accessing inconsistent grant table state such as current version, partially initialized active table pages, etc.
v3:
* Addressed gnttab_swap_grant_ref() comment from Andrew Cooper
v2:
* Add arm part per request from Julien Grall
--- docs/misc/grant-tables.txt | 28 +++++++++- xen/arch/arm/mm.c | 4 +- xen/arch/x86/mm.c | 4 +- xen/common/grant_table.c | 120 +++++++++++++++++++++++------------------ xen/include/xen/grant_table.h | 9 ++-- 5 files changed, 104 insertions(+), 61 deletions(-)
docs/misc/grant-tables.txt +27 -1
@@ -74,7 +74,33 @@ is complete. matching map track entry is then removed, as if unmap had been invoked. These are not used by the transfer mechanism. map->domid : owner of the mapped frame - map->ref_and_flags : grant reference, ro/rw, mapped for host or device access + map->ref : grant reference + map->flags : ro/rw, mapped for host or device access + +******************************************************************************** + Locking + ~~~~~~~ + Xen uses several locks to serialize access to the internal grant table state. + + grant_table->lock : rwlock used to prevent readers from accessing + inconsistent grant table state such as current + version, partially initialized active table pages, + etc. + grant_table->maptrack_lock : spinlock used to protect the maptrack state + + The primary lock for the grant table is a read/write spinlock. All + functions that access members of struct grant_table must acquire a + read lock around critical sections. Any modification to the members + of struct grant_table (e.g., nr_status_frames, nr_grant_frames, + active frames, etc.) must only be made if the write lock is + held. These elements are read-mostly, and read critical sections can + be large, which makes a rwlock a good choice. + + The maptrack state is protected by its own spinlock. Any access (read + or write) of struct grant_table members that have a "maptrack_" + prefix must be made while holding the maptrack lock. The maptrack + state can be rapidly modified under some workloads, and the critical + sections are very small, thus we use a spinlock to protect them. ********************************************************************************
xen/arch/arm/mm.c +2 -2
@@ -1037,7 +1037,7 @@ int xenmem_add_to_physmap_one(switch ( space ) {case XENMAPSPACE_grant_table: -spin_lock (&d->grant_table->lock); +write_lock (&d->grant_table->lock);if ( d->grant_table->gt_version ==0 ) d->grant_table->gt_version =1 ;
@@ -1067,7 +1067,7 @@ int xenmem_add_to_physmap_one( t = p2m_ram_rw; -spin_unlock (&d->grant_table->lock); +write_unlock (&d->grant_table->lock);break ;case XENMAPSPACE_shared_info:if ( idx !=0 )
xen/arch/x86/mm.c +2 -2
@@ -4565,7 +4565,7 @@ int xenmem_add_to_physmap_one( mfn =virt_to_mfn (d->shared_info);break ;case XENMAPSPACE_grant_table: -spin_lock (&d->grant_table->lock); +write_lock (&d->grant_table->lock);if ( d->grant_table->gt_version ==0 ) d->grant_table->gt_version =1 ;
@@ -4587,7 +4587,7 @@ int xenmem_add_to_physmap_one( mfn =virt_to_mfn (d->grant_table->shared_raw[idx]); } -spin_unlock (&d->grant_table->lock); +write_unlock (&d->grant_table->lock);break ;case XENMAPSPACE_gmfn_range:case XENMAPSPACE_gmfn:
xen/common/grant_table.c +67 -53
440 lines not on this page · raw message
xen/include/xen/grant_table.h +6 -3
@@ -82,8 +82,11 @@ struct grant_table {struct grant_mapping **maptrack;unsigned int maptrack_head;unsigned int maptrack_limit; -/* Lock protecting updates to active and shared grant tables. */ -spinlock_t lock; +/* Lock protecting the maptrack page list, head, and limit */ +spinlock_t maptrack_lock; +/* Lock protecting updates to grant table state (version, active +entry list, etc.) */ +rwlock_t lock;/* The defined versions are 1 and 2. Set to 0 if we don't know what version to use yet. */ unsigned gt_version;
@@ -101,7 +104,7 @@ gnttab_release_mappings( struct domain *d);/* Increase the size of a domain's grant table. -* Caller must hold d's grant table lock. +* Caller must hold d's grant table write lock. */ int gnttab_grow_table (struct domain *d,unsigned int req_nr_frames);
Signature · 1 line
1.7.9.5
Re: [PATCH v3 1/2] gnttab: Introduce rwlock to protect updates to grant table state
Quoted 11 lines
Quoted 1 line
Quoted 1 line · On 03.12.14 at 15:29, <[email protected]> wrote:
xen/common/grant_table.c +2
-2@@ -227,23 +227,23 @@ double_gt_lock(struct grant_table *lgt, struct grant_table *rgt) {if ( lgt < rgt ) { -spin_lock (&lgt->lock); -spin_lock (&rgt->lock); +spin_lock (&lgt->maptrack_lock); +spin_lock (&rgt->maptrack_lock);
xen/common/grant_table.c double_gt_lock(struct grant_table *lgt, struct grant_table *rgt) · line 239
Quoted 10 lines
xen/common/grant_table.c +2
-2}else {if ( lgt != rgt ) -spin_lock (&rgt->lock); -spin_lock (&lgt->lock); +spin_lock (&rgt->maptrack_lock); +spin_lock (&lgt->maptrack_lock); } }
I think this function needs to be renamed with this change, to clarify that it's not the general grant table locks which get acquired here. Same for the unlock then of course. That'll also make stand out the places where the function is used.
Quoted 6 lines
xen/common/grant_table.c +1
-0@@ -891,28 +891,28 @@ __gnttab_unmap_common( ld = current->domain; lgt = ld->grant_table; +read_lock (&lgt->lock); op->frame = (unsigned long )(op->dev_bus_addr >>PAGE_SHIFT );
xen/common/grant_table.c __gnttab_unmap_common( · line 905
Quoted 10 lines
xen/common/grant_table.c +2
-0if (unlikely (op->handle >= lgt->maptrack_limit) ) { +read_unlock (&lgt->lock);gdprintk (XENLOG_INFO ,"Bad handle (%d).\n" , op->handle); op->status = GNTST_bad_handle;return ; } +read_unlock (&lgt->lock);
The added locking here seems pointless, or else there would be a bug in the existing code (which should then be fixed by a separate, much smaller change).
xen/common/grant_table.c __gnttab_unmap_common( · line 915
Quoted 13 lines
xen/common/grant_table.c +0
-3op->map = &maptrack_entry (lgt, op->handle); -spin_lock (&lgt->lock);if (unlikely (!op->map->flags) ) { -spin_unlock (&lgt->lock);gdprintk (XENLOG_INFO ,"Zero flags for handle (%d).\n" , op->handle); op->status = GNTST_bad_handle;return ; } dom = op->map->domid; -spin_unlock (&lgt->lock);
And the removed locking here needs special mentioning in the commit message, explaining why no locking is needed.
xen/common/grant_table.c __gnttab_unmap_common( · line 950
Quoted 8 lines
xen/common/grant_table.c +1
-0@@ -944,6 +944,7 @@ __gnttab_unmap_common( } op->rd = rd; +read_lock (&rgt->lock); act = &active_entry (rgt, op->map->ref);if ( op->frame ==0 )
The nesting of the two locks should be mentioned in the doc change.
xen/common/grant_table.c __gnttab_unmap_common( · line 1008
Quoted 5 lines
xen/common/grant_table.c +1
-0@@ -1004,6 +1005,7 @@ __gnttab_unmap_common( unmap_out:double_gt_unlock (lgt, rgt); +read_unlock (&rgt->lock);
And I'd prefer unlocking sequence to be the inverse of the locking one, just to avoid confusion about their nesting.
xen/common/grant_table.c gnttab_unpopulate_status_frames(struct domain *d, struct grant_table *gt) · line 1291
Quoted 7 lines
xen/common/grant_table.c +3
-0@@ -1284,10 +1286,13 @@ gnttab_unpopulate_status_frames(struct domain *d, struct grant_table *gt) gt->nr_status_frames =0 ; } +/* Grow the grant table. The caller must hold the grant table's +* write lock before calling this function. +*/
Coding style.
xen/common/grant_table.c __acquire_grant_for_copy( · line 1976
Quoted 9 lines
xen/common/grant_table.c +4
-1@@ -1965,17 +1970,21 @@ __acquire_grant_for_copy(PIN_FAIL (unlock_out_clear, GNTST_general_error,"transitive grant referenced bad domain %d\n" , trans_domid); -spin_unlock (&rgt->lock); + +/* __acquire_grant_for_copy() could take the read lock on +* the right table (if rd == td), so we have to drop the +* lock here and reacquire */
Coding style again.
Quoted 10 lines
xen/common/grant_table.c +1
-1@@ -2900,7 +2910,7 @@ gnttab_release_mappings( } rgt = rd->grant_table; -spin_lock (&rgt->lock); +read_lock (&rgt->lock); act = &active_entry (rgt, ref); sha =shared_entry_header (rgt, ref);
xen/common/grant_table.c gnttab_release_mappings( · line 2973
Quoted 5 lines
xen/common/grant_table.c +1
-1@@ -2960,7 +2970,7 @@ gnttab_release_mappings(if ( act->pin ==0 )gnttab_clear_flag (_GTF_reading, status); -spin_unlock (&rgt->lock); +read_unlock (&rgt->lock);
The maptrack entries are being accessed between these two - don't you need both locks here?
Quoted 6 lines
xen/include/xen/grant_table.h +0
-0@@ -82,8 +82,11 @@ struct grant_table {struct grant_mapping **maptrack;unsigned int maptrack_head;unsigned int maptrack_limit;
xen/include/xen/grant_table.h struct grant_table { · line 92
Quoted 10 lines
xen/include/xen/grant_table.h +5
-2-/* Lock protecting updates to active and shared grant tables. */ -spinlock_t lock; +/* Lock protecting the maptrack page list, head, and limit */ +spinlock_t maptrack_lock; +/* Lock protecting updates to grant table state (version, active +entry list, etc.) */ +rwlock_t lock;/* The defined versions are 1 and 2. Set to 0 if we don't know what version to use yet. */ unsigned gt_version;
Coding style again - the malformed existing comment is no reason to add another malformed one.
Jan
Re: [PATCH v3 1/2] gnttab: Introduce rwlock to protect updates to grant table state
Quoted 3 lines · Jan Beulich
Quoted 3 lines
Quoted 1 line
Quoted 1 line · On 03.12.14 at 15:29, <[email protected]> wrote:
xen/common/grant_table.c +0
-0
[...]
Quoted 10 lines
xen/common/grant_table.c __gnttab_unmap_common( · line 950
Quoted 8 lines
xen/common/grant_table.c +1
-0@@ -944,6 +944,7 @@ __gnttab_unmap_common( } op->rd = rd; +read_lock (&rgt->lock); act = &active_entry (rgt, op->map->ref);if ( op->frame ==0 )The nesting of the two locks should be mentioned in the doc change.
Where do you see the nesting coming from?
Christoph
Re: [PATCH v3 1/2] gnttab: Introduce rwlock to protect updates to grant table state
Quoted 13 lines
Quoted 1 line
Quoted 1 line · On 09.01.15 at 14:05, <[email protected]> wrote:
Quoted 3 lines · Jan Beulich
Quoted 3 lines
Quoted 1 line
Quoted 1 line · On 03.12.14 at 15:29, <[email protected]> wrote:
xen/common/grant_table.c +0
-0[...]
Quoted 7 lines
Quoted 7 lines
xen/common/grant_table.c +1
-0@@ -944,6 +944,7 @@ __gnttab_unmap_common( } op->rd = rd; +read_lock (&rgt->lock); act = &active_entry (rgt, op->map->ref);
Quoted 5 lines
Quoted 3 lines
xen/common/grant_table.c __gnttab_unmap_common( · line 950
Quoted 1 line
xen/common/grant_table.c +0
-0if ( op->frame ==0 )The nesting of the two locks should be mentioned in the doc change.
Where do you see the nesting coming from?
Reproducing the subsequent hunk I had commented on
xen/common/grant_table.c __gnttab_unmap_common( · line 1008
Quoted 5 lines
xen/common/grant_table.c +1
-0@@ -1004,6 +1005,7 @@ __gnttab_unmap_common( unmap_out:double_gt_unlock (lgt, rgt); +read_unlock (&rgt->lock);
there obviously are two (or really three) locks in simultaneous use here.
Jan
Re: [PATCH v3 1/2] gnttab: Introduce rwlock to protect updates to grant table state
Quoted 21 lines · Jan Beulich
Quoted 13 lines
Quoted 1 line
Quoted 1 line · On 09.01.15 at 14:05, <[email protected]> wrote:
Quoted 3 lines · Jan Beulich
Quoted 3 lines
Quoted 1 line
Quoted 1 line · On 03.12.14 at 15:29, <[email protected]> wrote:
xen/common/grant_table.c +0
-0[...]
Quoted 7 lines
Quoted 7 lines
xen/common/grant_table.c +1
-0@@ -944,6 +944,7 @@ __gnttab_unmap_common( } op->rd = rd; +read_lock (&rgt->lock); act = &active_entry (rgt, op->map->ref);Quoted 5 lines
Quoted 3 lines
xen/common/grant_table.c __gnttab_unmap_common( · line 950
Quoted 1 line
xen/common/grant_table.c +0
-0if ( op->frame ==0 )The nesting of the two locks should be mentioned in the doc change.
Where do you see the nesting coming from?
Reproducing the subsequent hunk I had commented on
Quoted 8 lines
xen/common/grant_table.c __gnttab_unmap_common( · line 1008
Quoted 5 lines
xen/common/grant_table.c +1
-0@@ -1004,6 +1005,7 @@ __gnttab_unmap_common( unmap_out:double_gt_unlock (lgt, rgt); +read_unlock (&rgt->lock);there obviously are two (or really three) locks in simultaneous use here.
Ah, I see now. My mistake is I was looking for multiple subsequent read_lock(&rgt->lock); calls and found nothing.
Christoph
PATCH v3 2/2 gnttab: refactor locking for scalability
In reply to Christoph EggerFrom: Matt Wilson <[email protected]>
This patch refactors grant table locking. It splits the previous single spinlock per grant table into multiple locks. The heavily modified components of the grant table (the maptrack state and the active entries) are now protected by their own spinlocks. The remaining elements of the grant table are read-mostly, so the main grant table lock is modified to be a rwlock to improve concurrency.
Workloads with high grant table operation rates, especially map/unmap operations as used by blkback/blkfront when persistent grants are not supported, show marked improvement with these changes. A domU with 24 VBDs in a streaming 2M write workload achieved 1,400 MB/sec before this change. Performance more than doubles with this patch, reaching 3,000 MB/sec before tuning and 3,600 MB/sec after adjusting event channel vCPU bindings.
v3:
* Addressed gnttab_swap_grant_ref() comment from Andrew Cooper
--- docs/misc/grant-tables.txt | 21 +++++ xen/common/grant_table.c | 219 ++++++++++++++++++++++++++++---------------- 2 files changed, 163 insertions(+), 77 deletions(-)
docs/misc/grant-tables.txt +21 -0
@@ -63,6 +63,7 @@ is complete. act->domid : remote domain being granted rights act->frame : machine frame being granted act->pin : used to hold reference counts + act->lock : spinlock used to serialize access to active entry state Map tracking ~~~~~~~~~~~~
@@ -87,6 +88,8 @@ is complete. version, partially initialized active table pages, etc. grant_table->maptrack_lock : spinlock used to protect the maptrack state + active_grant_entry->lock : spinlock used to serialize modifications to + active entries The primary lock for the grant table is a read/write spinlock. All functions that access members of struct grant_table must acquire a
@@ -102,6 +105,24 @@ is complete. state can be rapidly modified under some workloads, and the critical sections are very small, thus we use a spinlock to protect them. + Active entries are obtained by calling active_entry_acquire(gt, ref). + This function returns a pointer to the active entry after locking its + spinlock. The caller must hold the rwlock for the gt in question + before calling active_entry_acquire(). This is because the grant + table can be dynamically extended via gnttab_grow_table() while a + domain is running and must be fully initialized. Once all access to + the active entry is complete, release the lock by calling + active_entry_release(act). + + Summary of rules for locking: + active_entry_acquire() and active_entry_release() can only be + called when holding the relevant grant table's lock. I.e.: + read_lock(>->lock); + act = active_entry_acquire(gt, ref); + ... + active_entry_release(act); + read_unlock(>->lock); + ******************************************************************************** Granting a foreign domain access to frames
xen/common/grant_table.c +142 -77
589 lines not on this page · raw message
Signature · 1 line
1.7.9.5
Re: [PATCH v3 2/2] gnttab: refactor locking for scalability
Quoted 6 lines
Quoted 1 line
Quoted 1 line · On 03.12.14 at 15:29, <[email protected]> wrote:
xen/common/grant_table.c +1
-0@@ -182,6 +185,29 @@ nr_active_grant_frames(struct grant_table *gt)return num_act_frames_from_sha_frames (nr_grant_frames (gt)); } +static inline struct active_grant_entry *
xen/common/grant_table.c nr_active_grant_frames(struct grant_table *gt) · line 198
Quoted 10 lines
xen/common/grant_table.c +10
-0+active_entry_acquire(struct grant_table *t, grant_ref_t e) +{ +struct active_grant_entry *act; + +#ifndef NDEBUG +/* not perfect, but better than nothing for a debug build +* sanity check +*/ +BUG_ON (!rw_is_locked (&t->lock)); +#endif
ASSERT() is the canonical way to achieve this.
Quoted 14 lines
xen/common/grant_table.c +4
-1@@ -508,17 +511,27 @@ static int grant_map_exists(const struct domain *ld, nr_grant_entries(rgt));for ( ref = *ref_count; ref < max_iter; ref++ ) { - act = &active_entry (rgt, ref); + act =active_entry_acquire (rgt, ref);if ( !act->pin ) + { +active_entry_release (act);continue ; + }if ( act->domid != ld->domain_id )
xen/common/grant_table.c static int grant_map_exists(const struct domain *ld, · line 532
Quoted 10 lines
xen/common/grant_table.c +6
-0+ { +active_entry_release (act);continue ; + }if ( act->frame != mfn ) + { +active_entry_release (act);continue ; + }
With this repeated 3 times, perhaps simpler and easier to read if you put this ahead of the increment inside the for() header?
Quoted 7 lines
xen/common/grant_table.c +3
-0@@ -531,24 +544,44 @@ static int grant_map_exists(const struct domain *ld,return -EINVAL ; } +/* Count the number of mapped ro or rw entries tracked in the left +* grant table given a provided mfn provided by a foreign domain. +*
xen/common/grant_table.c static int grant_map_exists(const struct domain *ld, · line 559
Quoted 10 lines
xen/common/grant_table.c +4
-0+* This function takes the maptrack lock from the left grant table and +* must be called with the right grant table's rwlock held. +*/ static void mapcount (struct grant_table *lgt,struct domain *rd,unsigned long mfn,unsigned int *wrc,unsigned int *rdc) {struct grant_mapping *map;grant_handle_t handle; +struct active_grant_entry *act;
I don't see the need for this new local variable.
xen/common/grant_table.c static int grant_map_exists(const struct domain *ld, · line 571
Quoted 11 lines
xen/common/grant_table.c +9
-0*wrc = *rdc =0 ; +/* N.B.: while taking the left side maptrack spinlock prevents +* any mapping changes, the right side active entries could be +* changing while we are counting. To be perfectly correct, the +* caller should hold the grant table write lock, or some other +* mechanism should be used to prevent concurrent changes during +* this operation. This is tricky because we can't promote a read +* lock into a write lock. +*/ +spin_lock (&lgt->maptrack_lock);
We can't afford being less than "perfectly correct" here, as the insertion/removal of IOMMU mappings is driven by the result of this function. Or wait, together with the comment ahead of the function, are you perhaps simply meaning "has to" instead of "should", and perhaps without the "To be perfectly correct" pre- condition? Also, to prevent changes holding the grant table lock for reading ought to be sufficient. Which you then should ASSERT() is the case just like you do in active_entry_acquire().
Quoted 14 lines
xen/common/grant_table.c +0
-3@@ -692,12 +724,9 @@ __gnttab_map_grant_ref( GNTPIN_hstr_inc : GNTPIN_hstw_inc; frame = act->frame; - act_pin = act->pin; cache_flags = (shah->flags & (GTF_PAT |GTF_PWT |GTF_PCD ) ); -read_unlock (&rgt->lock); -/* pg may be set, with a refcount included, from __get_paged_frame */ if ( !pg ) {
xen/common/grant_table.c __gnttab_map_grant_ref( · line 806
Quoted 8 lines
xen/common/grant_table.c +0
-2@@ -772,8 +801,6 @@ __gnttab_map_grant_ref( goto undo_out; } -double_gt_lock (lgt, rgt); -if (gnttab_need_iommu_mapping (ld) ) {unsigned int wrc, rdc;
Taking these two together - isn't patch 1 wrong then, in that it acquires both maptrack locks in double_gt_lock()?
xen/common/grant_table.c __gnttab_unmap_common( · line 925
Quoted 11 lines
xen/common/grant_table.c +1
-1@@ -891,9 +917,9 @@ __gnttab_unmap_common( ld = current->domain; lgt = ld->grant_table; -read_lock (&lgt->lock); op->frame = (unsigned long )(op->dev_bus_addr >>PAGE_SHIFT ); +read_lock (&lgt->lock);if (unlikely (op->handle >= lgt->maptrack_limit) ) {read_unlock (&lgt->lock);
This should be done this way in patch 1.
xen/common/grant_table.c __gnttab_unmap_common( · line 964
Quoted 8 lines
xen/common/grant_table.c +0
-1@@ -933,19 +959,18 @@ __gnttab_unmap_common(TRACE_1D (TRC_MEM_PAGE_GRANT_UNMAP , dom); rgt = rd->grant_table; -double_gt_lock (lgt, rgt); op->flags = op->map->flags;if (unlikely (!op->flags) ||unlikely (op->map->domid != dom) )
How are these checks guarded now?
xen/common/grant_table.c gnttab_grow_table(struct domain *d, unsigned int req_nr_frames) · line 1347
Quoted 8 lines
xen/common/grant_table.c +4
-0@@ -1310,6 +1341,10 @@ gnttab_grow_table(struct domain *d, unsigned int req_nr_frames)if ( (gt->active[i] =alloc_xenheap_page ()) ==NULL ) goto active_alloc_failed;clear_page (gt->active[i]); +for ( j =0 ; j <ACGNT_PER_PAGE ; j++ ) + { +spin_lock_init (>->active[i][j].lock); + }
Please omit the braces when the body is just a single statement.
Jan