dm log userspace: a fix and other improvements
v2 cover dm log userspace: a fix and other improvements
Alasdair,
I've reviewed all of these patches from Jon:
- cleaned up all patch headers and subjects a bit
- patch 1 is a fix that should go to 2.6.37
- the others are fine to go to linux-next (and queue for 2.6.38)
- made some changes to patch 3 (uninit 'type' warning, move 'group' from end of 'log_c' structure to flush_by_group's stack). Jon may clean up the flush_by_group() list processing further but I've added a useful comment about the final list_splice_init.
Thanks,
Mike
Jonathan Brassow (4):
dm log userspace: trap all errors from failed log construction dm log userspace: split flush queue
dm log userspace: group clear and mark requests
dm log userspace: add version number to communication structure
drivers/md/dm-log-userspace-base.c | 130 ++++++++++++++++++++++++++------ drivers/md/dm-log-userspace-transfer.c | 1 + include/linux/dm-log-userspace.h | 13 +++- 3 files changed, 119 insertions(+), 25 deletions(-)
Signature · 1 line
1.7.2.3
PATCH v2 1/4 dm log userspace: trap all errors from failed log construction
From: Jonathan Brassow <[email protected]>
When constructing a mirror log, it is possible for the initial request to fail for other reasons besides -ESRCH. These must be handled too.
--- drivers/md/dm-log-userspace-base.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-)
drivers/md/dm-log-userspace-base.c +5 -2
@@ -181,8 +181,11 @@ static int userspace_ctr(struct dm_dirty_log *log, struct dm_target *ti, r =dm_consult_userspace (lc->uuid, lc->luid,DM_ULOG_CTR , ctr_str, str_size,NULL ,NULL ); -if (r == -ESRCH ) { -DMERR ("Userspace log server not found" ); +if (r <0 ) { +if (r == -ESRCH ) +DMERR ("Userspace log server not found" ); +else +DMERR ("Userspace log server failed to create log" ); goto out; }
Signature · 1 line
1.7.2.3
PATCH v2 2/4 dm log userspace: split flush queue
In reply to Mike SnitzerFrom: Jonathan Brassow <[email protected]>
Split the 'flush_list', which contained a mix of both 'mark' and 'clear' requests, into two distinct lists ('mark_list' and 'clear_list').
The device mapper log implementations (used by various DM targets) are allowed to cache 'mark' and 'clear' requests until a 'flush' is received. Until now, these cached requests were kept in the same list. They will now be put into distinct lists to facilitate group processing of these requests (in the next patch).
--- drivers/md/dm-log-userspace-base.c | 41 ++++++++++++++++++++++++++++-------- 1 files changed, 32 insertions(+), 9 deletions(-)
drivers/md/dm-log-userspace-base.c +32 -9
@@ -37,8 +37,15 @@ struct log_c {*/ uint64_t in_sync_hint; +/* +* Mark and clear requests are held until a flush is issued +* so that we can group, and thereby limit, the amount of +* network traffic between kernel and userspace. The 'flush_lock' +* is used to protect these lists. +*/ spinlock_t flush_lock; -struct list_head flush_list;/* only for clear and mark requests */ +struct list_head mark_list; +struct list_head clear_list; };static mempool_t *flush_entry_pool;
@@ -169,7 +176,8 @@ static int userspace_ctr(struct dm_dirty_log *log, struct dm_target *ti,strncpy (lc->uuid, argv[0 ],DM_UUID_LEN );spin_lock_init (&lc->flush_lock); -INIT_LIST_HEAD (&lc->flush_list); +INIT_LIST_HEAD (&lc->mark_list); +INIT_LIST_HEAD (&lc->clear_list); str_size =build_constructor_string (ti, argc -1 , argv +1 , &ctr_str);if (str_size <0 ) {
@@ -363,14 +371,16 @@ static int userspace_flush(struct dm_dirty_log *log)int r =0 ;unsigned long flags;struct log_c *lc = log->context; -LIST_HEAD (flush_list); +LIST_HEAD (mark_list); +LIST_HEAD (clear_list);struct flush_entry *fe, *tmp_fe;spin_lock_irqsave (&lc->flush_lock, flags); -list_splice_init (&lc->flush_list, &flush_list); +list_splice_init (&lc->mark_list, &mark_list); +list_splice_init (&lc->clear_list, &clear_list);spin_unlock_irqrestore (&lc->flush_lock, flags); -if (list_empty (&flush_list)) +if (list_empty (&mark_list) &&list_empty (&clear_list))return 0 ;/*
@@ -380,7 +390,16 @@ static int userspace_flush(struct dm_dirty_log *log)* do it one by one. */ -list_for_each_entry (fe, &flush_list, list) { +list_for_each_entry (fe, &mark_list, list) { + r =userspace_do_request (lc, lc->uuid, fe->type, + (char *)&fe->region, +sizeof (fe->region), +NULL ,NULL ); +if (r) + goto fail; + } + +list_for_each_entry (fe, &clear_list, list) { r = userspace_do_request(lc, lc->uuid, fe->type, (char *)&fe->region, sizeof(fe->region),
@@ -398,7 +417,11 @@ fail:* Calling code will receive an error and will know that * the log facility has failed. */ -list_for_each_entry_safe (fe, tmp_fe, &flush_list, list) { +list_for_each_entry_safe (fe, tmp_fe, &mark_list, list) { +list_del (&fe->list); +mempool_free (fe, flush_entry_pool); + } +list_for_each_entry_safe (fe, tmp_fe, &clear_list, list) {list_del (&fe->list);mempool_free (fe, flush_entry_pool); }
@@ -428,7 +451,7 @@ static void userspace_mark_region(struct dm_dirty_log *log, region_t region)spin_lock_irqsave (&lc->flush_lock, flags); fe->type =DM_ULOG_MARK_REGION ; fe->region = region; -list_add (&fe->list, &lc->flush_list); +list_add (&fe->list, &lc->mark_list);spin_unlock_irqrestore (&lc->flush_lock, flags);return ;
@@ -465,7 +488,7 @@ static void userspace_clear_region(struct dm_dirty_log *log, region_t region)spin_lock_irqsave (&lc->flush_lock, flags); fe->type =DM_ULOG_CLEAR_REGION ; fe->region = region; -list_add (&fe->list, &lc->flush_list); +list_add (&fe->list, &lc->clear_list);spin_unlock_irqrestore (&lc->flush_lock, flags);return ;
Signature · 1 line
1.7.2.3
PATCH v2 3/4 dm log userspace: group clear and mark requests
In reply to Mike SnitzerFrom: Jonathan Brassow <[email protected]>
Allow the device-mapper log's 'mark' and 'clear' requests to be grouped and processed in a batch. This can significantly reduce the amount of traffic going between the kernel and userspace (where the processing daemon resides).
--- drivers/md/dm-log-userspace-base.c | 102 ++++++++++++++++++++++++++++-------- 1 files changed, 79 insertions(+), 23 deletions(-)
drivers/md/dm-log-userspace-base.c +79 -23
@@ -18,6 +18,14 @@ struct flush_entry {struct list_head list; }; +/* +* This limit on the number of mark and clear request is, to a degree, +* arbitrary. However, there is some basis for the choice in the limits +* imposed on the size of data payload by dm-log-userspace-transfer.c: +* dm_consult_userspace(). +*/ +#define MAX_FLUSH_GROUP_COUNT 32 +struct log_c {struct dm_target *ti;uint32_t region_size;
@@ -349,6 +357,71 @@ static int userspace_in_sync(struct dm_dirty_log *log, region_t region,return (r) ?0 : (int )in_sync; } +static int flush_one_by_one (struct log_c *lc,struct list_head *flush_list) +{ +int r =0 ; +struct flush_entry *fe; + +list_for_each_entry (fe, flush_list, list) { + r =userspace_do_request (lc, lc->uuid, fe->type, + (char *)&fe->region, +sizeof (fe->region), +NULL ,NULL ); +if (r) +break ; + } + +return r; +} + +static int flush_by_group (struct log_c *lc,struct list_head *flush_list) +{ +int r =0 ; +int count; +uint32_t type =0 ; +struct flush_entry *fe, *tmp_fe; +LIST_HEAD (tmp_list); +uint64_t group[MAX_FLUSH_GROUP_COUNT ]; + +/* +* Group process the requests +*/ +while (!list_empty (flush_list)) { + count =0 ; + +list_for_each_entry_safe (fe, tmp_fe, flush_list, list) { + group[count] = fe->region; + count++; + +list_del (&fe->list); +list_add (&fe->list, &tmp_list); + + type = fe->type; +if (count >=MAX_FLUSH_GROUP_COUNT ) +break ; + } + + r =userspace_do_request (lc, lc->uuid, type, + (char *)(group), + count *sizeof (uint64_t ), +NULL ,NULL ); +if (r) { +/* Group send failed. Attempt one-by-one. */ +list_splice_init (&tmp_list, flush_list); + r =flush_one_by_one (lc, flush_list); +break ; + } + } + +/* +* Must collect flush_entrys that were successfully processed +* as a group so that they will be free'd by the caller. +*/ +list_splice_init (&tmp_list, flush_list); + +return r; +} +/* * userspace_flush *
@@ -383,30 +456,13 @@ static int userspace_flush(struct dm_dirty_log *log)if (list_empty (&mark_list) &&list_empty (&clear_list))return 0 ; -/* -* FIXME: Count up requests, group request types, -* allocate memory to stick all requests in and -* send to server in one go. Failing the allocation, -* do it one by one. -*/ - -list_for_each_entry (fe, &mark_list, list) { - r =userspace_do_request (lc, lc->uuid, fe->type, - (char *)&fe->region, -sizeof (fe->region), -NULL ,NULL ); -if (r) - goto fail; - } + r =flush_by_group (lc, &mark_list); +if (r) + goto fail; -list_for_each_entry (fe, &clear_list, list) { - r =userspace_do_request (lc, lc->uuid, fe->type, - (char *)&fe->region, -sizeof (fe->region), -NULL ,NULL ); -if (r) - goto fail; - } + r =flush_by_group (lc, &clear_list); +if (r) + goto fail; r =userspace_do_request (lc, lc->uuid,DM_ULOG_FLUSH ,NULL ,0 ,NULL ,NULL );
Signature · 1 line
1.7.2.3
PATCH v2 4/4 dm log userspace: add version number to communication structure
In reply to Mike SnitzerFrom: Jonathan Brassow <[email protected]>
This patch adds a 'version' field to the 'dm_ulog_request' structure.
The 'version' field is taken from a portion of the unused 'padding' field in the 'dm_ulog_request' structure. This was done to avoid changing the size of the structure and possibly disrupting backwards compatibility.
The version number will help notify user-space daemons when a change has been made to the kernel/userspace log API.
--- drivers/md/dm-log-userspace-transfer.c | 1 + include/linux/dm-log-userspace.h | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletions(-)
drivers/md/dm-log-userspace-transfer.c +1 -0
@@ -198,6 +198,7 @@ resend:memset (tfr,0 ,DM_ULOG_PREALLOCED_SIZE -sizeof (struct cn_msg ));memcpy (tfr->uuid, uuid,DM_UUID_LEN ); + tfr->version =DM_ULOG_REQUEST_VERSION ; tfr->luid = luid; tfr->seq = dm_ulog_seq++;
include/linux/dm-log-userspace.h +12 -1
@@ -370,6 +370,16 @@#define DM_ULOG_REQUEST_TYPE (request_type) \ (DM_ULOG_REQUEST_MASK & (request_type)) +/* +* DM_ULOG_REQUEST_VERSION is incremented when there is a +* change to the way information is passed between kernel +* and userspace. This could be a structure change of +* dm_ulog_request or a change in the way requests are +* issued/handled. Changes are outlined here: +* version 1: Initial implementation +*/ +#define DM_ULOG_REQUEST_VERSION 1 +struct dm_ulog_request {/* * The local unique identifier (luid) and the universally unique
@@ -383,8 +393,9 @@ struct dm_ulog_request {*/ uint64_t luid;char uuid[DM_UUID_LEN ]; -char padding[7 ];/* Padding because DM_UUID_LEN = 129 */ +char padding[3 ];/* Padding because DM_UUID_LEN = 129 */ +uint32_t version;/* See DM_ULOG_REQUEST_VERSION */ int32_t error;/* Used to report back processing errors */ uint32_t seq;/* Sequence number for request */
Signature · 1 line
1.7.2.3