gpiolib: fix the speed of descriptor label setting with SRCU
PATCH 1/1 gpiolib: fix the speed of descriptor label setting with SRCU
From: Bartosz Golaszewski <[email protected]>
Commit 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU") caused a massive drop in performance of requesting GPIO lines due to the call to synchronize_srcu() on each label change. Rework the code to not wait until all read-only users are done with reading the label but instead atomically replace the label pointer and schedule its release after all read-only critical sections are done.
To that end wrap the descriptor label in a struct that also contains the rcu_head struct required for deferring tasks using call_srcu() and stop using kstrdup_const() as we're required to allocate memory anyway. Just allocate enough for the label string and rcu_head in one go.
--- drivers/gpio/gpiolib.c | 31 ++++++++++++++++++++++++------- drivers/gpio/gpiolib.h | 7 ++++++- 2 files changed, 30 insertions(+), 8 deletions(-)
drivers/gpio/gpiolib.c +24 -7
@@ -101,6 +101,7 @@ static bool gpiolib_initialized;const char *gpiod_get_label (struct gpio_desc *desc) { +struct gpio_desc_label *label;unsigned long flags; flags =READ_ONCE (desc->flags);
@@ -108,23 +109,36 @@ const char *gpiod_get_label(struct gpio_desc *desc) !test_bit(FLAG_REQUESTED, &flags))return "interrupt" ; -return test_bit (FLAG_REQUESTED , &flags) ? -srcu_dereference (desc->label, &desc->srcu) :NULL ; +if (!test_bit (FLAG_REQUESTED , &flags)) +return NULL ; + + label =srcu_dereference_check (desc->label, &desc->srcu, +srcu_read_lock_held (&desc->srcu)); + +return label->str; +} + +static void desc_free_label (struct rcu_head *rh) +{ +kfree (container_of(rh ,struct gpio_desc_label ,rh )); }static int desc_set_label (struct gpio_desc *desc,const char *label) { -const char *new =NULL , *old; +struct gpio_desc_label *new =NULL , *old;if (label) { - new =kstrdup_const (label,GFP_KERNEL ); + new =kzalloc (struct_size (new, str,strlen (label) +1 ), +GFP_KERNEL );if (!new)return -ENOMEM ; + +strcpy (new->str, label); } old =rcu_replace_pointer (desc->label, new,1 ); -synchronize_srcu (&desc->srcu); -kfree_const (old); +if (old) +call_srcu (&desc->srcu, &old->rh, desc_free_label);return 0 ; }
@@ -697,8 +711,11 @@ static void gpiodev_release(struct device *dev)struct gpio_device *gdev =to_gpio_device (dev);unsigned int i; -for (i =0 ; i < gdev->ngpio; i++) +for (i =0 ; i < gdev->ngpio; i++) { +/* Free pending label. */ +synchronize_srcu (&gdev->descs[i].srcu);cleanup_srcu_struct (&gdev->descs[i].srcu); + }ida_free (&gpio_ida, gdev->id);kfree_const (gdev->label);
drivers/gpio/gpiolib.h +6 -1
@@ -137,6 +137,11 @@ int gpiod_set_transitory(struct gpio_desc *desc, bool transitory);void gpiod_line_state_notify (struct gpio_desc *desc,unsigned long action); +struct gpio_desc_label { +struct rcu_head rh; +char str[]; +}; +/** * struct gpio_desc - Opaque descriptor for a GPIO *
@@ -177,7 +182,7 @@ struct gpio_desc {#define FLAG_EVENT_CLOCK_HTE 19/* GPIO CDEV reports hardware timestamps in events */ /* Connection label */ -const char __rcu *label; +struct gpio_desc_label __rcu *label;/* Name of the GPIO */ const char *name;#ifdef CONFIG_OF_DYNAMIC
Signature · 1 line
2.40.1
Quoted 109 lines · Bartosz Golaszewski
From: Bartosz Golaszewski <[email protected]>
Commit
1f2bcb8c8ccd("gpio: protect the descriptor label with SRCU") caused a massive drop in performance of requesting GPIO lines due to the call tosynchronize_srcu()on each label change. Rework the code to not wait until all read-only users are done with reading the label but instead atomically replace the label pointer and schedule its release after all read-only critical sections are done.To that end wrap the descriptor label in a struct that also contains the
rcu_headstruct required for deferring tasks usingcall_srcu()and stop usingkstrdup_const()as we're required to allocate memory anyway. Just allocate enough for the label string andrcu_headin one go.Reported-by Neil Armstrong <[email protected]>Closes https://lore.kernel.org/linux-gpio/CAMRc=Mfig2oooDQYTqo23W3PXSdzhVO4p=G4+P8y1ppBOrkrJQ@mail.gmail.com/ open hereFixes 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU")Suggested-by Paul E. McKenney <[email protected]>Signed-off-by Bartosz Golaszewski <[email protected]>--- drivers/gpio/gpiolib.c | 31 ++++++++++++++++++++++++------- drivers/gpio/gpiolib.h | 7 ++++++- 2 files changed, 30 insertions(+), 8 deletions(-)drivers/gpio/gpiolib.c +24
-7@@ -101,6 +101,7 @@ static bool gpiolib_initialized;const char *gpiod_get_label (struct gpio_desc *desc) { +struct gpio_desc_label *label;unsigned long flags; flags =READ_ONCE (desc->flags);@@ -108,23 +109,36 @@ const char *gpiod_get_label(struct gpio_desc *desc) !test_bit(FLAG_REQUESTED, &flags))return "interrupt" ; -return test_bit (FLAG_REQUESTED , &flags) ? -srcu_dereference (desc->label, &desc->srcu) :NULL ; +if (!test_bit (FLAG_REQUESTED , &flags)) +return NULL ; + + label =srcu_dereference_check (desc->label, &desc->srcu, +srcu_read_lock_held (&desc->srcu)); + +return label->str; +} + +static void desc_free_label (struct rcu_head *rh) +{ +kfree (container_of(rh ,struct gpio_desc_label ,rh )); }static int desc_set_label (struct gpio_desc *desc,const char *label) { -const char *new =NULL , *old; +struct gpio_desc_label *new =NULL , *old;if (label) { - new =kstrdup_const (label,GFP_KERNEL ); + new =kzalloc (struct_size (new, str,strlen (label) +1 ), +GFP_KERNEL );if (!new)return -ENOMEM ; + +strcpy (new->str, label); } old =rcu_replace_pointer (desc->label, new,1 ); -synchronize_srcu (&desc->srcu); -kfree_const (old); +if (old) +call_srcu (&desc->srcu, &old->rh, desc_free_label);return 0 ; }@@ -697,8 +711,11 @@ static void gpiodev_release(struct device *dev)struct gpio_device *gdev =to_gpio_device (dev);unsigned int i; -for (i =0 ; i < gdev->ngpio; i++) +for (i =0 ; i < gdev->ngpio; i++) { +/* Free pending label. */ +synchronize_srcu (&gdev->descs[i].srcu);cleanup_srcu_struct (&gdev->descs[i].srcu); + }ida_free (&gpio_ida, gdev->id);kfree_const (gdev->label);drivers/gpio/gpiolib.h +5
-0@@ -137,6 +137,11 @@ int gpiod_set_transitory(struct gpio_desc *desc, bool transitory);void gpiod_line_state_notify (struct gpio_desc *desc,unsigned long action); +struct gpio_desc_label { +struct rcu_head rh; +char str[]; +}; +/** * struct gpio_desc - Opaque descriptor for a GPIO *
drivers/gpio/gpiolib.h struct gpio_desc { · line 188
Quoted 8 lines
drivers/gpio/gpiolib.h +1
-1@@ -177,7 +182,7 @@ struct gpio_desc {#define FLAG_EVENT_CLOCK_HTE 19/* GPIO CDEV reports hardware timestamps in events */ /* Connection label */ -const char __rcu *label; +struct gpio_desc_label __rcu *label;/* Name of the GPIO */ const char *name;#ifdef CONFIG_OF_DYNAMIC
Reduces the penalty from 100x / 2985x to only ~2x
Thanks,
Neil
Quoted 14 lines · Bartosz Golaszewski
From: Bartosz Golaszewski <[email protected]>
Commit
1f2bcb8c8ccd("gpio: protect the descriptor label with SRCU") caused a massive drop in performance of requesting GPIO lines due to the call tosynchronize_srcu()on each label change. Rework the code to not wait until all read-only users are done with reading the label but instead atomically replace the label pointer and schedule its release after all read-only critical sections are done.To that end wrap the descriptor label in a struct that also contains the
rcu_headstruct required for deferring tasks usingcall_srcu()and stop usingkstrdup_const()as we're required to allocate memory anyway. Just allocate enough for the label string andrcu_headin one go.
Quoted 5 lines
Reported-by Neil Armstrong <[email protected]>Closes https://lore.kernel.org/linux-gpio/CAMRc=Mfig2oooDQYTqo23W3PXSdzhVO4p=G4+P8y1ppBOrkrJQ@mail.gmail.com/ open hereFixes 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU")Suggested-by Paul E. McKenney <[email protected]>Signed-off-by Bartosz Golaszewski <[email protected]>
Looks good to me!
One semi-related question... Why the per-descriptor srcu_struct? Please see below for more on this question.
Thanx, Paul
Quoted 61 lines
--- drivers/gpio/gpiolib.c | 31 ++++++++++++++++++++++++------- drivers/gpio/gpiolib.h | 7 ++++++- 2 files changed, 30 insertions(+), 8 deletions(-)drivers/gpio/gpiolib.c +20
-6@@ -101,6 +101,7 @@ static bool gpiolib_initialized;const char *gpiod_get_label (struct gpio_desc *desc) { +struct gpio_desc_label *label;unsigned long flags; flags =READ_ONCE (desc->flags);@@ -108,23 +109,36 @@ const char *gpiod_get_label(struct gpio_desc *desc) !test_bit(FLAG_REQUESTED, &flags))return "interrupt" ; -return test_bit (FLAG_REQUESTED , &flags) ? -srcu_dereference (desc->label, &desc->srcu) :NULL ; +if (!test_bit (FLAG_REQUESTED , &flags)) +return NULL ; + + label =srcu_dereference_check (desc->label, &desc->srcu, +srcu_read_lock_held (&desc->srcu)); + +return label->str; +} + +static void desc_free_label (struct rcu_head *rh) +{ +kfree (container_of(rh ,struct gpio_desc_label ,rh )); }static int desc_set_label (struct gpio_desc *desc,const char *label) { -const char *new =NULL , *old; +struct gpio_desc_label *new =NULL , *old;if (label) { - new =kstrdup_const (label,GFP_KERNEL ); + new =kzalloc (struct_size (new, str,strlen (label) +1 ), +GFP_KERNEL );if (!new)return -ENOMEM ; + +strcpy (new->str, label); } old =rcu_replace_pointer (desc->label, new,1 ); -synchronize_srcu (&desc->srcu); -kfree_const (old); +if (old) +call_srcu (&desc->srcu, &old->rh, desc_free_label);return 0 ; }
drivers/gpio/gpiolib.c static void gpiodev_release(struct device *dev) · line 718
Quoted 9 lines
drivers/gpio/gpiolib.c +4
-1@@ -697,8 +711,11 @@ static void gpiodev_release(struct device *dev)struct gpio_device *gdev =to_gpio_device (dev);unsigned int i; -for (i =0 ; i < gdev->ngpio; i++) +for (i =0 ; i < gdev->ngpio; i++) { +/* Free pending label. */ +synchronize_srcu (&gdev->descs[i].srcu);cleanup_srcu_struct (&gdev->descs[i].srcu); + }
If the srcu_struct was shared among all of these, you could just do one synchronize_srcu() and one cleanup_srcu_struct() instead of needing to do one per gdev->desc[] entry.
You might be able to go further and have one srcu_struct for all the gpio devices.
Or did you guys run tests and find some performance problem with sharingsrcu_struct structures? (I wouldn't expect one, but sometimes the
hardware has a better imagination than I do.)
Quoted 30 lines
drivers/gpio/gpiolib.c +0
-0ida_free (&gpio_ida, gdev->id);kfree_const (gdev->label);drivers/gpio/gpiolib.h +6
-1@@ -137,6 +137,11 @@ int gpiod_set_transitory(struct gpio_desc *desc, bool transitory);void gpiod_line_state_notify (struct gpio_desc *desc,unsigned long action); +struct gpio_desc_label { +struct rcu_head rh; +char str[]; +}; +/** * struct gpio_desc - Opaque descriptor for a GPIO * @@ -177,7 +182,7 @@ struct gpio_desc {#define FLAG_EVENT_CLOCK_HTE 19/* GPIO CDEV reports hardware timestamps in events */ /* Connection label */ -const char __rcu *label; +struct gpio_desc_label __rcu *label;/* Name of the GPIO */ const char *name;#ifdef CONFIG_OF_DYNAMIC Signature · 1 line
2.40.1
Quoted 32 lines · Paul E. McKenney
Quoted 14 lines · Bartosz Golaszewski
From: Bartosz Golaszewski <[email protected]>
Commit
1f2bcb8c8ccd("gpio: protect the descriptor label with SRCU") caused a massive drop in performance of requesting GPIO lines due to the call tosynchronize_srcu()on each label change. Rework the code to not wait until all read-only users are done with reading the label but instead atomically replace the label pointer and schedule its release after all read-only critical sections are done.To that end wrap the descriptor label in a struct that also contains the
rcu_headstruct required for deferring tasks usingcall_srcu()and stop usingkstrdup_const()as we're required to allocate memory anyway. Just allocate enough for the label string andrcu_headin one go.Quoted 5 lines
Reported-by Neil Armstrong <[email protected]>Closes https://lore.kernel.org/linux-gpio/CAMRc=Mfig2oooDQYTqo23W3PXSdzhVO4p=G4+P8y1ppBOrkrJQ@mail.gmail.com/ open hereFixes 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU")Suggested-by Paul E. McKenney <[email protected]>Signed-off-by Bartosz Golaszewski <[email protected]>Looks good to me!
Acked-by Paul E. McKenney <[email protected]>One semi-related question... Why the per-descriptor
srcu_struct?If the
srcu_structwas shared among all of these, you could just do onesynchronize_srcu()and onecleanup_srcu_struct()instead of needing to do one per gdev->desc[] entry.
Quoted 7 lines
You might be able to go further and have one
srcu_structfor all the gpio devices.Or did you guys run tests and find some performance problem with sharing
srcu_structstructures? (I wouldn't expect one, but sometimes the
hardware has a better imagination than I do.)
I guess my goal was not to make synchronize_srcu() for descriptor X wait for read-only operations on descriptor Y. But with that gone, I suppose you're right, we can improve this patch further by switching to a single SRCU descriptor.
I'll send a v2.
Bart
Quoted 41 lines · Bartosz Golaszewski
Quoted 32 lines · Paul E. McKenney
Quoted 14 lines · Bartosz Golaszewski
From: Bartosz Golaszewski <[email protected]>
Commit
1f2bcb8c8ccd("gpio: protect the descriptor label with SRCU") caused a massive drop in performance of requesting GPIO lines due to the call tosynchronize_srcu()on each label change. Rework the code to not wait until all read-only users are done with reading the label but instead atomically replace the label pointer and schedule its release after all read-only critical sections are done.To that end wrap the descriptor label in a struct that also contains the
rcu_headstruct required for deferring tasks usingcall_srcu()and stop usingkstrdup_const()as we're required to allocate memory anyway. Just allocate enough for the label string andrcu_headin one go.Quoted 5 lines
Reported-by Neil Armstrong <[email protected]>Closes https://lore.kernel.org/linux-gpio/CAMRc=Mfig2oooDQYTqo23W3PXSdzhVO4p=G4+P8y1ppBOrkrJQ@mail.gmail.com/ open hereFixes 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU")Suggested-by Paul E. McKenney <[email protected]>Signed-off-by Bartosz Golaszewski <[email protected]>Looks good to me!
Acked-by Paul E. McKenney <[email protected]>One semi-related question... Why the per-descriptor
srcu_struct?If the
srcu_structwas shared among all of these, you could just do onesynchronize_srcu()and onecleanup_srcu_struct()instead of needing to do one per gdev->desc[] entry.Quoted 7 lines
You might be able to go further and have one
srcu_structfor all the gpio devices.Or did you guys run tests and find some performance problem with sharing
srcu_structstructures? (I wouldn't expect one, but sometimes the
hardware has a better imagination than I do.)
Quoted 6 lines
I guess my goal was not to make
synchronize_srcu()for descriptor X wait for read-only operations on descriptor Y. But with that gone, I suppose you're right, we can improve this patch further by switching to a single SRCU descriptor.I'll send a v2.
My guess is that a separate patch for each of the two changes would be best, but I must defer to you guys on that.
Thanx, Paul
Quoted 48 lines · Paul E. McKenney
Quoted 46 lines · Bartosz Golaszewski
Quoted 32 lines · Paul E. McKenney
Quoted 14 lines · Bartosz Golaszewski
From: Bartosz Golaszewski <[email protected]>
Commit
1f2bcb8c8ccd("gpio: protect the descriptor label with SRCU") caused a massive drop in performance of requesting GPIO lines due to the call tosynchronize_srcu()on each label change. Rework the code to not wait until all read-only users are done with reading the label but instead atomically replace the label pointer and schedule its release after all read-only critical sections are done.To that end wrap the descriptor label in a struct that also contains the
rcu_headstruct required for deferring tasks usingcall_srcu()and stop usingkstrdup_const()as we're required to allocate memory anyway. Just allocate enough for the label string andrcu_headin one go.Quoted 5 lines
Reported-by Neil Armstrong <[email protected]>Closes https://lore.kernel.org/linux-gpio/CAMRc=Mfig2oooDQYTqo23W3PXSdzhVO4p=G4+P8y1ppBOrkrJQ@mail.gmail.com/ open hereFixes 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU")Suggested-by Paul E. McKenney <[email protected]>Signed-off-by Bartosz Golaszewski <[email protected]>Looks good to me!
Acked-by Paul E. McKenney <[email protected]>One semi-related question... Why the per-descriptor
srcu_struct?If the
srcu_structwas shared among all of these, you could just do onesynchronize_srcu()and onecleanup_srcu_struct()instead of needing to do one per gdev->desc[] entry.Quoted 7 lines
You might be able to go further and have one
srcu_structfor all the gpio devices.Or did you guys run tests and find some performance problem with sharing
srcu_structstructures? (I wouldn't expect one, but sometimes the
hardware has a better imagination than I do.)I guess my goal was not to make
synchronize_srcu()for descriptor X wait for read-only operations on descriptor Y. But with that gone, I suppose you're right, we can improve this patch further by switching to a single SRCU descriptor.
Quoted 6 lines
Quoted 1 line
I'll send a v2.
My guess is that a separate patch for each of the two changes would be best, but I must defer to you guys on that.
Thanx, Paul
Ok, makes sense, I'll apply this and send a follow-up.
Bart
Bartosz Golaszewski
AppliedFrom: Bartosz Golaszewski <[email protected]>
Quoted 8 lines · Bartosz Golaszewski
Commit
1f2bcb8c8ccd("gpio: protect the descriptor label with SRCU") caused a massive drop in performance of requesting GPIO lines due to the call tosynchronize_srcu()on each label change. Rework the code to not wait until all read-only users are done with reading the label but instead atomically replace the label pointer and schedule its release after all read-only critical sections are done.[...]
Applied, thanks!
[1/1] gpiolib: fix the speed of descriptor label setting with SRCU
commit: a86d27693066a34a29be86f394bbad847b2d1749
Best regards,
Signature · 1 line
Bartosz Golaszewski <[email protected]>
Quoted 9 lines · Bartosz Golaszewski
From: Bartosz Golaszewski <[email protected]>
Commit
1f2bcb8c8ccd("gpio: protect the descriptor label with SRCU") caused a massive drop in performance of requesting GPIO lines due to the call tosynchronize_srcu()on each label change. Rework the code to not wait until all read-only users are done with reading the label but instead atomically replace the label pointer and schedule its release after all read-only critical sections are done.
Quoted 3 lines
To that end wrap the descriptor label in a struct that also contains the
rcu_headstruct required for deferring tasks usingcall_srcu()and stop usingkstrdup_const()as we're required to allocate memory anyway.
If there is no label and we assign something like "?" (two bytes) we got with your patch the allocation of most likely 32 bytes (as next power of two for the SLAB) instead of 18..24.
OTOH, I dunno if SLAB supports 24-bytes. If not, it means that up to 16 bytes label there would be no difference. In any case, with a new update (as far as I understood the next move) it might return to kstrdup_const() or so.
Quoted 1 line
Just allocate enough for the label string and
rcu_headin one go.
Signature · 2 lines
With Best Regards, Andy Shevchenko
Quoted 15 lines · Andy Shevchenko
Quoted 9 lines · Bartosz Golaszewski
From: Bartosz Golaszewski <[email protected]>
Commit
1f2bcb8c8ccd("gpio: protect the descriptor label with SRCU") caused a massive drop in performance of requesting GPIO lines due to the call tosynchronize_srcu()on each label change. Rework the code to not wait until all read-only users are done with reading the label but instead atomically replace the label pointer and schedule its release after all read-only critical sections are done.Quoted 3 lines
To that end wrap the descriptor label in a struct that also contains the
rcu_headstruct required for deferring tasks usingcall_srcu()and stop usingkstrdup_const()as we're required to allocate memory anyway.
Quoted 8 lines
If there is no label and we assign something like "?" (two bytes) we got with your patch the allocation of most likely 32 bytes (as next power of two for the SLAB) instead of 18..24.
OTOH, I dunno if SLAB supports 24-bytes. If not, it means that up to 16 bytes label there would be no difference. In any case, with a new update (as far as I understood the next move) it might return to
kstrdup_const()or so.
Memory is cheap. This is just done for requested lines of which there are never that many. I wouldn't stress about it. The rcu_head struct is already 32 bytes on its own.
Bart