Add cpu_relax() calls in css_tryget() and cgroup_clear_css_refs()
PATCH 1/1 Add cpu_relax() calls in css_tryget() and cgroup_clear_css_refs()
Add cpu_relax() calls in css_tryget() and cgroup_clear_css_refs()
css_tryget() and cgroup_clear_css_refs() contain polling loops; these loops should have cpu_relax calls in them to reduce cross-cache traffic.
Signed-off-by Paul Menage <[email protected]>
---
include/linux/cgroup.h | 1 + kernel/cgroup.c | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-)
include/linux/cgroup.h +1 -0
@@ -99,6 +99,7 @@ static inline bool css_tryget(struct cgroup_subsys_state *css)while (!atomic_inc_not_zero (&css->refcnt)) {if (test_bit (CSS_REMOVED , &css->flags))return false; +cpu_relax (); }return true; }
kernel/cgroup.c +5 -2
@@ -2509,7 +2509,7 @@ static int cgroup_clear_css_refs(struct cgroup *cgrp)for_each_subsys (cgrp->root, ss) {struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];int refcnt; - do { +while (1 ) {/* We can only remove a CSS with a refcnt==1 */ refcnt =atomic_read (&css->refcnt);if (refcnt >1 ) {
@@ -2523,7 +2523,10 @@ static int cgroup_clear_css_refs(struct cgroup *cgrp)* css_tryget() to spin until we set the * CSS_REMOVED bits or abort */ - }while (atomic_cmpxchg (&css->refcnt, refcnt,0 ) != refcnt); +if (atomic_cmpxchg (&css->refcnt, refcnt,0 ) == refcnt) +break ; +cpu_relax (); + } } done:for_each_subsys (cgrp->root, ss) {