net/sched: prepare lockless qdisc dumps
Subsystems (6)
v2 cover net/sched: prepare lockless qdisc dumps
Goal is to no longer acquire RTNL in qdisc dumps.
This series annotate data-races, and change mq and mq_prio to no longer acquire children qdisc spinlocks.
v2: addressed sashiko (and Victor Nogueira) feedback.
Eric Dumazet (8):
net/sched: add READ_ONCE() in gnet_stats_add_queue[_cpu] net/sched: add qdisc_qlen_inc() and qdisc_qlen_dec() net/sched: annotate data-races around sch->qstats.backlog net/sched: add qdisc_qlen_lockless() helper
net/sched: add const qualifiers to gnet_stats helpers net/sched: mq: no longer acquire qdisc spinlocks in dump operations net/sched: mq_prio: no longer acquire qdisc spinlocks in mqprio_dump() net/sched: mq_prio: no longer acquire qdisc spinlocks in
mqprio_dump_class_stats()
include/net/gen_stats.h | 16 +++---- include/net/sch_generic.h | 64 ++++++++++++++++++++------- net/core/gen_stats.c | 51 +++++++++++---------- net/sched/sch_api.c | 4 +- net/sched/sch_cake.c | 18 ++++---- net/sched/sch_cbs.c | 6 +-- net/sched/sch_choke.c | 8 ++-- net/sched/sch_codel.c | 2 +- net/sched/sch_drr.c | 6 +-- net/sched/sch_dualpi2.c | 6 +-- net/sched/sch_etf.c | 8 ++-- net/sched/sch_ets.c | 6 +-- net/sched/sch_fq.c | 8 ++-- net/sched/sch_fq_codel.c | 11 ++--- net/sched/sch_fq_pie.c | 8 ++-- net/sched/sch_generic.c | 12 ++--- net/sched/sch_gred.c | 2 +- net/sched/sch_hfsc.c | 6 +-- net/sched/sch_hhf.c | 7 +-- net/sched/sch_htb.c | 6 +-- net/sched/sch_mq.c | 37 +++++++++++----- net/sched/sch_mqprio.c | 93 ++++++++++++++++++++------------------- net/sched/sch_multiq.c | 4 +- net/sched/sch_netem.c | 12 ++--- net/sched/sch_prio.c | 6 +-- net/sched/sch_qfq.c | 8 ++-- net/sched/sch_red.c | 6 +-- net/sched/sch_sfb.c | 8 ++-- net/sched/sch_sfq.c | 11 ++--- net/sched/sch_skbprio.c | 4 +- net/sched/sch_taprio.c | 4 +- net/sched/sch_tbf.c | 10 ++--- net/sched/sch_teql.c | 2 +- 33 files changed, 256 insertions(+), 204 deletions(-)
Signature · 1 line
2.54.0.563.g4f69b47b94-goog
PATCH v2 1/8 net/sched: add READ_ONCE() in gnet_stats_add_queue[_cpu]
Stats are read locklessly, add READ_ONCE() to prevent load-tearing.
Write side will be handled in separate patches.
--- net/core/gen_stats.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)
net/core/gen_stats.c +10 -10
@@ -345,11 +345,11 @@ static void gnet_stats_add_queue_cpu(struct gnet_stats_queue *qstats,for_each_possible_cpu (i) {const struct gnet_stats_queue *qcpu =per_cpu_ptr (q, i); - qstats->qlen += qcpu->qlen; - qstats->backlog += qcpu->backlog; - qstats->drops += qcpu->drops; - qstats->requeues += qcpu->requeues; - qstats->overlimits += qcpu->overlimits; + qstats->qlen +=READ_ONCE (qcpu->qlen); + qstats->backlog +=READ_ONCE (qcpu->backlog); + qstats->drops +=READ_ONCE (qcpu->drops); + qstats->requeues +=READ_ONCE (qcpu->requeues); + qstats->overlimits +=READ_ONCE (qcpu->overlimits); } }
@@ -360,11 +360,11 @@ void gnet_stats_add_queue(struct gnet_stats_queue *qstats,if (cpu) {gnet_stats_add_queue_cpu (qstats, cpu); }else { - qstats->qlen += q->qlen; - qstats->backlog += q->backlog; - qstats->drops += q->drops; - qstats->requeues += q->requeues; - qstats->overlimits += q->overlimits; + qstats->qlen +=READ_ONCE (q->qlen); + qstats->backlog +=READ_ONCE (q->backlog); + qstats->drops +=READ_ONCE (q->drops); + qstats->requeues +=READ_ONCE (q->requeues); + qstats->overlimits +=READ_ONCE (q->overlimits); } }EXPORT_SYMBOL (gnet_stats_add_queue);
Signature · 1 line
2.54.0.563.g4f69b47b94-goog
PATCH v2 2/8 net/sched: add qdisc_qlen_inc() and qdisc_qlen_dec()
In reply to Eric DumazetHelpers to increment or decrement sch->q.qlen, with appropriate WRITE_ONCE() to prevent store tearing.
Add other WRITE_ONCE() when sch->q.qlen is changed.
--- include/net/sch_generic.h | 32 +++++++++++++++++++++----------- net/sched/sch_api.c | 2 +- net/sched/sch_cake.c | 8 ++++---- net/sched/sch_cbs.c | 4 ++-- net/sched/sch_choke.c | 8 ++++---- net/sched/sch_drr.c | 4 ++-- net/sched/sch_dualpi2.c | 6 +++--- net/sched/sch_etf.c | 8 ++++---- net/sched/sch_ets.c | 4 ++-- net/sched/sch_fq.c | 6 +++--- net/sched/sch_fq_codel.c | 7 ++++--- net/sched/sch_fq_pie.c | 4 ++-- net/sched/sch_generic.c | 10 +++++----- net/sched/sch_hfsc.c | 4 ++-- net/sched/sch_hhf.c | 7 ++++--- net/sched/sch_htb.c | 4 ++-- net/sched/sch_mq.c | 5 +++-- net/sched/sch_mqprio.c | 18 ++++++++++-------- net/sched/sch_multiq.c | 4 ++-- net/sched/sch_netem.c | 10 +++++----- net/sched/sch_prio.c | 4 ++-- net/sched/sch_qfq.c | 6 +++--- net/sched/sch_red.c | 4 ++-- net/sched/sch_sfb.c | 4 ++-- net/sched/sch_sfq.c | 9 +++++---- net/sched/sch_skbprio.c | 4 ++-- net/sched/sch_taprio.c | 4 ++-- net/sched/sch_tbf.c | 6 +++--- net/sched/sch_teql.c | 2 +- 29 files changed, 107 insertions(+), 91 deletions(-)
include/net/sch_generic.h +21 -11
@@ -542,6 +542,16 @@ static inline int qdisc_qlen(const struct Qdisc *q)return q->q.qlen; } +static inline void qdisc_qlen_inc (struct Qdisc *q) +{ +WRITE_ONCE (q->q.qlen, q->q.qlen +1 ); +} + +static inline void qdisc_qlen_dec (struct Qdisc *q) +{ +WRITE_ONCE (q->q.qlen, q->q.qlen -1 ); +} +static inline int qdisc_qlen_sum (const struct Qdisc *q) {__u32 qlen = q->qstats.qlen;
@@ -549,9 +559,9 @@ static inline int qdisc_qlen_sum(const struct Qdisc *q)if (qdisc_is_percpu_stats (q)) {for_each_possible_cpu (i) - qlen +=per_cpu_ptr (q->cpu_qstats, i)->qlen; + qlen +=READ_ONCE (per_cpu_ptr (q->cpu_qstats, i)->qlen); }else { - qlen += q->q.qlen; + qlen +=READ_ONCE (q->q.qlen); }return qlen;
@@ -1068,7 +1078,7 @@ static inline void __qdisc_enqueue_tail(struct sk_buff *skb, qh->tail = skb; qh->head = skb; } - qh->qlen++; +WRITE_ONCE (qh->qlen, qh->qlen +1 ); }static inline int qdisc_enqueue_tail (struct sk_buff *skb,struct Qdisc *sch)
@@ -1086,7 +1096,7 @@ static inline void __qdisc_enqueue_head(struct sk_buff *skb,if (!qh->head) qh->tail = skb; qh->head = skb; - qh->qlen++; +WRITE_ONCE (qh->qlen, qh->qlen +1 ); }static inline struct sk_buff *__qdisc_dequeue_head (struct qdisc_skb_head *qh)
@@ -1095,7 +1105,7 @@ static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh)if (likely (skb !=NULL )) { qh->head = skb->next; - qh->qlen--; +WRITE_ONCE (qh->qlen, qh->qlen -1 );if (qh->head ==NULL ) qh->tail =NULL ; skb->next =NULL ;
@@ -1110,7 +1120,7 @@ static inline struct sk_buff *qdisc_dequeue_internal(struct Qdisc *sch, bool dir skb =__skb_dequeue (&sch->gso_skb);if (skb) { - sch->q.qlen--; +qdisc_qlen_dec (sch);qdisc_qstats_backlog_dec (sch, skb);return skb; }
@@ -1266,7 +1276,7 @@ static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch)__skb_queue_head (&sch->gso_skb, skb);/* it's still part of the queue */ qdisc_qstats_backlog_inc (sch, skb); - sch->q.qlen++; +qdisc_qlen_inc (sch); } }
@@ -1283,7 +1293,7 @@ static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch, }else {qdisc_qstats_backlog_dec (sch, skb);qdisc_bstats_update (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch); } }
@@ -1295,7 +1305,7 @@ static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch,this_cpu_add (sch->cpu_qstats->backlog, pkt_len); }else { sch->qstats.backlog += pkt_len; - sch->q.qlen++; +qdisc_qlen_inc (sch); } }
@@ -1311,7 +1321,7 @@ static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch)qdisc_qstats_cpu_qlen_dec (sch); }else {qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch); } }else { skb = sch->dequeue (sch);
@@ -1332,7 +1342,7 @@ static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) qh->head =NULL ; qh->tail =NULL ; - qh->qlen =0 ; +WRITE_ONCE (qh->qlen,0 ); } }
net/sched/sch_api.c +1 -1
@@ -805,7 +805,7 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) cl = cops->find (sch, parentid); cops->qlen_notify (sch, cl); } - sch->q.qlen -= n; +WRITE_ONCE (sch->q.qlen, sch->q.qlen - n); sch->qstats.backlog -= len;__qdisc_qstats_drop (sch, drops); }
net/sched/sch_cake.c +4 -4
@@ -1612,7 +1612,7 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free)cake_advance_shaper (q, b, skb, now, true);qdisc_drop_reason (skb, sch, to_free,QDISC_DROP_OVERLIMIT ); - sch->q.qlen--; +qdisc_qlen_dec (sch);cake_heapify (q,0 );
@@ -1822,7 +1822,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, segs);flow_queue_add (flow, segs); - sch->q.qlen++; +qdisc_qlen_inc (sch); numsegs++; slen += segs->len; q->buffer_used += segs->truesize;
@@ -1861,7 +1861,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,qdisc_tree_reduce_backlog (sch,1 , ack_pkt_len);consume_skb (ack); }else { - sch->q.qlen++; +qdisc_qlen_inc (sch); q->buffer_used += skb->truesize; }
@@ -1987,7 +1987,7 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch)WRITE_ONCE (b->tin_backlog, b->tin_backlog - len); sch->qstats.backlog -= len; q->buffer_used -= skb->truesize; - sch->q.qlen--; +qdisc_qlen_dec (sch);if (q->overflow_timeout)cake_heapify (q, b->overflow_idx[q->cur_flow]);
net/sched/sch_cbs.c +2 -2
@@ -97,7 +97,7 @@ static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch,return err; sch->qstats.backlog += len; - sch->q.qlen++; +qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ; }
@@ -168,7 +168,7 @@ static struct sk_buff *cbs_child_dequeue(struct Qdisc *sch, struct Qdisc *child)qdisc_qstats_backlog_dec (sch, skb);qdisc_bstats_update (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);return skb; }
net/sched/sch_choke.c +4 -4
@@ -123,7 +123,7 @@ static void choke_drop_by_idx(struct Qdisc *sch, unsigned int idx,if (idx == q->tail)choke_zap_tail_holes (q); - --sch->q.qlen; +qdisc_qlen_dec (sch);qdisc_qstats_backlog_dec (sch, skb);qdisc_tree_reduce_backlog (sch,1 ,qdisc_pkt_len (skb));qdisc_drop (skb, sch, to_free);
@@ -271,7 +271,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch,if (sch->q.qlen < q->limit) { q->tab[q->tail] = skb; q->tail = (q->tail +1 ) & q->tab_mask; - ++sch->q.qlen; +qdisc_qlen_inc (sch);qdisc_qstats_backlog_inc (sch, skb);return NET_XMIT_SUCCESS ; }
@@ -298,7 +298,7 @@ static struct sk_buff *choke_dequeue(struct Qdisc *sch) skb = q->tab[q->head]; q->tab[q->head] =NULL ;choke_zap_head_holes (q); - --sch->q.qlen; +qdisc_qlen_dec (sch);qdisc_qstats_backlog_dec (sch, skb);qdisc_bstats_update (sch, skb);
@@ -396,7 +396,7 @@ static int choke_change(struct Qdisc *sch, struct nlattr *opt, } dropped +=qdisc_pkt_len (skb);qdisc_qstats_backlog_dec (sch, skb); - --sch->q.qlen; +qdisc_qlen_dec (sch);rtnl_qdisc_drop (skb, sch); }qdisc_tree_reduce_backlog (sch, oqlen - sch->q.qlen, dropped);
net/sched/sch_drr.c +2 -2
@@ -366,7 +366,7 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; +qdisc_qlen_inc (sch);return err; }
@@ -399,7 +399,7 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch)bstats_update (&cl->bstats, skb);qdisc_bstats_update (sch, skb);qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);return skb; }
net/sched/sch_dualpi2.c +3 -3
@@ -415,7 +415,7 @@ static int dualpi2_enqueue_skb(struct sk_buff *skb, struct Qdisc *sch,dualpi2_skb_cb (skb)->apply_step =skb_apply_step (skb, q);/* Keep the overall qdisc stats consistent */ - ++sch->q.qlen; +qdisc_qlen_inc (sch);qdisc_qstats_backlog_inc (sch, skb); ++q->packets_in_l;if (!q->l_head_ts)
@@ -530,7 +530,7 @@ static struct sk_buff *dequeue_packet(struct Qdisc *sch,qdisc_qstats_backlog_dec (q->l_queue, skb);/* Keep the global queue size consistent */ - --sch->q.qlen; +qdisc_qlen_dec (sch); q->memory_used -= skb->truesize; }else if (c_len) { skb =__qdisc_dequeue_head (&sch->q);
@@ -888,7 +888,7 @@ static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt,* l_queue on enqueue; qdisc_dequeue_internal() * handled l_queue, so we further account for sch. */ - --sch->q.qlen; +qdisc_qlen_dec (sch);qdisc_qstats_backlog_dec (sch, skb); q->memory_used -= skb->truesize;rtnl_qdisc_drop (skb, q->l_queue);
net/sched/sch_etf.c +4 -4
@@ -189,7 +189,7 @@ static int etf_enqueue_timesortedlist(struct sk_buff *nskb, struct Qdisc *sch,rb_insert_color_cached (&nskb->rbnode, &q->head, leftmost);qdisc_qstats_backlog_inc (sch, nskb); - sch->q.qlen++; +qdisc_qlen_inc (sch);/* Now we may need to re-arm the qdisc watchdog for the next packet. */ reset_watchdog (sch);
@@ -222,7 +222,7 @@ static void timesortedlist_drop(struct Qdisc *sch, struct sk_buff *skb,qdisc_qstats_backlog_dec (sch, skb);qdisc_drop (skb, sch, &to_free);qdisc_qstats_overlimit (sch); - sch->q.qlen--; +qdisc_qlen_dec (sch); }kfree_skb_list (to_free);
@@ -247,7 +247,7 @@ static void timesortedlist_remove(struct Qdisc *sch, struct sk_buff *skb) q->last = skb->tstamp; - sch->q.qlen--; +qdisc_qlen_dec (sch); }static struct sk_buff *etf_dequeue_timesortedlist (struct Qdisc *sch)
@@ -426,7 +426,7 @@ static void timesortedlist_clear(struct Qdisc *sch)rb_erase_cached (&skb->rbnode, &q->head);rtnl_kfree_skbs (skb, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch); } }
net/sched/sch_ets.c +2 -2
@@ -449,7 +449,7 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; +qdisc_qlen_inc (sch);return err; }
@@ -458,7 +458,7 @@ ets_qdisc_dequeue_skb(struct Qdisc *sch, struct sk_buff *skb) {qdisc_bstats_update (sch, skb);qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);return skb; }
net/sched/sch_fq.c +3 -3
@@ -497,7 +497,7 @@ static void fq_dequeue_skb(struct Qdisc *sch, struct fq_flow *flow,fq_erase_head (sch, flow, skb);skb_mark_not_on_list (skb);qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);qdisc_bstats_update (sch, skb); }
@@ -597,7 +597,7 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch,flow_queue_add (f, skb);qdisc_qstats_backlog_inc (sch, skb); - sch->q.qlen++; +qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ; }
@@ -801,7 +801,7 @@ static void fq_reset(struct Qdisc *sch)struct fq_flow *f;unsigned int idx; - sch->q.qlen =0 ; +WRITE_ONCE (sch->q.qlen,0 ); sch->qstats.backlog =0 ;fq_flow_purge (&q->internal);
net/sched/sch_fq_codel.c +4 -3
@@ -178,7 +178,7 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets, q->memory_usage -= mem;__qdisc_qstats_drop (sch, i); sch->qstats.backlog -= len; - sch->q.qlen -= i; +WRITE_ONCE (sch->q.qlen, sch->q.qlen - i);return idx; }
@@ -215,7 +215,8 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch,get_codel_cb (skb)->mem_usage = skb->truesize; q->memory_usage +=get_codel_cb (skb)->mem_usage; memory_limited = q->memory_usage > q->memory_limit; -if (++sch->q.qlen <= sch->limit && !memory_limited) +qdisc_qlen_inc (sch); +if (sch->q.qlen <= sch->limit && !memory_limited)return NET_XMIT_SUCCESS ; prev_backlog = sch->qstats.backlog;
@@ -266,7 +267,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)WRITE_ONCE (q->backlogs[flow - q->flows], q->backlogs[flow - q->flows] -qdisc_pkt_len (skb)); q->memory_usage -=get_codel_cb (skb)->mem_usage; - sch->q.qlen--; +qdisc_qlen_dec (sch); sch->qstats.backlog -=qdisc_pkt_len (skb); }return skb;
net/sched/sch_fq_pie.c +2 -2
@@ -185,7 +185,7 @@ static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, q->stats.packets_in++; q->memory_usage += skb->truesize; sch->qstats.backlog += pkt_len; - sch->q.qlen++; +qdisc_qlen_inc (sch);flow_queue_add (sel_flow, skb);if (list_empty (&sel_flow->flowchain)) {list_add_tail (&sel_flow->flowchain, &q->new_flows);
@@ -263,7 +263,7 @@ static struct sk_buff *fq_pie_qdisc_dequeue(struct Qdisc *sch) skb =dequeue_head (flow); pkt_len =qdisc_pkt_len (skb); sch->qstats.backlog -= pkt_len; - sch->q.qlen--; +qdisc_qlen_dec (sch);qdisc_bstats_update (sch, skb); }
net/sched/sch_generic.c +5 -5
@@ -118,7 +118,7 @@ static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)qdisc_qstats_cpu_qlen_dec (q); }else {qdisc_qstats_backlog_dec (q, skb); - q->q.qlen--; +qdisc_qlen_dec (q); } }else { skb =SKB_XOFF_MAGIC ;
@@ -159,7 +159,7 @@ static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,qdisc_qstats_cpu_qlen_inc (q); }else {qdisc_qstats_backlog_inc (q, skb); - q->q.qlen++; +qdisc_qlen_inc (q); }if (lock)
@@ -188,7 +188,7 @@ static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) }else { q->qstats.requeues++;qdisc_qstats_backlog_inc (q, skb); - q->q.qlen++; +qdisc_qlen_inc (q); } skb = next;
@@ -294,7 +294,7 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,qdisc_qstats_cpu_qlen_dec (q); }else {qdisc_qstats_backlog_dec (q, skb); - q->q.qlen--; +qdisc_qlen_dec (q); } }else { skb =NULL ;
@@ -1062,7 +1062,7 @@ void qdisc_reset(struct Qdisc *qdisc)__skb_queue_purge (&qdisc->gso_skb);__skb_queue_purge (&qdisc->skb_bad_txq); - qdisc->q.qlen =0 ; +WRITE_ONCE (qdisc->q.qlen,0 ); qdisc->qstats.backlog =0 ; }EXPORT_SYMBOL (qdisc_reset);
net/sched/sch_hfsc.c +2 -2
@@ -1561,7 +1561,7 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) } sch->qstats.backlog += len; - sch->q.qlen++; +qdisc_qlen_inc (sch);if (first && !cl_in_el_or_vttree (cl)) {if (cl->cl_flags &HFSC_RSC )
@@ -1650,7 +1650,7 @@ hfsc_dequeue(struct Qdisc *sch)qdisc_bstats_update (sch, skb);qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);return skb; }
net/sched/sch_hhf.c +4 -3
@@ -360,7 +360,7 @@ static unsigned int hhf_drop(struct Qdisc *sch, struct sk_buff **to_free)if (bucket->head) {struct sk_buff *skb =dequeue_head (bucket); - sch->q.qlen--; +qdisc_qlen_dec (sch);qdisc_qstats_backlog_dec (sch, skb);qdisc_drop (skb, sch, to_free); }
@@ -400,7 +400,8 @@ static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch, } bucket->deficit = weight * q->quantum; } -if (++sch->q.qlen <= sch->limit) +qdisc_qlen_inc (sch); +if (sch->q.qlen <= sch->limit)return NET_XMIT_SUCCESS ; prev_backlog = sch->qstats.backlog;
@@ -443,7 +444,7 @@ static struct sk_buff *hhf_dequeue(struct Qdisc *sch)if (bucket->head) { skb =dequeue_head (bucket); - sch->q.qlen--; +qdisc_qlen_dec (sch);qdisc_qstats_backlog_dec (sch, skb); }
net/sched/sch_htb.c +2 -2
@@ -651,7 +651,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; +qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ; }
@@ -951,7 +951,7 @@ static struct sk_buff *htb_dequeue(struct Qdisc *sch) ok:qdisc_bstats_update (sch, skb);qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);return skb; }
net/sched/sch_mq.c +3 -2
@@ -143,10 +143,10 @@ EXPORT_SYMBOL_NS_GPL(mq_attach, "NET_SCHED_INTERNAL");void mq_dump_common (struct Qdisc *sch,struct sk_buff *skb) {struct net_device *dev =qdisc_dev (sch); +unsigned int qlen =0 ;struct Qdisc *qdisc;unsigned int ntx; - sch->q.qlen =0 ;gnet_stats_basic_sync_init (&sch->bstats);memset (&sch->qstats,0 ,sizeof (sch->qstats));
@@ -163,10 +163,11 @@ void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb) &qdisc->bstats, false);gnet_stats_add_queue (&sch->qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen +=qdisc_qlen (qdisc); + qlen +=qdisc_qlen (qdisc);spin_unlock_bh (qdisc_lock (qdisc)); } +WRITE_ONCE (sch->q.qlen, qlen); }EXPORT_SYMBOL_NS_GPL (mq_dump_common,"NET_SCHED_INTERNAL" );
net/sched/sch_mqprio.c +10 -8
@@ -555,10 +555,11 @@ static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb)struct mqprio_sched *priv =qdisc_priv (sch);struct nlattr *nla = (struct nlattr *)skb_tail_pointer (skb);struct tc_mqprio_qopt opt = {0 }; +unsigned int qlen =0 ;struct Qdisc *qdisc;unsigned int ntx; - sch->q.qlen =0 ; + qlen =0 ;gnet_stats_basic_sync_init (&sch->bstats);memset (&sch->qstats,0 ,sizeof (sch->qstats));
@@ -575,10 +576,11 @@ static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb) &qdisc->bstats, false);gnet_stats_add_queue (&sch->qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen +=qdisc_qlen (qdisc); + qlen +=qdisc_qlen (qdisc);spin_unlock_bh (qdisc_lock (qdisc)); } +WRITE_ONCE (sch->q.qlen, qlen);mqprio_qopt_reconstruct (dev, &opt); opt.hw = priv->hw_offload;
@@ -663,12 +665,12 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, __acquires(d->lock) {if (cl >=TC_H_MIN_PRIORITY ) { -int i; -__u32 qlen; -struct gnet_stats_queue qstats = {0 }; -struct gnet_stats_basic_sync bstats;struct net_device *dev =qdisc_dev (sch);struct netdev_tc_txq tc = dev->tc_to_txq[cl &TC_BITMASK ]; +struct gnet_stats_queue qstats = {0 }; +struct gnet_stats_basic_sync bstats; +u32 qlen =0 ; +int i;gnet_stats_basic_sync_init (&bstats);/* Drop lock here it will be reclaimed before touching
@@ -689,11 +691,11 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, &qdisc->bstats, false);gnet_stats_add_queue (&qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen +=qdisc_qlen (qdisc); + qlen +=qdisc_qlen (qdisc);spin_unlock_bh (qdisc_lock (qdisc)); } - qlen =qdisc_qlen (sch) + qstats.qlen; + qlen = qlen + qstats.qlen;/* Reclaim root sleeping lock before completing stats */ if (d->lock)
net/sched/sch_multiq.c +2 -2
@@ -76,7 +76,7 @@ multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch, ret =qdisc_enqueue (skb, qdisc, to_free);if (ret ==NET_XMIT_SUCCESS ) { - sch->q.qlen++; +qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ; }if (net_xmit_drop_count (ret))
@@ -106,7 +106,7 @@ static struct sk_buff *multiq_dequeue(struct Qdisc *sch) skb = qdisc->dequeue (qdisc);if (skb) {qdisc_bstats_update (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);return skb; } }
net/sched/sch_netem.c +5 -5
@@ -416,7 +416,7 @@ static void tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch)rb_insert_color (&nskb->rbnode, &q->t_root); } q->t_len++; - sch->q.qlen++; +qdisc_qlen_inc (sch); }/* netem can't properly corrupt a megapacket (like we get from GSO), so instead
@@ -751,19 +751,19 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch)if (net_xmit_drop_count (err))qdisc_qstats_drop (sch); sch->qstats.backlog -= pkt_len; - sch->q.qlen--; +qdisc_qlen_dec (sch);qdisc_tree_reduce_backlog (sch,1 , pkt_len); } goto tfifo_dequeue; } - sch->q.qlen--; +qdisc_qlen_dec (sch); goto deliver; }if (q->qdisc) { skb = q->qdisc->ops->dequeue (q->qdisc);if (skb) { - sch->q.qlen--; +qdisc_qlen_dec (sch); goto deliver; } }
@@ -776,7 +776,7 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch)if (q->qdisc) { skb = q->qdisc->ops->dequeue (q->qdisc);if (skb) { - sch->q.qlen--; +qdisc_qlen_dec (sch); goto deliver; } }
net/sched/sch_prio.c +2 -2
@@ -86,7 +86,7 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) ret =qdisc_enqueue (skb, qdisc, to_free);if (ret ==NET_XMIT_SUCCESS ) { sch->qstats.backlog += len; - sch->q.qlen++; +qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ; }if (net_xmit_drop_count (ret))
@@ -119,7 +119,7 @@ static struct sk_buff *prio_dequeue(struct Qdisc *sch)if (skb) {qdisc_bstats_update (sch, skb);qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);return skb; } }
net/sched/sch_qfq.c +3 -3
@@ -1152,12 +1152,12 @@ static struct sk_buff *qfq_dequeue(struct Qdisc *sch)if (!skb)return NULL ; - sch->q.qlen--; +qdisc_qlen_dec (sch); skb =agg_dequeue (in_serv_agg, cl, len);if (!skb) { - sch->q.qlen++; +qdisc_qlen_inc (sch);return NULL ; }
@@ -1265,7 +1265,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch,_bstats_update (&cl->bstats, len, gso_segs); sch->qstats.backlog += len; - ++sch->q.qlen; +qdisc_qlen_inc (sch); agg = cl->agg;/* if the class is active, then done here */
net/sched/sch_red.c +2 -2
@@ -139,7 +139,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, ret =qdisc_enqueue (skb, child, to_free);if (likely (ret ==NET_XMIT_SUCCESS )) { sch->qstats.backlog += len; - sch->q.qlen++; +qdisc_qlen_inc (sch); }else if (net_xmit_drop_count (ret)) {WRITE_ONCE (q->stats.pdrop, q->stats.pdrop +1 );
@@ -166,7 +166,7 @@ static struct sk_buff *red_dequeue(struct Qdisc *sch)if (skb) {qdisc_bstats_update (sch, skb);qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch); }else {if (!red_is_idling (&q->vars))red_start_of_idle_period (&q->vars);
net/sched/sch_sfb.c +2 -2
@@ -416,7 +416,7 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch, ret =qdisc_enqueue (skb, child, to_free);if (likely (ret ==NET_XMIT_SUCCESS )) { sch->qstats.backlog += len; - sch->q.qlen++; +qdisc_qlen_inc (sch);increment_qlen (&cb, q); }else if (net_xmit_drop_count (ret)) { WRITE_ONCE(q->stats.childdrop,
@@ -446,7 +446,7 @@ static struct sk_buff *sfb_dequeue(struct Qdisc *sch)if (skb) {qdisc_bstats_update (sch, skb);qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);decrement_qlen (skb, q); }
net/sched/sch_sfq.c +5 -4
@@ -302,7 +302,7 @@ static unsigned int sfq_drop(struct Qdisc *sch, struct sk_buff **to_free) len =qdisc_pkt_len (skb);WRITE_ONCE (slot->backlog, slot->backlog - len);sfq_dec (q, x); - sch->q.qlen--; +qdisc_qlen_dec (sch);qdisc_qstats_backlog_dec (sch, skb);qdisc_drop_reason (skb, sch, to_free,QDISC_DROP_OVERLIMIT );return len;
@@ -456,7 +456,8 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)/* We could use a bigger initial quantum for new flows */ WRITE_ONCE (slot->allot, q->quantum); } -if (++sch->q.qlen <= q->limit) +qdisc_qlen_inc (sch); +if (sch->q.qlen <= q->limit)return NET_XMIT_SUCCESS ; qlen = slot->qlen;
@@ -497,7 +498,7 @@ sfq_dequeue(struct Qdisc *sch) skb =slot_dequeue_head (slot);sfq_dec (q, a);qdisc_bstats_update (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);qdisc_qstats_backlog_dec (sch, skb);WRITE_ONCE (slot->backlog, slot->backlog -qdisc_pkt_len (skb));/* Is the slot empty? */
@@ -596,7 +597,7 @@ static void sfq_rehash(struct Qdisc *sch)WRITE_ONCE (slot->allot, q->quantum); } } - sch->q.qlen -= dropped; +WRITE_ONCE (sch->q.qlen, sch->q.qlen - dropped);qdisc_tree_reduce_backlog (sch, dropped, drop_len); }
net/sched/sch_skbprio.c +2 -2
@@ -93,7 +93,7 @@ static int skbprio_enqueue(struct sk_buff *skb, struct Qdisc *sch,if (prio < q->lowest_prio) q->lowest_prio = prio; - sch->q.qlen++; +qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ; }
@@ -145,7 +145,7 @@ static struct sk_buff *skbprio_dequeue(struct Qdisc *sch)if (unlikely (!skb))return NULL ; - sch->q.qlen--; +qdisc_qlen_dec (sch);qdisc_qstats_backlog_dec (sch, skb);qdisc_bstats_update (sch, skb);
net/sched/sch_taprio.c +2 -2
@@ -574,7 +574,7 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch, }qdisc_qstats_backlog_inc (sch, skb); - sch->q.qlen++; +qdisc_qlen_inc (sch);return qdisc_enqueue (skb, child, to_free); }
@@ -755,7 +755,7 @@ static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq,qdisc_bstats_update (sch, skb);qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);return skb; }
net/sched/sch_tbf.c +3 -3
@@ -231,7 +231,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch, len += seg_len; } } - sch->q.qlen += nb; +WRITE_ONCE (sch->q.qlen, sch->q.qlen + nb); sch->qstats.backlog += len;if (nb >0 ) {qdisc_tree_reduce_backlog (sch,1 - nb, prev_len - len);
@@ -264,7 +264,7 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; +qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ; }
@@ -309,7 +309,7 @@ static struct sk_buff *tbf_dequeue(struct Qdisc *sch) q->tokens = toks; q->ptokens = ptoks;qdisc_qstats_backlog_dec (sch, skb); - sch->q.qlen--; +qdisc_qlen_dec (sch);qdisc_bstats_update (sch, skb);return skb; }
net/sched/sch_teql.c +1 -1
@@ -107,7 +107,7 @@ teql_dequeue(struct Qdisc *sch) }else {qdisc_bstats_update (sch, skb); } - sch->q.qlen = dat->q.qlen + q->q.qlen; +WRITE_ONCE (sch->q.qlen, dat->q.qlen +READ_ONCE (q->q.qlen));return skb; }
Signature · 1 line
2.54.0.563.g4f69b47b94-goog
PATCH v2 3/8 net/sched: annotate data-races around sch->qstats.backlog
In reply to Eric DumazetAdd qstats_backlog_sub() and qstats_backlog_add() helpers and use them instead of open-coding them.
These helpers use WRITE_ONCE() to prevent store-tearing.
Also use WRITE_ONCE() in fq_reset() and qdisc_reset() when sch->qstats.backlog is cleared.
--- include/net/sch_generic.h | 16 +++++++++++++--- net/sched/sch_api.c | 2 +- net/sched/sch_cake.c | 10 +++++----- net/sched/sch_cbs.c | 2 +- net/sched/sch_codel.c | 2 +- net/sched/sch_drr.c | 2 +- net/sched/sch_ets.c | 2 +- net/sched/sch_fq.c | 2 +- net/sched/sch_fq_codel.c | 4 ++-- net/sched/sch_fq_pie.c | 4 ++-- net/sched/sch_generic.c | 2 +- net/sched/sch_gred.c | 2 +- net/sched/sch_hfsc.c | 2 +- net/sched/sch_htb.c | 2 +- net/sched/sch_netem.c | 2 +- net/sched/sch_prio.c | 2 +- net/sched/sch_qfq.c | 2 +- net/sched/sch_red.c | 2 +- net/sched/sch_sfb.c | 4 ++-- net/sched/sch_sfq.c | 2 +- net/sched/sch_tbf.c | 4 ++-- 21 files changed, 41 insertions(+), 31 deletions(-)
include/net/sch_generic.h +13 -3
@@ -965,10 +965,15 @@ static inline void qdisc_bstats_update(struct Qdisc *sch,bstats_update (&sch->bstats, skb); } +static inline void qstats_backlog_sub (struct Qdisc *sch,u32 val) +{ +WRITE_ONCE (sch->qstats.backlog, sch->qstats.backlog - val); +} +static inline void qdisc_qstats_backlog_dec (struct Qdisc *sch,const struct sk_buff *skb) { - sch->qstats.backlog -=qdisc_pkt_len (skb); +qstats_backlog_sub (sch,qdisc_pkt_len (skb)); } static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,
@@ -977,10 +982,15 @@ static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,this_cpu_sub (sch->cpu_qstats->backlog,qdisc_pkt_len (skb)); } +static inline void qstats_backlog_add (struct Qdisc *sch,u32 val) +{ +WRITE_ONCE (sch->qstats.backlog, sch->qstats.backlog + val); +} +static inline void qdisc_qstats_backlog_inc (struct Qdisc *sch,const struct sk_buff *skb) { - sch->qstats.backlog +=qdisc_pkt_len (skb); +qstats_backlog_add (sch,qdisc_pkt_len (skb)); } static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch,
@@ -1304,7 +1314,7 @@ static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch,qdisc_qstats_cpu_qlen_inc (sch);this_cpu_add (sch->cpu_qstats->backlog, pkt_len); }else { - sch->qstats.backlog += pkt_len; +qstats_backlog_add (sch, pkt_len);qdisc_qlen_inc (sch); } }
net/sched/sch_api.c +1 -1
@@ -806,7 +806,7 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) cops->qlen_notify (sch, cl); }WRITE_ONCE (sch->q.qlen, sch->q.qlen - n); - sch->qstats.backlog -= len; +qstats_backlog_sub (sch, len);__qdisc_qstats_drop (sch, drops); }rcu_read_unlock ();
net/sched/sch_cake.c +5 -5
@@ -1600,10 +1600,10 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free) b->unresponsive_flow_count + 1); len =qdisc_pkt_len (skb); - q->buffer_used -= skb->truesize; +qstats_backlog_sub (sch, len); + q->buffer_used -= skb->truesize;WRITE_ONCE (b->tin_backlog, b->tin_backlog - len);WRITE_ONCE (b->backlogs[idx], b->backlogs[idx] - len); - sch->qstats.backlog -= len;WRITE_ONCE (flow->dropped, flow->dropped +1 );WRITE_ONCE (b->tin_dropped, b->tin_dropped +1 );
@@ -1830,7 +1830,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, }/* stats */ - sch->qstats.backlog += slen; +qstats_backlog_add (sch, slen); q->avg_window_bytes += slen;WRITE_ONCE (b->bytes, b->bytes + slen);WRITE_ONCE (b->tin_backlog, b->tin_backlog + slen);
@@ -1867,7 +1867,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,/* stats */ WRITE_ONCE (b->packets, b->packets +1 ); - sch->qstats.backlog += len - ack_pkt_len; +qstats_backlog_add (sch, len - ack_pkt_len); q->avg_window_bytes += len - ack_pkt_len;WRITE_ONCE (b->bytes, b->bytes + len - ack_pkt_len);WRITE_ONCE (b->tin_backlog, b->tin_backlog + len - ack_pkt_len);
@@ -1985,7 +1985,7 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch) len =qdisc_pkt_len (skb);WRITE_ONCE (b->backlogs[q->cur_flow], b->backlogs[q->cur_flow] - len);WRITE_ONCE (b->tin_backlog, b->tin_backlog - len); - sch->qstats.backlog -= len; +qstats_backlog_sub (sch, len); q->buffer_used -= skb->truesize;qdisc_qlen_dec (sch);
net/sched/sch_cbs.c +1 -1
@@ -96,7 +96,7 @@ static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch,if (err !=NET_XMIT_SUCCESS )return err; - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ;
net/sched/sch_codel.c +1 -1
@@ -42,7 +42,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)struct sk_buff *skb =__qdisc_dequeue_head (&sch->q);if (skb) { - sch->qstats.backlog -=qdisc_pkt_len (skb); +qstats_backlog_sub (sch,qdisc_pkt_len (skb));prefetch (&skb->end);/* we'll need skb_shinfo() */ }return skb;
net/sched/sch_drr.c +1 -1
@@ -365,7 +365,7 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch, cl->deficit = cl->quantum; } - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);qdisc_qlen_inc (sch);return err; }
net/sched/sch_ets.c +1 -1
@@ -448,7 +448,7 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, cl->deficit = cl->quantum; } - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);qdisc_qlen_inc (sch);return err; }
net/sched/sch_fq.c +1 -1
@@ -802,7 +802,7 @@ static void fq_reset(struct Qdisc *sch)unsigned int idx;WRITE_ONCE (sch->q.qlen,0 ); - sch->qstats.backlog =0 ; +WRITE_ONCE (sch->qstats.backlog,0 );fq_flow_purge (&q->internal);
net/sched/sch_fq_codel.c +2 -2
@@ -177,7 +177,7 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets,WRITE_ONCE (q->backlogs[idx], q->backlogs[idx] - len); q->memory_usage -= mem;__qdisc_qstats_drop (sch, i); - sch->qstats.backlog -= len; +qstats_backlog_sub (sch, len);WRITE_ONCE (sch->q.qlen, sch->q.qlen - i);return idx; }
@@ -268,7 +268,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx) q->backlogs[flow - q->flows] - qdisc_pkt_len(skb)); q->memory_usage -=get_codel_cb (skb)->mem_usage;qdisc_qlen_dec (sch); - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog_dec (sch, skb); }return skb; }
net/sched/sch_fq_pie.c +2 -2
@@ -184,7 +184,7 @@ static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, pkt_len =qdisc_pkt_len (skb); q->stats.packets_in++; q->memory_usage += skb->truesize; - sch->qstats.backlog += pkt_len; +qstats_backlog_add (sch, pkt_len);qdisc_qlen_inc (sch);flow_queue_add (sel_flow, skb);if (list_empty (&sel_flow->flowchain)) {
@@ -262,7 +262,7 @@ static struct sk_buff *fq_pie_qdisc_dequeue(struct Qdisc *sch)if (flow->head) { skb =dequeue_head (flow); pkt_len =qdisc_pkt_len (skb); - sch->qstats.backlog -= pkt_len; +qstats_backlog_sub (sch, pkt_len);qdisc_qlen_dec (sch);qdisc_bstats_update (sch, skb); }
net/sched/sch_generic.c +1 -1
@@ -1063,7 +1063,7 @@ void qdisc_reset(struct Qdisc *qdisc)__skb_queue_purge (&qdisc->skb_bad_txq);WRITE_ONCE (qdisc->q.qlen,0 ); - qdisc->qstats.backlog =0 ; +WRITE_ONCE (qdisc->qstats.backlog,0 ); }EXPORT_SYMBOL (qdisc_reset);
net/sched/sch_gred.c +1 -1
@@ -388,7 +388,7 @@ static int gred_offload_dump_stats(struct Qdisc *sch) bytes +=u64_stats_read (&hw_stats->stats.bstats[i].bytes); packets +=u64_stats_read (&hw_stats->stats.bstats[i].packets); sch->qstats.qlen += hw_stats->stats.qstats[i].qlen; - sch->qstats.backlog += hw_stats->stats.qstats[i].backlog; +qstats_backlog_add (sch, hw_stats->stats.qstats[i].backlog);__qdisc_qstats_drop (sch, hw_stats->stats.qstats[i].drops); sch->qstats.requeues += hw_stats->stats.qstats[i].requeues; sch->qstats.overlimits += hw_stats->stats.qstats[i].overlimits;
net/sched/sch_hfsc.c +1 -1
@@ -1560,7 +1560,7 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)return err; } - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);qdisc_qlen_inc (sch);if (first && !cl_in_el_or_vttree (cl)) {
net/sched/sch_htb.c +1 -1
@@ -650,7 +650,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch,htb_activate (q, cl); } - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ; }
net/sched/sch_netem.c +1 -1
@@ -750,7 +750,7 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch)if (err !=NET_XMIT_SUCCESS ) {if (net_xmit_drop_count (err))qdisc_qstats_drop (sch); - sch->qstats.backlog -= pkt_len; +qstats_backlog_sub (sch, pkt_len);qdisc_qlen_dec (sch);qdisc_tree_reduce_backlog (sch,1 , pkt_len); }
net/sched/sch_prio.c +1 -1
@@ -85,7 +85,7 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) ret =qdisc_enqueue (skb, qdisc, to_free);if (ret ==NET_XMIT_SUCCESS ) { - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ; }
net/sched/sch_qfq.c +1 -1
@@ -1264,7 +1264,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, }_bstats_update (&cl->bstats, len, gso_segs); - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);qdisc_qlen_inc (sch); agg = cl->agg;
net/sched/sch_red.c +1 -1
@@ -138,7 +138,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, len =qdisc_pkt_len (skb); ret =qdisc_enqueue (skb, child, to_free);if (likely (ret ==NET_XMIT_SUCCESS )) { - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);qdisc_qlen_inc (sch); }else if (net_xmit_drop_count (ret)) { WRITE_ONCE(q->stats.pdrop,
net/sched/sch_sfb.c +2 -2
@@ -415,7 +415,7 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch,memcpy (&cb,sfb_skb_cb (skb),sizeof (cb)); ret =qdisc_enqueue (skb, child, to_free);if (likely (ret ==NET_XMIT_SUCCESS )) { - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);qdisc_qlen_inc (sch);increment_qlen (&cb, q); }else if (net_xmit_drop_count (ret)) {
@@ -592,7 +592,7 @@ static int sfb_dump(struct Qdisc *sch, struct sk_buff *skb) .penalty_burst = q->penalty_burst, }; - sch->qstats.backlog = q->qdisc->qstats.backlog; +WRITE_ONCE (sch->qstats.backlog,READ_ONCE (q->qdisc->qstats.backlog)); opts =nla_nest_start_noflag (skb,TCA_OPTIONS );if (opts ==NULL ) goto nla_put_failure;
net/sched/sch_sfq.c +1 -1
@@ -427,7 +427,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)/* We know we have at least one packet in queue */ head =slot_dequeue_head (slot); delta =qdisc_pkt_len (head) -qdisc_pkt_len (skb); - sch->qstats.backlog -= delta; +qstats_backlog_sub (sch, delta);WRITE_ONCE (slot->backlog, slot->backlog - delta);qdisc_drop_reason (head, sch, to_free,QDISC_DROP_FLOW_LIMIT );
net/sched/sch_tbf.c +2 -2
@@ -232,7 +232,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch, } }WRITE_ONCE (sch->q.qlen, sch->q.qlen + nb); - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);if (nb >0 ) {qdisc_tree_reduce_backlog (sch,1 - nb, prev_len - len);consume_skb (skb);
@@ -263,7 +263,7 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch,return ret; } - sch->qstats.backlog += len; +qstats_backlog_add (sch, len);qdisc_qlen_inc (sch);return NET_XMIT_SUCCESS ; }
Signature · 1 line
2.54.0.563.g4f69b47b94-goog
PATCH v2 4/8 net/sched: add qdisc_qlen_lockless() helper
In reply to Eric DumazetUsed in contexts were qdisc spinlock is not held.
--- include/net/sch_generic.h | 7 ++++++- net/sched/sch_mq.c | 2 +- net/sched/sch_mqprio.c | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-)
include/net/sch_generic.h +6 -1
@@ -542,6 +542,11 @@ static inline int qdisc_qlen(const struct Qdisc *q)return q->q.qlen; } +static inline int qdisc_qlen_lockless (const struct Qdisc *q) +{ +return READ_ONCE (q->q.qlen); +} +static inline void qdisc_qlen_inc (struct Qdisc *q) {WRITE_ONCE (q->q.qlen, q->q.qlen +1 );
@@ -561,7 +566,7 @@ static inline int qdisc_qlen_sum(const struct Qdisc *q)for_each_possible_cpu (i) qlen +=READ_ONCE (per_cpu_ptr (q->cpu_qstats, i)->qlen); }else { - qlen +=READ_ONCE (q->q.qlen); + qlen +=qdisc_qlen_lockless (q); }return qlen;
net/sched/sch_mq.c +1 -1
@@ -163,7 +163,7 @@ void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb) &qdisc->bstats, false);gnet_stats_add_queue (&sch->qstats, qdisc->cpu_qstats, &qdisc->qstats); - qlen +=qdisc_qlen (qdisc); + qlen +=qdisc_qlen_lockless (qdisc);spin_unlock_bh (qdisc_lock (qdisc)); }
net/sched/sch_mqprio.c +2 -2
@@ -576,7 +576,7 @@ static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb) &qdisc->bstats, false);gnet_stats_add_queue (&sch->qstats, qdisc->cpu_qstats, &qdisc->qstats); - qlen +=qdisc_qlen (qdisc); + qlen +=qdisc_qlen_lockless (qdisc);spin_unlock_bh (qdisc_lock (qdisc)); }
@@ -691,7 +691,7 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, &qdisc->bstats, false);gnet_stats_add_queue (&qstats, qdisc->cpu_qstats, &qdisc->qstats); - qlen +=qdisc_qlen (qdisc); + qlen +=qdisc_qlen_lockless (qdisc);spin_unlock_bh (qdisc_lock (qdisc)); }
Signature · 1 line
2.54.0.563.g4f69b47b94-goog
PATCH v2 5/8 net/sched: add const qualifiers to gnet_stats helpers
In reply to Eric DumazetIn preparation of lockless qdisc dumps, add const qualifiers to:
- gnet_stats_add_basic()
- gnet_stats_copy_basic()
- gnet_stats_copy_basic_hw()
- gnet_stats_copy_queue()
- gnet_stats_read_basic()
- ___gnet_stats_copy_basic()
- qdisc_qstats_copy()
--- include/net/gen_stats.h | 16 ++++++++-------- include/net/sch_generic.h | 2 +- net/core/gen_stats.c | 31 +++++++++++++++---------------- 3 files changed, 24 insertions(+), 25 deletions(-)
include/net/gen_stats.h +8 -8
@@ -47,19 +47,19 @@ int gnet_stats_start_copy_compat(struct sk_buff *skb, int type, int padattr);int gnet_stats_copy_basic (struct gnet_dump *d, -struct gnet_stats_basic_sync __percpu *cpu, -struct gnet_stats_basic_sync *b,bool running); +const struct gnet_stats_basic_sync __percpu *cpu, +const struct gnet_stats_basic_sync *b,bool running);void gnet_stats_add_basic (struct gnet_stats_basic_sync *bstats, -struct gnet_stats_basic_sync __percpu *cpu, -struct gnet_stats_basic_sync *b,bool running); +const struct gnet_stats_basic_sync __percpu *cpu, +const struct gnet_stats_basic_sync *b,bool running);int gnet_stats_copy_basic_hw (struct gnet_dump *d, -struct gnet_stats_basic_sync __percpu *cpu, -struct gnet_stats_basic_sync *b,bool running); +const struct gnet_stats_basic_sync __percpu *cpu, +const struct gnet_stats_basic_sync *b,bool running);int gnet_stats_copy_rate_est (struct gnet_dump *d,struct net_rate_estimator __rcu **ptr);int gnet_stats_copy_queue (struct gnet_dump *d, -struct gnet_stats_queue __percpu *cpu_q, -struct gnet_stats_queue *q,__u32 qlen); +const struct gnet_stats_queue __percpu *cpu_q, +const struct gnet_stats_queue *q,__u32 qlen);void gnet_stats_add_queue (struct gnet_stats_queue *qstats,const struct gnet_stats_queue __percpu *cpu_q,const struct gnet_stats_queue *q);
include/net/sch_generic.h +1 -1
@@ -1054,7 +1054,7 @@ static inline void qdisc_qstats_overlimit(struct Qdisc *sch)WRITE_ONCE (sch->qstats.overlimits, sch->qstats.overlimits +1 ); } -static inline int qdisc_qstats_copy (struct gnet_dump *d,struct Qdisc *sch) +static inline int qdisc_qstats_copy (struct gnet_dump *d,const struct Qdisc *sch) {__u32 qlen =qdisc_qlen_sum (sch);
net/core/gen_stats.c +15 -16
@@ -124,13 +124,13 @@ void gnet_stats_basic_sync_init(struct gnet_stats_basic_sync *b)EXPORT_SYMBOL (gnet_stats_basic_sync_init);static void gnet_stats_add_basic_cpu (struct gnet_stats_basic_sync *bstats, -struct gnet_stats_basic_sync __percpu *cpu) +const struct gnet_stats_basic_sync __percpu *cpu) {u64 t_bytes =0 , t_packets =0 ;int i;for_each_possible_cpu (i) { -struct gnet_stats_basic_sync *bcpu =per_cpu_ptr (cpu, i); +const struct gnet_stats_basic_sync *bcpu =per_cpu_ptr (cpu, i);unsigned int start;u64 bytes, packets;
@@ -147,8 +147,8 @@ static void gnet_stats_add_basic_cpu(struct gnet_stats_basic_sync *bstats, }void gnet_stats_add_basic (struct gnet_stats_basic_sync *bstats, -struct gnet_stats_basic_sync __percpu *cpu, -struct gnet_stats_basic_sync *b,bool running) +const struct gnet_stats_basic_sync __percpu *cpu, +const struct gnet_stats_basic_sync *b,bool running) {unsigned int start;u64 bytes =0 ;
@@ -172,8 +172,8 @@ void gnet_stats_add_basic(struct gnet_stats_basic_sync *bstats,EXPORT_SYMBOL (gnet_stats_add_basic);static void gnet_stats_read_basic (u64 *ret_bytes,u64 *ret_packets, -struct gnet_stats_basic_sync __percpu *cpu, -struct gnet_stats_basic_sync *b,bool running) +const struct gnet_stats_basic_sync __percpu *cpu, +const struct gnet_stats_basic_sync *b,bool running) {unsigned int start;
@@ -182,8 +182,7 @@ static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets,int i;for_each_possible_cpu (i) { -struct gnet_stats_basic_sync *bcpu =per_cpu_ptr (cpu, i); -unsigned int start; +const struct gnet_stats_basic_sync *bcpu =per_cpu_ptr (cpu, i);u64 bytes, packets; do {
@@ -209,8 +208,8 @@ static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets,static int ___gnet_stats_copy_basic (struct gnet_dump *d, -struct gnet_stats_basic_sync __percpu *cpu, -struct gnet_stats_basic_sync *b, +const struct gnet_stats_basic_sync __percpu *cpu, +const struct gnet_stats_basic_sync *b,int type,bool running) {u64 bstats_bytes, bstats_packets;
@@ -258,8 +257,8 @@ ___gnet_stats_copy_basic(struct gnet_dump *d,*/ int gnet_stats_copy_basic (struct gnet_dump *d, -struct gnet_stats_basic_sync __percpu *cpu, -struct gnet_stats_basic_sync *b, +const struct gnet_stats_basic_sync __percpu *cpu, +const struct gnet_stats_basic_sync *b,bool running) {return ___gnet_stats_copy_basic (d, cpu, b,TCA_STATS_BASIC , running);
@@ -285,8 +284,8 @@ EXPORT_SYMBOL(gnet_stats_copy_basic);*/ int gnet_stats_copy_basic_hw (struct gnet_dump *d, -struct gnet_stats_basic_sync __percpu *cpu, -struct gnet_stats_basic_sync *b, +const struct gnet_stats_basic_sync __percpu *cpu, +const struct gnet_stats_basic_sync *b,bool running) {return ___gnet_stats_copy_basic (d, cpu, b,TCA_STATS_BASIC_HW , running);
@@ -385,8 +384,8 @@ EXPORT_SYMBOL(gnet_stats_add_queue);*/ int gnet_stats_copy_queue (struct gnet_dump *d, -struct gnet_stats_queue __percpu *cpu_q, -struct gnet_stats_queue *q,__u32 qlen) +const struct gnet_stats_queue __percpu *cpu_q, +const struct gnet_stats_queue *q,__u32 qlen) {struct gnet_stats_queue qstats = {0 };
Signature · 1 line
2.54.0.563.g4f69b47b94-goog
PATCH v2 6/8 net/sched: mq: no longer acquire qdisc spinlocks in dump operations
In reply to Eric DumazetPrepare mq_dump_common() for RTNL avoidance.
Use RCU instead of RTNL, and no longer acquire each children spinlock.
--- include/net/sch_generic.h | 9 +++++++++ net/sched/sch_mq.c | 32 ++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 10 deletions(-)
include/net/sch_generic.h +9 -0
@@ -952,6 +952,15 @@ static inline void _bstats_update(struct gnet_stats_basic_sync *bstats,u64_stats_update_end (&bstats->syncp); } +static inline void _bstats_set (struct gnet_stats_basic_sync *bstats, +u64 bytes,u64 packets) +{ +u64_stats_update_begin (&bstats->syncp); +u64_stats_set (&bstats->bytes, bytes); +u64_stats_set (&bstats->packets, packets); +u64_stats_update_end (&bstats->syncp); +} +static inline void bstats_update (struct gnet_stats_basic_sync *bstats,const struct sk_buff *skb) {
net/sched/sch_mq.c +22 -10
@@ -143,30 +143,42 @@ EXPORT_SYMBOL_NS_GPL(mq_attach, "NET_SCHED_INTERNAL");void mq_dump_common (struct Qdisc *sch,struct sk_buff *skb) {struct net_device *dev =qdisc_dev (sch); +struct gnet_stats_queue qstats = {0 }; +struct gnet_stats_basic_sync bstats; +const struct Qdisc *qdisc;unsigned int qlen =0 ; -struct Qdisc *qdisc;unsigned int ntx; -gnet_stats_basic_sync_init (&sch->bstats); -memset (&sch->qstats,0 ,sizeof (sch->qstats)); +gnet_stats_basic_sync_init (&bstats);/* MQ supports lockless qdiscs. However, statistics accounting needs * to account for all, none, or a mix of locked and unlocked child * qdiscs. Percpu stats are added to counters in-band and locking * qdisc totals are added at end. */ +rcu_read_lock ();for (ntx =0 ; ntx < dev->num_tx_queues; ntx++) { - qdisc =rtnl_dereference (netdev_get_tx_queue (dev, ntx)->qdisc_sleeping); -spin_lock_bh (qdisc_lock (qdisc)); + qdisc =rcu_dereference (netdev_get_tx_queue (dev, ntx)->qdisc_sleeping); -gnet_stats_add_basic (&sch->bstats, qdisc->cpu_bstats, - &qdisc->bstats, false); -gnet_stats_add_queue (&sch->qstats, qdisc->cpu_qstats, +gnet_stats_add_basic (&bstats, qdisc->cpu_bstats, + &qdisc->bstats, true); +gnet_stats_add_queue (&qstats, qdisc->cpu_qstats, &qdisc->qstats); qlen +=qdisc_qlen_lockless (qdisc); - -spin_unlock_bh (qdisc_lock (qdisc)); } +rcu_read_unlock (); + +spin_lock_bh (qdisc_lock (sch)); +_bstats_set (&sch->bstats,u64_stats_read (&bstats.bytes), +u64_stats_read (&bstats.packets)); +spin_unlock_bh (qdisc_lock (sch)); + +WRITE_ONCE (sch->qstats.qlen, qstats.qlen); +WRITE_ONCE (sch->qstats.backlog, qstats.backlog); +WRITE_ONCE (sch->qstats.drops, qstats.drops); +WRITE_ONCE (sch->qstats.requeues, qstats.requeues); +WRITE_ONCE (sch->qstats.overlimits, qstats.overlimits); +WRITE_ONCE (sch->q.qlen, qlen); }EXPORT_SYMBOL_NS_GPL (mq_dump_common,"NET_SCHED_INTERNAL" );
Signature · 1 line
2.54.0.563.g4f69b47b94-goog
PATCH v2 7/8 net/sched: mq_prio: no longer acquire qdisc spinlocks in mqprio_dump()
In reply to Eric DumazetPrepare mqprio_dump() for RTNL avoidance.
Use RCU instead of RTNL, and no longer acquire each children spinlock.
--- net/sched/sch_mqprio.c | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-)
net/sched/sch_mqprio.c +24 -13
@@ -551,35 +551,46 @@ static int mqprio_dump_tc_entries(struct mqprio_sched *priv,static int mqprio_dump (struct Qdisc *sch,struct sk_buff *skb) { -struct net_device *dev =qdisc_dev (sch); -struct mqprio_sched *priv =qdisc_priv (sch);struct nlattr *nla = (struct nlattr *)skb_tail_pointer (skb); +struct mqprio_sched *priv =qdisc_priv (sch); +struct net_device *dev =qdisc_dev (sch); +struct gnet_stats_queue qstats = {0 }; +struct gnet_stats_basic_sync bstats;struct tc_mqprio_qopt opt = {0 }; +const struct Qdisc *qdisc;unsigned int qlen =0 ; -struct Qdisc *qdisc;unsigned int ntx; - qlen =0 ; -gnet_stats_basic_sync_init (&sch->bstats); -memset (&sch->qstats,0 ,sizeof (sch->qstats)); +gnet_stats_basic_sync_init (&bstats);/* MQ supports lockless qdiscs. However, statistics accounting needs * to account for all, none, or a mix of locked and unlocked child * qdiscs. Percpu stats are added to counters in-band and locking * qdisc totals are added at end. */ +rcu_read_lock ();for (ntx =0 ; ntx < dev->num_tx_queues; ntx++) { - qdisc =rtnl_dereference (netdev_get_tx_queue (dev, ntx)->qdisc_sleeping); -spin_lock_bh (qdisc_lock (qdisc)); + qdisc =rcu_dereference (netdev_get_tx_queue (dev, ntx)->qdisc_sleeping); -gnet_stats_add_basic (&sch->bstats, qdisc->cpu_bstats, - &qdisc->bstats, false); -gnet_stats_add_queue (&sch->qstats, qdisc->cpu_qstats, +gnet_stats_add_basic (&bstats, qdisc->cpu_bstats, + &qdisc->bstats, true); +gnet_stats_add_queue (&qstats, qdisc->cpu_qstats, &qdisc->qstats); qlen +=qdisc_qlen_lockless (qdisc); - -spin_unlock_bh (qdisc_lock (qdisc)); } +rcu_read_unlock (); + +spin_lock_bh (qdisc_lock (sch)); +_bstats_set (&sch->bstats,u64_stats_read (&bstats.bytes), +u64_stats_read (&bstats.packets)); +spin_unlock_bh (qdisc_lock (sch)); + +WRITE_ONCE (sch->qstats.qlen, qstats.qlen); +WRITE_ONCE (sch->qstats.backlog, qstats.backlog); +WRITE_ONCE (sch->qstats.drops, qstats.drops); +WRITE_ONCE (sch->qstats.requeues, qstats.requeues); +WRITE_ONCE (sch->qstats.overlimits, qstats.overlimits); +WRITE_ONCE (sch->q.qlen, qlen);mqprio_qopt_reconstruct (dev, &opt);
Signature · 1 line
2.54.0.563.g4f69b47b94-goog
PATCH v2 8/8 net/sched: mq_prio: no longer acquire qdisc spinlocks in mqprio_dump_class_stats()
In reply to Eric DumazetPrepare mqprio_dump_class_stats() for RTNL avoidance.
Use RCU instead of RTNL, and no longer acquire each children spinlock.
As a bonus we no longer have to release/acquire d->lock.
--- net/sched/sch_mqprio.c | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-)
net/sched/sch_mqprio.c +15 -25
@@ -672,9 +672,11 @@ static int mqprio_dump_class(struct Qdisc *sch, unsigned long cl,static int mqprio_dump_class_stats (struct Qdisc *sch,unsigned long cl,struct gnet_dump *d) - __releases(d->lock) - __acquires(d->lock) { +const struct Qdisc *qdisc; +int res =0 ; + +rcu_read_lock ();if (cl >=TC_H_MIN_PRIORITY ) {struct net_device *dev =qdisc_dev (sch);struct netdev_tc_txq tc = dev->tc_to_txq[cl &TC_BITMASK ];
@@ -684,46 +686,34 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,int i;gnet_stats_basic_sync_init (&bstats); -/* Drop lock here it will be reclaimed before touching -* statistics this is required because the d->lock we -* hold here is the look on dev_queue->qdisc_sleeping -* also acquired below. -*/ -if (d->lock) -spin_unlock_bh (d->lock);for (i = tc.offset; i < tc.offset + tc.count; i++) {struct netdev_queue *q =netdev_get_tx_queue (dev, i); -struct Qdisc *qdisc =rtnl_dereference (q->qdisc); - -spin_lock_bh (qdisc_lock (qdisc)); + qdisc =rcu_dereference (q->qdisc);gnet_stats_add_basic (&bstats, qdisc->cpu_bstats, - &qdisc->bstats, false); + &qdisc->bstats, true);gnet_stats_add_queue (&qstats, qdisc->cpu_qstats, &qdisc->qstats); qlen +=qdisc_qlen_lockless (qdisc); - -spin_unlock_bh (qdisc_lock (qdisc)); } + qlen = qlen + qstats.qlen; -/* Reclaim root sleeping lock before completing stats */ -if (d->lock) -spin_lock_bh (d->lock);if (gnet_stats_copy_basic (d,NULL , &bstats, false) <0 ||gnet_stats_copy_queue (d,NULL , &qstats, qlen) <0 ) -return -1 ; + res =-1 ; }else {struct netdev_queue *dev_queue =mqprio_queue_get (sch, cl); - sch =rtnl_dereference (dev_queue->qdisc_sleeping); -if (gnet_stats_copy_basic (d, sch->cpu_bstats, - &sch->bstats, true) <0 || -qdisc_qstats_copy (d, sch) <0 ) -return -1 ; + qdisc =rcu_dereference (dev_queue->qdisc_sleeping); +if (gnet_stats_copy_basic (d, qdisc->cpu_bstats, + &qdisc->bstats, true) <0 || +qdisc_qstats_copy (d, qdisc) <0 ) + res =-1 ; } -return 0 ; +rcu_read_unlock (); +return res; }static void mqprio_walk (struct Qdisc *sch,struct qdisc_walker *arg)
Signature · 1 line
2.54.0.563.g4f69b47b94-goog
Quoted 4 lines · Eric Dumazet
Goal is to no longer acquire RTNL in qdisc dumps.
This series annotate data-races, and change mq and
mq_prioto no longer acquire children qdisc spinlocks.
Nice!
For the series:
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:
Hello:
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:
Quoted 8 lines · On Sun, 10 May 2026 09:14:47 +0000 you wrote:
Goal is to no longer acquire RTNL in qdisc dumps.
This series annotate data-races, and change mq and
mq_prioto no longer acquire children qdisc spinlocks.v2: addressed sashiko (and Victor Nogueira) feedback.
[...]
Here is the summary with links:
- [v2,net-next,1/8] net/sched: add READ_ONCE() in gnet_stats_add_queue[_cpu]
https://git.kernel.org/netdev/net-next/c/a00073324266 - [v2,net-next,2/8] net/sched: add qdisc_qlen_inc() and qdisc_qlen_dec() https://git.kernel.org/netdev/net-next/c/863cd7865201 - [v2,net-next,3/8] net/sched: annotate data-races around sch->qstats.backlog https://git.kernel.org/netdev/net-next/c/0e5b05f44d81 - [v2,net-next,4/8] net/sched: add qdisc_qlen_lockless() helper https://git.kernel.org/netdev/net-next/c/554e96a47a84 - [v2,net-next,5/8] net/sched: add const qualifiers to gnet_stats helpers https://git.kernel.org/netdev/net-next/c/5b12bd5c474b - [v2,net-next,6/8] net/sched: mq: no longer acquire qdisc spinlocks in dump operations https://git.kernel.org/netdev/net-next/c/4f07179f4006 - [v2,net-next,7/8] net/sched: mq_prio: no longer acquire qdisc spinlocks in mqprio_dump() https://git.kernel.org/netdev/net-next/c/bb19ce4c4333 - [v2,net-next,8/8] net/sched: mq_prio: no longer acquire qdisc spinlocks in mqprio_dump_class_stats() https://git.kernel.org/netdev/net-next/c/881f7c5eff30
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html