net: sched: make bstats per cpu and estimator RCU safe
Subsystems (5)
PATCH v2 1/4 net: sched: make bstats per cpu and estimator RCU safe
From: John Fastabend <[email protected]>
In order to run qdisc's without locking statistics and estimators need to be handled correctly.
To resolve bstats make the statistics per cpu. And because this is only needed for qdiscs that are running without locks which is not the case for most qdiscs in the near future only create percpu stats when qdiscs set the TCQ_F_CPUSTATS flag.
Next because estimators use the bstats to calculate packets per second and bytes per second the estimator code paths are updated to use the per cpu statistics.
--- include/net/gen_stats.h | 11 +++++++++ include/net/sch_generic.h | 22 +++++++++++++++++- net/core/gen_estimator.c | 29 +++++++++++++++--------- net/core/gen_stats.c | 53 ++++++++++++++++++++++++++++++++++++++++---- net/netfilter/xt_RATEEST.c | 2 +- net/sched/act_api.c | 5 ++-- net/sched/act_police.c | 2 +- net/sched/sch_api.c | 29 +++++++++++++++++++----- net/sched/sch_atm.c | 2 +- net/sched/sch_cbq.c | 7 +++--- net/sched/sch_drr.c | 7 +++--- net/sched/sch_generic.c | 3 ++ net/sched/sch_hfsc.c | 13 +++++++---- net/sched/sch_htb.c | 12 +++++++--- net/sched/sch_mq.c | 2 +- net/sched/sch_mqprio.c | 4 ++- net/sched/sch_multiq.c | 2 +- net/sched/sch_prio.c | 2 +- net/sched/sch_qfq.c | 8 ++++--- 19 files changed, 164 insertions(+), 51 deletions(-)
include/net/gen_stats.h +11 -0
@@ -6,6 +6,11 @@#include <linux/rtnetlink.h> #include <linux/pkt_sched.h> +struct gnet_stats_basic_cpu { +struct gnet_stats_basic_packed bstats; +struct u64_stats_sync syncp; +}; +struct gnet_dump {spinlock_t * lock;struct sk_buff * skb;
@@ -27,7 +32,11 @@ int gnet_stats_start_copy_compat(struct sk_buff *skb, int type, spinlock_t *lock, struct gnet_dump *d);int gnet_stats_copy_basic (struct gnet_dump *d, +struct gnet_stats_basic_cpu __percpu *cpu,struct gnet_stats_basic_packed *b); +void __gnet_stats_copy_basic (struct gnet_stats_basic_packed *bstats, +struct gnet_stats_basic_cpu __percpu *cpu, +struct gnet_stats_basic_packed *b);int gnet_stats_copy_rate_est (struct gnet_dump *d,const struct gnet_stats_basic_packed *b,struct gnet_stats_rate_est64 *r);
@@ -37,11 +46,13 @@ int gnet_stats_copy_app(struct gnet_dump *d, void *st, int len);int gnet_stats_finish_copy (struct gnet_dump *d);int gen_new_estimator (struct gnet_stats_basic_packed *bstats, +struct gnet_stats_basic_cpu __percpu *cpu_bstats,struct gnet_stats_rate_est64 *rate_est,spinlock_t *stats_lock,struct nlattr *opt);void gen_kill_estimator (struct gnet_stats_basic_packed *bstats,struct gnet_stats_rate_est64 *rate_est);int gen_replace_estimator (struct gnet_stats_basic_packed *bstats, +struct gnet_stats_basic_cpu __percpu *cpu_bstats,struct gnet_stats_rate_est64 *rate_est,spinlock_t *stats_lock,struct nlattr *opt); bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
include/net/sch_generic.h +21 -1
@@ -6,6 +6,7 @@#include <linux/rcupdate.h> #include <linux/pkt_sched.h> #include <linux/pkt_cls.h> +#include <linux/percpu.h> #include <net/gen_stats.h> #include <net/rtnetlink.h>
@@ -58,6 +59,7 @@ struct Qdisc {* multiqueue device. */ #define TCQ_F_WARN_NONWC (1 << 16) +#define TCQ_F_CPUSTATS 0x20/* run using percpu statistics */ u32 limit;const struct Qdisc_ops *ops;struct qdisc_size_table __rcu *stab;
@@ -83,7 +85,10 @@ struct Qdisc {*/ unsigned long state;struct sk_buff_head q; -struct gnet_stats_basic_packed bstats; +union { +struct gnet_stats_basic_packed bstats; +struct gnet_stats_basic_cpu __percpu *cpu_bstats; + } __packed;unsigned int __state;struct gnet_stats_queue qstats;struct rcu_head rcu_head;
@@ -487,6 +492,10 @@ static inline int qdisc_enqueue_root(struct sk_buff *skb, struct Qdisc *sch)return qdisc_enqueue (skb, sch) &NET_XMIT_MASK ; } +static inline bool qdisc_is_percpu_stats (const struct Qdisc *q) +{ +return q->flags &TCQ_F_CPUSTATS ; +}static inline void bstats_update (struct gnet_stats_basic_packed *bstats,const struct sk_buff *skb)
@@ -495,6 +504,17 @@ static inline void bstats_update(struct gnet_stats_basic_packed *bstats, bstats->packets +=skb_is_gso (skb) ?skb_shinfo (skb)->gso_segs :1 ; } +static inline void qdisc_bstats_update_cpu (struct Qdisc *sch, +const struct sk_buff *skb) +{ +struct gnet_stats_basic_cpu *bstats = +this_cpu_ptr (sch->cpu_bstats); + +u64_stats_update_begin (&bstats->syncp); +bstats_update (&bstats->bstats, skb); +u64_stats_update_end (&bstats->syncp); +} +static inline void qdisc_bstats_update (struct Qdisc *sch,const struct sk_buff *skb) {
net/core/gen_estimator.c +18 -11
@@ -91,6 +91,8 @@ struct gen_estimatoru32 avpps;struct rcu_head e_rcu;struct rb_node node; +struct gnet_stats_basic_cpu __percpu *cpu_bstats; +struct rcu_head head; };struct gen_estimator_head
@@ -115,9 +117,8 @@ static void est_timer(unsigned long arg)rcu_read_lock ();list_for_each_entry_rcu (e, &elist[idx].list, list) { -u64 nbytes; +struct gnet_stats_basic_packed b = {0 };u64 brate; -u32 npackets;u32 rate;spin_lock (e->stats_lock);
@@ -125,15 +126,15 @@ static void est_timer(unsigned long arg)if (e->bstats ==NULL ) goto skip; - nbytes = e->bstats->bytes; - npackets = e->bstats->packets; - brate = (nbytes - e->last_bytes)<<(7 - idx); - e->last_bytes = nbytes; +__gnet_stats_copy_basic (&b, e->cpu_bstats, e->bstats); + + brate = (b.bytes - e->last_bytes)<<(7 - idx); + e->last_bytes = b.bytes; e->avbps += (brate >> e->ewma_log) - (e->avbps >> e->ewma_log); e->rate_est->bps = (e->avbps+0xF )>>5 ; - rate = (npackets - e->last_packets)<<(12 - idx); - e->last_packets = npackets; + rate = (b.packets - e->last_packets)<<(12 - idx); + e->last_packets = b.packets; e->avpps += (rate >> e->ewma_log) - (e->avpps >> e->ewma_log); e->rate_est->pps = (e->avpps+0x1FF )>>10 ; skip:
@@ -203,12 +204,14 @@ struct gen_estimator *gen_find_node(const struct gnet_stats_basic_packed *bstats* */ int gen_new_estimator (struct gnet_stats_basic_packed *bstats, +struct gnet_stats_basic_cpu __percpu *cpu_bstats,struct gnet_stats_rate_est64 *rate_est,spinlock_t *stats_lock,struct nlattr *opt) {struct gen_estimator *est;struct gnet_estimator *parm =nla_data (opt); +struct gnet_stats_basic_packed b = {0 };int idx;if (nla_len (opt) <sizeof (*parm))
@@ -221,15 +224,18 @@ int gen_new_estimator(struct gnet_stats_basic_packed *bstats,if (est ==NULL )return -ENOBUFS ; +__gnet_stats_copy_basic (&b, cpu_bstats, bstats); + idx = parm->interval +2 ; est->bstats = bstats; est->rate_est = rate_est; est->stats_lock = stats_lock; est->ewma_log = parm->ewma_log; - est->last_bytes = bstats->bytes; + est->last_bytes = b.bytes; est->avbps = rate_est->bps<<5 ; - est->last_packets = bstats->packets; + est->last_packets = b.packets; est->avpps = rate_est->pps<<10 ; + est->cpu_bstats = cpu_bstats;spin_lock_bh (&est_tree_lock);if (!elist[idx].timer.function) {
@@ -290,11 +296,12 @@ EXPORT_SYMBOL(gen_kill_estimator);* Returns 0 on success or a negative error code. */ int gen_replace_estimator (struct gnet_stats_basic_packed *bstats, +struct gnet_stats_basic_cpu __percpu *cpu_bstats,struct gnet_stats_rate_est64 *rate_est,spinlock_t *stats_lock,struct nlattr *opt) {gen_kill_estimator (bstats, rate_est); -return gen_new_estimator (bstats, rate_est, stats_lock, opt); +return gen_new_estimator (bstats, cpu_bstats, rate_est, stats_lock, opt); }EXPORT_SYMBOL (gen_replace_estimator);
net/core/gen_stats.c +48 -5
73 lines not on this page · raw message
net/netfilter/xt_RATEEST.c +1 -1
@@ -136,7 +136,7 @@ static int xt_rateest_tg_checkentry(const struct xt_tgchk_param *par) cfg.est.interval = info->interval; cfg.est.ewma_log = info->ewma_log; - ret =gen_new_estimator (&est->bstats, &est->rstats, + ret =gen_new_estimator (&est->bstats,NULL , &est->rstats, &est->lock, &cfg.opt);if (ret <0 ) goto err2;
net/sched/act_api.c +3 -2
@@ -252,7 +252,8 @@ int tcf_hash_create(u32 index, struct nlattr *est, struct tc_action *a, p->tcfc_tm.install = jiffies; p->tcfc_tm.lastuse = jiffies;if (est) { -int err =gen_new_estimator (&p->tcfc_bstats, &p->tcfc_rate_est, +int err =gen_new_estimator (&p->tcfc_bstats,NULL , + &p->tcfc_rate_est, &p->tcfc_lock, est);if (err) {kfree (p);
@@ -619,7 +620,7 @@ int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *a,if (err <0 ) goto errout; -if (gnet_stats_copy_basic (&d, &p->tcfc_bstats) <0 || +if (gnet_stats_copy_basic (&d,NULL , &p->tcfc_bstats) <0 ||gnet_stats_copy_rate_est (&d, &p->tcfc_bstats, &p->tcfc_rate_est) <0 ||gnet_stats_copy_queue (&d, &p->tcfc_qstats) <0 )
net/sched/act_police.c +1 -1
@@ -178,7 +178,7 @@ override:spin_lock_bh (&police->tcf_lock);if (est) { - err =gen_replace_estimator (&police->tcf_bstats, + err =gen_replace_estimator (&police->tcf_bstats,NULL , &police->tcf_rate_est, &police->tcf_lock, est);if (err)
net/sched/sch_api.c +23 -6
@@ -942,6 +942,13 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue, sch->handle = handle;if (!ops->init || (err = ops->init (sch, tca[TCA_OPTIONS ])) ==0 ) { +if (qdisc_is_percpu_stats (sch)) { + sch->cpu_bstats = + alloc_percpu(struct gnet_stats_basic_cpu); +if (!sch->cpu_bstats) + goto err_out4; + } +if (tca[TCA_STAB ]) { stab =qdisc_get_stab (tca[TCA_STAB ]);if (IS_ERR (stab)) {
@@ -964,8 +971,11 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,else root_lock =qdisc_lock (sch); - err =gen_new_estimator (&sch->bstats, &sch->rate_est, - root_lock, tca[TCA_RATE ]); + err =gen_new_estimator (&sch->bstats, + sch->cpu_bstats, + &sch->rate_est, + root_lock, + tca[TCA_RATE ]);if (err) goto err_out4; }
@@ -984,6 +994,7 @@ err_out:return NULL ; err_out4: +free_percpu (sch->cpu_bstats);/* * Any broken qdiscs that would require a ops->reset() here? * The qdisc was never in action so it shouldn't be necessary.
@@ -1022,9 +1033,11 @@ static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)because change can't be undone. */ if (sch->flags &TCQ_F_MQROOT ) goto out; -gen_replace_estimator (&sch->bstats, &sch->rate_est, -qdisc_root_sleeping_lock (sch), - tca[TCA_RATE ]); +gen_replace_estimator (&sch->bstats, + sch->cpu_bstats, + &sch->rate_est, +qdisc_root_sleeping_lock (sch), + tca[TCA_RATE ]); } out:return 0 ;
@@ -1299,6 +1312,7 @@ graft:static int tc_fill_qdisc (struct sk_buff *skb,struct Qdisc *q,u32 clid,u32 portid,u32 seq,u16 flags,int event) { +struct gnet_stats_basic_cpu __percpu *cpu_bstats =NULL ;struct tcmsg *tcm;struct nlmsghdr *nlh;unsigned char *b =skb_tail_pointer (skb);
@@ -1334,7 +1348,10 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,if (q->ops->dump_stats && q->ops->dump_stats (q, &d) <0 ) goto nla_put_failure; -if (gnet_stats_copy_basic (&d, &q->bstats) <0 || +if (qdisc_is_percpu_stats (q)) + cpu_bstats = q->cpu_bstats; + +if (gnet_stats_copy_basic (&d, cpu_bstats, &q->bstats) <0 ||gnet_stats_copy_rate_est (&d, &q->bstats, &q->rate_est) <0 ||gnet_stats_copy_queue (&d, &q->qstats) <0 ) goto nla_put_failure;
net/sched/sch_atm.c +1 -1
@@ -639,7 +639,7 @@ atm_tc_dump_class_stats(struct Qdisc *sch, unsigned long arg, flow->qstats.qlen = flow->q->q.qlen; -if (gnet_stats_copy_basic (d, &flow->bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &flow->bstats) <0 ||gnet_stats_copy_queue (d, &flow->qstats) <0 )return -1 ;
net/sched/sch_cbq.c +4 -3
@@ -1601,7 +1601,7 @@ cbq_dump_class_stats(struct Qdisc *sch, unsigned long arg,if (cl->undertime !=PSCHED_PASTPERFECT ) cl->xstats.undertime = cl->undertime - q->now; -if (gnet_stats_copy_basic (d, &cl->bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 ||gnet_stats_copy_queue (d, &cl->qstats) <0 )return -1 ;
@@ -1759,7 +1759,8 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t }if (tca[TCA_RATE ]) { - err =gen_replace_estimator (&cl->bstats, &cl->rate_est, + err =gen_replace_estimator (&cl->bstats,NULL , + &cl->rate_est,qdisc_root_sleeping_lock (sch), tca[TCA_RATE ]);if (err) {
@@ -1852,7 +1853,7 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t goto failure;if (tca[TCA_RATE ]) { - err =gen_new_estimator (&cl->bstats, &cl->rate_est, + err =gen_new_estimator (&cl->bstats,NULL , &cl->rate_est,qdisc_root_sleeping_lock (sch), tca[TCA_RATE ]);if (err) {
net/sched/sch_drr.c +4 -3
@@ -88,7 +88,8 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,if (cl !=NULL ) {if (tca[TCA_RATE ]) { - err =gen_replace_estimator (&cl->bstats, &cl->rate_est, + err =gen_replace_estimator (&cl->bstats,NULL , + &cl->rate_est,qdisc_root_sleeping_lock (sch), tca[TCA_RATE ]);if (err)
@@ -116,7 +117,7 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid, cl->qdisc = &noop_qdisc;if (tca[TCA_RATE ]) { - err =gen_replace_estimator (&cl->bstats, &cl->rate_est, + err =gen_replace_estimator (&cl->bstats,NULL , &cl->rate_est,qdisc_root_sleeping_lock (sch), tca[TCA_RATE ]);if (err) {
@@ -282,7 +283,7 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg, cl->qdisc->qstats.qlen = cl->qdisc->q.qlen; } -if (gnet_stats_copy_basic (d, &cl->bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 ||gnet_stats_copy_queue (d, &cl->qdisc->qstats) <0 )return -1 ;
net/sched/sch_generic.c +3 -0
@@ -632,6 +632,9 @@ static void qdisc_rcu_free(struct rcu_head *head) { struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head); +if (qdisc_is_percpu_stats (qdisc)) +free_percpu (qdisc->cpu_bstats); +kfree ((char *) qdisc - qdisc->padded); }
net/sched/sch_hfsc.c +8 -5
@@ -1014,9 +1014,12 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid, cur_time =psched_get_time ();if (tca[TCA_RATE ]) { - err =gen_replace_estimator (&cl->bstats, &cl->rate_est, -qdisc_root_sleeping_lock (sch), - tca[TCA_RATE ]); +spinlock_t *lock =qdisc_root_sleeping_lock (sch); + + err =gen_replace_estimator (&cl->bstats,NULL , + &cl->rate_est, + lock, + tca[TCA_RATE ]);if (err)return err; }
@@ -1063,7 +1066,7 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid,return -ENOBUFS ;if (tca[TCA_RATE ]) { - err =gen_new_estimator (&cl->bstats, &cl->rate_est, + err =gen_new_estimator (&cl->bstats,NULL , &cl->rate_est,qdisc_root_sleeping_lock (sch), tca[TCA_RATE ]);if (err) {
@@ -1374,7 +1377,7 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg, xstats.work = cl->cl_total; xstats.rtwork = cl->cl_cumul; -if (gnet_stats_copy_basic (d, &cl->bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 ||gnet_stats_copy_queue (d, &cl->qstats) <0 )return -1 ;
net/sched/sch_htb.c +8 -4
@@ -1144,7 +1144,7 @@ htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d) cl->xstats.tokens =PSCHED_NS2TICKS (cl->tokens); cl->xstats.ctokens =PSCHED_NS2TICKS (cl->ctokens); -if (gnet_stats_copy_basic (d, &cl->bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d,NULL , &cl->rate_est) <0 ||gnet_stats_copy_queue (d, &cl->qstats) <0 )return -1 ;
@@ -1402,7 +1402,8 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, goto failure;if (htb_rate_est || tca[TCA_RATE ]) { - err =gen_new_estimator (&cl->bstats, &cl->rate_est, + err =gen_new_estimator (&cl->bstats,NULL , + &cl->rate_est,qdisc_root_sleeping_lock (sch), tca[TCA_RATE ] ? : &est.nla);if (err) {
@@ -1464,8 +1465,11 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, parent->children++; }else {if (tca[TCA_RATE ]) { - err =gen_replace_estimator (&cl->bstats, &cl->rate_est, -qdisc_root_sleeping_lock (sch), +spinlock_t *lock =qdisc_root_sleeping_lock (sch); + + err =gen_replace_estimator (&cl->bstats,NULL , + &cl->rate_est, + lock, tca[TCA_RATE ]);if (err)return err;
net/sched/sch_mq.c +1 -1
@@ -201,7 +201,7 @@ static int mq_dump_class_stats(struct Qdisc *sch, unsigned long cl, sch = dev_queue->qdisc_sleeping; sch->qstats.qlen = sch->q.qlen; -if (gnet_stats_copy_basic (d, &sch->bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &sch->bstats) <0 ||gnet_stats_copy_queue (d, &sch->qstats) <0 )return -1 ;return 0 ;
net/sched/sch_mqprio.c +2 -2
@@ -355,7 +355,7 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, }/* Reclaim root sleeping lock before completing stats */ spin_lock_bh (d->lock); -if (gnet_stats_copy_basic (d, &bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &bstats) <0 ||gnet_stats_copy_queue (d, &qstats) <0 )return -1 ; }else {
@@ -363,7 +363,7 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, sch = dev_queue->qdisc_sleeping; sch->qstats.qlen = sch->q.qlen; -if (gnet_stats_copy_basic (d, &sch->bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &sch->bstats) <0 ||gnet_stats_copy_queue (d, &sch->qstats) <0 )return -1 ; }
net/sched/sch_multiq.c +1 -1
@@ -361,7 +361,7 @@ static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl, cl_q = q->queues[cl -1 ]; cl_q->qstats.qlen = cl_q->q.qlen; -if (gnet_stats_copy_basic (d, &cl_q->bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &cl_q->bstats) <0 ||gnet_stats_copy_queue (d, &cl_q->qstats) <0 )return -1 ;
net/sched/sch_prio.c +1 -1
@@ -325,7 +325,7 @@ static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl, cl_q = q->queues[cl -1 ]; cl_q->qstats.qlen = cl_q->q.qlen; -if (gnet_stats_copy_basic (d, &cl_q->bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &cl_q->bstats) <0 ||gnet_stats_copy_queue (d, &cl_q->qstats) <0 )return -1 ;
net/sched/sch_qfq.c +5 -3
@@ -459,7 +459,8 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid,if (cl !=NULL ) {/* modify existing class */ if (tca[TCA_RATE ]) { - err =gen_replace_estimator (&cl->bstats, &cl->rate_est, + err =gen_replace_estimator (&cl->bstats,NULL , + &cl->rate_est,qdisc_root_sleeping_lock (sch), tca[TCA_RATE ]);if (err)
@@ -484,7 +485,8 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, cl->qdisc = &noop_qdisc;if (tca[TCA_RATE ]) { - err =gen_new_estimator (&cl->bstats, &cl->rate_est, + err =gen_new_estimator (&cl->bstats,NULL , + &cl->rate_est,qdisc_root_sleeping_lock (sch), tca[TCA_RATE ]);if (err)
@@ -667,7 +669,7 @@ static int qfq_dump_class_stats(struct Qdisc *sch, unsigned long arg, xstats.weight = cl->agg->class_weight; xstats.lmax = cl->agg->lmax; -if (gnet_stats_copy_basic (d, &cl->bstats) <0 || +if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 ||gnet_stats_copy_queue (d, &cl->qdisc->qstats) <0 )return -1 ;
PATCH v2 2/4 net: sched: implement qstat helper routines
This adds helpers to manipulate qstats logic and replaces locations that touch the counters directly. This simplifies future patches to push qstats onto per cpu counters.
--- include/net/sch_generic.h | 43 +++++++++++++++++++++++++++++++++++-------- net/sched/sch_api.c | 2 +- net/sched/sch_atm.c | 2 +- net/sched/sch_cbq.c | 10 +++++----- net/sched/sch_choke.c | 14 +++++++------- net/sched/sch_codel.c | 2 +- net/sched/sch_drr.c | 4 ++-- net/sched/sch_dsmark.c | 2 +- net/sched/sch_fifo.c | 2 +- net/sched/sch_fq.c | 4 ++-- net/sched/sch_fq_codel.c | 8 ++++---- net/sched/sch_gred.c | 4 ++-- net/sched/sch_hfsc.c | 8 ++++---- net/sched/sch_hhf.c | 8 ++++---- net/sched/sch_htb.c | 6 +++--- net/sched/sch_ingress.c | 2 +- net/sched/sch_multiq.c | 4 ++-- net/sched/sch_netem.c | 15 +++++++-------- net/sched/sch_pie.c | 2 +- net/sched/sch_prio.c | 4 ++-- net/sched/sch_qfq.c | 4 ++-- net/sched/sch_red.c | 8 ++++---- net/sched/sch_sfb.c | 10 +++++----- net/sched/sch_sfq.c | 17 +++++++++-------- net/sched/sch_tbf.c | 8 ++++---- 25 files changed, 110 insertions(+), 83 deletions(-)
include/net/sch_generic.h +35 -8
@@ -521,11 +521,39 @@ static inline void qdisc_bstats_update(struct Qdisc *sch,bstats_update (&sch->bstats, skb); } +static inline void qdisc_qstats_backlog (struct Qdisc *sch, +const struct sk_buff *skb) +{ + sch->qstats.backlog -=qdisc_pkt_len (skb); +} + +static inline void qdisc_qstats_backlog_inc (struct Qdisc *sch, +const struct sk_buff *skb) +{ + + sch->qstats.backlog +=qdisc_pkt_len (skb); +} + +static inline void __qdisc_qstats_drop (struct Qdisc *sch,int count) +{ + sch->qstats.drops += count; +} + +static inline void qdisc_qstats_drop (struct Qdisc *sch) +{ + sch->qstats.drops++; +} + +static inline void qdisc_qstats_overlimit (struct Qdisc *sch) +{ + sch->qstats.overlimits++; +} +static inline int __qdisc_enqueue_tail (struct sk_buff *skb,struct Qdisc *sch,struct sk_buff_head *list) {__skb_queue_tail (list, skb); - sch->qstats.backlog +=qdisc_pkt_len (skb); +qdisc_qstats_backlog_inc (sch, skb);return NET_XMIT_SUCCESS ; }
@@ -541,7 +569,7 @@ static inline struct sk_buff *__qdisc_dequeue_head(struct Qdisc *sch,struct sk_buff *skb =__skb_dequeue (list);if (likely (skb !=NULL )) { - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb);qdisc_bstats_update (sch, skb); }
@@ -559,10 +587,9 @@ static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch,struct sk_buff *skb =__skb_dequeue (list);if (likely (skb !=NULL )) { -unsigned int len =qdisc_pkt_len (skb); - sch->qstats.backlog -= len; +qdisc_qstats_backlog (sch, skb);kfree_skb (skb); -return len; +return qdisc_pkt_len (skb); }return 0 ;
@@ -579,7 +606,7 @@ static inline struct sk_buff *__qdisc_dequeue_tail(struct Qdisc *sch,struct sk_buff *skb =__skb_dequeue_tail (list);if (likely (skb !=NULL )) - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb);return skb; }
@@ -661,14 +688,14 @@ static inline unsigned int qdisc_queue_drop(struct Qdisc *sch)static inline int qdisc_drop (struct sk_buff *skb,struct Qdisc *sch) {kfree_skb (skb); - sch->qstats.drops++; +qdisc_qstats_drop (sch);return NET_XMIT_DROP ; }static inline int qdisc_reshape_fail (struct sk_buff *skb,struct Qdisc *sch) { - sch->qstats.drops++; +qdisc_qstats_drop (sch);#ifdef CONFIG_NET_CLS_ACT if (sch->reshape_fail ==NULL || sch->reshape_fail (skb, sch))
net/sched/sch_api.c +1 -1
@@ -763,7 +763,7 @@ void qdisc_tree_decrease_qlen(struct Qdisc *sch, unsigned int n) cops->put (sch, cl); } sch->q.qlen -= n; - sch->qstats.drops += drops; +__qdisc_qstats_drop (sch, drops); } }EXPORT_SYMBOL (qdisc_tree_decrease_qlen);
net/sched/sch_atm.c +1 -1
@@ -417,7 +417,7 @@ done:if (ret !=NET_XMIT_SUCCESS ) { drop:__maybe_unused if (net_xmit_drop_count (ret)) { - sch->qstats.drops++; +qdisc_qstats_drop (sch);if (flow) flow->qstats.drops++; }
net/sched/sch_cbq.c +5 -5
@@ -377,7 +377,7 @@ cbq_enqueue(struct sk_buff *skb, struct Qdisc *sch)#endif if (cl ==NULL ) {if (ret & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return ret; }
@@ -395,7 +395,7 @@ cbq_enqueue(struct sk_buff *skb, struct Qdisc *sch) }if (net_xmit_drop_count (ret)) { - sch->qstats.drops++; +qdisc_qstats_drop (sch);cbq_mark_toplevel (q, cl); cl->qstats.drops++; }
@@ -650,11 +650,11 @@ static int cbq_reshape_fail(struct sk_buff *skb, struct Qdisc *child) return 0; }if (net_xmit_drop_count (ret)) - sch->qstats.drops++; +qdisc_qstats_drop (sch);return 0 ; } - sch->qstats.drops++; +qdisc_qstats_drop (sch);return -1 ; }#endif
@@ -995,7 +995,7 @@ cbq_dequeue(struct Qdisc *sch)*/ if (sch->q.qlen) { - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);if (q->wd_expires)qdisc_watchdog_schedule (&q->watchdog, now + q->wd_expires);
net/sched/sch_choke.c +7 -7
@@ -127,7 +127,7 @@ static void choke_drop_by_idx(struct Qdisc *sch, unsigned int idx)if (idx == q->tail)choke_zap_tail_holes (q); - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb);qdisc_drop (skb, sch);qdisc_tree_decrease_qlen (sch,1 ); --sch->q.qlen;
@@ -302,7 +302,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch)if (q->vars.qavg > p->qth_max) { q->vars.qcount =-1 ; - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);if (use_harddrop (q) || !use_ecn (q) || !INET_ECN_set_ce (skb)) { q->stats.forced_drop++;
@@ -315,7 +315,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch) q->vars.qcount =0 ; q->vars.qR =red_random (p); - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);if (!use_ecn (q) || !INET_ECN_set_ce (skb)) { q->stats.prob_drop++; goto congestion_drop;
@@ -332,7 +332,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch) q->tab[q->tail] = skb; q->tail = (q->tail +1 ) & q->tab_mask; ++sch->q.qlen; - sch->qstats.backlog +=qdisc_pkt_len (skb); +qdisc_qstats_backlog_inc (sch, skb);return NET_XMIT_SUCCESS ; }
@@ -345,7 +345,7 @@ congestion_drop: other_drop:if (ret & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return ret; }
@@ -365,7 +365,7 @@ static struct sk_buff *choke_dequeue(struct Qdisc *sch) q->tab[q->head] =NULL ;choke_zap_head_holes (q); --sch->q.qlen; - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb);qdisc_bstats_update (sch, skb);return skb;
@@ -460,7 +460,7 @@ static int choke_change(struct Qdisc *sch, struct nlattr *opt) ntab[tail++] = skb;continue ; } - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb); --sch->q.qlen;qdisc_drop (skb, sch); }
net/sched/sch_codel.c +1 -1
@@ -149,7 +149,7 @@ static int codel_change(struct Qdisc *sch, struct nlattr *opt)while (sch->q.qlen > sch->limit) {struct sk_buff *skb =__skb_dequeue (&sch->q); - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb);qdisc_drop (skb, sch); }qdisc_tree_decrease_qlen (sch, qlen - sch->q.qlen);
net/sched/sch_drr.c +2 -2
@@ -360,7 +360,7 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch) cl =drr_classify (skb, sch, &err);if (cl ==NULL ) {if (err & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return err; }
@@ -369,7 +369,7 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch)if (unlikely (err !=NET_XMIT_SUCCESS )) {if (net_xmit_drop_count (err)) { cl->qstats.drops++; - sch->qstats.drops++; +qdisc_qstats_drop (sch); }return err; }
net/sched/sch_dsmark.c +1 -1
@@ -258,7 +258,7 @@ static int dsmark_enqueue(struct sk_buff *skb, struct Qdisc *sch) err =qdisc_enqueue (skb, p->q);if (err !=NET_XMIT_SUCCESS ) {if (net_xmit_drop_count (err)) - sch->qstats.drops++; +qdisc_qstats_drop (sch);return err; }
net/sched/sch_fifo.c +1 -1
@@ -42,7 +42,7 @@ static int pfifo_tail_enqueue(struct sk_buff *skb, struct Qdisc *sch)/* queue full, remove one skb to fulfill the limit */ __qdisc_queue_drop_head (sch, &sch->q); - sch->qstats.drops++; +qdisc_qstats_drop (sch);qdisc_enqueue_tail (skb, sch);return NET_XMIT_CN ;
net/sched/sch_fq.c +2 -2
@@ -290,7 +290,7 @@ static struct sk_buff *fq_dequeue_head(struct Qdisc *sch, struct fq_flow *flow) flow->head = skb->next; skb->next =NULL ; flow->qlen--; - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb); sch->q.qlen--; }return skb;
@@ -371,7 +371,7 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch) f->qlen++;if (skb_is_retransmit (skb)) q->stat_tcp_retrans++; - sch->qstats.backlog +=qdisc_pkt_len (skb); +qdisc_qstats_backlog_inc (sch, skb);if (fq_flow_is_detached (f)) {fq_flow_add_tail (&q->new_flows, f);if (time_after (jiffies, f->age + q->flow_refill_delay))
net/sched/sch_fq_codel.c +4 -4
@@ -164,8 +164,8 @@ static unsigned int fq_codel_drop(struct Qdisc *sch) q->backlogs[idx] -= len;kfree_skb (skb); sch->q.qlen--; - sch->qstats.drops++; - sch->qstats.backlog -= len; +qdisc_qstats_drop (sch); +qdisc_qstats_backlog (sch, skb); flow->dropped++;return idx; }
@@ -180,7 +180,7 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch) idx =fq_codel_classify (skb, sch, &ret);if (idx ==0 ) {if (ret & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return ret; }
@@ -190,7 +190,7 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch) flow = &q->flows[idx];flow_queue_add (flow, skb); q->backlogs[idx] +=qdisc_pkt_len (skb); - sch->qstats.backlog +=qdisc_pkt_len (skb); +qdisc_qstats_backlog_inc (sch, skb);if (list_empty (&flow->flowchain)) {list_add_tail (&flow->flowchain, &q->new_flows);
net/sched/sch_gred.c +2 -2
@@ -209,7 +209,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch)break ;case RED_PROB_MARK : - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);if (!gred_use_ecn (t) || !INET_ECN_set_ce (skb)) { q->stats.prob_drop++; goto congestion_drop;
@@ -219,7 +219,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch)break ;case RED_HARD_MARK : - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);if (gred_use_harddrop (t) || !gred_use_ecn (t) || !INET_ECN_set_ce (skb)) { q->stats.forced_drop++;
net/sched/sch_hfsc.c +4 -4
@@ -1591,7 +1591,7 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch) cl =hfsc_classify (skb, sch, &err);if (cl ==NULL ) {if (err & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return err; }
@@ -1600,7 +1600,7 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch)if (unlikely (err !=NET_XMIT_SUCCESS )) {if (net_xmit_drop_count (err)) { cl->qstats.drops++; - sch->qstats.drops++; +qdisc_qstats_drop (sch); }return err; }
@@ -1643,7 +1643,7 @@ hfsc_dequeue(struct Qdisc *sch)*/ cl =vttree_get_minvt (&q->root, cur_time);if (cl ==NULL ) { - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);hfsc_schedule_watchdog (sch);return NULL ; }
@@ -1698,7 +1698,7 @@ hfsc_drop(struct Qdisc *sch)list_move_tail (&cl->dlist, &q->droplist); } cl->qstats.drops++; - sch->qstats.drops++; +qdisc_qstats_drop (sch); sch->q.qlen--;return len; }
net/sched/sch_hhf.c +4 -4
@@ -376,8 +376,8 @@ static unsigned int hhf_drop(struct Qdisc *sch)struct sk_buff *skb =dequeue_head (bucket); sch->q.qlen--; - sch->qstats.drops++; - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_drop (sch); +qdisc_qstats_backlog (sch, skb);kfree_skb (skb); }
@@ -395,7 +395,7 @@ static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch) bucket = &q->buckets[idx];bucket_add (bucket, skb); - sch->qstats.backlog +=qdisc_pkt_len (skb); +qdisc_qstats_backlog_inc (sch, skb);if (list_empty (&bucket->bucketchain)) {unsigned int weight;
@@ -457,7 +457,7 @@ begin:if (bucket->head) { skb =dequeue_head (bucket); sch->q.qlen--; - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb); }if (!skb) {
net/sched/sch_htb.c +3 -3
@@ -586,13 +586,13 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch) #ifdef CONFIG_NET_CLS_ACT }else if (!cl) {if (ret & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return ret;#endif }else if ((ret =qdisc_enqueue (skb, cl->un.leaf.q)) !=NET_XMIT_SUCCESS ) {if (net_xmit_drop_count (ret)) { - sch->qstats.drops++; +qdisc_qstats_drop (sch); cl->qstats.drops++; }return ret;
@@ -925,7 +925,7 @@ ok: goto ok; } } - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);if (likely (next_event > q->now)) {if (!test_bit (__QDISC_STATE_DEACTIVATED, &qdisc_root_sleeping (q->watchdog.qdisc)->state)) {
net/sched/sch_ingress.c +1 -1
@@ -69,7 +69,7 @@ static int ingress_enqueue(struct sk_buff *skb, struct Qdisc *sch)switch (result) {case TC_ACT_SHOT : result =TC_ACT_SHOT ; - sch->qstats.drops++; +qdisc_qstats_drop (sch);break ;case TC_ACT_STOLEN :case TC_ACT_QUEUED :
net/sched/sch_multiq.c +2 -2
@@ -75,7 +75,7 @@ multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch)if (qdisc ==NULL ) {if (ret & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return ret; }
@@ -87,7 +87,7 @@ multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch)return NET_XMIT_SUCCESS ; }if (net_xmit_drop_count (ret)) - sch->qstats.drops++; +qdisc_qstats_drop (sch);return ret; }
net/sched/sch_netem.c +7 -8
@@ -429,12 +429,12 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)/* Drop packet? */ if (loss_event (q)) {if (q->ecn &&INET_ECN_set_ce (skb)) - sch->qstats.drops++;/* mark packet */ +qdisc_qstats_drop (sch);/* mark packet */ else --count; }if (count ==0 ) { - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; }
@@ -478,7 +478,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)if (unlikely (skb_queue_len (&sch->q) >= sch->limit))return qdisc_reshape_fail (skb, sch); - sch->qstats.backlog +=qdisc_pkt_len (skb); +qdisc_qstats_backlog_inc (sch, skb); cb =netem_skb_cb (skb); if (q->gap == 0 || /* not doing reordering */
@@ -549,15 +549,14 @@ static unsigned int netem_drop(struct Qdisc *sch) sch->q.qlen--; skb->next =NULL ; skb->prev =NULL ; - len =qdisc_pkt_len (skb); - sch->qstats.backlog -= len; +qdisc_qstats_backlog (sch, skb);kfree_skb (skb); } }if (!len && q->qdisc && q->qdisc->ops->drop) len = q->qdisc->ops->drop (q->qdisc);if (len) - sch->qstats.drops++; +qdisc_qstats_drop (sch);return len; }
@@ -575,7 +574,7 @@ tfifo_dequeue: skb =__skb_dequeue (&sch->q);if (skb) { deliver: - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb);qdisc_unthrottled (sch);qdisc_bstats_update (sch, skb);return skb;
@@ -610,7 +609,7 @@ deliver:if (unlikely (err !=NET_XMIT_SUCCESS )) {if (net_xmit_drop_count (err)) { - sch->qstats.drops++; +qdisc_qstats_drop (sch);qdisc_tree_decrease_qlen (sch,1 ); } }
net/sched/sch_pie.c +1 -1
@@ -232,7 +232,7 @@ static int pie_change(struct Qdisc *sch, struct nlattr *opt)while (sch->q.qlen > sch->limit) {struct sk_buff *skb =__skb_dequeue (&sch->q); - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb);qdisc_drop (skb, sch); }qdisc_tree_decrease_qlen (sch, qlen - sch->q.qlen);
net/sched/sch_prio.c +2 -2
@@ -77,7 +77,7 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch)if (qdisc ==NULL ) {if (ret & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return ret; }
@@ -89,7 +89,7 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch)return NET_XMIT_SUCCESS ; }if (net_xmit_drop_count (ret)) - sch->qstats.drops++; +qdisc_qstats_drop (sch);return ret; }
net/sched/sch_qfq.c +2 -2
@@ -1229,7 +1229,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch) cl =qfq_classify (skb, sch, &err);if (cl ==NULL ) {if (err & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return err; }
@@ -1249,7 +1249,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)pr_debug ("qfq_enqueue: enqueue failed %d\n" , err);if (net_xmit_drop_count (err)) { cl->qstats.drops++; - sch->qstats.drops++; +qdisc_qstats_drop (sch); }return err; }
net/sched/sch_red.c +4 -4
@@ -74,7 +74,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch)break ;case RED_PROB_MARK : - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);if (!red_use_ecn (q) || !INET_ECN_set_ce (skb)) { q->stats.prob_drop++; goto congestion_drop;
@@ -84,7 +84,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch)break ;case RED_HARD_MARK : - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);if (red_use_harddrop (q) || !red_use_ecn (q) || !INET_ECN_set_ce (skb)) { q->stats.forced_drop++;
@@ -100,7 +100,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch) sch->q.qlen++; }else if (net_xmit_drop_count (ret)) { q->stats.pdrop++; - sch->qstats.drops++; +qdisc_qstats_drop (sch); }return ret;
@@ -142,7 +142,7 @@ static unsigned int red_drop(struct Qdisc *sch)if (child->ops->drop && (len = child->ops->drop (child)) >0 ) { q->stats.other++; - sch->qstats.drops++; +qdisc_qstats_drop (sch); sch->q.qlen--;return len; }
net/sched/sch_sfb.c +5 -5
@@ -290,7 +290,7 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch)struct flow_keys keys;if (unlikely (sch->q.qlen >= q->limit)) { - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch); q->stats.queuedrop++; goto drop; }
@@ -348,7 +348,7 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch)sfb_skb_cb (skb)->hashes[slot] =0 ;if (unlikely (minqlen >= q->max)) { - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch); q->stats.bucketdrop++; goto drop; }
@@ -376,7 +376,7 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch) } }if (sfb_rate_limit (skb, q)) { - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch); q->stats.penaltydrop++; goto drop; }
@@ -411,7 +411,7 @@ enqueue:increment_qlen (skb, q); }else if (net_xmit_drop_count (ret)) { q->stats.childdrop++; - sch->qstats.drops++; +qdisc_qstats_drop (sch); }return ret;
@@ -420,7 +420,7 @@ drop:return NET_XMIT_CN ; other_drop:if (ret & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return ret; }
net/sched/sch_sfq.c +9 -8
@@ -331,8 +331,8 @@ drop:sfq_dec (q, x);kfree_skb (skb); sch->q.qlen--; - sch->qstats.drops++; - sch->qstats.backlog -= len; +qdisc_qstats_drop (sch); +qdisc_qstats_backlog (sch, skb);return len; }
@@ -379,7 +379,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch) hash =sfq_classify (skb, sch, &ret);if (hash ==0 ) {if (ret & __NET_XMIT_BYPASS) - sch->qstats.drops++; +qdisc_qstats_drop (sch);kfree_skb (skb);return ret; }
@@ -409,7 +409,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)break ;case RED_PROB_MARK : - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);if (sfq_prob_mark (q)) {/* We know we have at least one packet in queue */ if (sfq_headdrop(q) &&
@@ -426,7 +426,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch) goto congestion_drop;case RED_HARD_MARK : - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch);if (sfq_hard_mark (q)) {/* We know we have at least one packet in queue */ if (sfq_headdrop(q) &&
@@ -461,7 +461,7 @@ congestion_drop: } enqueue: - sch->qstats.backlog +=qdisc_pkt_len (skb); +qdisc_qstats_backlog_inc (sch, skb); slot->backlog +=qdisc_pkt_len (skb);slot_queue_add (slot, skb);sfq_inc (q, x);
@@ -520,7 +520,7 @@ next_slot:sfq_dec (q, a);qdisc_bstats_update (sch, skb); sch->q.qlen--; - sch->qstats.backlog -=qdisc_pkt_len (skb); +qdisc_qstats_backlog (sch, skb); slot->backlog -=qdisc_pkt_len (skb);/* Is the slot empty? */ if (slot->qlen ==0 ) {
@@ -586,7 +586,8 @@ static void sfq_rehash(struct Qdisc *sch)if (x ==SFQ_EMPTY_SLOT ) { x = q->dep[0 ].next;/* get a free slot */ if (x >=SFQ_MAX_FLOWS ) { -drop: sch->qstats.backlog -=qdisc_pkt_len (skb); +drop: +qdisc_qstats_backlog (sch, skb);kfree_skb (skb); dropped++;continue ;
net/sched/sch_tbf.c +4 -4
@@ -175,7 +175,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch) ret =qdisc_enqueue (segs, q->qdisc);if (ret !=NET_XMIT_SUCCESS ) {if (net_xmit_drop_count (ret)) - sch->qstats.drops++; +qdisc_qstats_drop (sch); }else { nb++; }
@@ -201,7 +201,7 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch) ret =qdisc_enqueue (skb, q->qdisc);if (ret !=NET_XMIT_SUCCESS ) {if (net_xmit_drop_count (ret)) - sch->qstats.drops++; +qdisc_qstats_drop (sch);return ret; }
@@ -216,7 +216,7 @@ static unsigned int tbf_drop(struct Qdisc *sch)if (q->qdisc->ops->drop && (len = q->qdisc->ops->drop (q->qdisc)) !=0 ) { sch->q.qlen--; - sch->qstats.drops++; +qdisc_qstats_drop (sch); }return len; }
@@ -281,7 +281,7 @@ static struct sk_buff *tbf_dequeue(struct Qdisc *sch)(cf. CSZ, HPFQ, HFSC) */ - sch->qstats.overlimits++; +qdisc_qstats_overlimit (sch); }return NULL ; }
PATCH v2 3/4 net: sched: restrict use of qstats qlen
In reply to John FastabendThis removes the use of qstats->qlen variable from the classifiers and makes it an explicit argument to gnet_stats_copy_queue().
The qlen represents the qdisc queue length and is packed into the qstats at the last moment before passnig to user space. By handling it explicitely we avoid, in the percpu stats case, having to figure out which per_cpu variable to put it in.
It would probably be best to remove it from qstats completely but qstats is a user space ABI and can't be broken. A future patch could make an internal only qstats structure that would avoid having to allocate an additional u32 variable on the Qdisc struct. This would make the qstats struct 128bits instead of 128+32.
--- include/net/gen_stats.h | 3 ++- net/core/gen_stats.c | 6 +++++- net/sched/act_api.c | 4 +++- net/sched/sch_api.c | 5 +++-- net/sched/sch_atm.c | 4 +--- net/sched/sch_cbq.c | 3 +-- net/sched/sch_drr.c | 7 +++---- net/sched/sch_fq_codel.c | 2 +- net/sched/sch_hfsc.c | 3 +-- net/sched/sch_htb.c | 5 +++-- net/sched/sch_mq.c | 4 +--- net/sched/sch_mqprio.c | 9 ++++----- net/sched/sch_multiq.c | 3 +-- net/sched/sch_prio.c | 3 +-- net/sched/sch_qfq.c | 3 +-- net/sched/sch_sfq.c | 2 +- 16 files changed, 32 insertions(+), 34 deletions(-)
include/net/gen_stats.h +2 -1
@@ -40,7 +40,8 @@ void __gnet_stats_copy_basic(struct gnet_stats_basic_packed *bstats,int gnet_stats_copy_rate_est (struct gnet_dump *d,const struct gnet_stats_basic_packed *b,struct gnet_stats_rate_est64 *r); -int gnet_stats_copy_queue (struct gnet_dump *d,struct gnet_stats_queue *q); +int gnet_stats_copy_queue (struct gnet_dump *d, +struct gnet_stats_queue *q,__u32 len);int gnet_stats_copy_app (struct gnet_dump *d,void *st,int len);int gnet_stats_finish_copy (struct gnet_dump *d);
net/core/gen_stats.c +5 -1
@@ -219,6 +219,7 @@ EXPORT_SYMBOL(gnet_stats_copy_rate_est);* gnet_stats_copy_queue - copy queue statistics into statistics TLV * @d: dumping handle * @q: queue statistics +* @qlen: queue length statistics * * Appends the queue statistics to the top level TLV created by * gnet_stats_start_copy().
@@ -227,8 +228,11 @@ EXPORT_SYMBOL(gnet_stats_copy_rate_est);* if the room in the socket buffer was not sufficient. */ int -gnet_stats_copy_queue (struct gnet_dump *d,struct gnet_stats_queue *q) +gnet_stats_copy_queue (struct gnet_dump *d, +struct gnet_stats_queue *q,__u32 qlen) { + q->qlen = qlen; +if (d->compat_tc_stats) { d->tc_stats.drops = q->drops; d->tc_stats.qlen = q->qlen;
net/sched/act_api.c +3 -1
@@ -623,7 +623,9 @@ int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *a,if (gnet_stats_copy_basic (&d,NULL , &p->tcfc_bstats) <0 ||gnet_stats_copy_rate_est (&d, &p->tcfc_bstats, &p->tcfc_rate_est) <0 || -gnet_stats_copy_queue (&d, &p->tcfc_qstats) <0 ) +gnet_stats_copy_queue (&d, + &p->tcfc_qstats, + p->tcfc_qstats.qlen) <0 ) goto errout;if (gnet_stats_finish_copy (&d) <0 )
net/sched/sch_api.c +3 -2
@@ -1318,6 +1318,7 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,unsigned char *b =skb_tail_pointer (skb);struct gnet_dump d;struct qdisc_size_table *stab; +__u32 qlen;cond_resched (); nlh =nlmsg_put (skb, portid, seq, event,sizeof (*tcm), flags);
@@ -1335,7 +1336,7 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid, goto nla_put_failure;if (q->ops->dump && q->ops->dump (q, skb) <0 ) goto nla_put_failure; - q->qstats.qlen = q->q.qlen; + qlen = q->q.qlen; stab =rtnl_dereference (q->stab);if (stab &&qdisc_dump_stab (skb, stab) <0 )
@@ -1353,7 +1354,7 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,if (gnet_stats_copy_basic (&d, cpu_bstats, &q->bstats) <0 ||gnet_stats_copy_rate_est (&d, &q->bstats, &q->rate_est) <0 || -gnet_stats_copy_queue (&d, &q->qstats) <0 ) +gnet_stats_copy_queue (&d, &q->qstats, qlen) <0 ) goto nla_put_failure;if (gnet_stats_finish_copy (&d) <0 )
net/sched/sch_atm.c +1 -3
@@ -637,10 +637,8 @@ atm_tc_dump_class_stats(struct Qdisc *sch, unsigned long arg, {struct atm_flow_data *flow = (struct atm_flow_data *)arg; - flow->qstats.qlen = flow->q->q.qlen; -if (gnet_stats_copy_basic (d,NULL , &flow->bstats) <0 || -gnet_stats_copy_queue (d, &flow->qstats) <0 ) +gnet_stats_copy_queue (d, &flow->qstats, flow->q->q.qlen) <0 )return -1 ;return 0 ;
net/sched/sch_cbq.c +1 -2
@@ -1594,7 +1594,6 @@ cbq_dump_class_stats(struct Qdisc *sch, unsigned long arg,struct cbq_sched_data *q =qdisc_priv (sch);struct cbq_class *cl = (struct cbq_class *)arg; - cl->qstats.qlen = cl->q->q.qlen; cl->xstats.avgidle = cl->avgidle; cl->xstats.undertime =0 ;
@@ -1603,7 +1602,7 @@ cbq_dump_class_stats(struct Qdisc *sch, unsigned long arg,if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 || -gnet_stats_copy_queue (d, &cl->qstats) <0 ) +gnet_stats_copy_queue (d, &cl->qstats, cl->q->q.qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &cl->xstats,sizeof (cl->xstats));
net/sched/sch_drr.c +3 -4
@@ -275,17 +275,16 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d) {struct drr_class *cl = (struct drr_class *)arg; +__u32 qlen = cl->qdisc->q.qlen;struct tc_drr_stats xstats;memset (&xstats,0 ,sizeof (xstats)); -if (cl->qdisc->q.qlen) { +if (qlen) xstats.deficit = cl->deficit; - cl->qdisc->qstats.qlen = cl->qdisc->q.qlen; - }if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 || -gnet_stats_copy_queue (d, &cl->qdisc->qstats) <0 ) +gnet_stats_copy_queue (d, &cl->qdisc->qstats, qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &xstats,sizeof (xstats));
net/sched/sch_fq_codel.c +1 -1
@@ -550,7 +550,7 @@ static int fq_codel_dump_class_stats(struct Qdisc *sch, unsigned long cl, qs.backlog = q->backlogs[idx]; qs.drops = flow->dropped; } -if (gnet_stats_copy_queue (d, &qs) <0 ) +if (gnet_stats_copy_queue (d, &qs,0 ) <0 )return -1 ;if (idx < q->flows_cnt)return gnet_stats_copy_app (d, &xstats,sizeof (xstats));
net/sched/sch_hfsc.c +1 -2
@@ -1370,7 +1370,6 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg,struct hfsc_class *cl = (struct hfsc_class *)arg;struct tc_hfsc_stats xstats; - cl->qstats.qlen = cl->qdisc->q.qlen; cl->qstats.backlog = cl->qdisc->qstats.backlog; xstats.level = cl->level; xstats.period = cl->cl_vtperiod;
@@ -1379,7 +1378,7 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg,if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 || -gnet_stats_copy_queue (d, &cl->qstats) <0 ) +gnet_stats_copy_queue (d, &cl->qstats, cl->qdisc->q.qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &xstats,sizeof (xstats));
net/sched/sch_htb.c +3 -2
@@ -1138,15 +1138,16 @@ static int htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d) { struct htb_class *cl = (struct htb_class *)arg; +__u32 qlen =0 ;if (!cl->level && cl->un.leaf.q) - cl->qstats.qlen = cl->un.leaf.q->q.qlen; + qlen = cl->un.leaf.q->q.qlen; cl->xstats.tokens =PSCHED_NS2TICKS (cl->tokens); cl->xstats.ctokens =PSCHED_NS2TICKS (cl->ctokens);if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d,NULL , &cl->rate_est) <0 || -gnet_stats_copy_queue (d, &cl->qstats) <0 ) +gnet_stats_copy_queue (d, &cl->qstats, qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &cl->xstats,sizeof (cl->xstats));
net/sched/sch_mq.c +1 -3
@@ -112,7 +112,6 @@ static int mq_dump(struct Qdisc *sch, struct sk_buff *skb) sch->q.qlen += qdisc->q.qlen; sch->bstats.bytes += qdisc->bstats.bytes; sch->bstats.packets += qdisc->bstats.packets; - sch->qstats.qlen += qdisc->qstats.qlen; sch->qstats.backlog += qdisc->qstats.backlog; sch->qstats.drops += qdisc->qstats.drops; sch->qstats.requeues += qdisc->qstats.requeues;
@@ -200,9 +199,8 @@ static int mq_dump_class_stats(struct Qdisc *sch, unsigned long cl,struct netdev_queue *dev_queue =mq_queue_get (sch, cl); sch = dev_queue->qdisc_sleeping; - sch->qstats.qlen = sch->q.qlen;if (gnet_stats_copy_basic (d,NULL , &sch->bstats) <0 || -gnet_stats_copy_queue (d, &sch->qstats) <0 ) +gnet_stats_copy_queue (d, &sch->qstats, sch->q.qlen) <0 )return -1 ;return 0 ; }
net/sched/sch_mqprio.c +4 -5
@@ -236,7 +236,6 @@ static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb) sch->q.qlen += qdisc->q.qlen; sch->bstats.bytes += qdisc->bstats.bytes; sch->bstats.packets += qdisc->bstats.packets; - sch->qstats.qlen += qdisc->qstats.qlen; sch->qstats.backlog += qdisc->qstats.backlog; sch->qstats.drops += qdisc->qstats.drops; sch->qstats.requeues += qdisc->qstats.requeues;
@@ -327,6 +326,7 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,if (cl <=netdev_get_num_tc (dev)) {int i; +__u32 qlen =0 ;struct Qdisc *qdisc;struct gnet_stats_queue qstats = {0 };struct gnet_stats_basic_packed bstats = {0 };
@@ -344,9 +344,9 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, qdisc =rtnl_dereference (q->qdisc);spin_lock_bh (qdisc_lock (qdisc)); + qlen += qdisc->q.qlen; bstats.bytes += qdisc->bstats.bytes; bstats.packets += qdisc->bstats.packets; - qstats.qlen += qdisc->qstats.qlen; qstats.backlog += qdisc->qstats.backlog; qstats.drops += qdisc->qstats.drops; qstats.requeues += qdisc->qstats.requeues;
@@ -356,15 +356,14 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,/* Reclaim root sleeping lock before completing stats */ spin_lock_bh (d->lock);if (gnet_stats_copy_basic (d,NULL , &bstats) <0 || -gnet_stats_copy_queue (d, &qstats) <0 ) +gnet_stats_copy_queue (d, &qstats, qlen) <0 )return -1 ; }else {struct netdev_queue *dev_queue =mqprio_queue_get (sch, cl); sch = dev_queue->qdisc_sleeping; - sch->qstats.qlen = sch->q.qlen;if (gnet_stats_copy_basic (d,NULL , &sch->bstats) <0 || -gnet_stats_copy_queue (d, &sch->qstats) <0 ) +gnet_stats_copy_queue (d, &sch->qstats, sch->q.qlen) <0 )return -1 ; }return 0 ;
net/sched/sch_multiq.c +1 -2
@@ -360,9 +360,8 @@ static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl,struct Qdisc *cl_q; cl_q = q->queues[cl -1 ]; - cl_q->qstats.qlen = cl_q->q.qlen;if (gnet_stats_copy_basic (d,NULL , &cl_q->bstats) <0 || -gnet_stats_copy_queue (d, &cl_q->qstats) <0 ) +gnet_stats_copy_queue (d, &cl_q->qstats, cl_q->q.qlen) <0 )return -1 ;return 0 ;
net/sched/sch_prio.c +1 -2
@@ -324,9 +324,8 @@ static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl,struct Qdisc *cl_q; cl_q = q->queues[cl -1 ]; - cl_q->qstats.qlen = cl_q->q.qlen;if (gnet_stats_copy_basic (d,NULL , &cl_q->bstats) <0 || -gnet_stats_copy_queue (d, &cl_q->qstats) <0 ) +gnet_stats_copy_queue (d, &cl_q->qstats, cl_q->q.qlen) <0 )return -1 ;return 0 ;
net/sched/sch_qfq.c +1 -2
@@ -664,14 +664,13 @@ static int qfq_dump_class_stats(struct Qdisc *sch, unsigned long arg,struct tc_qfq_stats xstats;memset (&xstats,0 ,sizeof (xstats)); - cl->qdisc->qstats.qlen = cl->qdisc->q.qlen; xstats.weight = cl->agg->class_weight; xstats.lmax = cl->agg->lmax;if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 || -gnet_stats_copy_queue (d, &cl->qdisc->qstats) <0 ) +gnet_stats_copy_queue (d, &cl->qdisc->qstats, cl->qdisc->q.qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &xstats,sizeof (xstats));
net/sched/sch_sfq.c +1 -1
@@ -871,7 +871,7 @@ static int sfq_dump_class_stats(struct Qdisc *sch, unsigned long cl, qs.qlen = slot->qlen; qs.backlog = slot->backlog; } -if (gnet_stats_copy_queue (d, &qs) <0 ) +if (gnet_stats_copy_queue (d, &qs, qs.qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &xstats,sizeof (xstats)); }
PATCH v2 4/4 net: sched: enable per cpu qstats
In reply to John FastabendAfter previous patches to simplify qstats the qstats can be made per cpu with a packed union in Qdisc struct.
--- include/net/gen_stats.h | 3 ++ include/net/sch_generic.h | 12 +++++++++- net/core/gen_stats.c | 55 +++++++++++++++++++++++++++++++++++++++------ net/sched/act_api.c | 2 +- net/sched/sch_api.c | 12 ++++++++-- net/sched/sch_atm.c | 2 +- net/sched/sch_cbq.c | 2 +- net/sched/sch_drr.c | 2 +- net/sched/sch_fq_codel.c | 2 +- net/sched/sch_hfsc.c | 2 +- net/sched/sch_htb.c | 2 +- net/sched/sch_mq.c | 2 +- net/sched/sch_mqprio.c | 5 ++-- net/sched/sch_multiq.c | 2 +- net/sched/sch_prio.c | 2 +- net/sched/sch_qfq.c | 3 ++ net/sched/sch_sfq.c | 2 +- 17 files changed, 87 insertions(+), 25 deletions(-)
include/net/gen_stats.h +2 -1
@@ -41,7 +41,8 @@ int gnet_stats_copy_rate_est(struct gnet_dump *d,const struct gnet_stats_basic_packed *b, struct gnet_stats_rate_est64 *r);int gnet_stats_copy_queue (struct gnet_dump *d, -struct gnet_stats_queue *q,__u32 len); +struct gnet_stats_queue __percpu *cpu_q, +struct gnet_stats_queue *q,__u32 qlen);int gnet_stats_copy_app (struct gnet_dump *d,void *st,int len);int gnet_stats_finish_copy (struct gnet_dump *d);
include/net/sch_generic.h +11 -1
@@ -90,7 +90,10 @@ struct Qdisc {struct gnet_stats_basic_cpu __percpu *cpu_bstats; } __packed;unsigned int __state; -struct gnet_stats_queue qstats; +union { +struct gnet_stats_queue qstats; +struct gnet_stats_queue __percpu *cpu_qstats; + } __packed;struct rcu_head rcu_head;int padded;atomic_t refcnt;
@@ -544,6 +547,13 @@ static inline void qdisc_qstats_drop(struct Qdisc *sch) sch->qstats.drops++; } +static inline void qdisc_qstats_drop_cpu (struct Qdisc *sch) +{ +struct gnet_stats_queue *qstats =this_cpu_ptr (sch->cpu_qstats); + + qstats->drops++; +} +static inline void qdisc_qstats_overlimit (struct Qdisc *sch) { sch->qstats.overlimits++;
net/core/gen_stats.c +48 -7
81 lines not on this page · raw message
net/sched/act_api.c +1 -1
@@ -623,7 +623,7 @@ int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *a,if (gnet_stats_copy_basic (&d,NULL , &p->tcfc_bstats) <0 ||gnet_stats_copy_rate_est (&d, &p->tcfc_bstats, &p->tcfc_rate_est) <0 || -gnet_stats_copy_queue (&d, +gnet_stats_copy_queue (&d,NULL , &p->tcfc_qstats, p->tcfc_qstats.qlen) <0 ) goto errout;
net/sched/sch_api.c +10 -2
@@ -947,6 +947,10 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,alloc_percpu (struct gnet_stats_basic_cpu );if (!sch->cpu_bstats) goto err_out4; + + sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue); +if (!sch->cpu_qstats) + goto err_out4; }if (tca[TCA_STAB ]) {
@@ -995,6 +999,7 @@ err_out: err_out4:free_percpu (sch->cpu_bstats); +free_percpu (sch->cpu_qstats);/* * Any broken qdiscs that would require a ops->reset() here? * The qdisc was never in action so it shouldn't be necessary.
@@ -1313,6 +1318,7 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid, u32 portid, u32 seq, u16 flags, int event) { struct gnet_stats_basic_cpu __percpu *cpu_bstats = NULL; +struct gnet_stats_queue __percpu *cpu_qstats =NULL ;struct tcmsg *tcm;struct nlmsghdr *nlh;unsigned char *b =skb_tail_pointer (skb);
@@ -1349,12 +1355,14 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,if (q->ops->dump_stats && q->ops->dump_stats (q, &d) <0 ) goto nla_put_failure; -if (qdisc_is_percpu_stats (q)) +if (qdisc_is_percpu_stats (q)) { cpu_bstats = q->cpu_bstats; + cpu_qstats = q->cpu_qstats; + }if (gnet_stats_copy_basic (&d, cpu_bstats, &q->bstats) <0 ||gnet_stats_copy_rate_est (&d, &q->bstats, &q->rate_est) <0 || -gnet_stats_copy_queue (&d, &q->qstats, qlen) <0 ) +gnet_stats_copy_queue (&d, cpu_qstats, &q->qstats, qlen) <0 ) goto nla_put_failure;if (gnet_stats_finish_copy (&d) <0 )
net/sched/sch_atm.c +1 -1
@@ -638,7 +638,7 @@ atm_tc_dump_class_stats(struct Qdisc *sch, unsigned long arg,struct atm_flow_data *flow = (struct atm_flow_data *)arg;if (gnet_stats_copy_basic (d,NULL , &flow->bstats) <0 || -gnet_stats_copy_queue (d, &flow->qstats, flow->q->q.qlen) <0 ) +gnet_stats_copy_queue (d,NULL , &flow->qstats, flow->q->q.qlen) <0 )return -1 ;return 0 ;
net/sched/sch_cbq.c +1 -1
@@ -1602,7 +1602,7 @@ cbq_dump_class_stats(struct Qdisc *sch, unsigned long arg,if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 || -gnet_stats_copy_queue (d, &cl->qstats, cl->q->q.qlen) <0 ) +gnet_stats_copy_queue (d,NULL , &cl->qstats, cl->q->q.qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &cl->xstats,sizeof (cl->xstats));
net/sched/sch_drr.c +1 -1
@@ -284,7 +284,7 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg,if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 || -gnet_stats_copy_queue (d, &cl->qdisc->qstats, qlen) <0 ) +gnet_stats_copy_queue (d,NULL , &cl->qdisc->qstats, qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &xstats,sizeof (xstats));
net/sched/sch_fq_codel.c +1 -1
@@ -550,7 +550,7 @@ static int fq_codel_dump_class_stats(struct Qdisc *sch, unsigned long cl, qs.backlog = q->backlogs[idx]; qs.drops = flow->dropped; } -if (gnet_stats_copy_queue (d, &qs,0 ) <0 ) +if (gnet_stats_copy_queue (d,NULL , &qs,0 ) <0 )return -1 ;if (idx < q->flows_cnt)return gnet_stats_copy_app (d, &xstats,sizeof (xstats));
net/sched/sch_hfsc.c +1 -1
@@ -1378,7 +1378,7 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg,if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 || -gnet_stats_copy_queue (d, &cl->qstats, cl->qdisc->q.qlen) <0 ) +gnet_stats_copy_queue (d,NULL , &cl->qstats, cl->qdisc->q.qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &xstats,sizeof (xstats));
net/sched/sch_htb.c +1 -1
@@ -1147,7 +1147,7 @@ htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d)if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d,NULL , &cl->rate_est) <0 || -gnet_stats_copy_queue (d, &cl->qstats, qlen) <0 ) +gnet_stats_copy_queue (d,NULL , &cl->qstats, qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &cl->xstats,sizeof (cl->xstats));
net/sched/sch_mq.c +1 -1
@@ -200,7 +200,7 @@ static int mq_dump_class_stats(struct Qdisc *sch, unsigned long cl, sch = dev_queue->qdisc_sleeping;if (gnet_stats_copy_basic (d,NULL , &sch->bstats) <0 || -gnet_stats_copy_queue (d, &sch->qstats, sch->q.qlen) <0 ) +gnet_stats_copy_queue (d,NULL , &sch->qstats, sch->q.qlen) <0 )return -1 ;return 0 ; }
net/sched/sch_mqprio.c +3 -2
@@ -356,14 +356,15 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,/* Reclaim root sleeping lock before completing stats */ spin_lock_bh (d->lock);if (gnet_stats_copy_basic (d,NULL , &bstats) <0 || -gnet_stats_copy_queue (d, &qstats, qlen) <0 ) +gnet_stats_copy_queue (d,NULL , &qstats, qlen) <0 )return -1 ; }else {struct netdev_queue *dev_queue =mqprio_queue_get (sch, cl); sch = dev_queue->qdisc_sleeping;if (gnet_stats_copy_basic (d,NULL , &sch->bstats) <0 || -gnet_stats_copy_queue (d, &sch->qstats, sch->q.qlen) <0 ) +gnet_stats_copy_queue (d,NULL , + &sch->qstats, sch->q.qlen) <0 )return -1 ; }return 0 ;
net/sched/sch_multiq.c +1 -1
@@ -361,7 +361,7 @@ static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl, cl_q = q->queues[cl -1 ];if (gnet_stats_copy_basic (d,NULL , &cl_q->bstats) <0 || -gnet_stats_copy_queue (d, &cl_q->qstats, cl_q->q.qlen) <0 ) +gnet_stats_copy_queue (d,NULL , &cl_q->qstats, cl_q->q.qlen) <0 )return -1 ;return 0 ;
net/sched/sch_prio.c +1 -1
@@ -325,7 +325,7 @@ static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl, cl_q = q->queues[cl -1 ];if (gnet_stats_copy_basic (d,NULL , &cl_q->bstats) <0 || -gnet_stats_copy_queue (d, &cl_q->qstats, cl_q->q.qlen) <0 ) +gnet_stats_copy_queue (d,NULL , &cl_q->qstats, cl_q->q.qlen) <0 )return -1 ;return 0 ;
net/sched/sch_qfq.c +2 -1
@@ -670,7 +670,8 @@ static int qfq_dump_class_stats(struct Qdisc *sch, unsigned long arg,if (gnet_stats_copy_basic (d,NULL , &cl->bstats) <0 ||gnet_stats_copy_rate_est (d, &cl->bstats, &cl->rate_est) <0 || -gnet_stats_copy_queue (d, &cl->qdisc->qstats, cl->qdisc->q.qlen) <0 ) +gnet_stats_copy_queue (d,NULL , + &cl->qdisc->qstats, cl->qdisc->q.qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &xstats,sizeof (xstats));
net/sched/sch_sfq.c +1 -1
@@ -871,7 +871,7 @@ static int sfq_dump_class_stats(struct Qdisc *sch, unsigned long cl, qs.qlen = slot->qlen; qs.backlog = slot->backlog; } -if (gnet_stats_copy_queue (d, &qs, qs.qlen) <0 ) +if (gnet_stats_copy_queue (d,NULL , &qs, qs.qlen) <0 )return -1 ;return gnet_stats_copy_app (d, &xstats,sizeof (xstats)); }