pktgen patches
Subsystems (2)
RFC 3/4 pktgen: clock optimizations
In reply to Stephen HemmingerThis optimizes pktgen to avoid calling ktime_get_ts unless it is needed.
* if delay is 0, then no need to update next_tx
* if queue is stopped, then keep old value will already force tx on next cycle.
--- net/core/pktgen.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
net/core/pktgen.c +7 -7
@@ -3425,11 +3425,12 @@ static __inline__ void pktgen_xmit(struc txq =netdev_get_tx_queue (odev, queue_map);__netif_tx_lock_bh (txq); -if (!netif_tx_queue_stopped (txq) && - !netif_tx_queue_frozen (txq)) { +if (netif_tx_queue_stopped (txq) ||netif_tx_queue_frozen (txq)) + pkt_dev->last_ok =0 ;/* Retry it next time */ +else {atomic_inc (&(pkt_dev->skb->users)); - retry_now: +retry_now: ret = (*xmit)(pkt_dev->skb, odev);if (likely (ret ==NETDEV_TX_OK )) {txq_trans_update (txq);
@@ -3453,10 +3454,9 @@ static __inline__ void pktgen_xmit(struc pkt_dev->last_ok =0 ; } - pkt_dev->next_tx =ktime_add_ns (ktime_now (), pkt_dev->delay); - }else {/* Retry it next time */ - pkt_dev->last_ok =0 ; - pkt_dev->next_tx =ktime_now (); +if (pkt_dev->delay) + pkt_dev->next_tx =ktime_add_ns (ktime_now (), + pkt_dev->delay); }__netif_tx_unlock_bh (txq);
RFC 2/4 pktgen: spin using hrtimer
In reply to Stephen HemmingerThis changes how the pktgen thread spins/waits between packets if delay is configured. The new code is basically an revised version of what the nanosleep system call does.
It requires exporting hrtimer_init_sleeper from standard hrtimer code, since pktgen can be a module.
--- kernel/hrtimer.c | 1 + net/core/pktgen.c | 43 ++++++++++++++++++++++++++----------------- 2 files changed, 27 insertions(+), 17 deletions(-)
net/core/pktgen.c +26 -22
@@ -131,6 +131,7 @@#include <linux/ioport.h> #include <linux/interrupt.h> #include <linux/capability.h> +#include <linux/hrtimer.h> #include <linux/freezer.h> #include <linux/delay.h> #include <linux/timer.h>
@@ -2084,33 +2085,32 @@ static void pktgen_setup_inject(struct p pkt_dev->nflows =0 ; } -static inline s64 delta_ns (ktime_t a,ktime_t b) -{ -return ktime_to_ns (ktime_sub (a, b)); -} -static void spin (struct pktgen_dev *pkt_dev,ktime_t spin_until) { -ktime_t start, now; -s64 dt; +struct hrtimer_sleeper t; +enum hrtimer_mode mode =HRTIMER_MODE_REL ; +unsigned long slack =rt_task (current) ?0 : current->timer_slack_ns; + +hrtimer_init_on_stack (&t.timer,CLOCK_MONOTONIC , mode); +hrtimer_set_expires_range_ns (&t.timer, spin_until, slack); +hrtimer_init_sleeper (&t, current); - start = now =ktime_now (); +/* based on do_nanosleep() */ +do { +set_current_state (TASK_INTERRUPTIBLE ); +hrtimer_start_expires (&t.timer, mode); +if (!hrtimer_active (&t.timer)) + t.task =NULL ; -while ((dt =delta_ns (spin_until, now)) >0 ) { -/* TODO: optimize sleeping behavior */ -if (dt >TICK_NSEC ) -schedule_timeout_interruptible (1 ); -else if (dt >100 *NSEC_PER_USEC ) { -if (!pkt_dev->running) -return ; -if (need_resched ()) -schedule (); - } +if (likely (t.task)) +schedule (); - now =ktime_now (); - } +hrtimer_cancel (&t.timer); + mode =HRTIMER_MODE_ABS ; + }while (t.task && pkt_dev->running && !signal_pending (current)); - pkt_dev->idle_acc +=ktime_to_ns (ktime_sub (now, start)); +__set_current_state (TASK_RUNNING ); +destroy_hrtimer_on_stack (&t.timer); }static inline void set_pkt_overhead (struct pktgen_dev *pkt_dev)
@@ -3352,8 +3352,12 @@ static __inline__ void pktgen_xmit(strucint ret;if (pkt_dev->delay) { -if (ktime_lt (ktime_now (), pkt_dev->next_tx)) +ktime_t start =ktime_now (); +if (ktime_lt (start, pkt_dev->next_tx)) {spin (pkt_dev, pkt_dev->next_tx); + pkt_dev->idle_acc +=ktime_to_ns (ktime_sub (ktime_now (), + start)); + }/* This is max DELAY, this has special meaning of * "never transmit"
kernel/hrtimer.c +1 -0
@@ -1477,6 +1477,7 @@ void hrtimer_init_sleeper(struct hrtimer sl->timer.function = hrtimer_wakeup; sl->task = task; } +EXPORT_SYMBOL_GPL (hrtimer_init_sleeper); static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) {
RFC 1/4 pktgen: convert to use ktime_t
In reply to Stephen HemmingerThe kernel ktime_t is a nice generic infrastructure for mananging high resolution times, as is done in pktgen. It also switchs from using gettimeofday() to a monotonic clock which avoids any NTP disturbance. In the process base resolution is increased to nanoseconds.
net/core/pktgen.c +86 -106
@@ -246,16 +246,14 @@ struct pktgen_dev {int max_pkt_size;/* = ETH_ZLEN; */ int pkt_overhead;/* overhead for MPLS, VLANs, IPSEC etc */ int nfrags; -__u32 delay_us;/* Default delay */ -__u32 delay_ns; +u64 delay;/* nano-seconds */ +__u64 count;/* Default No packets to send */ __u64 sofar;/* How many pkts we've sent so far */ __u64 tx_bytes;/* How many bytes we've transmitted */ __u64 errors;/* Errors when trying to transmit, pkts will be re-sent */ /* runtime counters relating to clone_skb */ -__u64 next_tx_us;/* timestamp of when to tx next */ -__u32 next_tx_ns;__u64 allocated_skbs;__u32 clone_count;
@@ -263,9 +261,11 @@ struct pktgen_dev {* Or a failed transmit of some sort? This will keep * sequence numbers in order, for example. */ -__u64 started_at;/* micro-seconds */ -__u64 stopped_at;/* micro-seconds */ -__u64 idle_acc;/* micro-seconds */ +ktime_t next_tx; +ktime_t started_at; +ktime_t stopped_at; +u64 idle_acc;/* nano-seconds */ +__u32 seq_num;int clone_skb;/* Use multiple SKBs during packet gen. If this number
@@ -397,23 +397,20 @@ struct pktgen_thread {#define REMOVE 1#define FIND 0 -/** Convert to micro-seconds */ -static inline __u64 tv_to_us (const struct timeval *tv) +static inline ktime_t ktime_now (void ) { -__u64 us = tv->tv_usec; - us += (__u64 ) tv->tv_sec * (__u64 )1000000 ; -return us; +struct timespec ts; +ktime_get_ts (&ts); + +return timespec_to_ktime (ts); } -static __u64 getCurUs (void ) +/* This works even if 32 bit because of careful byte order choice */ +static inline int ktime_lt (const ktime_t cmp1,const ktime_t cmp2) { -struct timeval tv; -do_gettimeofday (&tv); -return tv_to_us (&tv); +return cmp1.tv64 < cmp2.tv64; } -/* old include end */ -static char version[]__initdata =VERSION ;static int pktgen_remove_device (struct pktgen_thread *t,struct pktgen_dev *i);
@@ -510,9 +507,8 @@ static const struct file_operations pktgstatic int pktgen_if_show (struct seq_file *seq,void *v) {struct pktgen_dev *pkt_dev = seq->private; -__u64 sa; -__u64 stopped; -__u64 now =getCurUs (); +ktime_t stopped; +struct timeval trun, idle; seq_printf(seq, "Params: count %llu min_pkt_size: %u max_pkt_size: %u\n",
@@ -520,9 +516,8 @@ static int pktgen_if_show(struct seq_fil pkt_dev->max_pkt_size);seq_printf (seq, -" frags: %d delay: %u clone_skb: %d ifname: %s\n" , - pkt_dev->nfrags, -1000 * pkt_dev->delay_us + pkt_dev->delay_ns, +" frags: %d delay: %llu clone_skb: %d ifname: %s\n" , + pkt_dev->nfrags, (unsigned long long ) pkt_dev->delay, pkt_dev->clone_skb, pkt_dev->odev->name); seq_printf(seq, " flows: %u flowlen: %u\n", pkt_dev->cflows,
@@ -654,17 +649,19 @@ static int pktgen_if_show(struct seq_filseq_puts (seq,"\n" ); - sa = pkt_dev->started_at; - stopped = pkt_dev->stopped_at; -if (pkt_dev->running) - stopped = now;/* not really stopped, more like last-running-at */ +/* not really stopped, more like last-running-at */ + stopped = pkt_dev->running ?ktime_now () : pkt_dev->stopped_at; + trun =ktime_to_timeval (ktime_sub (stopped, pkt_dev->started_at)); + idle =ns_to_timeval (pkt_dev->idle_acc);seq_printf (seq, -"Current:\n pkts-sofar: %llu errors: %llu\n started: %lluus stopped: %lluus idle: %lluus\n" , +"Current:\n pkts-sofar: %llu errors: %llu\n" , (unsigned long long )pkt_dev->sofar, - (unsigned long long )pkt_dev->errors, (unsigned long long )sa, - (unsigned long long )stopped, - (unsigned long long )pkt_dev->idle_acc); + (unsigned long long )pkt_dev->errors); + +seq_printf (seq," running: %d.%06ds idle: %d.%06ds\n" , + (int )trun.tv_sec, (int )trun.tv_usec, + (int )idle.tv_sec, (int )idle.tv_usec); seq_printf(seq, " seq_num: %d cur_dst_mac_offset: %d cur_src_mac_offset: %d\n",
@@ -950,15 +947,13 @@ static ssize_t pktgen_if_write(struct fireturn len; } i += len; -if (value ==0x7FFFFFFF ) { - pkt_dev->delay_us =0x7FFFFFFF ; - pkt_dev->delay_ns =0 ; - }else { - pkt_dev->delay_us = value /1000 ; - pkt_dev->delay_ns = value %1000 ; - } -sprintf (pg_result,"OK: delay=%u" , -1000 * pkt_dev->delay_us + pkt_dev->delay_ns); +if (value ==0x7FFFFFFF ) + pkt_dev->delay =ULLONG_MAX ; +else + pkt_dev->delay = (u64 )value *NSEC_PER_USEC ; + +sprintf (pg_result,"OK: delay=%llu" , + (unsigned long long ) pkt_dev->delay);return count; }if (!strcmp (name,"udp_src_min" )) {
@@ -2089,27 +2084,33 @@ static void pktgen_setup_inject(struct p pkt_dev->nflows =0 ; } -static void spin (struct pktgen_dev *pkt_dev,__u64 spin_until_us) +static inline s64 delta_ns (ktime_t a,ktime_t b) { -__u64 start; -__u64 now; +return ktime_to_ns (ktime_sub (a, b)); +} - start = now =getCurUs (); -while (now < spin_until_us) { +static void spin (struct pktgen_dev *pkt_dev,ktime_t spin_until) +{ +ktime_t start, now; +s64 dt; + + start = now =ktime_now (); + +while ((dt =delta_ns (spin_until, now)) >0 ) {/* TODO: optimize sleeping behavior */ -if (spin_until_us - now >jiffies_to_usecs (1 ) +1 ) +if (dt >TICK_NSEC )schedule_timeout_interruptible (1 ); -else if (spin_until_us - now >100 ) { +else if (dt >100 *NSEC_PER_USEC ) {if (!pkt_dev->running)return ;if (need_resched ())schedule (); } - now =getCurUs (); + now =ktime_now (); } - pkt_dev->idle_acc += now - start; + pkt_dev->idle_acc +=ktime_to_ns (ktime_sub (now, start)); }static inline void set_pkt_overhead (struct pktgen_dev *pkt_dev)
@@ -3072,9 +3073,9 @@ static void pktgen_run(struct pktgen_thrpktgen_clear_counters (pkt_dev); pkt_dev->running =1 ;/* Cranke yeself! */ pkt_dev->skb =NULL ; - pkt_dev->started_at =getCurUs (); - pkt_dev->next_tx_us =getCurUs ();/* Transmit immediately */ - pkt_dev->next_tx_ns =0 ; + pkt_dev->started_at = + pkt_dev->next_tx =ktime_now (); +set_pkt_overhead (pkt_dev);strcpy (pkt_dev->result,"Starting" );
@@ -3193,28 +3194,21 @@ static void pktgen_reset_all_threads(voistatic void show_results (struct pktgen_dev *pkt_dev,int nr_frags) { -__u64 total_us, bps, mbps, pps, idle; +__u64 bps, mbps, pps;char *p = pkt_dev->result; - - total_us = pkt_dev->stopped_at - pkt_dev->started_at; - - idle = pkt_dev->idle_acc; - - p +=sprintf (p,"OK: %llu(c%llu+d%llu) usec, %llu (%dbyte,%dfrags)\n" , - (unsigned long long )total_us, - (unsigned long long )(total_us - idle), - (unsigned long long )idle, +ktime_t elapsed =ktime_sub (pkt_dev->stopped_at, + pkt_dev->started_at); +ktime_t idle =ns_to_ktime (pkt_dev->idle_acc); + + p +=sprintf (p,"OK: %llu(c%llu+d%llu) nsec, %llu (%dbyte,%dfrags)\n" , + (unsigned long long )ktime_to_us (elapsed), + (unsigned long long )ktime_to_us (ktime_sub (elapsed, idle)), + (unsigned long long )ktime_to_us (idle), (unsigned long long )pkt_dev->sofar, pkt_dev->cur_pkt_size, nr_frags); - pps = pkt_dev->sofar *USEC_PER_SEC ; - -while ((total_us >>32 ) !=0 ) { - pps >>=1 ; - total_us >>=1 ; - } - -do_div (pps, total_us); + pps =div64_u64 (pkt_dev->sofar *NSEC_PER_SEC , +ktime_to_ns (elapsed)); bps = pps *8 * pkt_dev->cur_pkt_size;
@@ -3239,7 +3233,7 @@ static int pktgen_stop_device(struct pktreturn -EINVAL ; } - pkt_dev->stopped_at =getCurUs (); + pkt_dev->stopped_at =ktime_now (); pkt_dev->running =0 ;show_results (pkt_dev, nr_frags);
@@ -3258,7 +3252,7 @@ static struct pktgen_dev *next_to_run(stcontinue ;if (best ==NULL ) best = pkt_dev; -else if (pkt_dev->next_tx_us < best->next_tx_us) +else if (ktime_lt (pkt_dev->next_tx, best->next_tx)) best = pkt_dev; }if_unlock (t);
@@ -3354,23 +3348,19 @@ static __inline__ void pktgen_xmit(strucint (*xmit)(struct sk_buff *,struct net_device *) = odev->netdev_ops->ndo_start_xmit;struct netdev_queue *txq; -__u64 idle_start =0 ;u16 queue_map;int ret; -if (pkt_dev->delay_us || pkt_dev->delay_ns) { -u64 now; - - now =getCurUs (); -if (now < pkt_dev->next_tx_us) -spin (pkt_dev, pkt_dev->next_tx_us); +if (pkt_dev->delay) { +if (ktime_lt (ktime_now (), pkt_dev->next_tx)) +spin (pkt_dev, pkt_dev->next_tx);/* This is max DELAY, this has special meaning of * "never transmit" */ -if (pkt_dev->delay_us ==0x7FFFFFFF ) { - pkt_dev->next_tx_us =getCurUs () + pkt_dev->delay_us; - pkt_dev->next_tx_ns = pkt_dev->delay_ns; +if (pkt_dev->delay ==ULLONG_MAX ) { + pkt_dev->next_tx =ktime_add_ns (pkt_dev->next_tx, +LONG_MAX ); goto out; } }
@@ -3386,7 +3376,7 @@ static __inline__ void pktgen_xmit(strucif (netif_tx_queue_stopped (txq) ||netif_tx_queue_frozen (txq) ||need_resched ()) { - idle_start =getCurUs (); +ktime_t idle_start =ktime_now ();if (!netif_running (odev)) {pktgen_stop_device (pkt_dev);
@@ -3397,12 +3387,12 @@ static __inline__ void pktgen_xmit(strucif (need_resched ())schedule (); - pkt_dev->idle_acc +=getCurUs () - idle_start; + pkt_dev->idle_acc +=ktime_to_ns (ktime_sub (ktime_now (), + idle_start));if (netif_tx_queue_stopped (txq) ||netif_tx_queue_frozen (txq)) { - pkt_dev->next_tx_us =getCurUs ();/* TODO */ - pkt_dev->next_tx_ns =0 ; + pkt_dev->next_tx =ktime_now (); goto out;/* Try the next interface */ } }
@@ -3459,22 +3449,10 @@ static __inline__ void pktgen_xmit(struc pkt_dev->last_ok =0 ; } - pkt_dev->next_tx_us =getCurUs (); - pkt_dev->next_tx_ns =0 ; - - pkt_dev->next_tx_us += pkt_dev->delay_us; - pkt_dev->next_tx_ns += pkt_dev->delay_ns; - -if (pkt_dev->next_tx_ns >1000 ) { - pkt_dev->next_tx_us++; - pkt_dev->next_tx_ns -=1000 ; - } - } - -else {/* Retry it next time */ + pkt_dev->next_tx =ktime_add_ns (ktime_now (), pkt_dev->delay); + }else {/* Retry it next time */ pkt_dev->last_ok =0 ; - pkt_dev->next_tx_us =getCurUs ();/* TODO */ - pkt_dev->next_tx_ns =0 ; + pkt_dev->next_tx =ktime_now (); }__netif_tx_unlock_bh (txq);
@@ -3482,14 +3460,17 @@ static __inline__ void pktgen_xmit(struc/* If pkt_dev->count is zero, then run forever */ if ((pkt_dev->count !=0 ) && (pkt_dev->sofar >= pkt_dev->count)) {if (atomic_read (&(pkt_dev->skb->users)) !=1 ) { - idle_start =getCurUs (); +ktime_t idle_start =ktime_now (); +while (atomic_read (&(pkt_dev->skb->users)) !=1 ) {if (signal_pending (current)) {break ; }schedule (); } - pkt_dev->idle_acc +=getCurUs () - idle_start; + pkt_dev->idle_acc +=ktime_to_ns (ktime_sub (ktime_now (), + idle_start)); + }/* Done with this */
@@ -3651,8 +3632,7 @@ static int pktgen_add_device(struct pktg pkt_dev->max_pkt_size =ETH_ZLEN ; pkt_dev->nfrags =0 ; pkt_dev->clone_skb = pg_clone_skb_d; - pkt_dev->delay_us = pg_delay_d /1000 ; - pkt_dev->delay_ns = pg_delay_d %1000 ; + pkt_dev->delay = pg_delay_d; pkt_dev->count = pg_count_d; pkt_dev->sofar =0 ; pkt_dev->udp_src_min =9 ;/* sink port */
cover pktgen patches
These are some experimental (not ready yet), patches to try and improve the pktgen timer code. Need some review by the hrtimer folks as well.
RFC 4/4 pktgen: minor cleanup
A couple of minor functions can be written more compactly.
--- net/core/pktgen.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
net/core/pktgen.c +9 -16
@@ -507,7 +507,7 @@ static const struct file_operations pktgstatic int pktgen_if_show (struct seq_file *seq,void *v) { -struct pktgen_dev *pkt_dev = seq->private; +const struct pktgen_dev *pkt_dev = seq->private;ktime_t stopped;struct timeval trun, idle;
@@ -1666,7 +1666,7 @@ static const struct file_operations pktgstatic int pktgen_thread_show (struct seq_file *seq,void *v) {struct pktgen_thread *t = seq->private; -struct pktgen_dev *pkt_dev; +const struct pktgen_dev *pkt_dev;BUG_ON (!t);
@@ -2121,13 +2121,9 @@ static inline void set_pkt_overhead(stru pkt_dev->pkt_overhead +=SVLAN_TAG_SIZE (pkt_dev); } -static inline int f_seen (struct pktgen_dev *pkt_dev,int flow) +static inline int f_seen (const struct pktgen_dev *pkt_dev,int flow) { - -if (pkt_dev->flows[flow].flags &F_INIT ) -return 1 ; -else -return 0 ; +return !!(pkt_dev->flows[flow].flags &F_INIT ); }static inline int f_pick (struct pktgen_dev *pkt_dev)
@@ -3102,17 +3098,14 @@ static void pktgen_stop_all_threads_ifs(mutex_unlock (&pktgen_thread_lock); } -static int thread_is_running (struct pktgen_thread *t) +static int thread_is_running (const struct pktgen_thread *t) { -struct pktgen_dev *pkt_dev; -int res =0 ; +const struct pktgen_dev *pkt_dev;list_for_each_entry (pkt_dev, &t->if_list, list) -if (pkt_dev->running) { - res =1 ; -break ; - } -return res; +if (pkt_dev->running) +return 1 ; +return 0 ; }static int pktgen_wait_thread_run (struct pktgen_thread *t)
Quoted 5 lines · Stephen Hemminger
These are some experimental (not ready yet), patches to try and improve the pktgen timer code. Need some review by the hrtimer folks as well.
I'll try these in a couple of hours thanks.
As a followup on our other pktgen thread, if I switch my 3GHz cpu to 2GHz (powersave cpufreq), then I get litle bit better pktgen throughput from my 1Gb/s tg3 link.
So getting pktgen looping faster (spending less time in kernel time services) might actually slow down the trafic.
We might try to avoid any memory ops while waiting txq to become available, and cpu_relax()/ndelay(500)/udelay(1) as much as possible, to free bus bandwidth ?