ath9k: txctl and/or TID corruption in ath_tx_start_dma.

This system is running 84 VIFs, WPA encryption, wpa-supplicant scan-sharing and some scan-avoidance logic in mac80211.

All stations are trying to run a 56kbps TCP data flow as soon as they associate.

I have seen and reported this problem previously, but it seems I finally got my debug code right, because now it prints useful information and recovers.

Something is corrupting txctl->an, at the least.

The method with debug looks like this:

/* FIXME: tx power */
static int ath_tx_start_dma(struct ath_softc *sc, struct ath_buf *bf,
struct ath_tx_control *txctl)

{
	struct sk_buff *skb = bf->bf_mpdu;
	struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb);
	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
	struct list_head bf_head;
	struct ath_atx_tid *tid;
	u8 tidno;
	int rv = 0;

	spin_lock_bh(&txctl->txq->axq_lock);

	if ((tx_info->flags & IEEE80211_TX_CTL_AMPDU) && txctl->an) {
		tidno = ieee80211_get_qos_ctl(hdr)[0] &
			IEEE80211_QOS_CTL_TID_MASK;
		BUG_ON(tidno < 0);
		BUG_ON(tidno >= WME_NUM_TID);
		tid = ATH_AN_2_TID(txctl->an, tidno);

		if ((!tid) || ((unsigned long)(tid) < 4096)) {
			printk("ERROR: ath9k: tid is NULL, tid: %p tidno: %i txctl->an: %p\n",
			       tid, tidno, txctl->an);
			WARN_ON(1);
			rv = -EINVAL;
			goto out;
		}
		if (!tid->ac) {
			printk("ERROR: ath9k: tid->ac is NULL, tid: %p tidno: %i\n",
			       tid, tidno);
			WARN_ON(tid->ac == NULL);
			rv = -EINVAL;
			goto out;
		}
		else {
			if (tid->ac->txq != txctl->txq) {
				printk("ERROR: ath9k: tid->ac->txq (%p) != txctl->txq (%p), tidno: %i\n",
				       tid->ac->txq, txctl->txq, tidno);
				WARN_ON(tid->ac->txq != txctl->txq);
				rv = -EINVAL;
				goto out;
			}
		}

		/*
		 * Try aggregation if it's a unicast data frame
		 * and the destination is HT capable.
		 */
		ath_tx_send_ampdu(sc, tid, bf, txctl);
	} else {
		INIT_LIST_HEAD(&bf_head);
		list_add_tail(&bf->list, &bf_head);

		bf->bf_state.bfs_ftype = txctl->frame_type;
		bf->bf_state.bfs_paprd = txctl->paprd;

		if (bf->bf_state.bfs_paprd)
			ar9003_hw_set_paprd_txdesc(sc->sc_ah, bf->bf_desc,
						   bf->bf_state.bfs_paprd);

		ath_tx_send_normal(sc, txctl->txq, NULL, &bf_head);
	}
out:
	spin_unlock_bh(&txctl->txq->axq_lock);
	return rv;
}

Dec  6 15:20:56 localhost kernel: ADDRCONF(NETDEV_UP): sta67: link is not ready
Dec  6 15:20:56 localhost kernel: start_sw_scan: running-other-vifs: 0  running-station-vifs: 69, associated-stations: 67 scanning current channel: 2437 MHz
Dec  6 15:20:56 localhost kernel: ADDRCONF(NETDEV_UP): sta68: link is not ready
Dec  6 15:20:56 localhost kernel: ERROR: ath9k: tid is NULL, tid: 0000002c tidno: 0 txctl->an: 00000028
Dec  6 15:20:56 localhost kernel: ------------[ cut here ]------------
Dec  6 15:20:56 localhost kernel: WARNING: at /home/greearb/git/linux.wireless-testing/drivers/net/wireless/ath/ath9k/xmit.c:1708 ath_tx_start+0x4cd/0x69a [ath9k]()
Dec  6 15:20:56 localhost kernel: Hardware name: PDSBM
Dec  6 15:20:56 localhost kernel: Modules linked in: aes_i586 aes_generic 8021q garp stp llc michael_mic fuse macvlan pktgen nfs lockd fscache nfs_acl 
auth_rpcgss sunrpc ipv6 uinput arc4 ecb ath9k mac80211 ath9k_common ath9k_hw ath microcode e1000e iTCO_wdt cfg80211 iTCO_vendor_support pcspkr i2c_i801 i915 
drm_kms_helper drm i2c_algo_bit i2c_core video output [last unloaded: michael_mic]
Dec  6 15:20:56 localhost kernel: Pid: 7652, comm: sh Tainted: P            2.6.37-rc4-wl+ #55
Dec  6 15:20:56 localhost kernel: Call Trace:
Dec  6 15:20:56 localhost kernel: [<78436fbd>] warn_slowpath_common+0x77/0x8c
Dec  6 15:20:56 localhost kernel: [<f9260932>] ? ath_tx_start+0x4cd/0x69a [ath9k]
Dec  6 15:20:56 localhost kernel: [<f9260932>] ? ath_tx_start+0x4cd/0x69a [ath9k]
Dec  6 15:20:56 localhost kernel: [<78436fef>] warn_slowpath_null+0x1d/0x1f
Dec  6 15:20:56 localhost kernel: [<f9260932>] ath_tx_start+0x4cd/0x69a [ath9k]
Dec  6 15:20:56 localhost kernel: [<f925ac69>] ath9k_tx+0x197/0x1c8 [ath9k]
Dec  6 15:20:56 localhost kernel: [<f91a93fe>] __ieee80211_tx+0x102/0x167 [mac80211]
Dec  6 15:20:56 localhost kernel: [<f91aad9b>] ieee80211_tx_pending+0x108/0x1fe [mac80211]
Dec  6 15:20:57 localhost kernel: [<7843bba5>] tasklet_action+0x88/0xe3
Dec  6 15:20:57 localhost kernel: [<7843c121>] __do_softirq+0x85/0x142
Dec  6 15:20:57 localhost kernel: [<7843c09c>] ? __do_softirq+0x0/0x142
Dec  6 15:20:57 localhost kernel: <IRQ>  [<7843bf43>] ? irq_exit+0x35/0x69
Dec  6 15:20:57 localhost kernel: [<78404245>] ? do_IRQ+0x8e/0xa2
Dec  6 15:20:57 localhost kernel: [<784cfade>] ? __d_lookup+0x6a/0x10d
Dec  6 15:20:57 localhost kernel: [<784036ae>] ? common_interrupt+0x2e/0x40
Dec  6 15:20:57 localhost kernel: [<784c00d8>] ? do_loop_readv_writev+0x20/0x50
Dec  6 15:20:57 localhost kernel: [<784c7bbb>] ? path_get+0x1/0x23
Dec  6 15:20:57 localhost kernel: [<784bf42b>] ? nameidata_to_filp+0x2a/0x52
Dec  6 15:20:57 localhost kernel: [<784c9026>] ? may_open+0x87/0xf9
Dec  6 15:20:57 localhost kernel: [<784c93d9>] ? do_last+0x341/0x42d
Dec  6 15:20:57 localhost kernel: [<784cab6b>] ? do_filp_open+0x364/0x409
Dec  6 15:20:57 localhost kernel: [<784c01e5>] ? fsnotify_access+0x65/0x6d
Dec  6 15:20:57 localhost kernel: [<784c557d>] ? open_exec+0x28/0x7c
Dec  6 15:20:57 localhost kernel: [<784f25cd>] ? load_elf_binary+0x1b6/0xf61
Dec  6 15:20:57 localhost kernel: [<784c49fa>] ? search_binary_handler+0x14d/0x223
Dec  6 15:20:57 localhost kernel: [<784c4956>] ? search_binary_handler+0xa9/0x223
Dec  6 15:20:57 localhost kernel: [<784f2417>] ? load_elf_binary+0x0/0xf61
Dec  6 15:20:57 localhost kernel: [<784c5d6e>] ? do_execve+0x159/0x225
Dec  6 15:20:57 localhost kernel: [<78408ccb>] ? sys_execve+0x31/0x53
Dec  6 15:20:57 localhost kernel: [<784031d2>] ? ptregs_execve+0x12/0x20
Dec  6 15:20:57 localhost kernel: [<784030dc>] ? sysenter_do_call+0x12/0x38
Dec  6 15:20:57 localhost kernel: ---[ end trace 445c2a46f5f18fe7 ]---
Dec  6 15:20:57 localhost kernel: ERROR: ath9k: tid is NULL, tid: 0000002c tidno: 0 txctl->an: 00000028
Dec  6 15:20:57 localhost kernel: ------------[ cut here ]------------
Dec  6 15:20:57 localhost kernel: WARNING: at /home/greearb/git/linux.wireless-testing/drivers/net/wireless/ath/ath9k/xmit.c:1708 ath_tx_start+0x4cd/0x69a [ath9k]()
Dec  6 15:20:57 localhost kernel: Hardware name: PDSBM
Dec  6 15:20:57 localhost kernel: Modules linked in: aes_i586 aes_generic 8021q garp stp llc michael_mic fuse macvlan pktgen nfs lockd fscache nfs_acl 
auth_rpcgss sunrpc ipv6 uinput arc4 ecb ath9k mac80211 ath9k_common ath9k_hw ath microcode e1000e iTCO_wdt cfg80211 iTCO_vendor_support pcspkr i2c_i801 i915 
drm_kms_helper drm i2c_algo_bit i2c_core video output [last unloaded: michael_mic]
Dec  6 15:20:57 localhost kernel: Pid: 7652, comm: sh Tainted: P        W   2.6.37-rc4-wl+ #55
Dec  6 15:20:57 localhost kernel: Call Trace:
Dec  6 15:20:57 localhost kernel: [<78436fbd>] warn_slowpath_common+0x77/0x8c
Dec  6 15:20:57 localhost kernel: [<f9260932>] ? ath_tx_start+0x4cd/0x69a [ath9k]
Dec  6 15:20:57 localhost kernel: [<f9260932>] ? ath_tx_start+0x4cd/0x69a [ath9k]
Dec  6 15:20:57 localhost kernel: [<78436fef>] warn_slowpath_null+0x1d/0x1f
Dec  6 15:20:57 localhost kernel: [<f9260932>] ath_tx_start+0x4cd/0x69a [ath9k]
Dec  6 15:20:57 localhost kernel: [<f925ac69>] ath9k_tx+0x197/0x1c8 [ath9k]
Dec  6 15:20:57 localhost kernel: [<f91a93fe>] __ieee80211_tx+0x102/0x167 [mac80211]
Dec  6 15:20:57 localhost kernel: [<f91aad9b>] ieee80211_tx_pending+0x108/0x1fe [mac80211]
Dec  6 15:20:57 localhost kernel: [<7843bba5>] tasklet_action+0x88/0xe3
Dec  6 15:20:57 localhost kernel: [<7843c121>] __do_softirq+0x85/0x142
Dec  6 15:20:57 localhost kernel: [<7843c09c>] ? __do_softirq+0x0/0x142
Dec  6 15:20:57 localhost kernel: <IRQ>  [<7843bf43>] ? irq_exit+0x35/0x69
Dec  6 15:20:57 localhost kernel: [<78404245>] ? do_IRQ+0x8e/0xa2
Dec  6 15:20:57 localhost kernel: [<784cfade>] ? __d_lookup+0x6a/0x10d
Dec  6 15:20:57 localhost kernel: [<784036ae>] ? common_interrupt+0x2e/0x40
Dec  6 15:20:57 localhost kernel: [<784c00d8>] ? do_loop_readv_writev+0x20/0x50
Dec  6 15:20:57 localhost kernel: [<784c7bbb>] ? path_get+0x1/0x23
Dec  6 15:20:57 localhost kernel: [<784bf42b>] ? nameidata_to_filp+0x2a/0x52
Dec  6 15:20:57 localhost kernel: [<784c9026>] ? may_open+0x87/0xf9
Dec  6 15:20:57 localhost kernel: [<784c93d9>] ? do_last+0x341/0x42d
Dec  6 15:20:57 localhost kernel: [<784cab6b>] ? do_filp_open+0x364/0x409
Dec  6 15:20:57 localhost kernel: [<784c01e5>] ? fsnotify_access+0x65/0x6d
Dec  6 15:20:57 localhost kernel: [<784c557d>] ? open_exec+0x28/0x7c
Dec  6 15:20:57 localhost kernel: [<784f25cd>] ? load_elf_binary+0x1b6/0xf61
Dec  6 15:20:57 localhost kernel: [<784c49fa>] ? search_binary_handler+0x14d/0x223
Dec  6 15:20:57 localhost kernel: [<784c4956>] ? search_binary_handler+0xa9/0x223
Dec  6 15:20:57 localhost kernel: [<784f2417>] ? load_elf_binary+0x0/0xf61
Dec  6 15:20:57 localhost kernel: [<784c5d6e>] ? do_execve+0x159/0x225
Dec  6 15:20:57 localhost kernel: [<78408ccb>] ? sys_execve+0x31/0x53
Dec  6 15:20:57 localhost kernel: [<784031d2>] ? ptregs_execve+0x12/0x20
Dec  6 15:20:57 localhost kernel: [<784030dc>] ? sysenter_do_call+0x12/0x38
Dec  6 15:20:57 localhost kernel: ---[ end trace 445c2a46f5f18fe8 ]---
Dec  6 15:20:57 localhost kernel: ERROR: ath9k: tid is NULL, tid: 0000002c tidno: 0 txctl->an: 00000028

Ben Greear <[email protected]>
Candela Technologies Inc  http://www.candelatech.com

I think maybe I found a cause for this. According to net/mac80211.h, sta can be null, but this code never checks for that. If the compiler is being clever, it may not actually do a dereference, so maybe that's why it never crashed when assigning txctl->an.

I'm going to test this patch. This one isn't always easy to hit, so going to leave lots of debugging in mine...but if this looks correct, I or someone else can provide a cleaned up patch...

index 8b0b076..6cdb1d2 100644
@@ -1764,7 +1764,20 @@ int ath_tx_start(struct ieee80211_hw *hw, struct sk_buff *skb,
         int frmlen = skb->len + FCS_LEN;
         int q;
 
-       txctl->an = (struct ath_node *)sta->drv_priv;
+       /* NOTE:  sta can be NULL according to net/mac80211.h */
+       if (sta) {
+               txctl->an = (struct ath_node *)sta->drv_priv;
+               if (((unsigned long)(txctl->an)) < 4096) {
+                       printk("invalid txctl->an: %p  sta: %p  sta->drv_priv: %p\n",
+                              txctl->an, sta, sta->drv_priv);
+                       WARN_ON(1);
+                       return -EINVAL;
+               }
+       }
+       else {
+               printk("ath9k: sta was NULL in ath_tx_start.\n");
+       }
+
         if (info->control.hw_key)
                 frmlen += info->control.hw_key->icv_len;
 

Ben Greear <[email protected]>
Candela Technologies Inc  http://www.candelatech.com