Skip to contentSkip to conversation

tools/workqueue/wq_dump.py: Backward compatibility and live worker inspection

PATCH Applied Aaron Tomlin 2 participants 5 messages Aug 31

lore
Share & tools
lore
#0 lore raw

cover tools/workqueue/wq_dump.py: Backward compatibility and live worker inspection

Hi Tejun, Breno,

The Drgn program wq_dump.py is used to inspect workqueue configurations, affinity scopes, and pool associations. However, it currently focuses on static topology and lacks visibility into live, in-flight work items and transient pool states. Furthermore, recent refactoring broke compatibility when running against older kernels.

This 2-part series addresses these limitations:

Patch 1 introduces a wq_attrs() helper to support both older and newer kernel versions seamlessly.

Patch 2 provides live busy worker inspection (via option -b|--busy). It iterates through each pool's busy_hash table to dump active worker PIDs, command names, workqueues, callback function symbols, descriptions, and elapsed durations.

Feedback and comments are welcome.

Aaron Tomlin (2):
tools/workqueue/wq_dump.py: Support backward compatibility for

    wq->attrs rename

tools/workqueue/wq_dump.py: Add busy worker inspection and BH pool
states

 tools/workqueue/wq_dump.py | 60 ++++++++++++++++++++++++++++++++++----
 1 file changed, 55 insertions(+), 5 deletions(-)
Signature · 1 line
2.55.0
#1 lore raw

PATCH 1/2 tools/workqueue/wq_dump.py: Support backward compatibility for wq->attrs rename

Commit 464e454e1cb4 ("workqueue: rename wq->unbound_attrs to wq->attrs") renamed wq->unbound_attrs to wq->attrs. When running wq_dump.py against older running kernels or vmcores where struct workqueue_struct still contains unbound_attrs, drgn raises an AttributeError.

Add a wq_attrs() helper to allow wq_dump.py to inspect both older and newer kernel versions seamlessly.

Fixes 464e454e1cb4 ("workqueue: rename wq->unbound_attrs to wq->attrs")
---
 tools/workqueue/wq_dump.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
tools/workqueue/wq_dump.py +8 -2
index 31afc24ef17b..9313ebe0c525 100644
@@ -78,6 +78,12 @@ def cpumask_str(cpumask):
 
 wq_type_len = 9
 
+def wq_attrs(wq):
+    try:
+        return wq.attrs
+    except AttributeError:
+        return wq.unbound_attrs
+
 def wq_type_str(wq):
     if wq.flags & WQ_BH:
         return f'{"bh":{wq_type_len}}'
@@ -85,7 +91,7 @@ def wq_type_str(wq):
         if wq.flags & WQ_ORDERED:
             return f'{"ordered":{wq_type_len}}'
         else:
-            if wq.attrs.affn_strict:
+            if wq_attrs(wq).affn_strict:
                 return f'{"unbound,S":{wq_type_len}}'
             else:
                 return f'{"unbound":{wq_type_len}}'
@@ -206,7 +212,7 @@ for wq in list_for_each_entry('struct workqueue_struct', workqueues.address_of_(
 
     print(f'{wq.name.string_().decode():{WQ_NAME_LEN}}', end='')
     if wq.flags & WQ_UNBOUND:
-        print(f' {cpumask_str(wq.attrs.cpumask):{ucpus_len}}', end='')
+        print(f' {cpumask_str(wq_attrs(wq).cpumask):{ucpus_len}}', end='')
     else:
         print(f' {"":{ucpus_len}}', end='')
 
Signature · 1 line
2.55.0
Applied
#2 lore raw

Re: [PATCH 1/2] tools/workqueue/wq_dump.py: Support backward compatibility for wq->attrs rename

Applied to wq/for-7.3-fixes.

Thanks.

Signature · 1 line
tejun
#3 lore raw

PATCH 2/2 tools/workqueue/wq_dump.py: Add busy worker inspection and BH pool states

In reply to Aaron Tomlin

Currently, wq_dump.py displays static affinity scopes and pool topology, offering no visibility into in-flight work items or transient pool states.

Enhance wq_dump.py with live busy worker inspection (i.e., option -b|--busy) to dump active worker tasks, callback functions, workqueues, descriptions, and elapsed durations.

---
 tools/workqueue/wq_dump.py | 50 +++++++++++++++++++++++++++++++++++---
 1 file changed, 47 insertions(+), 3 deletions(-)
tools/workqueue/wq_dump.py +47 -3
index 9313ebe0c525..1659bba6c0dd 100644
@@ -29,6 +29,10 @@ Lists all worker pools indexed by their ID. For each pool:
   workers   number of all workers
   cpu       CPU the pool is associated with (per-cpu pool)
   cpus      CPUs the workers in the pool can run on (unbound pool)
+  flags     pool flags (bh, draining, disassociated)
+
+  If -b|--busy is specified, lists all busy workers currently executing
+  work items, their task PID/comm, workqueue, callback function, and duration.
 
 Workqueue CPU -> pool
 =====================
@@ -49,12 +53,14 @@ import sys
 import argparse
 parser = argparse.ArgumentParser(description=desc,
                                  formatter_class=argparse.RawTextHelpFormatter)
+parser.add_argument('-b', '--busy', action='store_true',
+                    help='Show busy workers currently executing work items')
 args = parser.parse_args()
 
 import drgn
-from drgn.helpers.linux.list import list_for_each_entry,list_empty
+from drgn.helpers.linux.list import list_for_each_entry, list_empty, hlist_for_each_entry
 from drgn.helpers.linux.percpu import per_cpu_ptr
-from drgn.helpers.linux.cpumask import for_each_cpu,for_each_possible_cpu
+from drgn.helpers.linux.cpumask import for_each_cpu, for_each_possible_cpu
 from drgn.helpers.linux.nodemask import for_each_node
 from drgn.helpers.linux.idr import idr_for_each
 
@@ -62,6 +68,10 @@ def err(s):
     print(s, file=sys.stderr, flush=True)
     sys.exit(1)
 
+def get_hz():
+    cs = prog['clocksource_jiffies']
+    return round(1000000000 / (cs.mult.value_() >> cs.shift.value_()))
+
 def cpumask_str(cpumask):
     output = ""
     base = 0
@@ -118,9 +128,13 @@ WQ_AFFN_NUMA            = prog['WQ_AFFN_NUMA']
 WQ_AFFN_SYSTEM          = prog['WQ_AFFN_SYSTEM']
 
 POOL_BH                 = prog['POOL_BH']
+POOL_BH_DRAINING        = prog['POOL_BH_DRAINING']
+POOL_DISASSOCIATED      = prog['POOL_DISASSOCIATED']
+HIGHPRI_NICE_LEVEL      = prog['HIGHPRI_NICE_LEVEL']
 
 WQ_NAME_LEN             = prog['WQ_NAME_LEN'].value_()
 cpumask_str_len         = len(cpumask_str(wq_unbound_cpumask))
+hz                      = get_hz()
 
 print('Affinity Scopes')
 print('===============')
@@ -168,14 +182,44 @@ for pi, pool in idr_for_each(worker_pool_idr):
     if pool.cpu >= 0:
         print(f'cpu={pool.cpu.value_():3}', end='')
         if pool.flags & POOL_BH:
-            print(' bh', end='')
+            bh_type = 'bh-hi' if pool.attrs.nice == HIGHPRI_NICE_LEVEL else 'bh'
+            print(f' {bh_type}', end='')
+            if pool.flags & POOL_BH_DRAINING:
+                print(' draining', end='')
+        if pool.flags & POOL_DISASSOCIATED:
+            print(' disassociated', end='')
     else:
         print(f'cpus={cpumask_str(pool.attrs.cpumask)}', end='')
         print(f' pod_cpus={cpumask_str(pool.attrs.__pod_cpumask)}', end='')
         if pool.attrs.affn_strict:
             print(' strict', end='')
+        if pool.flags & POOL_DISASSOCIATED:
+            print(' disassociated', end='')
     print('')
 
+    if args.busy:
+        for bkt in pool.busy_hash:
+            for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
+                wq_name = worker.current_pwq.wq.name.string_().decode()
+                fn_name = prog.symbol(worker.current_func.value_()).name
+
+                dur_str = ''
+                if 'jiffies' in prog and worker.current_start.value_():
+                    jiffies = prog['jiffies'].value_()
+                    dur_s = max(0, (jiffies - worker.current_start.value_()) // hz)
+                    dur_str = f' for {dur_s}s'
+
+                if pool.flags & POOL_BH:
+                    w_id = 'bh' if pool.attrs.nice != HIGHPRI_NICE_LEVEL else 'bh-hi'
+                elif worker.task.value_():
+                    w_id = f'PID {worker.task.pid.value_():<6} ({worker.task.comm.string_().decode()})'
+                else:
+                    w_id = f'worker[{worker.id.value_()}]'
+
+                desc = worker.desc.string_().decode()
+                desc_str = f' desc="{desc}"' if desc and desc != wq_name else ''
+                print(f'    busy: {w_id}: {wq_name}:{fn_name}{dur_str}{desc_str}')
+
 print('')
 print('Workqueue CPU -> pool')
 print('=====================')
Signature · 1 line
2.55.0
#4 lore raw

Re: [PATCH 2/2] tools/workqueue/wq_dump.py: Add busy worker inspection and BH pool states

Hello, Aaron.

tools/workqueue/wq_dump.py for pi, pool in idr_for_each(worker_pool_idr): · line 190

Quoted 4 lines · Aaron Tomlin
tools/workqueue/wq_dump.py +4 -0
+            if pool.flags & POOL_BH_DRAINING:
+                print(' draining', end='')
+        if pool.flags & POOL_DISASSOCIATED:
+            print(' disassociated', end='')

Note that POOL_DISASSOCIATED is set by init_worker_pool() and cleared only for per-cpu non-BH pools, so BH and unbound pools carry it for their whole lifetime. This would print " disassociated" on every one of their lines. It's only meaningful on a per-cpu non-BH pool whose CPU is offline.

tools/workqueue/wq_dump.py for pi, pool in idr_for_each(worker_pool_idr): · line 204

Quoted 5 lines
tools/workqueue/wq_dump.py +5 -0
+    if args.busy:
+        for bkt in pool.busy_hash:
+            for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
+                wq_name = worker.current_pwq.wq.name.string_().decode()
+                fn_name = prog.symbol(worker.current_func.value_()).name

process_one_work() hashes the worker before setting current_func and current_pwq, and clears them right after unhashing, all under pool->lock. A worker caught in either window reads back NULL, so worker.current_pwq.wq raises and prog.symbol(0) can raise errors and kill the rest of the dump. Work items can finish at a very high rate, so this wouldn't be difficult to hit on a busy machine. Maybe catch the exceptions and retry the worker?

tools/workqueue/wq_dump.py for pi, pool in idr_for_each(worker_pool_idr): · line 210

Quoted 5 lines
tools/workqueue/wq_dump.py +5 -0
+                dur_str = ''
+                if 'jiffies' in prog and worker.current_start.value_():
+                    jiffies = prog['jiffies'].value_()
+                    dur_s = max(0, (jiffies - worker.current_start.value_()) // hz)
+                    dur_str = f' for {dur_s}s'

worker->current_start was added in v7.0 by e8e14ac7cfe4 ("workqueue: Show in-flight work item duration in stall diagnostics"). The 'jiffies' in prog test doesn't protect the member access, so on an older kernel or vmcore the first busy worker aborts the dump, the same failure mode the previous patch fixes for wq->attrs. Please handle it the same way.

Also, the values are plain Python integers, so on a 32bit kernel the subtraction goes negative when jiffies wraps (the first wrap is five minutes after boot due to INITIAL_JIFFIES) and the max() turns the duration into "for 0s". Mask the difference to the target's word size instead, e.g.:

mask = (1 << (prog['jiffies'].type_.size * 8)) - 1 dur_s = ((jiffies - worker.current_start.value_()) & mask) // hz

Thanks.

Signature · 1 line
tejun