Skip to contentSkip to conversation

gpu: nova-core: add formatted trace events

PATCH New Zhi Wang 1 participant 4 messages Aug 5

Subsystems (5)
lore
Share & tools
lore

Zhi Wang

author
#0 lore raw

cover gpu: nova-core: add formatted trace events

Tracking driver and firmware activity is useful for debugging and bug reports, e.g. GSP and FSP message sends and receives, vGPU states, lifecycle events[1]. Trace events provide a low-overhead interface for frequent diagnostics and allow each component stream to be enabled independently.

This series adds a shared formatted event class and independent events for general driver, FSP, GSP and vGPU messages. It replaces selected dev_dbg!() calls with their corresponding trace events and adds FSP NVDM send and receive records with the message type and length.

Use the module selector to enable the events before nova-core probes. With tracefs mounted at /sys/kernel/tracing, run:

cd /sys/kernel/tracing
echo 'nova_core:*:mod:nova_core' > set_event
echo 1 > tracing_on
insmod /path/to/nova-core.ko
cat trace

[1] https://github.com/zhiwang-nvidia/nova-core/tree/zhi/nova-gpu-wip-nova-gsp-20260803

Zhi Wang (3):
gpu: nova-core: add formatted tracing
gpu: nova-core: trace driver probe and firmware messages
gpu: nova-core: trace vGPU state

 drivers/gpu/Makefile               |   4 +-
 drivers/gpu/nova-core/driver.rs    |   7 +-
 drivers/gpu/nova-core/fsp.rs       |  16 ++-
 drivers/gpu/nova-core/gsp/cmdq.rs  |  11 ++-
 drivers/gpu/nova-core/nova_core.rs |   1 +
 drivers/gpu/nova-core/trace.c      |   5 +
 drivers/gpu/nova-core/trace.h      |  52 ++++++++++
 drivers/gpu/nova-core/trace.rs     | 151 +++++++++++++++++++++++++++++
 drivers/gpu/nova-core/vgpu.rs      |   5 +-
 rust/bindings/bindings_helper.h    |   4 +
 10 files changed, 245 insertions(+), 11 deletions(-)
 create mode 100644 drivers/gpu/nova-core/trace.c
 create mode 100644 drivers/gpu/nova-core/trace.h
 create mode 100644 drivers/gpu/nova-core/trace.rs

base-commit: 44e7e7f7cffb10a93bb88e7cb59b7b8b3e2deb1c

Signature · 1 line
2.53.0
author
#1 lore raw

PATCH 1/3 gpu: nova-core: add formatted tracing

Provide formatted trace events for general driver, FSP, GSP and vGPU messages. Add a shared event class and a formatting helper that avoids formatting disabled events and uses a fixed-size stack buffer.

---
 drivers/gpu/Makefile               |   4 +-
 drivers/gpu/nova-core/nova_core.rs |   1 +
 drivers/gpu/nova-core/trace.c      |   5 +
 drivers/gpu/nova-core/trace.h      |  52 +++++++++
 drivers/gpu/nova-core/trace.rs     | 162 +++++++++++++++++++++++++++++
 rust/bindings/bindings_helper.h    |   4 +
 6 files changed, 227 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/nova-core/trace.c
 create mode 100644 drivers/gpu/nova-core/trace.h
 create mode 100644 drivers/gpu/nova-core/trace.rs
drivers/gpu/Makefile +3 -1
index e372fc02139f..d26e7dea9e66 100644
@@ -14,7 +14,9 @@ obj-$(CONFIG_TRACE_GPU_MEM)		+= trace/
 # system supports cross-crate dependencies natively.
 
 obj-$(CONFIG_NOVA_CORE) += nova-core.o
-nova-core-y := nova-core/nova_core.o nova-core/nova_core_exports.o
+nova-core-y := nova-core/nova_core.o \
+	       nova-core/trace.o \
+	       nova-core/nova_core_exports.o
 
 obj-$(CONFIG_DRM_NOVA) += nova-drm.o
 nova-drm-y := drm/nova/nova.o
drivers/gpu/nova-core/nova_core.rs +1 -0
index 35a8b1214b0e..85ee5067ef56 100644
@@ -22,6 +22,7 @@
 mod num;
 mod regs;
 mod sbuffer;
+mod trace;
 mod vbios;
 mod vgpu;
 
drivers/gpu/nova-core/trace.c new file +5 -0
new file mode 100644
index 000000000000..c72cb0f65b47
@@ -0,0 +1,5 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#define CREATE_TRACE_POINTS
+#define CREATE_RUST_TRACE_POINTS
+#include "trace.h"
drivers/gpu/nova-core/trace.h new file +52 -0

52 lines not on this page · raw message

drivers/gpu/nova-core/trace.rs new file +162 -0

162 lines not on this page · raw message

rust/bindings/bindings_helper.h +4 -0
index 1124785e210b..d1e0a60e370f 100644
@@ -165,3 +165,7 @@ const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TRIM_DISABLE = GPU_BUDDY_TRIM_DI
 #include "../../drivers/android/binder/rust_binder.h"
 #include "../../drivers/android/binder/rust_binder_events.h"
 #endif
+
+#if IS_ENABLED(CONFIG_NOVA_CORE)
+#include "../../drivers/gpu/nova-core/trace.h"
+#endif
Signature · 1 line
2.53.0
author
#2 lore raw

PATCH 2/3 gpu: nova-core: trace driver probe and firmware messages

In reply to Zhi Wang

Use independently selectable trace streams for driver, GSP and FSP activity. Replace the probe and GSP dev_dbg() calls with formatted traces, and add FSP NVDM send and receive traces with the message type and length.

---
 drivers/gpu/nova-core/driver.rs   |  7 +++++--
 drivers/gpu/nova-core/fsp.rs      | 16 +++++++++++++++-
 drivers/gpu/nova-core/gsp/cmdq.rs | 11 ++++++-----
 drivers/gpu/nova-core/trace.rs    |  9 ---------
 4 files changed, 26 insertions(+), 17 deletions(-)
drivers/gpu/nova-core/driver.rs +5 -2
index 5738d4ac521b..cb92ce9a594e 100644
@@ -18,7 +18,10 @@
     types::ForLt,
 };
 
-use crate::gpu::Gpu;
+use crate::{
+    gpu::Gpu,
+    trace::nova_core_trace_driver, //
+};
 
 /// Counter for generating unique auxiliary device IDs.
 static AUXILIARY_ID_COUNTER: Atomic<u32> = Atomic::new(0);
@@ -73,7 +76,7 @@ fn probe<'bound>(
         _info: &'bound Self::IdInfo,
     ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound {
         pin_init::pin_init_scope(move || {
-            dev_dbg!(pdev, "Probe Nova Core GPU driver.\n");
+            nova_core_trace_driver!(pdev, "Probe Nova Core GPU driver.");
 
             pdev.enable_device_mem()?;
             pdev.set_master();
drivers/gpu/nova-core/fsp.rs +15 -1
index ba4544210e40..9dc919de2200 100644
@@ -51,7 +51,8 @@
         NvdmType, //
     },
     num,
-    regs, //
+    regs,
+    trace::nova_core_trace_fsp, //
 };
 
 mod hal;
@@ -434,6 +435,12 @@ fn send_sync_fsp<M>(&mut self, dev: &device::Device, msg: &M) -> Result<KVec<u8>
     where
         M: MessageToFsp,
     {
+        nova_core_trace_fsp!(
+            dev,
+            "FSP NVDM: send: type={:?}, length=0x{:x}",
+            M::NVDM_TYPE,
+            msg.as_bytes().len(),
+        );
         self.falcon.send_msg(msg.as_bytes())?;
 
         let response_buf = self.falcon.recv_msg().inspect_err(|e| {
@@ -489,6 +496,13 @@ fn send_sync_fsp<M>(&mut self, dev: &device::Device, msg: &M) -> Result<KVec<u8>
             return Err(EIO);
         }
 
+        nova_core_trace_fsp!(
+            dev,
+            "FSP NVDM: receive: type={:?}, length=0x{:x}",
+            M::NVDM_TYPE,
+            response_buf.len(),
+        );
+
         Ok(response_buf)
     }
 
drivers/gpu/nova-core/gsp/cmdq.rs +6 -5
index cd844fe48f05..6137c60b6231 100644
@@ -58,7 +58,8 @@
         GSP_PAGE_SIZE, //
     },
     num,
-    sbuffer::SBufferIter, //
+    sbuffer::SBufferIter,
+    trace::nova_core_trace_gsp, //
 };
 
 use super::regs;
@@ -675,9 +676,9 @@ fn send_single_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result
                 dst.contents.1,
             ])));
 
-        dev_dbg!(
+        nova_core_trace_gsp!(
             &self.dev,
-            "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}\n",
+            "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}",
             self.seq,
             M::FUNCTION,
             dst.header.length(),
@@ -754,9 +755,9 @@ fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> {
         // Extract the `GspMsgElement`.
         let (header, slice_1) = GspMsgElement::from_bytes_prefix(slice_1).ok_or(EIO)?;
 
-        dev_dbg!(
+        nova_core_trace_gsp!(
             &self.dev,
-            "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}\n",
+            "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}",
             header.sequence(),
             header.function(),
             header.length(),
drivers/gpu/nova-core/trace.rs +0 -9
index 893479b3fcb5..db975804f855 100644
@@ -55,7 +55,6 @@ pub(crate) unsafe fn $event(
 ///
 /// `trace` must synchronously consume `dev`, `message`, and `message_len`
 /// according to the `nova_core_trace_class` event prototype.
-#[expect(dead_code)]
 pub(crate) unsafe fn nova_core_trace_fmt(
     dev: &CStr,
     args: fmt::Arguments<'_>,
@@ -81,7 +80,6 @@ pub(crate) unsafe fn nova_core_trace_fmt(
     }
 }
 
-#[expect(unused_macros)]
 macro_rules! nova_core_trace_impl {
     ($event:ident, $dev:expr, $($arg:tt)*) => {{
         #[cfg(CONFIG_TRACEPOINTS)]
@@ -121,24 +119,20 @@ macro_rules! nova_core_trace_impl {
 
 // Frontend macros expand in their caller's module and invoke this helper by
 // path, so the re-export must be visible from the parent module.
-#[expect(unused_imports)]
 pub(super) use nova_core_trace_impl;
 
-#[expect(unused_macros)]
 macro_rules! nova_core_trace_driver {
     ($dev:expr, $($arg:tt)*) => {
         $crate::trace::nova_core_trace_impl!(nova_core_trace_driver, $dev, $($arg)*)
     };
 }
 
-#[expect(unused_macros)]
 macro_rules! nova_core_trace_fsp {
     ($dev:expr, $($arg:tt)*) => {
         $crate::trace::nova_core_trace_impl!(nova_core_trace_fsp, $dev, $($arg)*)
     };
 }
 
-#[expect(unused_macros)]
 macro_rules! nova_core_trace_gsp {
     ($dev:expr, $($arg:tt)*) => {
         $crate::trace::nova_core_trace_impl!(nova_core_trace_gsp, $dev, $($arg)*)
@@ -152,11 +146,8 @@ macro_rules! nova_core_trace_vgpu {
     };
 }
 
-#[expect(unused_imports)]
 pub(crate) use nova_core_trace_driver;
-#[expect(unused_imports)]
 pub(crate) use nova_core_trace_fsp;
-#[expect(unused_imports)]
 pub(crate) use nova_core_trace_gsp;
 #[expect(unused_imports)]
 pub(crate) use nova_core_trace_vgpu;
Signature · 1 line
2.53.0
author
#3 lore raw

PATCH 3/3 gpu: nova-core: trace vGPU state

In reply to Zhi Wang

Record the vGPU state in an independently selectable vGPU trace stream. Replace the vGPU state dev_dbg() call with the formatted vGPU trace event.

---
 drivers/gpu/nova-core/trace.rs | 2 --
 drivers/gpu/nova-core/vgpu.rs  | 5 +++--
 2 files changed, 3 insertions(+), 4 deletions(-)
drivers/gpu/nova-core/trace.rs +0 -2
index db975804f855..17967a686fa2 100644
@@ -139,7 +139,6 @@ macro_rules! nova_core_trace_gsp {
     };
 }
 
-#[expect(unused_macros)]
 macro_rules! nova_core_trace_vgpu {
     ($dev:expr, $($arg:tt)*) => {
         $crate::trace::nova_core_trace_impl!(nova_core_trace_vgpu, $dev, $($arg)*)
@@ -149,5 +148,4 @@ macro_rules! nova_core_trace_vgpu {
 pub(crate) use nova_core_trace_driver;
 pub(crate) use nova_core_trace_fsp;
 pub(crate) use nova_core_trace_gsp;
-#[expect(unused_imports)]
 pub(crate) use nova_core_trace_vgpu;
drivers/gpu/nova-core/vgpu.rs +3 -2
index 6b7e045acea8..e4fc7e101c93 100644
@@ -13,7 +13,8 @@
         Fsp,
         VgpuMode, //
     },
-    gpu::Chipset, //
+    gpu::Chipset,
+    trace::nova_core_trace_vgpu, //
 };
 
 mod hal;
@@ -50,7 +51,7 @@ pub(crate) fn new(
             );
             VgpuState::Disabled
         });
-        dev_dbg!(pdev, "vGPU state: {:?}\n", state);
+        nova_core_trace_vgpu!(pdev, "state: {:?}", state);
 
         Self { state }
     }
Signature · 1 line
2.53.0