Introduce NVIDIA vGPU manager

index 5ceb7760c78e..26e94d908b77 100644
@@ -174,13 +174,9 @@ pub(crate) struct FbRanges {
 
 impl FbRanges {
     /// Computes concrete framebuffer ranges required on non-FSP booting architectures.
-    pub(crate) fn new(
-        chipset: Chipset,
-        bar: Bar0<'_>,
-        gsp_fw: &GspFirmware<'_>,
-        vgpu_state: VgpuState,
-    ) -> Result<Self> {
-        let hal = hal::fb_hal(chipset);
+    pub(crate) fn new(ctx: &gsp::GspBootContext<'_, '_>, gsp_fw: &GspFirmware<'_>) -> Result<Self> {
+        let bar = ctx.bar;
+        let hal = hal::fb_hal(ctx.chipset);
 
         let fb = {
             let fb_size = hal.vidmem_size(bar);

@@ -242,7 +238,7 @@ pub(crate) fn new(
             FbRange(fw_image_addr..fw_image_addr + fw_image_size)
         };
 
-        let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb.end)?;
+        let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(ctx, fb.end)?;
 
         let wpr2_heap = {
             const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::SZ_1M;

@@ -298,11 +294,11 @@ pub(crate) fn wpr2_range(bar: Bar0<'_>) -> Option<Range<u64>> {
 }
 
 /// Computes the number of VF partitions and the WPR2 heap size from the vGPU state.
-fn wpr2_heap_params(chipset: Chipset, vgpu_state: VgpuState, fb_size: u64) -> Result<(u8, u64)> {
-    Ok(match vgpu_state {
+fn wpr2_heap_params(ctx: &gsp::GspBootContext<'_, '_>, fb_size: u64) -> Result<(u8, u64)> {
+    Ok(match ctx.vgpu.state() {
         VgpuState::Disabled => (
             0,
-            gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb_size)?,
+            gsp::LibosParams::from_chipset(ctx.chipset).wpr_heap_size(ctx.chipset, fb_size)?,
         ),
         VgpuState::Enabled { total_vfs } => (
             u8::try_from(total_vfs.get()).map_err(|_| EINVAL)?,

@@ -328,10 +324,10 @@ pub(crate) struct FbSizes {
 
 impl FbSizes {
     /// Computes the framebuffer region sizes for GSP-FMC boot.
-    pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, vgpu_state: VgpuState) -> Result<Self> {
-        let hal = hal::fb_hal(chipset);
-        let fb_size = hal.vidmem_size(bar);
-        let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb_size)?;
+    pub(crate) fn new(ctx: &gsp::GspBootContext<'_, '_>) -> Result<Self> {
+        let hal = hal::fb_hal(ctx.chipset);
+        let fb_size = hal.vidmem_size(ctx.bar);
+        let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(ctx, fb_size)?;
 
         Ok(Self {
             frts_size: hal.frts_size(),

index 8848feebb9db..510d32212d41 100644
@@ -401,7 +401,7 @@ pub(crate) fn boot(
 
         dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(),);
 
-        let init_payload = commands::build_gsp_init_payload(pdev, chipset, ctx.vgpu.state())?;
+        let init_payload = commands::build_gsp_init_payload(ctx)?;
         let load_exec = LoadExecContext {
             bootloader: generic_bootloader.as_ref(),
             gsp_falcon,

index d12f90d94eb6..d6895dc5207e 100644
@@ -2,14 +2,12 @@
 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 use kernel::{
-    device,
     pci,
     prelude::*,
     transmute::AsBytes, //
 };
 
 use crate::{
-    gpu::Chipset,
     gsp::{
         cmdq::Cmdq,
         fw::{

@@ -30,6 +28,7 @@
             Encoder,
             UnknownKeyPolicy, //
         },
+        GspBootContext, //
     },
     sbuffer::SBufferIter,
     vgpu::VgpuState, //

@@ -42,28 +41,25 @@
 /// # Errors
 ///
 /// - `ENOMEM` if the request or the encoder buffer cannot be allocated.
-pub(crate) fn build_gsp_init_payload(
-    pdev: &pci::Device<device::Bound>,
-    chipset: Chipset,
-    vgpu_state: VgpuState,
-) -> Result<EncodedStream> {
+/// - `ENODEV` if vGPU mode is enabled but the SR-IOV capability is missing.
+///
+/// Errors reading the PCI configuration or decoding the VF BAR layout are propagated as-is.
+pub(super) fn build_gsp_init_payload(ctx: &GspBootContext<'_, '_>) -> Result<EncodedStream> {
     let mut encoder = Encoder::new();
-    let vf_info = build_vf_info(pdev, vgpu_state)?;
-    GspInitRequest::new(pdev, chipset, vgpu_state, vf_info)?.encode(&mut encoder)?;
+    let vf_info = build_vf_info(ctx)?;
+    GspInitRequest::new(ctx.pdev, ctx.chipset, ctx.vgpu.state(), vf_info)?.encode(&mut encoder)?;
 
     Ok(encoder.finish())
 }
 
 /// Builds the optional VF topology portion of the `GSP_INIT` request.
-fn build_vf_info(
-    pdev: &pci::Device<device::Bound>,
-    vgpu_state: VgpuState,
-) -> Result<Option<VfInfo>> {
-    let VgpuState::Enabled { total_vfs } = vgpu_state else {
+fn build_vf_info(ctx: &GspBootContext<'_, '_>) -> Result<Option<VfInfo>> {
+    let VgpuState::Enabled { total_vfs } = ctx.vgpu.state() else {
         return Ok(None);
     };
 
-    let sriov = pdev
+    let sriov = ctx
+        .pdev
         .config_space_extended()?
         .find_ext_capability::<pci::ExtSriovRegs>()?
         .ok_or(ENODEV)?;

index 91201b51030e..f91cdd061706 100644
@@ -151,7 +151,7 @@ fn boot<'gpu>(
         let chipset = ctx.chipset;
         let gsp_falcon = ctx.gsp_falcon;
 
-        let fb_sizes = FbSizes::new(chipset, ctx.bar, ctx.vgpu.state())?;
+        let fb_sizes = FbSizes::new(ctx)?;
         dev_dbg!(dev, "{:#x?}\n", fb_sizes);
 
         let wpr_meta =

index 265039b8ff55..c221797d6f3c 100644
@@ -260,7 +260,7 @@ fn boot<'gpu>(
         let gsp_falcon = ctx.gsp_falcon;
         let sec2_falcon = ctx.sec2_falcon;
 
-        let fb_ranges = FbRanges::new(chipset, bar, gsp_fw, ctx.vgpu.state())?;
+        let fb_ranges = FbRanges::new(ctx, gsp_fw)?;
         dev_dbg!(dev, "{:#x?}\n", fb_ranges);
 
         // Declared before the unload guard so that if Booter fails while running, SEC2 is reset

index 26e94d908b77..882827bcfd3a 100644
@@ -295,7 +295,7 @@ pub(crate) fn wpr2_range(bar: Bar0<'_>) -> Option<Range<u64>> {
 
 /// Computes the number of VF partitions and the WPR2 heap size from the vGPU state.
 fn wpr2_heap_params(ctx: &gsp::GspBootContext<'_, '_>, fb_size: u64) -> Result<(u8, u64)> {
-    Ok(match ctx.vgpu.state() {
+    Ok(match *ctx.vgpu_state {
         VgpuState::Disabled => (
             0,
             gsp::LibosParams::from_chipset(ctx.chipset).wpr_heap_size(ctx.chipset, fb_size)?,

index be212bbf724f..3272afdb7d1e 100644
@@ -372,7 +372,7 @@ fn drop(self: Pin<&mut Self>) {
                     gsp_falcon: &*this.gsp_falcon,
                     sec2_falcon: &*this.sec2_falcon,
                     fsp: this.fsp.as_mut(),
-                    vgpu: &*this.vgpu,
+                    vgpu_state: this.vgpu.state(),
                 },
                 bundle,
             )

@@ -445,7 +445,7 @@ pub(crate) fn new<'a>(
                     gsp_falcon,
                     sec2_falcon,
                     fsp: fsp.as_mut(),
-                    vgpu,
+                    vgpu_state: vgpu.state(),
                 })?,
             }),
 

index 5b57be7f2d42..3dbc37a3b35b 100644
@@ -66,7 +66,7 @@
         fw::GspArgumentsPadded, //
     },
     num,
-    vgpu::VgpuManager, //
+    vgpu::VgpuState, //
 };
 
 pub(crate) const GSP_PAGE_SHIFT: usize = 12;

@@ -85,7 +85,7 @@ pub(crate) struct GspBootContext<'ctx, 'gpu> {
     pub(crate) gsp_falcon: &'ctx Falcon<'gpu, GspFalcon>,
     pub(crate) sec2_falcon: &'ctx Falcon<'gpu, Sec2Falcon>,
     pub(crate) fsp: Option<&'ctx mut Fsp<'gpu>>,
-    pub(crate) vgpu: &'ctx VgpuManager,
+    pub(crate) vgpu_state: &'ctx VgpuState,
 }
 
 impl<'ctx, 'gpu> GspBootContext<'ctx, 'gpu> {

index d6895dc5207e..9473004a0921 100644
@@ -47,14 +47,14 @@
 pub(super) fn build_gsp_init_payload(ctx: &GspBootContext<'_, '_>) -> Result<EncodedStream> {
     let mut encoder = Encoder::new();
     let vf_info = build_vf_info(ctx)?;
-    GspInitRequest::new(ctx.pdev, ctx.chipset, ctx.vgpu.state(), vf_info)?.encode(&mut encoder)?;
+    GspInitRequest::new(ctx.pdev, ctx.chipset, *ctx.vgpu_state, vf_info)?.encode(&mut encoder)?;
 
     Ok(encoder.finish())
 }
 
 /// Builds the optional VF topology portion of the `GSP_INIT` request.
 fn build_vf_info(ctx: &GspBootContext<'_, '_>) -> Result<Option<VfInfo>> {
-    let VgpuState::Enabled { total_vfs } = ctx.vgpu.state() else {
+    let VgpuState::Enabled { total_vfs } = *ctx.vgpu_state else {
         return Ok(None);
     };
 

index 6b7e045acea8..905b3a7331fd 100644
@@ -85,7 +85,7 @@ fn detect_state(
     }
 
     /// Returns the detected vGPU state for this boot.
-    pub(crate) fn state(&self) -> VgpuState {
-        self.state
+    pub(crate) fn state(&self) -> &VgpuState {
+        &self.state
     }
 }

index 3272afdb7d1e..650bef8b5f44 100644
@@ -48,7 +48,7 @@
         GpuMm,
         VramAddress, //
     },
-    vgpu::VgpuManager, //
+    vgpu::VgpuState, //
 };
 
 #[cfg_attr(not(CONFIG_KUNIT = "y"), expect(dead_code))]

@@ -299,8 +299,7 @@ struct GspResources<'gpu> {
     // TODO: use different resource types for each boot method, and make the relevant Gsp methods
     // generic against them.
     fsp: Option<Fsp<'gpu>>,
-    /// vGPU state detected before GSP boot.
-    vgpu: VgpuManager,
+    vgpu_state: VgpuState,
     /// GSP runtime data.
     #[pin]
     gsp: Gsp<'gpu>,

@@ -372,7 +371,7 @@ fn drop(self: Pin<&mut Self>) {
                     gsp_falcon: &*this.gsp_falcon,
                     sec2_falcon: &*this.sec2_falcon,
                     fsp: this.fsp.as_mut(),
-                    vgpu_state: this.vgpu.state(),
+                    vgpu_state: this.vgpu_state,
                 },
                 bundle,
             )

@@ -431,7 +430,7 @@ pub(crate) fn new<'a>(
 
                 fsp: Fsp::try_new(dev, bar, spec.chipset)?,
 
-                vgpu: VgpuManager::new(pdev, spec.chipset, fsp.as_mut()),
+                vgpu_state: VgpuState::detect(pdev, spec.chipset, fsp.as_mut()),
 
                 gsp <- Gsp::new(pdev, *spec, bar),
 

@@ -445,7 +444,7 @@ pub(crate) fn new<'a>(
                     gsp_falcon,
                     sec2_falcon,
                     fsp: fsp.as_mut(),
-                    vgpu_state: vgpu.state(),
+                    vgpu_state,
                 })?,
             }),
 

index 905b3a7331fd..b405ba49c490 100644
@@ -30,19 +30,16 @@ pub(crate) enum VgpuState {
     },
 }
 
-/// vGPU state manager.
-pub(crate) struct VgpuManager {
-    state: VgpuState,
-}
-
-impl VgpuManager {
-    /// Creates a vGPU manager by querying SR-IOV and the FSP PRC vGPU knob.
-    pub(crate) fn new(
+impl VgpuState {
+    /// Detects the boot mode, falling back to disabled if querying the device fails.
+    ///
+    /// Call after creating the FSP and before allocating GSP firmware resources.
+    pub(crate) fn detect(
         pdev: &pci::Device<device::Core<'_>>,
         chipset: Chipset,
         fsp: Option<&mut Fsp<'_>>,
     ) -> Self {
-        let state = Self::detect_state(pdev, chipset, fsp).unwrap_or_else(|e| {
+        let state = Self::query_state(pdev, chipset, fsp).unwrap_or_else(|e| {
             dev_warn!(
                 pdev,
                 "vGPU state detection failed: {:?}; disabling vGPU\n",

@@ -51,12 +48,11 @@ pub(crate) fn new(
             VgpuState::Disabled
         });
         dev_dbg!(pdev, "vGPU state: {:?}\n", state);
-
-        Self { state }
+        state
     }
 
     /// Detects the vGPU state from the chipset, SR-IOV capability and FSP PRC knob.
-    fn detect_state(
+    fn query_state(
         pdev: &pci::Device<device::Core<'_>>,
         chipset: Chipset,
         fsp: Option<&mut Fsp<'_>>,

@@ -83,9 +79,4 @@ fn detect_state(
             VgpuMode::Disabled => Ok(VgpuState::Disabled),
         }
     }
-
-    /// Returns the detected vGPU state for this boot.
-    pub(crate) fn state(&self) -> &VgpuState {
-        &self.state
-    }
 }

index 650bef8b5f44..c239b6614d44 100644
@@ -48,6 +48,7 @@
         GpuMm,
         VramAddress, //
     },
+    num,
     vgpu::VgpuState, //
 };
 

@@ -56,6 +57,10 @@
 mod hal;
 mod regs;
 
+use self::channel::TOTAL_CHANNELS;
+
+pub(crate) use self::channel::ChannelIdPool;
+
 macro_rules! define_chipset {
     ({ $($variant:ident = $value:expr),* $(,)* }) =>
     {

@@ -335,6 +340,8 @@ pub(crate) struct Gpu<'gpu> {
     /// GSP and its resources.
     #[pin]
     gsp_resources: GspResources<'gpu>,
+    #[pin]
+    chid_pool: ChannelIdPool,
     /// System memory page required for flushing all pending GPU-side memory writes done through
     /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation).
     ///

@@ -417,6 +424,8 @@ pub(crate) fn new<'a>(
             // Initialize this early because `gsp_resources` depends on it.
             sysmem_flush: SysmemFlush::register(dev, bar, spec.chipset)?,
 
+            chid_pool <- ChannelIdPool::new(cv!(num::u32_as_usize(TOTAL_CHANNELS))),
+
             gsp_resources <- try_pin_init!(GspResources {
                 device: pdev,
 

index 485efaba059d..75666c6768e6 100644
@@ -21,6 +21,10 @@
     }, //
 };
 
+// TODO: Query the channel capacity through GMCAPI once an equivalent of
+// NV2080_CTRL_CMD_INTERNAL_FIFO_GET_NUM_CHANNELS is available.
+pub(super) const TOTAL_CHANNELS: u32 = 2048;
+
 /// Pool for tracking reservations of channel IDs.
 #[pin_data]
 pub(crate) struct ChannelIdPool {

index 9473004a0921..436ea52689b9 100644
@@ -126,7 +126,7 @@ pub(crate) fn gsp_init(
 /// # Errors
 ///
 /// - `EINVAL` if the payload is not a whole number of NVKV words, or if the stream is malformed
-///   or omits a required key.
+///   or omits a required key, or the FIFO engine count exceeds the supported table capacity.
 /// - `ENOMEM` if the words or the decoded regions cannot be allocated.
 fn decode_gsp_init_reply(payload_0: &[u8], payload_1: &[u8]) -> Result<GspStaticInfo> {
     const WORD_SIZE: usize = size_of::<u64>();

116 lines not on this page · raw message

index c239b6614d44..11d9986c4b19 100644
@@ -49,7 +49,10 @@
         VramAddress, //
     },
     num,
-    vgpu::VgpuState, //
+    vgpu::{
+        VgpuManager,
+        VgpuState, //
+    },
 };
 
 #[cfg_attr(not(CONFIG_KUNIT = "y"), expect(dead_code))]

@@ -323,6 +326,7 @@ fn static_info(&self) -> &gsp::commands::GspStaticInfo {
 #[pin_data]
 pub(crate) struct Gpu<'gpu> {
     spec: Spec,
+    vgpu: Option<VgpuManager<'gpu>>,
     /// GSP event interrupt registration.
     ///
     /// Must be kept declared *before* `gsp_resources`, so that the handler is unregistered, and

@@ -457,6 +461,24 @@ pub(crate) fn new<'a>(
                 })?,
             }),
 
+            vgpu: {
+                let info = &gsp_resources.boot_result.static_info;
+                match gsp_resources.vgpu_state {
+                    VgpuState::Disabled => None,
+                    VgpuState::Enabled { .. } => Some(VgpuManager::new(
+                        // SAFETY: `chid_pool` is initialized above at its final pinned address.
+                        // The private manager and its pool borrow cannot escape this `Gpu`.
+                        // Completed field drop order drops the manager before the pool; on failure,
+                        // pin-init drops it before the earlier-initialized pool.
+                        unsafe { &*core::ptr::from_ref(chid_pool.as_ref().get_ref()) },
+                        &info.fifo_engine_list(),
+                        info.vmmu_segment_size,
+                        TOTAL_CHANNELS,
+                    )),
+                }
+            },
+
+            // GSP boot left the SWGEN0 latch set and pending bits in the tree.
             _: {
                 irq::gsp::quiesce(bar, gsp_resources.spec.chipset, vectors_ref)?;
             },

index 436ea52689b9..97e32f81a39b 100644
@@ -34,7 +34,10 @@
     vgpu::VgpuState, //
 };
 
-pub(crate) use fw::commands::GspStaticInfo;
+pub(crate) use fw::commands::{
+    FifoEngineList,
+    GspStaticInfo, //
+};
 
 /// Builds the NVKV-encoded payload of a `GSP_INIT` request for `pdev`.
 ///

index 17ae4224e214..7381b30f3680 100644
@@ -394,8 +394,7 @@ pub(crate) struct GspStaticInfo {
     gpu_name: ArrayVec<u8, { Self::MAX_GPU_NAME_LEN }>,
     fb_regions: KVVec<FbRegion>,
     bar1_pde_base: u64,
-    #[cfg_attr(not(CONFIG_KUNIT = "y"), expect(dead_code))]
-    vmmu_segment_size: u64,
+    pub(crate) vmmu_segment_size: u64,
     fifo_engine_count: FifoEngineCount,
     fifo_engine_gmc_ids: [u32; MAX_FIFO_ENGINES],
     fifo_engine_flags: [u32; MAX_FIFO_ENGINES],

@@ -464,7 +463,6 @@ pub(crate) fn total_fb_end(&self) -> Option<u64> {
     }
 
     /// Returns the host-driven engines in their firmware FIFO order.
-    #[expect(dead_code)]
     pub(crate) fn fifo_engine_list(&self) -> FifoEngineList {
         // INVARIANT: The list starts empty and appends at most one ID per supported input slot.
         let mut fifo_engine_list = FifoEngineList {

index b405ba49c490..5cd82adeb0f8 100644
@@ -13,7 +13,11 @@
         Fsp,
         VgpuMode, //
     },
-    gpu::Chipset, //
+    gpu::{
+        ChannelIdPool,
+        Chipset, //
+    },
+    gsp::commands::FifoEngineList, //
 };
 
 mod hal;

@@ -80,3 +84,33 @@ fn query_state(
         }
     }
 }
+
+/// Runtime resources for an enabled vGPU boot.
+pub(crate) struct VgpuManager<'gpu> {
+    #[expect(dead_code)]
+    chid_pool: &'gpu ChannelIdPool,
+    /// VMMU segment size in bytes, or zero if GSP-RM omitted it.
+    #[expect(dead_code)]
+    vmmu_segment_size: u64,
+    #[expect(dead_code)]
+    total_channels: u32,
+    #[expect(dead_code)]
+    fifo_engine_list: FifoEngineList,
+}
+
+impl<'gpu> VgpuManager<'gpu> {
+    /// Retains runtime parameters from a completed vGPU-enabled GSP boot.
+    pub(crate) fn new(
+        chid_pool: &'gpu ChannelIdPool,
+        fifo_engine_list: &FifoEngineList,
+        vmmu_segment_size: u64,
+        total_channels: u32,
+    ) -> Self {
+        Self {
+            chid_pool,
+            vmmu_segment_size,
+            total_channels,
+            fifo_engine_list: *fifo_engine_list,
+        }
+    }
+}

index 882827bcfd3a..ba68a812b095 100644
@@ -302,7 +302,7 @@ fn wpr2_heap_params(ctx: &gsp::GspBootContext<'_, '_>, fb_size: u64) -> Result<(
         ),
         VgpuState::Enabled { total_vfs } => (
             u8::try_from(total_vfs.get()).map_err(|_| EINVAL)?,
-            gsp::LibosParams::vgpu_wpr_heap_size(),
+            gsp::LibosParams::vgpu_wpr_heap_size(total_vfs.get()),
         ),
     })
 }

index 717ba1cc4bf3..90bb324666e5 100644
@@ -133,8 +133,11 @@ pub(crate) fn from_chipset(chipset: Chipset) -> &'static LibosParams {
     }
 
     /// Returns the WPR heap size to reserve when vGPU is enabled.
-    pub(crate) fn vgpu_wpr_heap_size() -> u64 {
-        u64::from(bindings::GSP_FW_HEAP_SIZE_VGPU_DEFAULT)
+    pub(crate) fn vgpu_wpr_heap_size(total_vfs: u16) -> u64 {
+        u64::from(match total_vfs {
+            2..=32 => bindings::GSP_FW_HEAP_SIZE_VGPU_DEFAULT,
+            _ => r000_00::GSP_FW_HEAP_SIZE_VGPU_48VMS,
+        })
     }
 
     /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size

index 454d0ea9360f..fa17d3e2edf6 100644
@@ -84,6 +84,7 @@ impl<T> ::core::cmp::Eq for __BindgenUnionField<T> {}
 pub const GSP_FW_HEAP_PARAM_SIZE_PER_GB: u32 = 98304;
 pub const GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE: u32 = 100663296;
 pub const GSP_FW_HEAP_SIZE_VGPU_DEFAULT: u32 = 609222656;
+pub const GSP_FW_HEAP_SIZE_VGPU_48VMS: u32 = 1436549120;
 pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB: u32 = 64;
 pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB: u32 = 256;
 pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB: u32 = 88;

index 11d9986c4b19..97b33d4b2631 100644
@@ -16,7 +16,6 @@
         SizeConstants,
         SZ_4K, //
     },
-    sync::Arc,
 };
 
 use crate::{

@@ -340,7 +339,8 @@ pub(crate) struct Gpu<'gpu> {
     /// the GSP is still operational.
     mm: GpuMm<'gpu>,
     /// BAR1 user interface for CPU access to GPU virtual memory.
-    bar_user: Arc<BarUser<'gpu>>,
+    #[pin]
+    bar_user: BarUser<'gpu>,
     /// GSP and its resources.
     #[pin]
     gsp_resources: GspResources<'gpu>,

@@ -545,18 +545,15 @@ pub(crate) fn new<'a>(
             },
 
             // Create BAR1 user interface for CPU access to GPU virtual memory.
-            bar_user: {
+            bar_user <- {
                 let pdb_addr = VramAddress::from_raw(gsp_resources.static_info().bar1_pde_base());
                 let bar1_idx = crate::driver::bar1_resource_index(pdev)?;
                 let bar1_size = pdev.resource_len(bar1_idx)?;
-                Arc::pin_init(
-                    BarUser::new(
-                        pdb_addr,
-                        gsp_resources.spec.chipset,
-                        bar1_size,
-                        bar1,
-                    )?,
-                    GFP_KERNEL,
+                BarUser::new(
+                    pdb_addr,
+                    gsp_resources.spec.chipset,
+                    bar1_size,
+                    bar1,
                 )?
             },
         })

@@ -573,7 +570,7 @@ pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device<device::Bou
             dev,
             this.mm,
             info.usable_fb_regions(),
-            this.bar_user,
+            this.bar_user.as_ref().get_ref(),
             info.bar1_pde_base(),
             this.spec.chipset,
         ) {

index ea85c821f0e2..a835180db6c5 100644
@@ -298,8 +298,7 @@ pub(crate) mod selftest {
 
     use kernel::{
         device,
-        sizes::SizeConstants,
-        sync::Arc, //
+        sizes::SizeConstants, //
     };
 
     use super::*;

@@ -309,7 +308,7 @@ pub(crate) fn run(
         dev: &device::Device<device::Bound>,
         mm: &mut GpuMm<'_>,
         mut usable_fb_regions: impl Iterator<Item = Range<u64>>,
-        bar_user: &Arc<bar_user::BarUser<'_>>,
+        bar_user: &bar_user::BarUser<'_>,
         bar1_pdb: u64,
         chipset: Chipset,
     ) -> Result {

index cd2d0271e64c..414c69e037ee 100644
@@ -7,10 +7,7 @@
     io::Io,
     new_mutex,
     prelude::*,
-    sync::{
-        Arc,
-        Mutex, //
-    },
+    sync::Mutex, //
 };
 
 use crate::{

@@ -60,12 +57,12 @@ pub(crate) fn new(
     }
 
     /// Map physical pages to a contiguous BAR1 virtual range.
-    pub(crate) fn map(
-        self: &Arc<Self>,
+    pub(crate) fn map<'access>(
+        &'access self,
         mm: &mut GpuMm<'_>,
         pfns: &[Pfn],
         writable: bool,
-    ) -> Result<BarUserAccess<'gpu>> {
+    ) -> Result<BarUserAccess<'access, 'gpu>> {
         if pfns.is_empty() {
             return Err(EINVAL);
         }

@@ -73,21 +70,21 @@ pub(crate) fn map(
         let mapped = vmm.map_pages(mm, pfns, None, writable)?;
 
         Ok(BarUserAccess {
-            bar_user: self.clone(),
+            bar_user: self,
             mapped: Some(mapped),
         })
     }
 }
 
 /// Access object for a mapped BAR1 region.
-pub(crate) struct BarUserAccess<'gpu> {
-    bar_user: Arc<BarUser<'gpu>>,
+pub(crate) struct BarUserAccess<'access, 'gpu> {
+    bar_user: &'access BarUser<'gpu>,
     /// Cleared only after PTE invalidation and its TLB flush succeed.
     mapped: Option<MappedRange>,
 }
 
 #[expect(dead_code)]
-impl BarUserAccess<'_> {
+impl BarUserAccess<'_, '_> {
     /// Tear down the BAR1 mapping.
     pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result {
         self.unmap(mm)

@@ -167,7 +164,7 @@ pub(crate) fn try_write64(&self, value: u64, offset: usize) -> Result {
     }
 }
 
-impl Drop for BarUserAccess<'_> {
+impl Drop for BarUserAccess<'_, '_> {
     fn drop(&mut self) {
         if self.mapped.is_some() {
             kernel::pr_warn!(

@@ -185,10 +182,10 @@ fn drop(&mut self) {
 /// address space. Uses the `GpuMm`'s buddy allocator to allocate page tables
 /// and test pages as needed.
 #[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
-pub(crate) fn run_self_test(
+pub(super) fn run_self_test(
     dev: &device::Device<device::Bound>,
     mm: &mut GpuMm<'_>,
-    bar_user: &Arc<BarUser<'_>>,
+    bar_user: &BarUser<'_>,
     bar1_pdb: u64,
     chipset: Chipset,
 ) -> Result {

@@ -401,7 +398,7 @@ pub(crate) fn run_self_test(
     drop(vmm);
 
     // Test 4: Exercise `BarUser::map()` end-to-end.
-    let bar_user = Arc::pin_init(
+    let bar_user = KBox::pin_init(
         BarUser::new(pdb_addr, chipset, SZ_64K.into_safe_cast(), bar1)?,
         GFP_KERNEL,
     )?;

index a835180db6c5..a1e0bbffc460 100644
@@ -67,6 +67,8 @@ macro_rules! impl_pfn_bounded {
 mod regs;
 pub(super) mod tlb;
 pub(super) mod vmm;
+#[expect(dead_code)]
+pub(crate) mod vram;
 
 /// GPU Memory Manager - owns all core MM components.
 ///

75 lines not on this page · raw message

index b6cc177825d7..bed90a3de8bd 100644
@@ -34,6 +34,62 @@ impl VramBlock {
     pub(crate) const fn address(&self) -> u64 {
         self.address
     }
+
+    /// Creates a byte range relative to this allocation, retaining its backing storage.
+    pub(crate) fn region(self: &Arc<Self>, range: Range<u64>) -> Result<VramRegion> {
+        VramRegion::new(self.clone(), range)
+    }
+}
+
+/// A byte range that keeps its VRAM allocation alive.
+#[derive(Clone)]
+pub(crate) struct VramRegion {
+    backing: Arc<VramBlock>,
+    address: u64,
+    size: u64,
+}
+
+impl VramRegion {
+    fn new(backing: Arc<VramBlock>, range: Range<u64>) -> Result<Self> {
+        if range.start >= range.end || range.end > backing.size {
+            return Err(EINVAL);
+        }
+
+        let size = range.end - range.start;
+        let address = backing.address.checked_add(range.start).ok_or(EOVERFLOW)?;
+        backing.address.checked_add(range.end).ok_or(EOVERFLOW)?;
+
+        Ok(Self {
+            backing,
+            address,
+            size,
+        })
+    }
+
+    pub(crate) const fn address(&self) -> u64 {
+        self.address
+    }
+
+    pub(crate) const fn size(&self) -> u64 {
+        self.size
+    }
+
+    /// Creates a nonempty subregion relative to this view's start.
+    pub(crate) fn subregion(&self, range: Range<u64>) -> Result<Self> {
+        if range.start >= range.end || range.end > self.size {
+            return Err(EINVAL);
+        }
+
+        let size = range.end - range.start;
+        let address = self.address.checked_add(range.start).ok_or(EOVERFLOW)?;
+        address.checked_add(size).ok_or(EOVERFLOW)?;
+
+        Ok(Self {
+            backing: self.backing.clone(),
+            address,
+            size,
+        })
+    }
 }
 
 impl GpuMm<'_> {

index 414c69e037ee..964aee8bfdb1 100644
@@ -7,6 +7,10 @@
     io::Io,
     new_mutex,
     prelude::*,
+    ptr::{
+        Alignable,
+        Alignment, //
+    },
     sync::Mutex, //
 };
 

@@ -18,6 +22,7 @@
             MappedRange,
             Vmm, //
         },
+        vram::VramRegion,
         GpuMm,
         Pfn,
         Vfn,

@@ -176,6 +181,146 @@ fn drop(&mut self) {
     }
 }
 
+/// A BAR mapping that retains its backing VRAM region.
+#[must_use]
+pub(crate) struct BarMapping<'map, 'gpu> {
+    access: BarUserAccess<'map, 'gpu>,
+    mm: &'map Mutex<GpuMm<'gpu>>,
+    region: VramRegion,
+    region_offset: usize,
+    region_size: usize,
+    unmap_error: Option<Error>,
+}
+
+#[expect(dead_code)]
+impl<'map, 'gpu> BarMapping<'map, 'gpu> {
+    /// Maps the containing pages while restricting CPU access to the requested byte range.
+    pub(crate) fn new(
+        bar_user: &'map BarUser<'gpu>,
+        mm: &'map Mutex<GpuMm<'gpu>>,
+        region: VramRegion,
+        writable: bool,
+    ) -> Result<Self> {
+        let page_size: u64 = PAGE_SIZE.into_safe_cast();
+        let page_align = Alignment::new::<PAGE_SIZE>();
+
+        let region_start = region.address();
+        let region_end = region_start.checked_add(region.size()).ok_or(EOVERFLOW)?;
+
+        let map_start = region_start.align_down(page_align);
+        let map_end = region_end.align_up(page_align).ok_or(EOVERFLOW)?;
+        let map_size = map_end.checked_sub(map_start).ok_or(EINVAL)?;
+        let num_pages = usize::try_from(map_size / page_size).map_err(|_| EOVERFLOW)?;
+        if num_pages == 0 {
+            return Err(EINVAL);
+        }
+
+        let region_offset = usize::try_from(region_start - map_start).map_err(|_| EOVERFLOW)?;
+        let region_size = usize::try_from(region.size()).map_err(|_| EOVERFLOW)?;
+
+        let mut pfns = KVec::with_capacity(num_pages, GFP_KERNEL)?;
+        for page in 0..num_pages {
+            let page = u64::try_from(page).map_err(|_| EOVERFLOW)?;
+            let byte_offset = page.checked_mul(page_size).ok_or(EOVERFLOW)?;
+            let address = map_start.checked_add(byte_offset).ok_or(EOVERFLOW)?;
+            pfns.push(Pfn::from(VramAddress::from_raw(address)), GFP_KERNEL)?;
+        }
+
+        let access = bar_user.map(&mut mm.lock(), &pfns, writable)?;
+
+        Ok(Self {
+            access,
+            mm,
+            region,
+            region_offset,
+            region_size,
+            unmap_error: None,
+        })
+    }
+
+    pub(crate) fn region(&self) -> &VramRegion {
+        &self.region
+    }
+
+    pub(crate) fn gpu_va_addr(&self) -> Result<u64> {
+        self.access()?
+            .base()
+            .into_raw()
+            .checked_add(u64::try_from(self.region_offset).map_err(|_| EOVERFLOW)?)
+            .ok_or(EOVERFLOW)
+    }
+
+    pub(crate) const fn size(&self) -> usize {
+        self.region_size
+    }
+
+    fn access(&self) -> Result<&BarUserAccess<'map, 'gpu>> {
+        if self.access.mapped.is_none() {
+            return Err(ENODEV);
+        }
+        if let Some(error) = self.unmap_error {
+            return Err(error);
+        }
+        Ok(&self.access)
+    }
+
+    fn access_offset(&self, offset: usize, width: usize) -> Result<usize> {
+        let region_end = offset.checked_add(width).ok_or(EOVERFLOW)?;
+        if region_end > self.region_size {
+            return Err(EINVAL);
+        }
+
+        let access_offset = self.region_offset.checked_add(offset).ok_or(EOVERFLOW)?;
+        if !access_offset.is_multiple_of(width) {
+            return Err(EINVAL);
+        }
+
+        Ok(access_offset)
+    }
+
+    pub(crate) fn try_read32(&self, offset: usize) -> Result<u32> {
+        self.access()?
+            .try_read32(self.access_offset(offset, size_of::<u32>())?)
+    }
+
+    pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result {
+        self.access()?
+            .try_write32(value, self.access_offset(offset, size_of::<u32>())?)
+    }
+
+    pub(crate) fn try_write64(&self, value: u64, offset: usize) -> Result {
+        self.access()?
+            .try_write64(value, self.access_offset(offset, size_of::<u64>())?)
+    }
+
+    /// Unmap while retaining the mapping and its backing storage on failure.
+    ///
+    /// The first error is retained; later calls do not retry the operation.
+    pub(crate) fn unmap(&mut self) -> Result {
+        if let Some(error) = self.unmap_error {
+            return Err(error);
+        }
+        let result = self.access.unmap(&mut self.mm.lock());
+        if let Err(error) = result {
+            self.unmap_error = Some(error);
+        }
+        result
+    }
+}
+
+impl Drop for BarMapping<'_, '_> {
+    /// Drop unmaps the region and may sleep. Call [`Self::unmap`] to observe errors;
+    /// on failure the owner must retain this object until device teardown.
+    fn drop(&mut self) {
+        if self.unmap_error.is_some() {
+            return;
+        }
+        if let Err(error) = self.unmap() {
+            kernel::pr_err!("failed to unmap BAR1 region: {:?}\n", error);
+        }
+    }
+}
+
 /// Run MM subsystem self-tests during probe.
 ///
 /// Tests page table infrastructure and `BAR1` MMIO access using the `BAR1`

200 lines not on this page · raw message

index 90bb324666e5..71732418e7a5 100644
@@ -814,7 +814,7 @@ pub(crate) fn status(&self) -> u32 {
     /// sequence number `sequence`.
     pub(crate) fn is_response_to(&self, command_id: u32, sequence: u32) -> bool {
         self.is_response()
-            && self.command_id() == command_id
+            && self.command_id() == (command_id & GMCAPI_COMMAND_ID_MASK)
             && self.sequence == u64::from(sequence)
     }
 }

@@ -911,8 +911,8 @@ impl GspGmcMsgElement {
     /// Creates the queue element header and the GMC API header of a request that carries
     /// `payload_size` bytes of payload under the RPC sequence number `sequence`.
     ///
-    /// `max_response_size` is the largest response that the sender accepts, and zero for a request
-    /// that GSP-RM does not answer.
+    /// `max_response_size` is the response payload capacity in bytes. Zero permits a status-only
+    /// response; whether GSP-RM sends a response at all is determined by the command's protocol.
     ///
     /// # Errors
     ///

61 lines not on this page · raw message

index 71732418e7a5..a416826737bb 100644
@@ -802,6 +802,13 @@ pub(crate) fn sequence_number(&self) -> u64 {
         self.sequence & !GMC_EVENT_SEQUENCE_BASE
     }
 
+    /// Returns the raw `max_resp_or_status` word carried by a GMC header.
+    ///
+    /// Its meaning for an event is defined by that event's command.
+    pub(crate) fn raw_status_word(&self) -> u32 {
+        self.max_resp_or_status
+    }
+
     /// Returns the `NV_STATUS` that a response carries.
     ///
     /// The value is meaningful only when [`Self::is_response`] is `true`. In a request, the same

index fa17d3e2edf6..3cc718bb2ab8 100644
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 #[repr(C)]
 #[derive(Default)]

@@ -850,3 +851,175 @@ fn default() -> Self {
         }
     }
 }
+pub const GSP_PLUGIN_BOOTLOADED: u32 = 1315261039;
+pub const VGPU_CPU_GSP_CTRL_BUFF_VERSION: u32 = 2;
+pub const VGPU_CPU_GSP_CTRL_BUFF_REGION_SIZE: u32 = 4096;
+pub const VGPU_CPU_GSP_RESPONSE_BUFF_REGION_SIZE: u32 = 4096;
+pub const VGPU_CPU_GSP_MESSAGE_BUFF_REGION_SIZE: u32 = 4096;
+pub const VGPU_CPU_GSP_MIGRATION_BUFF_REGION_SIZE: u32 = 2097152;
+pub const VGPU_CPU_GSP_ERROR_BUFF_REGION_SIZE: u32 = 4096;
+pub const VGPU_CPU_GSP_INIT_TASK_LOG_BUFF_REGION_SIZE: u32 = 131072;
+pub const VGPU_CPU_GSP_VGPU_TASK_LOG_BUFF_REGION_SIZE: u32 = 262144;
+pub const VGPU_CPU_GSP_KERNEL_TASK_LOG_BUFF_REGION_SIZE: u32 = 65536;
+pub const VGPU_CPU_GSP_GUEST_RPC_TRACE_BUFF_REGION_SIZE: u32 = 65536;
+pub const VGPU_CPU_GSP_COMMUNICATION_BUFF_TOTAL_SIZE: u32 = 2637824;
+pub type VGPU_CPU_GSP_BOOL = u32_;
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
+pub struct VGPU_CPU_GSP_VGX_VERSION {
+    pub major_number: u32_,
+    pub minor_number: u32_,
+}
+#[repr(C)]
+#[derive(Debug, Copy, Clone, MaybeZeroable)]
+pub struct VGPU_CPU_GSP_GUEST_INFO {
+    pub vgx_version: VGPU_CPU_GSP_VGX_VERSION,
+    pub guest_driver_version_buffer_length: u32_,
+    pub guest_version_buffer_length: u32_,
+    pub guest_title_buffer_length: u32_,
+    pub guest_changelist_number: u32_,
+    pub guest_driver_version_buffer: [ffi::c_char; 256usize],
+    pub guest_version_buffer: [ffi::c_char; 256usize],
+    pub guest_title_buffer: [ffi::c_char; 256usize],
+    pub guest_branch_buffer: [ffi::c_char; 256usize],
+}
+impl Default for VGPU_CPU_GSP_GUEST_INFO {
+    fn default() -> Self {
+        let mut s = ::core::mem::MaybeUninit::<Self>::uninit();
+        unsafe {
+            ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1);
+            s.assume_init()
+        }
+    }
+}
+#[repr(C)]
+#[derive(Copy, Clone, MaybeZeroable)]
+pub union VGPU_CPU_GSP_CTRL_BUFF_REGION {
+    pub buf: [u8_; 4096usize],
+    pub __bindgen_anon_1: VGPU_CPU_GSP_CTRL_BUFF_REGION__bindgen_ty_1,
+}
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
+pub struct VGPU_CPU_GSP_CTRL_BUFF_REGION__bindgen_ty_1 {
+    pub version: u32_,
+    pub message_type: u32_,
+    pub message_seq_num: u32_,
+    pub __bindgen_padding_0: [u8; 4usize],
+    pub response_buff_offset: u64_,
+    pub message_buff_offset: u64_,
+    pub migration_buff_offset: u64_,
+    pub error_buff_offset: u64_,
+    pub guest_rpc_trace_buff_offset: u64_,
+    pub migration_buf_cpu_access_offset: u32_,
+    pub is_migration_in_progress: u8_,
+    pub __bindgen_padding_1: [u8; 3usize],
+    pub error_buff_cpu_get_idx: u32_,
+    pub guest_rpc_trace_buff_cpu_get_idx: u32_,
+    pub attached_vgpu_count: u32_,
+    pub is_gr_init_done: u8_,
+    pub __bindgen_padding_2: [u8; 3usize],
+    pub host_info: [VGPU_CPU_GSP_CTRL_BUFF_REGION__bindgen_ty_1__bindgen_ty_1; 16usize],
+}
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
+pub struct VGPU_CPU_GSP_CTRL_BUFF_REGION__bindgen_ty_1__bindgen_ty_1 {
+    pub vgpu_type_id: u32_,
+    pub host_gpu_pci_id: u32_,
+    pub pci_dev_id: u32_,
+    pub vgpu_uuid: [u8_; 16usize],
+}
+impl Default for VGPU_CPU_GSP_CTRL_BUFF_REGION {
+    fn default() -> Self {
+        let mut s = ::core::mem::MaybeUninit::<Self>::uninit();
+        unsafe {
+            ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1);
+            s.assume_init()
+        }
+    }
+}
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_VERSION_NEGOTIATION: MESSAGE = 1;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_SETUP_CONFIG_PARAMS_AND_INIT: MESSAGE = 2;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_RESET: MESSAGE = 3;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_MIGRATION_STOP_WORK: MESSAGE = 4;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_MIGRATION_CANCEL_STOP: MESSAGE = 5;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_MIGRATION_SAVE_STATE: MESSAGE = 6;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_MIGRATION_CANCEL_SAVE: MESSAGE = 7;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_MIGRATION_RESTORE_STATE: MESSAGE = 8;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_MIGRATION_RESTORE_DEFERRED_STATE: MESSAGE = 9;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_MIGRATION_RESUME_WORK: MESSAGE = 10;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_CONSOLE_VNC_STATE: MESSAGE = 11;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_VF_BAR0_REG_ACCESS: MESSAGE = 12;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_UPDATE_BME_STATE: MESSAGE = 13;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_RESET_MIGRATION_BUFFER_PTR: MESSAGE = 14;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_RELEASE_CLIENT_DATABASE: MESSAGE = 15;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_CHECK_IS_ALIVE: MESSAGE = 16;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_SEND_STATIC_INFO: MESSAGE = 17;
+pub const MESSAGE_NV_VGPU_CPU_RPC_MSG_MAX: MESSAGE = 18;
+pub type MESSAGE = ffi::c_uint;
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
+pub struct VGPU_CPU_GSP_DISPLAYLESS_SURFACE {
+    pub sequence_update_start: u64_,
+    pub sequence_update_end: u64_,
+    pub effective_fb_page_size: u32_,
+    pub rect_width: u32_,
+    pub rect_height: u32_,
+    pub surface_width: u32_,
+    pub surface_height: u32_,
+    pub surface_size: u32_,
+    pub surface_offset: u32_,
+    pub surface_format: u32_,
+    pub surface_kind: u32_,
+    pub surface_pitch: u32_,
+    pub surface_type: u32_,
+    pub surface_block_height: u8_,
+    pub __bindgen_padding_0: [u8; 3usize],
+    pub is_blanking_enabled: VGPU_CPU_GSP_BOOL,
+    pub is_flip_pending: VGPU_CPU_GSP_BOOL,
+    pub is_free_pending: VGPU_CPU_GSP_BOOL,
+    pub is_memory_blocklinear: VGPU_CPU_GSP_BOOL,
+}
+#[repr(C)]
+#[derive(Copy, Clone, MaybeZeroable)]
+pub union VGPU_CPU_GSP_RESPONSE_BUFF_REGION {
+    pub buf: [u8_; 4096usize],
+    pub __bindgen_anon_1: VGPU_CPU_GSP_RESPONSE_BUFF_REGION__bindgen_ty_1,
+}
+#[repr(C)]
+#[derive(Debug, Copy, Clone, MaybeZeroable)]
+pub struct VGPU_CPU_GSP_RESPONSE_BUFF_REGION__bindgen_ty_1 {
+    pub message_seq_num_received: u32_,
+    pub message_seq_num_processed: u32_,
+    pub result_code: u32_,
+    pub guest_rpc_version: u32_,
+    pub migration_buf_gsp_access_offset: u32_,
+    pub migration_state_save_complete: u32_,
+    pub is_migration_allowed: VGPU_CPU_GSP_BOOL,
+    pub __bindgen_padding_0: [u8; 4usize],
+    pub surface: [VGPU_CPU_GSP_DISPLAYLESS_SURFACE; 4usize],
+    pub error_buff_gsp_put_idx: u32_,
+    pub grid_license_state: u32_,
+    pub guest_os_type: u32_,
+    pub frl_config: u32_,
+    pub guest_info: VGPU_CPU_GSP_GUEST_INFO,
+    pub is_guest_info_populated: VGPU_CPU_GSP_BOOL,
+    pub guest_rpc_trace_buff_gsp_put_idx: u32_,
+}
+impl Default for VGPU_CPU_GSP_RESPONSE_BUFF_REGION__bindgen_ty_1 {
+    fn default() -> Self {
+        let mut s = ::core::mem::MaybeUninit::<Self>::uninit();
+        unsafe {
+            ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1);
+            s.assume_init()
+        }
+    }
+}
+impl Default for VGPU_CPU_GSP_RESPONSE_BUFF_REGION {
+    fn default() -> Self {
+        let mut s = ::core::mem::MaybeUninit::<Self>::uninit();
+        unsafe {
+            ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1);
+            s.assume_init()
+        }
+    }
+}

index 5cd82adeb0f8..1402e37541b7 100644
@@ -21,6 +21,7 @@
 };
 
 mod hal;
+mod vram;
 
 /// vGPU state detected during GPU construction.
 #[derive(Debug, Clone, Copy)]

181 lines not on this page · raw message

index 3dbc37a3b35b..25b4bf6eb848 100644
@@ -28,7 +28,7 @@
 pub(crate) mod cmdq;
 pub(crate) mod commands;
 mod fw;
-mod nvkv;
+pub(crate) mod nvkv;
 mod regs;
 
 pub(crate) use fw::{

index 97e32f81a39b..742e46efd413 100644
@@ -22,6 +22,7 @@
             GMCAPI_CMD_GSP_SUSPEND, //
         },
         nvkv::{
+            nvkv_words,
             Decoder,
             Encodable,
             EncodedStream,

@@ -30,7 +31,6 @@
         },
         GspBootContext, //
     },
-    sbuffer::SBufferIter,
     vgpu::VgpuState, //
 };
 

@@ -132,20 +132,7 @@ pub(crate) fn gsp_init(
 ///   or omits a required key, or the FIFO engine count exceeds the supported table capacity.
 /// - `ENOMEM` if the words or the decoded regions cannot be allocated.
 fn decode_gsp_init_reply(payload_0: &[u8], payload_1: &[u8]) -> Result<GspStaticInfo> {
-    const WORD_SIZE: usize = size_of::<u64>();
-
-    let len = payload_0.len() + payload_1.len();
-    if len % WORD_SIZE != 0 {
-        return Err(EINVAL);
-    }
-
-    let mut words = KVVec::with_capacity(len / WORD_SIZE, GFP_KERNEL)?;
-    let mut bytes = SBufferIter::new_reader([payload_0, payload_1]);
-    for _ in 0..len / WORD_SIZE {
-        let mut word = [0u8; WORD_SIZE];
-        bytes.read_exact(&mut word)?;
-        words.push(u64::from_le_bytes(word), GFP_KERNEL)?;
-    }
+    let words = nvkv_words(payload_0, payload_1)?;
 
     let decoder = Decoder::new(&words, UnknownKeyPolicy::Ignore);
     let mut schema = GspInitResponseSchema::default();

index e7a9549919fb..05b1ba5a4f39 100644
@@ -27,6 +27,34 @@
 };
 use zerocopy::Immutable;
 
+use crate::sbuffer::SBufferIter;
+
+/// Joins the two halves of a wrapped payload into the `u64` words an NVKV stream is made of.
+///
+/// # Errors
+///
+/// - `EINVAL` if the combined length is not a whole number of words.
+/// - `ENOMEM` if the buffer cannot be allocated.
+pub(crate) fn nvkv_words(payload_0: &[u8], payload_1: &[u8]) -> Result<KVVec<u64>> {
+    const WORD_SIZE: usize = size_of::<u64>();
+
+    // Each byte slice is at most `isize::MAX` bytes, so their sum fits in `usize`.
+    let len = payload_0.len() + payload_1.len();
+    if len % WORD_SIZE != 0 {
+        return Err(EINVAL);
+    }
+
+    let mut words = KVVec::with_capacity(len / WORD_SIZE, GFP_KERNEL)?;
+    let mut bytes = SBufferIter::new_reader([payload_0, payload_1]);
+    for _ in 0..len / WORD_SIZE {
+        let mut word = [0u8; WORD_SIZE];
+        bytes.read_exact(&mut word)?;
+        words.push(u64::from_le_bytes(word), GFP_KERNEL)?;
+    }
+
+    Ok(words)
+}
+
 mod encode;
 pub(crate) use encode::*;
 

index 25b4bf6eb848..45645bc7c9d8 100644
@@ -32,6 +32,7 @@
 mod regs;
 
 pub(crate) use fw::{
+    r000_00 as bindings,
     GspFmcBootParams,
     GspFwWprMeta,
     LibosMemoryRegionInitArgument,

index 43038c7fe823..abd9edadff64 100644
@@ -551,10 +551,8 @@ fn payload_length(&self) -> Option<usize> {
 /// Response from a GMC API command.
 pub(crate) struct GmcResponse {
     /// Response status (`NV_STATUS` code). Zero means success.
-    #[expect(dead_code)]
     pub(crate) status: u32,
     /// Response payload copied out of the message queue.
-    #[expect(dead_code)]
     pub(crate) payload: KVec<u8>,
 }
 

@@ -708,7 +706,6 @@ pub(crate) fn send_gmc_no_wait(
     /// # Errors
     ///
     /// Returns the same errors as [`Self::send_gmc_and_receive_timeout`].
-    #[expect(dead_code)]
     pub(crate) fn send_gmc_and_receive(
         &self,
         command_id: u32,

index a416826737bb..aef283dda087 100644
@@ -2,7 +2,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 pub(crate) mod commands;
-mod r000_00;
+pub(crate) mod r000_00;
 
 // Alias to avoid repeating the version number with every use.
 use r000_00 as bindings;

index 1402e37541b7..d910445dd2b0 100644
@@ -20,6 +20,8 @@
     gsp::commands::FifoEngineList, //
 };
 
+mod commands;
+mod fw;
 mod hal;
 mod vram;
 

91 lines not on this page · raw message

new file mode 100644
index 000000000000..23097cb13121
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! vGPU firmware interface.
+//!
+//! Exposes command constants and communication buffer layout types from
+//! raw firmware bindings.
+
+pub(super) mod commands;
+
+use crate::gsp::bindings;
+
+pub(super) const GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE: u32 =
+    bindings::GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE;
+
+pub(super) const GMCAPI_CMD_QUERY_VGPU_PROPERTIES: u32 =
+    bindings::GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_VGPU_PROPERTIES;

95 lines not on this page · raw message

index 97b33d4b2631..b41a9921381d 100644
@@ -54,14 +54,16 @@
     },
 };
 
-#[cfg_attr(not(CONFIG_KUNIT = "y"), expect(dead_code))]
 mod channel;
 mod hal;
 mod regs;
 
 use self::channel::TOTAL_CHANNELS;
 
-pub(crate) use self::channel::ChannelIdPool;
+pub(crate) use self::channel::{
+    ChannelIdPool,
+    ChannelIdReservation, //
+};
 
 macro_rules! define_chipset {
     ({ $($variant:ident = $value:expr),* $(,)* }) =>

@@ -325,7 +327,7 @@ fn static_info(&self) -> &gsp::commands::GspStaticInfo {
 #[pin_data]
 pub(crate) struct Gpu<'gpu> {
     spec: Spec,
-    vgpu: Option<VgpuManager<'gpu>>,
+    vgpu: Option<Pin<KBox<VgpuManager<'gpu>>>>,
     /// GSP event interrupt registration.
     ///
     /// Must be kept declared *before* `gsp_resources`, so that the handler is unregistered, and

@@ -465,7 +467,7 @@ pub(crate) fn new<'a>(
                 let info = &gsp_resources.boot_result.static_info;
                 match gsp_resources.vgpu_state {
                     VgpuState::Disabled => None,
-                    VgpuState::Enabled { .. } => Some(VgpuManager::new(
+                    VgpuState::Enabled { .. } => Some(KBox::pin_init(VgpuManager::new(
                         // SAFETY: `chid_pool` is initialized above at its final pinned address.
                         // The private manager and its pool borrow cannot escape this `Gpu`.
                         // Completed field drop order drops the manager before the pool; on failure,

@@ -474,7 +476,7 @@ pub(crate) fn new<'a>(
                         &info.fifo_engine_list(),
                         info.vmmu_segment_size,
                         TOTAL_CHANNELS,
-                    )),
+                    ), GFP_KERNEL)?),
                 }
             },
 

index d910445dd2b0..fb4f1b5f7754 100644
@@ -4,8 +4,10 @@
 
 use kernel::{
     device,
+    new_mutex,
     pci,
-    prelude::*, //
+    prelude::*,
+    sync::Mutex, //
 };
 
 use crate::{

@@ -23,6 +25,7 @@
 mod commands;
 mod fw;
 mod hal;
+mod instance;
 mod vram;
 
 /// vGPU state detected during GPU construction.

@@ -88,16 +91,17 @@ fn query_state(
     }
 }
 
+use self::instance::VgpuInstances;
+
 /// Runtime resources for an enabled vGPU boot.
+#[pin_data]
 pub(crate) struct VgpuManager<'gpu> {
-    #[expect(dead_code)]
+    #[pin]
+    instances: Mutex<VgpuInstances<'gpu>>,
     chid_pool: &'gpu ChannelIdPool,
     /// VMMU segment size in bytes, or zero if GSP-RM omitted it.
-    #[expect(dead_code)]
     vmmu_segment_size: u64,
-    #[expect(dead_code)]
     total_channels: u32,
-    #[expect(dead_code)]
     fifo_engine_list: FifoEngineList,
 }
 

@@ -108,12 +112,14 @@ pub(crate) fn new(
         fifo_engine_list: &FifoEngineList,
         vmmu_segment_size: u64,
         total_channels: u32,
-    ) -> Self {
-        Self {
+    ) -> impl PinInit<Self> + use<'gpu> {
+        let fifo_engine_list = *fifo_engine_list;
+        pin_init!(Self {
+            instances <- new_mutex!(VgpuInstances::new(), "nova-core::vgpu-instances"),
             chid_pool,
             vmmu_segment_size,
             total_channels,
-            fifo_engine_list: *fifo_engine_list,
-        }
+            fifo_engine_list,
+        })
     }
 }

218 lines not on this page · raw message

index 65e6a945ea6f..faaeea106009 100644
@@ -116,7 +116,6 @@ pub(super) struct VgpuVramSlotAllocator {
     slot_bitmap: Arc<SlotBitmap>,
 }
 
-#[expect(dead_code)]
 impl VgpuVramSlotAllocator {
     pub(super) fn new(mm: &GpuMm<'_>, layout: VgpuVramLayout) -> Result<Self> {
         let layout = layout.validated()?;

index 05b1ba5a4f39..02a12bd831d6 100644
@@ -172,6 +172,17 @@ pub(crate) struct Array<T: Default + Copy, const N: usize, const KEY_ID: KeyId>
     vec: ArrayVec<T, N>,
 }
 
+impl<T: Default + Copy, const N: usize, const KEY_ID: KeyId> Array<T, N, KEY_ID> {
+    /// Creates an array filled from `values`.
+    ///
+    /// Fails with `EINVAL` if `values` is longer than `N`.
+    pub(crate) fn new(values: &[T]) -> Result<Self> {
+        let mut vec = ArrayVec::default();
+        vec.extend_from_slice(values)?;
+        Ok(Self { vec })
+    }
+}
+
 bitfield! {
     /// The op word that starts each NVKV operation.
     struct Op(u64) {

index 0047be65e8a9..1f4df24d8c9c 100644
@@ -6,6 +6,7 @@
 use kernel::prelude::*;
 
 use super::{
+    Array,
     EncodedStream,
     Index,
     Key,

@@ -145,6 +146,17 @@ fn encode(&self, encoder: &mut Encoder) -> Result {
     }
 }
 
+impl<T, const N: usize, const KEY_ID: KeyId> Encodable for Array<T, N, KEY_ID>
+where
+    T: Default + Copy,
+    for<'a> IndexedKey<&'a [T], KEY_ID>: Encodable,
+{
+    #[inline(always)]
+    fn encode(&self, encoder: &mut Encoder) -> Result {
+        IndexedKey::<&[T], KEY_ID>::new(Index::new::<0>(), self.vec.as_slice()).encode(encoder)
+    }
+}
+
 impl<T: Encodable> Encodable for Option<T> {
     #[inline(always)]
     fn encode(&self, encoder: &mut Encoder) -> Result {

189 lines not on this page · raw message

index a1e0bbffc460..72e96a1dde5a 100644
@@ -67,7 +67,6 @@ macro_rules! impl_pfn_bounded {
 mod regs;
 pub(super) mod tlb;
 pub(super) mod vmm;
-#[expect(dead_code)]
 pub(crate) mod vram;
 
 /// GPU Memory Manager - owns all core MM components.

index fb4f1b5f7754..024cb13694f4 100644
@@ -24,6 +24,7 @@
 
 mod commands;
 mod fw;
+mod gsp_plugin_comm;
 mod hal;
 mod instance;
 mod vram;

index 23097cb13121..aabe114b55c2 100644
@@ -10,6 +10,21 @@
 
 use crate::gsp::bindings;
 
+pub(super) use bindings::{
+    GSP_PLUGIN_BOOTLOADED,
+    VGPU_CPU_GSP_COMMUNICATION_BUFF_TOTAL_SIZE,
+    VGPU_CPU_GSP_CTRL_BUFF_REGION as RawControlRegion,
+    VGPU_CPU_GSP_CTRL_BUFF_REGION_SIZE,
+    VGPU_CPU_GSP_ERROR_BUFF_REGION_SIZE,
+    VGPU_CPU_GSP_GUEST_RPC_TRACE_BUFF_REGION_SIZE,
+    VGPU_CPU_GSP_INIT_TASK_LOG_BUFF_REGION_SIZE,
+    VGPU_CPU_GSP_KERNEL_TASK_LOG_BUFF_REGION_SIZE,
+    VGPU_CPU_GSP_MESSAGE_BUFF_REGION_SIZE,
+    VGPU_CPU_GSP_MIGRATION_BUFF_REGION_SIZE,
+    VGPU_CPU_GSP_RESPONSE_BUFF_REGION_SIZE,
+    VGPU_CPU_GSP_VGPU_TASK_LOG_BUFF_REGION_SIZE, //
+};
+
 pub(super) const GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE: u32 =
     bindings::GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE;
 

205 lines not on this page · raw message

index b41a9921381d..012c7e6abca6 100644
@@ -8,6 +8,7 @@
     fmt,
     gpu::buddy::GpuBuddyParams,
     io::Io,
+    new_mutex,
     num::Bounded,
     pci,
     prelude::*,

@@ -16,6 +17,7 @@
         SizeConstants,
         SZ_4K, //
     },
+    sync::Mutex,
 };
 
 use crate::{

@@ -327,6 +329,7 @@ fn static_info(&self) -> &gsp::commands::GspStaticInfo {
 #[pin_data]
 pub(crate) struct Gpu<'gpu> {
     spec: Spec,
+    /// Drops before the MM, BAR1 mappings and GSP needed for instance teardown.
     vgpu: Option<Pin<KBox<VgpuManager<'gpu>>>>,
     /// GSP event interrupt registration.
     ///

@@ -339,7 +342,8 @@ pub(crate) struct Gpu<'gpu> {
     ///
     /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while
     /// the GSP is still operational.
-    mm: GpuMm<'gpu>,
+    #[pin]
+    mm: Mutex<GpuMm<'gpu>>,
     /// BAR1 user interface for CPU access to GPU virtual memory.
     #[pin]
     bar_user: BarUser<'gpu>,

@@ -463,22 +467,7 @@ pub(crate) fn new<'a>(
                 })?,
             }),
 
-            vgpu: {
-                let info = &gsp_resources.boot_result.static_info;
-                match gsp_resources.vgpu_state {
-                    VgpuState::Disabled => None,
-                    VgpuState::Enabled { .. } => Some(KBox::pin_init(VgpuManager::new(
-                        // SAFETY: `chid_pool` is initialized above at its final pinned address.
-                        // The private manager and its pool borrow cannot escape this `Gpu`.
-                        // Completed field drop order drops the manager before the pool; on failure,
-                        // pin-init drops it before the earlier-initialized pool.
-                        unsafe { &*core::ptr::from_ref(chid_pool.as_ref().get_ref()) },
-                        &info.fifo_engine_list(),
-                        info.vmmu_segment_size,
-                        TOTAL_CHANNELS,
-                    ), GFP_KERNEL)?),
-                }
-            },
+
 
             // GSP boot left the SWGEN0 latch set and pending bits in the tree.
             _: {

@@ -529,7 +518,7 @@ pub(crate) fn new<'a>(
             },
 
             // Create GPU memory manager owning memory management resources.
-            mm: {
+            mm <- {
                 let info = gsp_resources.static_info();
                 let usable_vram = info.usable_fb_regions().next().ok_or(ENODEV)?;
                 let buddy_params = GpuBuddyParams {

@@ -538,12 +527,15 @@ pub(crate) fn new<'a>(
                     chunk_size: Alignment::new::<SZ_4K>(),
                 };
 
-                GpuMm::new(
-                    bar,
-                    gsp_resources.spec.chipset,
-                    buddy_params,
-                    VramAddress::from_raw(info.total_fb_end().ok_or(ENODEV)?),
-                )?
+                new_mutex!(
+                    GpuMm::new(
+                        bar,
+                        gsp_resources.spec.chipset,
+                        buddy_params,
+                        VramAddress::from_raw(info.total_fb_end().ok_or(ENODEV)?),
+                    )?,
+                    "nova-core::gpu-mm",
+                )
             },
 
             // Create BAR1 user interface for CPU access to GPU virtual memory.

@@ -558,6 +550,34 @@ pub(crate) fn new<'a>(
                     bar1,
                 )?
             },
+
+            vgpu: {
+                match gsp_resources.vgpu_state {
+                    VgpuState::Disabled => None,
+                    VgpuState::Enabled { .. } => {
+                        // SAFETY: These sibling fields are initialized at their final pinned
+                        // addresses. The private manager cannot escape this `Gpu`, and is dropped
+                        // before all its dependencies, both here on failure and on normal removal.
+                        let (cmdq, bar_user, mm, chid_pool) = unsafe {
+                            (
+                                &*core::ptr::from_ref(&gsp_resources.gsp.cmdq),
+                                &*core::ptr::from_ref(bar_user.as_ref().get_ref()),
+                                &*core::ptr::from_ref(mm.as_ref().get_ref()),
+                                &*core::ptr::from_ref(chid_pool.as_ref().get_ref()),
+                            )
+                        };
+                        Some(KBox::pin_init(VgpuManager::new(
+                            dev,
+                            cmdq,
+                            bar_user,
+                            mm,
+                            chid_pool,
+                            gsp_resources.static_info(),
+                            TOTAL_CHANNELS,
+                        ), GFP_KERNEL)?)
+                    }
+                }
+            },
         })
     }
 

@@ -567,10 +587,11 @@ pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device<device::Bou
         let this = self.project();
         let dev = pdev.as_ref();
         let info = this.gsp_resources.static_info();
+        let mut mm = this.mm.lock();
 
         if let Err(err) = crate::mm::selftest::run(
             dev,
-            this.mm,
+            &mut mm,
             info.usable_fb_regions(),
             this.bar_user.as_ref().get_ref(),
             info.bar1_pde_base(),

index 7381b30f3680..b1087d950b30 100644
@@ -335,7 +335,6 @@ pub(crate) struct FifoEngineList {
 }
 
 impl FifoEngineList {
-    #[expect(dead_code)]
     pub(crate) fn gmc_ids(&self) -> &[u32] {
         // PANIC: The type invariant bounds `count` by the array capacity.
         &self.gmc_ids[..self.count]

index 024cb13694f4..84c4d0b3839d 100644
@@ -19,7 +19,17 @@
         ChannelIdPool,
         Chipset, //
     },
-    gsp::commands::FifoEngineList, //
+    gsp::{
+        cmdq::Cmdq,
+        commands::{
+            FifoEngineList,
+            GspStaticInfo, //
+        }, //
+    },
+    mm::{
+        bar_user::BarUser,
+        GpuMm, //
+    }, //
 };
 
 mod commands;

@@ -99,6 +109,10 @@ fn query_state(
 pub(crate) struct VgpuManager<'gpu> {
     #[pin]
     instances: Mutex<VgpuInstances<'gpu>>,
+    dev: &'gpu device::Device<device::Bound>,
+    cmdq: &'gpu Cmdq<'gpu>,
+    bar_user: &'gpu BarUser<'gpu>,
+    mm: &'gpu Mutex<GpuMm<'gpu>>,
     chid_pool: &'gpu ChannelIdPool,
     /// VMMU segment size in bytes, or zero if GSP-RM omitted it.
     vmmu_segment_size: u64,

@@ -109,14 +123,22 @@ pub(crate) struct VgpuManager<'gpu> {
 impl<'gpu> VgpuManager<'gpu> {
     /// Retains runtime parameters from a completed vGPU-enabled GSP boot.
     pub(crate) fn new(
+        dev: &'gpu device::Device<device::Bound>,
+        cmdq: &'gpu Cmdq<'gpu>,
+        bar_user: &'gpu BarUser<'gpu>,
+        mm: &'gpu Mutex<GpuMm<'gpu>>,
         chid_pool: &'gpu ChannelIdPool,
-        fifo_engine_list: &FifoEngineList,
-        vmmu_segment_size: u64,
+        info: &GspStaticInfo,
         total_channels: u32,
     ) -> impl PinInit<Self> + use<'gpu> {
-        let fifo_engine_list = *fifo_engine_list;
+        let fifo_engine_list = info.fifo_engine_list();
+        let vmmu_segment_size = info.vmmu_segment_size;
         pin_init!(Self {
             instances <- new_mutex!(VgpuInstances::new(), "nova-core::vgpu-instances"),
+            dev,
+            cmdq,
+            bar_user,
+            mm,
             chid_pool,
             vmmu_segment_size,
             total_channels,

index 679c22b0381a..2e9c250b65ab 100644
@@ -8,7 +8,9 @@
 
 use kernel::{
     device,
-    prelude::*, //
+    prelude::*,
+    time::Delta,
+    transmute::AsBytes, //
 };
 
 use crate::gsp::{

@@ -25,6 +27,7 @@
         commands::{
             VgpuPropertiesSchema, //
         },
+        GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK,
         GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE,
         GMCAPI_CMD_QUERY_VGPU_PROPERTIES, //
     }, //

@@ -89,3 +92,19 @@ pub(super) fn query_vgpu_properties(
         Ok(properties)
     }
 }
+
+/// Send BOOTLOAD and check its firmware status.
+pub(super) fn send_bootload(
+    dev: &device::Device<device::Bound>,
+    cmdq: &Cmdq<'_>,
+    payload: &[u64],
+) -> Result {
+    let command_id = GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK;
+    let response = cmdq.send_gmc_and_receive_timeout(
+        command_id,
+        AsBytes::as_bytes(payload),
+        0,
+        Delta::from_secs(10),
+    )?;
+    check_status(dev, command_id, response.status)
+}

index aabe114b55c2..82c29dfb467a 100644
@@ -30,3 +30,6 @@
 
 pub(super) const GMCAPI_CMD_QUERY_VGPU_PROPERTIES: u32 =
     bindings::GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_VGPU_PROPERTIES;
+
+pub(super) const GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK: u32 =
+    bindings::GMCAPI_COMMANDS_GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK;

index 83048473d35f..bf8d44819e13 100644
@@ -61,7 +61,6 @@ pub(crate) struct ChannelMapEntry(u64) {
 impl ChannelMapEntry {
     const KEY: KeyId = 0x1001;
 
-    #[expect(dead_code)]
     pub(crate) fn new(engine_type: u16, index: u16, chid_offset: u32) -> Self {
         Self::zeroed()
             .with_engine_type(engine_type)

@@ -159,7 +158,6 @@ pub(crate) struct BootloadInfo<'a> {
 }
 
 /// Encodes a `VGPU_BOOTLOAD` request using the typed NVKV schema.
-#[expect(dead_code)]
 pub(crate) fn encode_vgpu_bootload(info: BootloadInfo<'_>) -> Result<EncodedStream> {
     let request = VgpuBootloadRequest {
         dbdf: info.dbdf.into(),

index b945e216a1d6..0b4d03c5f0fc 100644
@@ -39,7 +39,6 @@
     size_of::<RawControlRegion>() == u32_as_usize(fw::VGPU_CPU_GSP_CTRL_BUFF_REGION_SIZE)
 );
 /// Physical VRAM regions containing the vGPU plugin logs.
-#[expect(dead_code)]
 pub(super) struct PluginLogRegions {
     pub(super) init: VramRegion,
     pub(super) vgpu: VramRegion,

@@ -86,7 +85,6 @@ pub(super) struct CommBufferRegion<'map, 'gpu> {
     kernel_log: VramRegion,
 }
 
-#[expect(dead_code)]
 impl<'map, 'gpu> CommBufferRegion<'map, 'gpu> {
     /// Map the communication portion of a plugin management heap.
     pub(super) fn new(

@@ -188,6 +186,19 @@ pub(super) fn plugin_logs(&self) -> PluginLogRegions {
         }
     }
 
+    /// Clear a previous boot marker before starting the plugin.
+    pub(super) fn clear_plugin_ready(&self) -> Result {
+        let offset = self.io_offset(
+            &self.control,
+            core::mem::offset_of!(RawControlRegion, __bindgen_anon_1.message_seq_num),
+            size_of::<u32>(),
+        )?;
+        self.map.try_write32(0, offset)?;
+        // Complete the posted clear before firmware can publish its new marker.
+        self.map.try_read32(offset)?;
+        Ok(())
+    }
+
     /// Return whether firmware has published the plugin boot marker.
     pub(super) fn is_plugin_ready(&self) -> Result<bool> {
         let value = self.read_u32(

@@ -199,6 +210,7 @@ pub(super) fn is_plugin_ready(&self) -> Result<bool> {
     }
 
     /// Invalidate the PTEs and release the communication mapping.
+    #[expect(dead_code)]
     pub(super) fn unmap(&mut self) -> Result {
         self.map.unmap()
     }

216 lines not on this page · raw message

index faaeea106009..bd7d1b4bf039 100644
@@ -102,7 +102,6 @@ fn drop(&mut self) {
 /// All device accesses and mappings of its regions must end before dropping the slot.
 /// Clearing the entry locks a sleeping mutex, so dropping the slot may sleep.
 #[must_use]
-#[expect(dead_code)]
 pub(super) struct VgpuVramSlot {
     pub(super) fbmem: VramRegion,
     pub(super) mgmt_heap: VramRegion,

index abd9edadff64..8730fd0079c4 100644
@@ -812,7 +812,6 @@ pub(crate) fn send_gmc_and_receive_timeout(
     ///
     /// Errors from initializing the request headers and from either callback are propagated
     /// as-is. The current valid element is consumed before a callback error is returned.
-    #[expect(dead_code)]
     pub(crate) fn send_gmc_and_wait_event(
         &self,
         command_id: u32,

index 84c4d0b3839d..f7184e7009a5 100644
@@ -105,7 +105,7 @@ fn query_state(
 use self::instance::VgpuInstances;
 
 /// Runtime resources for an enabled vGPU boot.
-#[pin_data]
+#[pin_data(PinnedDrop)]
 pub(crate) struct VgpuManager<'gpu> {
     #[pin]
     instances: Mutex<VgpuInstances<'gpu>>,

@@ -146,3 +146,10 @@ pub(crate) fn new(
         })
     }
 }
+
+#[pinned_drop]
+impl PinnedDrop for VgpuManager<'_> {
+    fn drop(self: Pin<&mut Self>) {
+        self.instances.lock().release_all(&self);
+    }
+}

65 lines not on this page · raw message

index 82c29dfb467a..1528cc56ce74 100644
@@ -33,3 +33,12 @@
 
 pub(super) const GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK: u32 =
     bindings::GMCAPI_COMMANDS_GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK;
+
+pub(super) const GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK: u32 =
+    bindings::GMCAPI_COMMANDS_GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK;
+
+pub(super) const GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_COMPLETE: u32 =
+    bindings::GMCAPI_COMMANDS_GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_COMPLETE;
+
+pub(super) const GMCAPI_CMD_CLEANUP_GSP_VGPU_PLUGIN_RESOURCES: u32 =
+    bindings::GMCAPI_COMMANDS_GMCAPI_CMD_CLEANUP_GSP_VGPU_PLUGIN_RESOURCES;

index 0b4d03c5f0fc..a00f4d2affc7 100644
@@ -210,7 +210,6 @@ pub(super) fn is_plugin_ready(&self) -> Result<bool> {
     }
 
     /// Invalidate the PTEs and release the communication mapping.
-    #[expect(dead_code)]
     pub(super) fn unmap(&mut self) -> Result {
         self.map.unmap()
     }

177 lines not on this page · raw message

index 964aee8bfdb1..bcbef1571fb9 100644
@@ -150,6 +150,11 @@ pub(crate) fn try_read32(&self, offset: usize) -> Result<u32> {
         self.bar_user.bar1.try_read32(off)
     }
 
+    fn try_write8(&self, value: u8, offset: usize) -> Result {
+        let off = self.bar_offset(offset)?;
+        self.bar_user.bar1.try_write8(value, off)
+    }
+
     /// Write a 32-bit value at the given offset.
     pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result {
         let off = self.bar_offset(offset)?;

@@ -283,6 +288,11 @@ pub(crate) fn try_read32(&self, offset: usize) -> Result<u32> {
             .try_read32(self.access_offset(offset, size_of::<u32>())?)
     }
 
+    pub(crate) fn try_write8(&self, value: u8, offset: usize) -> Result {
+        self.access()?
+            .try_write8(value, self.access_offset(offset, size_of::<u8>())?)
+    }
+
     pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result {
         self.access()?
             .try_write32(value, self.access_offset(offset, size_of::<u32>())?)

index 1528cc56ce74..03225b9833c3 100644
@@ -15,12 +15,14 @@
     VGPU_CPU_GSP_COMMUNICATION_BUFF_TOTAL_SIZE,
     VGPU_CPU_GSP_CTRL_BUFF_REGION as RawControlRegion,
     VGPU_CPU_GSP_CTRL_BUFF_REGION_SIZE,
+    VGPU_CPU_GSP_CTRL_BUFF_VERSION,
     VGPU_CPU_GSP_ERROR_BUFF_REGION_SIZE,
     VGPU_CPU_GSP_GUEST_RPC_TRACE_BUFF_REGION_SIZE,
     VGPU_CPU_GSP_INIT_TASK_LOG_BUFF_REGION_SIZE,
     VGPU_CPU_GSP_KERNEL_TASK_LOG_BUFF_REGION_SIZE,
     VGPU_CPU_GSP_MESSAGE_BUFF_REGION_SIZE,
     VGPU_CPU_GSP_MIGRATION_BUFF_REGION_SIZE,
+    VGPU_CPU_GSP_RESPONSE_BUFF_REGION as RawResponseRegion,
     VGPU_CPU_GSP_RESPONSE_BUFF_REGION_SIZE,
     VGPU_CPU_GSP_VGPU_TASK_LOG_BUFF_REGION_SIZE, //
 };

207 lines not on this page · raw message

index ec8e05dc3351..cadaa3495b4e 100644
@@ -7,13 +7,17 @@
         Io,
         Mmio, //
     },
+    prelude::*,
     sizes::SizeConstants,
     time, //
 };
 use pin_init::Zeroable;
 
 use crate::{
-    driver::NovaRegisters,
+    driver::{
+        Bar0,
+        NovaRegisters, //
+    },
     falcon::{
         DmaTrfCmdSize,
         FalconCoreRev,

@@ -39,6 +43,31 @@
     pub(crate) NV_PBUS_SW_SCRATCH(u32)[64] @ 0x00001400 {}
 }
 
+// VIRTUAL_FUNCTION
+
+register! {
+    base: NovaRegisters;
+
+    // PF BAR0 exposes the virtual-function register window at 0x00b8_0000.
+    pub(crate) NV_VIRTUAL_FUNCTION_PRIV_DOORBELL(u32) @ 0x00b8_2200 {
+        31:0 handle;
+    }
+}
+
+impl NV_VIRTUAL_FUNCTION_PRIV_DOORBELL {
+    const DOORBELL_STRIDE: u32 = 32;
+    const DOORBELL_VECTOR: u32 = 17;
+
+    /// Notify the GSP plugin for the given guest function and read back the doorbell.
+    pub(crate) fn ring_gsp_plugin(bar0: Bar0<'_>, gfid: u16) -> Result {
+        // A `u16` GFID produces a handle of at most 0x1f_fff1, which fits in `u32`.
+        let value = u32::from(gfid) * Self::DOORBELL_STRIDE + Self::DOORBELL_VECTOR;
+        bar0.try_write_reg(Self::zeroed().with_handle(value))?;
+        bar0.try_read(NV_VIRTUAL_FUNCTION_PRIV_DOORBELL)?;
+        Ok(())
+    }
+}
+
 // PGC6 register space.
 //
 // `GC6` is a GPU low-power state where VRAM is in self-refresh and the GPU is powered down (except

index f7184e7009a5..d6bcdabafb42 100644
@@ -35,6 +35,7 @@
 mod commands;
 mod fw;
 mod gsp_plugin_comm;
+mod gsp_plugin_rpc;
 mod hal;
 mod instance;
 mod vram;

index 03225b9833c3..d80f36465296 100644
@@ -10,6 +10,8 @@
 
 use crate::gsp::bindings;
 
+pub(super) use commands::RpcMessage;
+
 pub(super) use bindings::{
     GSP_PLUGIN_BOOTLOADED,
     VGPU_CPU_GSP_COMMUNICATION_BUFF_TOTAL_SIZE,

@@ -44,3 +46,14 @@
 
 pub(super) const GMCAPI_CMD_CLEANUP_GSP_VGPU_PLUGIN_RESOURCES: u32 =
     bindings::GMCAPI_COMMANDS_GMCAPI_CMD_CLEANUP_GSP_VGPU_PLUGIN_RESOURCES;
+
+/// State observed in the response buffer for an expected RPC sequence.
+pub(super) enum RpcResponse {
+    Pending {
+        /// Last sequence completed by firmware.
+        sequence: u32,
+    },
+    Complete {
+        status: u32,
+    },
+}

index bf8d44819e13..91d43cf6dd85 100644
@@ -28,6 +28,16 @@
     mm::vram::VramRegion, //
 };
 
+use super::bindings;
+
+/// Message types supported by the nova-core plugin RPC channel.
+#[expect(dead_code)]
+#[derive(Clone, Copy)]
+#[repr(u32)]
+pub(crate) enum RpcMessage {
+    VersionNegotiation = bindings::MESSAGE_NV_VGPU_CPU_RPC_MSG_VERSION_NEGOTIATION,
+}
+
 bitfield! {
     pub(crate) struct Dbdf(u32) {
         2:0 function;

69 lines not on this page · raw message

155 lines not on this page · raw message

index bf02595590e1..8a2d8e3970a3 100644
@@ -20,6 +20,7 @@
 };
 
 use crate::{
+    driver::Bar0,
     gpu::ChannelIdReservation,
     gsp::{
         cmdq::Cmdq,

@@ -41,6 +42,7 @@
         ChannelMapEntry, //
     },
     gsp_plugin_comm::CommBufferRegion,
+    gsp_plugin_rpc::PluginRpc,
     vram::{
         VgpuVramLayout,
         VgpuVramSlot,

@@ -141,7 +143,7 @@ struct VgpuInstance<'gpu> {
     vgpu_type: VgpuType,
     vm_pid: u32,
     num_plugin_channels: u32,
-    comm: CommBufferRegion<'gpu, 'gpu>,
+    plugin_rpc: PluginRpc<'gpu, 'gpu>,
     // Unmap the communication region before returning its slot and channel IDs.
     vram_slot: VgpuVramSlot,
     chids: ChannelIdReservation<'gpu>,

@@ -170,7 +172,7 @@ fn teardown(&mut self, vgpu: &VgpuManager<'gpu>) -> Result {
                 send_cleanup(vgpu.dev, vgpu.cmdq, self.gfid)?;
                 self.needs_teardown = false;
             }
-            self.comm.unmap()
+            self.plugin_rpc.unmap()
         })();
         if let Err(error) = result {
             self.failure = Some(error);

@@ -187,7 +189,7 @@ fn bootload(
     ) -> Result {
         let fb = &self.vram_slot.fbmem;
         let mgmt = &self.vram_slot.mgmt_heap;
-        let logs = self.comm.plugin_logs();
+        let logs = self.plugin_rpc.comm().plugin_logs();
 
         let payload = encode_vgpu_bootload(BootloadInfo {
             dbdf: self.dbdf,

@@ -215,11 +217,11 @@ fn bootload(
             payload.len() * size_of::<u64>(),
         );
 
-        self.comm.clear_plugin_ready()?;
+        self.plugin_rpc.comm().clear_plugin_ready()?;
         self.needs_teardown = true;
         send_bootload(dev, cmdq, &payload)?;
 
-        wait_plugin_ready(dev, &self.comm)?;
+        wait_plugin_ready(dev, self.plugin_rpc.comm())?;
 
         dev_dbg!(dev, "bootload: gfid={} plugin ready\n", self.gfid.get());
         Ok(())

@@ -293,6 +295,7 @@ fn alloc_vram_slot(&mut self, mm: &GpuMm<'_>, layout: VgpuVramLayout) -> Result<
     fn allocate_instance<'a>(
         &'a mut self,
         vgpu: &'a VgpuManager<'gpu>,
+        bar0: Bar0<'gpu>,
         info: InstanceInfo,
     ) -> Result<PendingInstance<'a, 'gpu>> {
         let InstanceInfo {

@@ -351,7 +354,7 @@ fn allocate_instance<'a>(
             vgpu_type,
             vm_pid,
             num_plugin_channels: PLUGIN_CHANNELS_PER_ENGINE,
-            comm,
+            plugin_rpc: PluginRpc::new(comm, bar0, gfid),
             vram_slot,
             chids,
             needs_teardown: false,

index 52c6fc0ee335..268c3043d88b 100644
@@ -27,13 +27,15 @@
         commands::{
             VgpuPropertiesSchema, //
         },
+        RpcMessage, //
         GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK,
         GMCAPI_CMD_CLEANUP_GSP_VGPU_PLUGIN_RESOURCES,
         GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE,
         GMCAPI_CMD_QUERY_VGPU_PROPERTIES,
         GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK,
-        GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_COMPLETE, //
+        GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_COMPLETE,
     },
+    gsp_plugin_rpc::PluginRpc,
     instance::Gfid, //
 };
 

@@ -159,3 +161,11 @@ pub(super) fn send_cleanup(
     dev_dbg!(dev, "cleanup: gfid={} done\n", gfid.get());
     Ok(())
 }
+
+/// Negotiate the host protocol with a bootloaded GSP plugin.
+pub(super) fn negotiate_plugin_version(
+    dev: &device::Device<device::Bound>,
+    rpc: &mut PluginRpc<'_, '_>,
+) -> Result {
+    rpc.rpc_call(dev, RpcMessage::VersionNegotiation, &[])
+}

index d80f36465296..6cec92c9b505 100644
@@ -10,8 +10,6 @@
 
 use crate::gsp::bindings;
 
-pub(super) use commands::RpcMessage;
-
 pub(super) use bindings::{
     GSP_PLUGIN_BOOTLOADED,
     VGPU_CPU_GSP_COMMUNICATION_BUFF_TOTAL_SIZE,

@@ -29,6 +27,8 @@
     VGPU_CPU_GSP_VGPU_TASK_LOG_BUFF_REGION_SIZE, //
 };
 
+pub(super) use commands::RpcMessage;
+
 pub(super) const GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE: u32 =
     bindings::GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE;
 

index 91d43cf6dd85..915585706174 100644
@@ -31,7 +31,6 @@
 use super::bindings;
 
 /// Message types supported by the nova-core plugin RPC channel.
-#[expect(dead_code)]
 #[derive(Clone, Copy)]
 #[repr(u32)]
 pub(crate) enum RpcMessage {

index 0ee32f011f63..696127c7169f 100644
@@ -65,7 +65,6 @@ pub(super) fn comm(&self) -> &CommBufferRegion<'map, 'gpu> {
     }
 
     /// Initialize the control and response buffers for the first RPC.
-    #[expect(dead_code)]
     pub(super) fn init_rpc(&mut self) -> Result {
         self.comm.initialize()?;
         self.message_sequence = 0;

@@ -82,7 +81,6 @@ fn next_sequence(&self) -> u32 {
     }
 
     /// Write one RPC message, ring the VF doorbell, and wait for its response.
-    #[expect(dead_code)]
     pub(super) fn rpc_call(
         &mut self,
         dev: &device::Device<device::Bound>,

index 8a2d8e3970a3..445d7f513fdc 100644
@@ -31,6 +31,7 @@
 
 use super::{
     commands::{
+        negotiate_plugin_version,
         send_bootload,
         send_cleanup,
         send_shutdown,

@@ -156,6 +157,8 @@ impl<'gpu> VgpuInstance<'gpu> {
     fn activate(&mut self, vgpu: &VgpuManager<'gpu>) -> Result {
         let dev = vgpu.dev;
         self.bootload(dev, vgpu.cmdq, &vgpu.fifo_engine_list)?;
+        self.plugin_rpc.init_rpc()?;
+        negotiate_plugin_version(dev, &mut self.plugin_rpc)?;
 
         Ok(())
     }

index 268c3043d88b..ec40a960069b 100644
@@ -169,3 +169,12 @@ pub(super) fn negotiate_plugin_version(
 ) -> Result {
     rpc.rpc_call(dev, RpcMessage::VersionNegotiation, &[])
 }
+
+/// Send an instance's encoded configuration to the GSP plugin.
+pub(super) fn send_plugin_config(
+    dev: &device::Device<device::Bound>,
+    rpc: &mut PluginRpc<'_, '_>,
+    config: &[u64],
+) -> Result {
+    rpc.rpc_call_nvkv(dev, RpcMessage::SetupConfigParamsAndInit, config)
+}

150 lines not on this page · raw message

index 696127c7169f..48977cd67792 100644
@@ -19,6 +19,7 @@
 
 use kernel::{
     device,
+    num::casts::usize_as_u64,
     prelude::*,
     time::{
         delay::fsleep,

@@ -26,6 +27,7 @@
         Instant,
         Monotonic, //
     },
+    transmute::AsBytes, //
 };
 
 use crate::{

@@ -105,6 +107,21 @@ pub(super) fn rpc_call(
         self.wait_response(dev, sequence)
     }
 
+    /// Send an NVKV stream prefixed by its word count.
+    pub(super) fn rpc_call_nvkv(
+        &mut self,
+        dev: &device::Device<device::Bound>,
+        message_type: RpcMessage,
+        encoded: &[u64],
+    ) -> Result {
+        let word_count = usize_as_u64(encoded.len());
+        let mut payload = KVec::new();
+        payload.extend_from_slice(&word_count.to_le_bytes(), GFP_KERNEL)?;
+        payload.extend_from_slice(AsBytes::as_bytes(encoded), GFP_KERNEL)?;
+
+        self.rpc_call(dev, message_type, &payload)
+    }
+
     fn wait_response(&self, dev: &device::Device<device::Bound>, expected_sequence: u32) -> Result {
         let start = Instant::<Monotonic>::now();
         let timeout = Delta::from_secs(120);

index 445d7f513fdc..247e88bdc82f 100644
@@ -34,10 +34,12 @@
         negotiate_plugin_version,
         send_bootload,
         send_cleanup,
+        send_plugin_config,
         send_shutdown,
         Dbdf, //
     },
     fw::commands::{
+        encode_plugin_config_params,
         encode_vgpu_bootload,
         BootloadInfo,
         ChannelMapEntry, //

@@ -159,6 +161,7 @@ fn activate(&mut self, vgpu: &VgpuManager<'gpu>) -> Result {
         self.bootload(dev, vgpu.cmdq, &vgpu.fifo_engine_list)?;
         self.plugin_rpc.init_rpc()?;
         negotiate_plugin_version(dev, &mut self.plugin_rpc)?;
+        self.configure_plugin(dev)?;
 
         Ok(())
     }

@@ -230,6 +233,19 @@ fn bootload(
         Ok(())
     }
 
+    fn configure_plugin(&mut self, dev: &device::Device<device::Bound>) -> Result {
+        let config = encode_plugin_config_params(
+            [0; 16],
+            self.dbdf,
+            self.vgpu_type.vgpu_type_id,
+            self.vm_pid,
+            u32::try_from(self.chids.len()).map_err(|_| EOVERFLOW)?,
+            self.num_plugin_channels,
+        )?;
+
+        send_plugin_config(dev, &mut self.plugin_rpc, &config)
+    }
+
     /// Stop the plugin when firmware may own instance resources.
     fn shutdown(&mut self, dev: &device::Device<device::Bound>, cmdq: &Cmdq<'_>) -> Result {
         if self.needs_teardown {

index ec40a960069b..bfb033e5eb37 100644
@@ -25,6 +25,7 @@
 use super::{
     fw::{
         commands::{
+            encode_plugin_set_bme,
             VgpuPropertiesSchema, //
         },
         RpcMessage, //

@@ -178,3 +179,13 @@ pub(super) fn send_plugin_config(
 ) -> Result {
     rpc.rpc_call_nvkv(dev, RpcMessage::SetupConfigParamsAndInit, config)
 }
+
+/// Update the bus-mastering state reported to the GSP plugin.
+pub(super) fn set_plugin_bme(
+    dev: &device::Device<device::Bound>,
+    rpc: &mut PluginRpc<'_, '_>,
+    enable: bool,
+) -> Result {
+    let bme = encode_plugin_set_bme(enable)?;
+    rpc.rpc_call_nvkv(dev, RpcMessage::UpdateBmeState, &bme)
+}

index e3b5198e9c80..cc2298c3679c 100644
@@ -37,6 +37,7 @@
 pub(crate) enum RpcMessage {
     VersionNegotiation = bindings::MESSAGE_NV_VGPU_CPU_RPC_MSG_VERSION_NEGOTIATION,
     SetupConfigParamsAndInit = bindings::MESSAGE_NV_VGPU_CPU_RPC_MSG_SETUP_CONFIG_PARAMS_AND_INIT,
+    UpdateBmeState = bindings::MESSAGE_NV_VGPU_CPU_RPC_MSG_UPDATE_BME_STATE,
 }
 
 bitfield! {

@@ -398,3 +399,24 @@ pub(crate) fn encode_plugin_config_params(
     request.encode(&mut encoder)?;
     Ok(encoder.finish())
 }
+
+nvkv_encode! {
+    struct PluginSetBmeRequest {
+        bme_enable: Key<bool, { Self::BME_ENABLE_KEY }, u32>,
+    }
+}
+
+impl PluginSetBmeRequest {
+    const BME_ENABLE_KEY: KeyId = 0x0100;
+}
+
+/// Encodes a plugin BME state update using the typed NVKV schema.
+pub(crate) fn encode_plugin_set_bme(enable: bool) -> Result<EncodedStream> {
+    let request = PluginSetBmeRequest {
+        bme_enable: enable.into(),
+    };
+
+    let mut encoder = Encoder::new();
+    request.encode(&mut encoder)?;
+    Ok(encoder.finish())
+}

index 247e88bdc82f..23c2b791aa25 100644
@@ -36,6 +36,7 @@
         send_cleanup,
         send_plugin_config,
         send_shutdown,
+        set_plugin_bme,
         Dbdf, //
     },
     fw::commands::{

@@ -162,6 +163,7 @@ fn activate(&mut self, vgpu: &VgpuManager<'gpu>) -> Result {
         self.plugin_rpc.init_rpc()?;
         negotiate_plugin_version(dev, &mut self.plugin_rpc)?;
         self.configure_plugin(dev)?;
+        set_plugin_bme(dev, &mut self.plugin_rpc, true)?;
 
         Ok(())
     }

index 3cc718bb2ab8..333256ea8a04 100644
@@ -851,6 +851,7 @@ fn default() -> Self {
         }
     }
 }
+pub const NV_ADDR_FBMEM: u32 = 2;
 pub const GSP_PLUGIN_BOOTLOADED: u32 = 1315261039;
 pub const VGPU_CPU_GSP_CTRL_BUFF_VERSION: u32 = 2;
 pub const VGPU_CPU_GSP_CTRL_BUFF_REGION_SIZE: u32 = 4096;

171 lines not on this page · raw message

index 6cec92c9b505..795ce4855586 100644
@@ -27,6 +27,8 @@
     VGPU_CPU_GSP_VGPU_TASK_LOG_BUFF_REGION_SIZE, //
 };
 
+pub(super) use bindings::NV_ADDR_FBMEM;
+
 pub(super) use commands::RpcMessage;
 
 pub(super) const GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE: u32 =

@@ -47,6 +49,15 @@
 pub(super) const GMCAPI_CMD_CLEANUP_GSP_VGPU_PLUGIN_RESOURCES: u32 =
     bindings::GMCAPI_COMMANDS_GMCAPI_CMD_CLEANUP_GSP_VGPU_PLUGIN_RESOURCES;
 
+pub(super) const GMCAPI_CMD_VGPU_MGR_ALLOC_GSP_CEUTILS: u32 =
+    bindings::GMCAPI_COMMANDS_GMCAPI_CMD_VGPU_MGR_ALLOC_GSP_CEUTILS;
+
+pub(super) const GMCAPI_CMD_VGPU_MGR_FREE_GSP_CEUTILS: u32 =
+    bindings::GMCAPI_COMMANDS_GMCAPI_CMD_VGPU_MGR_FREE_GSP_CEUTILS;
+
+pub(super) const GMCAPI_CMD_VGPU_MGR_SCRUB_GUEST_FB: u32 =
+    bindings::GMCAPI_COMMANDS_GMCAPI_CMD_VGPU_MGR_SCRUB_GUEST_FB;
+
 /// State observed in the response buffer for an expected RPC sequence.
 pub(super) enum RpcResponse {
     Pending {

63 lines not on this page · raw message

index d6bcdabafb42..f78a4621984a 100644
@@ -38,6 +38,7 @@
 mod gsp_plugin_rpc;
 mod hal;
 mod instance;
+mod scrubber;
 mod vram;
 
 /// vGPU state detected during GPU construction.

index 704d13259cf3..52e8339504c6 100644
@@ -207,7 +207,6 @@ pub(super) fn set_plugin_bme(
 }
 
 /// Whether a failed allocation may still have transferred CHID ownership to firmware.
-#[expect(dead_code)]
 pub(super) enum CeUtilsAllocError {
     /// A matching firmware response explicitly rejected the allocation.
     NotOwned(Error),

@@ -216,7 +215,6 @@ pub(super) enum CeUtilsAllocError {
 }
 
 /// Allocate a CeUtils channel and validate its semaphore description.
-#[expect(dead_code)]
 pub(super) fn alloc_ceutils(
     dev: &device::Device<device::Bound>,
     cmdq: &Cmdq<'_>,

@@ -265,7 +263,6 @@ pub(super) fn alloc_ceutils(
 }
 
 /// Release a CeUtils allocation, including one whose allocation reply was lost.
-#[expect(dead_code)]
 pub(super) fn free_ceutils(
     dev: &device::Device<device::Bound>,
     cmdq: &Cmdq<'_>,

@@ -282,7 +279,6 @@ pub(super) fn free_ceutils(
 }
 
 /// Submit an asynchronous guest FB scrub and return its work identifier.
-#[expect(dead_code)]
 pub(super) fn submit_ceutils_scrub(
     dev: &device::Device<device::Bound>,
     cmdq: &Cmdq<'_>,

111 lines not on this page · raw message

173 lines not on this page · raw message

index 012c7e6abca6..d5ac779e3b0b 100644
@@ -558,9 +558,10 @@ pub(crate) fn new<'a>(
                         // SAFETY: These sibling fields are initialized at their final pinned
                         // addresses. The private manager cannot escape this `Gpu`, and is dropped
                         // before all its dependencies, both here on failure and on normal removal.
-                        let (cmdq, bar_user, mm, chid_pool) = unsafe {
+                        // The GSP build ID is not modified after initialization.
+                        let (gsp, bar_user, mm, chid_pool) = unsafe {
                             (
-                                &*core::ptr::from_ref(&gsp_resources.gsp.cmdq),
+                                &*core::ptr::from_ref(&gsp_resources.gsp),
                                 &*core::ptr::from_ref(bar_user.as_ref().get_ref()),
                                 &*core::ptr::from_ref(mm.as_ref().get_ref()),
                                 &*core::ptr::from_ref(chid_pool.as_ref().get_ref()),

@@ -568,7 +569,9 @@ pub(crate) fn new<'a>(
                         };
                         Some(KBox::pin_init(VgpuManager::new(
                             dev,
-                            cmdq,
+                            &gsp.cmdq,
+                            gsp_resources.spec,
+                            gsp.build_id(),
                             bar_user,
                             mm,
                             chid_pool,

index 45645bc7c9d8..ab3a7adbff69 100644
@@ -180,6 +180,21 @@ fn new(spec: Spec, build_id: &BuildId, task_prefix: &str) -> Self {
     }
 }
 
+/// Size of the header prepended to debugfs log buffer dumps.
+pub(crate) const LOG_BUFFER_HEADER_SIZE: usize = size_of::<LogBufferHeader>();
+
+/// Builds a log header using the GPU implementation reported by the hardware.
+pub(crate) fn build_log_buffer_header(
+    spec: Spec,
+    build_id: &BuildId,
+    task_prefix: &str,
+) -> [u8; LOG_BUFFER_HEADER_SIZE] {
+    let header = LogBufferHeader::new(spec, build_id, task_prefix);
+    let mut bytes = [0; LOG_BUFFER_HEADER_SIZE];
+    bytes.copy_from_slice(header.as_bytes());
+    bytes
+}
+
 /// The logging buffers are byte queues that contain encoded printf-like
 /// messages from GSP-RM.  They need to be decoded by a special application
 /// that can parse the buffers.

@@ -368,6 +383,8 @@ fn register_debugfs<'data>(&'data self, dir: &debugfs::ScopedDir<'data, '_>) {
 pub(crate) struct Gsp<'gsp> {
     /// The GSP firmware's TLV.
     gsp_tlv: firmware::Firmware,
+    /// Build identifier of the firmware whose log buffers are exposed.
+    build_id: Option<BuildId>,
     /// Libos arguments.
     pub(crate) libos: Coherent<'gsp, [LibosMemoryRegionInitArgument]>,
     /// Log buffers, optionally exposed via debugfs.

@@ -383,6 +400,11 @@ pub(crate) struct Gsp<'gsp> {
 }
 
 impl<'gsp> Gsp<'gsp> {
+    /// Returns the GSP firmware build identifier, when available.
+    pub(crate) fn build_id(&self) -> Option<&BuildId> {
+        self.build_id.as_ref()
+    }
+
     // Creates an in-place initializer for a `Gsp` manager for `pdev`.
     pub(crate) fn new(
         pdev: &'gsp pci::Device<device::Bound>,

@@ -405,6 +427,7 @@ pub(crate) fn new(
 
             Ok(try_pin_init!(Self {
                 gsp_tlv,
+                build_id,
                 cmdq <- Cmdq::new(dev, bar),
                 rm_state_monitor: Coherent::zeroed(dev, GFP_KERNEL)?,
                 rmargs: Coherent::init(

index bcbef1571fb9..72c65702c58b 100644
@@ -197,7 +197,6 @@ pub(crate) struct BarMapping<'map, 'gpu> {
     unmap_error: Option<Error>,
 }
 
-#[expect(dead_code)]
 impl<'map, 'gpu> BarMapping<'map, 'gpu> {
     /// Maps the containing pages while restricting CPU access to the requested byte range.
     pub(crate) fn new(

@@ -243,6 +242,10 @@ pub(crate) fn new(
         })
     }
 
+    pub(crate) fn bar1(&self) -> &'gpu Bar1<'gpu> {
+        self.access.bar_user.bar1
+    }
+
     pub(crate) fn region(&self) -> &VramRegion {
         &self.region
     }

index f78a4621984a..2dbf95222d4d 100644
@@ -11,13 +11,15 @@
 };
 
 use crate::{
+    firmware::gsp::BuildId,
     fsp::{
         Fsp,
         VgpuMode, //
     },
     gpu::{
         ChannelIdPool,
-        Chipset, //
+        Chipset,
+        Spec, //
     },
     gsp::{
         cmdq::Cmdq,

@@ -38,6 +40,7 @@
 mod gsp_plugin_rpc;
 mod hal;
 mod instance;
+mod log;
 mod scrubber;
 mod vram;
 

@@ -113,6 +116,8 @@ pub(crate) struct VgpuManager<'gpu> {
     instances: Mutex<VgpuInstances<'gpu>>,
     dev: &'gpu device::Device<device::Bound>,
     cmdq: &'gpu Cmdq<'gpu>,
+    spec: Spec,
+    build_id: Option<&'gpu BuildId>,
     bar_user: &'gpu BarUser<'gpu>,
     mm: &'gpu Mutex<GpuMm<'gpu>>,
     chid_pool: &'gpu ChannelIdPool,

@@ -127,6 +132,8 @@ impl<'gpu> VgpuManager<'gpu> {
     pub(crate) fn new(
         dev: &'gpu device::Device<device::Bound>,
         cmdq: &'gpu Cmdq<'gpu>,
+        spec: Spec,
+        build_id: Option<&'gpu BuildId>,
         bar_user: &'gpu BarUser<'gpu>,
         mm: &'gpu Mutex<GpuMm<'gpu>>,
         chid_pool: &'gpu ChannelIdPool,

@@ -139,6 +146,8 @@ pub(crate) fn new(
             instances <- new_mutex!(VgpuInstances::new(), "nova-core::vgpu-instances"),
             dev,
             cmdq,
+            spec,
+            build_id,
             bar_user,
             mm,
             chid_pool,

140 lines not on this page · raw message

113 lines not on this page · raw message

171 lines not on this page · raw message

index d5ac779e3b0b..df9037d8965c 100644
@@ -1,6 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0
 
-use core::ops::Range;
+use core::{
+    num::NonZero,
+    ops::Range, //
+};
 
 use kernel::{
     device,

@@ -35,6 +38,7 @@
     fsp::Fsp,
     gsp::{
         self,
+        cmdq::Cmdq,
         Gsp,
         GspBootContext, //
     },

@@ -397,6 +401,25 @@ fn drop(self: Pin<&mut Self>) {
 }
 
 impl<'gpu> Gpu<'gpu> {
+    pub(crate) fn cmdq(&self) -> &Cmdq<'gpu> {
+        &self.gsp_resources.gsp.cmdq
+    }
+
+    pub(crate) fn vgpu_manager(&self) -> Option<&VgpuManager<'gpu>> {
+        self.vgpu.as_ref().map(|vgpu| vgpu.as_ref().get_ref())
+    }
+
+    pub(crate) fn vgpu_total_vfs(&self) -> Option<NonZero<u16>> {
+        match self.gsp_resources.vgpu_state {
+            VgpuState::Disabled => None,
+            VgpuState::Enabled { total_vfs } => Some(total_vfs),
+        }
+    }
+
+    pub(crate) fn bar0(&self) -> Bar0<'gpu> {
+        self.gsp_resources.bar
+    }
+
     pub(crate) fn new<'a>(
         pdev: &'gpu pci::Device<device::Core<'a>>,
         bar: Bar0<'gpu>,

@@ -468,7 +491,6 @@ pub(crate) fn new<'a>(
             }),
 
 
-
             // GSP boot left the SWGEN0 latch set and pending bits in the tree.
             _: {
                 irq::gsp::quiesce(bar, gsp_resources.spec.chipset, vectors_ref)?;

index cbaef6d3d9f1..4e074e014f8b 100644
@@ -29,6 +29,12 @@
 mod vbios;
 mod vgpu;
 
+#[cfg(CONFIG_PCI_IOV)]
+pub use vgpu::vgpu_api::{
+    NovaCoreVfApi,
+    VgpuTypeInfo, //
+};
+
 pub(crate) const MODULE_NAME: &core::ffi::CStr = <LocalModule as kernel::ModuleMetadata>::NAME;
 
 // TODO: Move this into per-module data once that exists.

index 2dbf95222d4d..47a27236a086 100644
@@ -42,6 +42,8 @@
 mod instance;
 mod log;
 mod scrubber;
+#[cfg_attr(not(CONFIG_PCI_IOV), expect(dead_code, unreachable_pub))]
+pub(crate) mod vgpu_api;
 mod vram;
 
 /// vGPU state detected during GPU construction.