Rust VFIO PCI support and NVIDIA vGPU driver

index d83492e06cee..10864ffda4ea 100644
@@ -206,6 +206,13 @@ macro_rules! module_pci_driver {
 impl DeviceId {
     const PCI_ANY_ID: u32 = !0;
 
+    /// Equivalent to C's `PCI_DRIVER_OVERRIDE_DEVICE_VFIO` macro.
+    pub const fn from_id_vfio_override(vendor: Vendor, device: u32) -> Self {
+        let mut id = Self::from_id(vendor, device);
+        id.0.override_only = bindings::PCI_ID_F_VFIO_DRIVER_OVERRIDE;
+        id
+    }
+
     /// Equivalent to C's `PCI_DEVICE` macro.
     ///
     /// Create a new `pci::DeviceId` from a vendor and device ID.

index 9d9bd3ac7025..fac134d1ebab 100644
@@ -40,6 +40,14 @@ pub fn is_virtfn(&self) -> bool {
         // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`.
         unsafe { (*self.as_raw()).is_virtfn() != 0 }
     }
+
+    /// Return the zero-based VF index within its PF, or an error for a non-VF device.
+    pub fn vf_id(&self) -> Result<u32> {
+        // SAFETY: `self.as_raw()` points to a live PCI device; the helper checks VF membership.
+        let id = unsafe { bindings::pci_iov_vf_id(self.as_raw()) };
+        to_result(id)?;
+        Ok(id as u32)
+    }
 }
 
 impl Device<device::Core<'_>> {

index 10864ffda4ea..a156927e23d5 100644
@@ -108,6 +108,7 @@ unsafe fn register(
             (*pdrv.get()).probe = Some(Self::probe_callback);
             (*pdrv.get()).remove = Some(Self::remove_callback);
             (*pdrv.get()).id_table = T::ID_TABLE.as_ptr();
+            (*pdrv.get()).driver_managed_dma = T::DRIVER_MANAGED_DMA;
             #[cfg(CONFIG_PCI_IOV)]
             if T::HAS_SRIOV_ENABLE {
                 (*pdrv.get()).sriov_configure = Some(Self::sriov_configure_callback);

@@ -345,6 +346,9 @@ pub trait Driver {
     /// The table of device ids supported by the driver.
     const ID_TABLE: IdTable<Self::IdInfo>;
 
+    /// Whether the driver manages DMA ownership instead of using the default DMA domain.
+    const DRIVER_MANAGED_DMA: bool = false;
+
     /// PCI driver probe.
     ///
     /// Called when a new pci device is added or discovered. Implementers should

index 930e63290cdd..25ae5d30a93f 100644
@@ -93,6 +93,8 @@
 #include <linux/task_work.h>
 #include <linux/tracepoint.h>
 #include <linux/usb.h>
+#include <linux/vfio.h>
+#include <linux/vfio_pci_core.h>
 #include <linux/wait.h>
 #include <linux/workqueue.h>
 #include <linux/xarray.h>

@@ -115,6 +117,8 @@
 const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN;
 const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN;
 const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE;
+const unsigned int RUST_CONST_HELPER_VFIO_DEVICE_RESET = VFIO_DEVICE_RESET;
+const unsigned int RUST_CONST_HELPER_VFIO_DEVICE_PCI_HOT_RESET = VFIO_DEVICE_PCI_HOT_RESET;
 const size_t RUST_CONST_HELPER_GENLMSG_DEFAULT_SIZE = GENLMSG_DEFAULT_SIZE;
 const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC;
 const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL;

index f9ef36217bb5..261a411ab506 100644
@@ -146,6 +146,8 @@
 pub mod uaccess;
 #[cfg(CONFIG_USB = "y")]
 pub mod usb;
+#[cfg(CONFIG_VFIO)]
+pub mod vfio;
 pub mod workqueue;
 pub mod xarray;
 

100 lines not on this page · raw message

1,109 lines not on this page · raw message

index ed93066e2da5..a0bfbc7c59a0 100644
@@ -1,14 +1,29 @@
 // SPDX-License-Identifier: GPL-2.0
 
-//! Virtual Function I/O (VFIO) abstractions.
+//! Virtual Function I/O (VFIO) devices and callback data.
+//!
+//! Bus-specific device types borrow their embedded VFIO device through [`AsRef`].
+//! C header: [`include/linux/vfio.h`](srctree/include/linux/vfio.h)
 
 use crate::{
     bindings,
     prelude::*,
+    sync::aref::AlwaysRefCounted,
+    types::{
+        CovariantForLt,
+        ForLt,
+        NotThreadSafe,
+        Opaque, //
+    },
     uaccess::{
         UserPtr,
         UserSlice, //
-    },
+    }, //
+};
+use core::{
+    cell::UnsafeCell,
+    marker::PhantomData,
+    ptr::NonNull, //
 };
 
 #[cfg(CONFIG_VFIO_PCI_CORE)]

@@ -17,12 +32,187 @@
 /// Reset the VFIO device.
 pub const DEVICE_RESET: u32 = bindings::VFIO_DEVICE_RESET;
 
-/// Reset the PCI slot or bus containing a VFIO device.
-pub const DEVICE_PCI_HOT_RESET: u32 = bindings::VFIO_DEVICE_PCI_HOT_RESET;
+/// The context in which a VFIO device reference is valid.
+///
+/// Callback views can only be borrowed from the corresponding VFIO trampoline.
+/// They cannot be refcounted or shared across threads.
+pub trait DeviceContext: private::Sealed {}
+
+/// An ordinary device reference, without callback-specific operations.
+pub struct Normal;
+
+/// The device is undergoing its exclusive first-open callback.
+pub struct Open;
+
+/// The device is borrowed for a VFIO ioctl callback on the current thread.
+pub struct Ioctl;
+
+/// The device is borrowed for a VFIO read callback on the current thread.
+pub struct Read;
+
+/// The device is borrowed for a VFIO write callback on the current thread.
+pub struct Write;
+
+/// The device is borrowed for a VFIO mmap callback on the current thread.
+pub struct Mmap;
+
+/// The device is borrowed for a VFIO region-info callback on the current thread.
+pub struct GetRegionInfo;
+
+mod private {
+    pub trait Sealed {}
+
+    impl Sealed for super::Normal {}
+    impl Sealed for super::Open {}
+    impl Sealed for super::Write {}
+    impl Sealed for super::Mmap {}
+    impl Sealed for super::Ioctl {}
+    impl Sealed for super::Read {}
+    impl Sealed for super::GetRegionInfo {}
+}
+
+impl DeviceContext for Normal {}
+impl DeviceContext for Open {}
+impl DeviceContext for Write {}
+impl DeviceContext for Mmap {}
+impl DeviceContext for Ioctl {}
+impl DeviceContext for Read {}
+impl DeviceContext for GetRegionInfo {}
+
+/// A VFIO device, independent of its bus-specific wrapper.
+///
+/// # Invariants
+///
+/// The underlying `vfio_device` is initialized and remains alive while borrowed.
+/// Callback contexts are borrowed only for the corresponding callback and thread;
+/// only [`Normal`] references can be reference-counted.
+#[repr(transparent)]
+pub struct Device<Ctx: DeviceContext = Normal> {
+    raw: Opaque<bindings::vfio_device>,
+    _context: PhantomData<(Ctx, NotThreadSafe)>,
+}
+
+impl<Ctx: DeviceContext> Device<Ctx> {
+    /// # Safety
+    ///
+    /// `raw` must point to an initialized `vfio_device` that remains alive for
+    /// `'a`. The caller must provide the guarantees of `Ctx` throughout the borrow.
+    #[cfg_attr(not(CONFIG_VFIO_PCI_CORE), expect(dead_code))]
+    unsafe fn from_raw<'a>(raw: *mut bindings::vfio_device) -> &'a Self {
+        // SAFETY: Self is a transparent wrapper; the caller guarantees validity and context.
+        unsafe { &*raw.cast() }
+    }
+
+    fn as_raw(&self) -> *mut bindings::vfio_device {
+        self.raw.get()
+    }
+}
+
+// SAFETY: VFIO uses the embedded class device's refcount to own the full allocation.
+unsafe impl AlwaysRefCounted for Device {
+    fn inc_ref(&self) {
+        // SAFETY: A shared reference keeps the initialized VFIO device alive.
+        unsafe { bindings::get_device(&raw mut (*self.as_raw()).device) };
+    }
+
+    unsafe fn dec_ref(obj: NonNull<Self>) {
+        // SAFETY: The caller owns the VFIO reference being released.
+        unsafe { bindings::put_device(&raw mut (*obj.as_ref().as_raw()).device) };
+    }
+}
+
+// SAFETY: Ordinary VFIO references expose no unsynchronized callback operations.
+unsafe impl Send for Device {}
+// SAFETY: Shared access only changes the embedded class device's reference count.
+unsafe impl Sync for Device {}
+
+/// Callback data stored after the bus-specific C device structure.
+///
+/// # Invariants
+///
+/// `reg_data` is dangling when unpublished; otherwise it borrows pinned data kept
+/// alive until callback access has ended. `open_data` is null outside a successful
+/// first open, otherwise owns a pinned allocation until the last close.
+/// The bus-specific layer drains callbacks before releasing either allocation.
+struct CallbackData<R: CovariantForLt, O: ForLt> {
+    reg_data: UnsafeCell<NonNull<R::Of<'static>>>,
+    open_data: UnsafeCell<*mut O::Of<'static>>,
+}
+
+#[cfg_attr(not(CONFIG_VFIO_PCI_CORE), expect(dead_code))]
+impl<R: CovariantForLt, O: ForLt> CallbackData<R, O> {
+    fn new() -> Self {
+        Self {
+            reg_data: UnsafeCell::new(NonNull::dangling()),
+            open_data: UnsafeCell::new(core::ptr::null_mut()),
+        }
+    }
+
+    /// Access registration data without allowing its erased lifetime to escape.
+    ///
+    /// # Safety
+    ///
+    /// Registration data must be published and remain valid through this call.
+    /// The caller must exclude concurrent changes to `reg_data`.
+    unsafe fn registration_data_with<V>(&self, f: impl for<'a> FnOnce(&'a R::Of<'a>) -> V) -> V {
+        // SAFETY: The caller keeps registration data alive and excludes mutation.
+        // Covariance permits shortening its erased lifetime to this borrow.
+        let reg_data = unsafe { (*self.reg_data.get()).cast::<R::Of<'_>>().as_ref() };
+        f(reg_data)
+    }
+
+    /// # Safety
+    ///
+    /// The caller must be an I/O callback while VFIO keeps the device open.
+    /// Registration and open data must remain valid, with their pointer slots
+    /// unchanged throughout the call.
+    unsafe fn callback_data_with<V>(
+        &self,
+        f: impl for<'borrow, 'data> FnOnce(&'borrow R::Of<'data>, Pin<&'borrow O::Of<'data>>) -> V,
+    ) -> V {
+        // SAFETY: The caller excludes open/close and unregister while borrowing the data.
+        unsafe {
+            self.registration_data_with(|reg_data| {
+                let ptr = (*self.open_data.get()).cast::<O::Of<'_>>();
+                // The data lifetime stays independent of the callback borrow,
+                // so borrowed OpenData fields cannot be stored back into OpenData.
+                f(reg_data, Pin::new_unchecked(&*ptr))
+            })
+        }
+    }
+
+    /// # Safety
+    ///
+    /// Call only during exclusive first open, with no existing open data.
+    /// All resources borrowed for `'a` must remain valid until `close()` returns.
+    unsafe fn open<'a, I: PinInit<O::Of<'a>, Error>>(&self, init: impl FnOnce() -> I) -> Result {
+        // Allocate before calling the driver so allocation failure cannot follow open.
+        let data = KBox::<O::Of<'a>>::new_uninit(GFP_KERNEL)?;
+        let data = data.write_pin_init(init())?;
+
+        // SAFETY: Only the owning pointer moves; close drops the allocation in place.
+        let raw = KBox::into_raw(unsafe { Pin::into_inner_unchecked(data) });
+        // SAFETY: First open is exclusive; the caller keeps borrowed resources alive
+        // through close. Lifetimes do not affect the allocation's layout.
+        unsafe { *self.open_data.get() = raw.cast::<O::Of<'static>>() };
+        Ok(())
+    }
+
+    /// # Safety
+    ///
+    /// Call once after a successful open, with all callbacks drained and the next
+    /// open excluded. Registration data and borrowed resources must remain alive.
+    unsafe fn close(&self) {
+        // SAFETY: Last close exclusively takes the allocation published by open.
+        let raw = unsafe { core::mem::replace(&mut *self.open_data.get(), core::ptr::null_mut()) };
+        // SAFETY: The allocation came from KBox::into_raw and is destroyed without moving it.
+        unsafe { drop(KBox::from_raw(raw)) };
+    }
+}
 
 /// Capability buffer supplied by VFIO for a region-info callback.
-#[cfg(CONFIG_VFIO_PCI_CORE)]
 pub struct InfoCap<'a> {
+    #[cfg_attr(not(CONFIG_VFIO_PCI_CORE), expect(dead_code))]
     raw: &'a mut bindings::vfio_info_cap,
 }
 

357 lines not on this page · raw message

index dc6febeee656..99beab189879 100644
@@ -35,6 +35,12 @@
     }, //
 };
 
+#[cfg(CONFIG_PCI_IOV)]
+use kernel::types::ForLt;
+
+#[cfg(CONFIG_PCI_IOV)]
+use crate::vgpu::vgpu_api::NovaCoreVfApi;
+
 /// Counter for generating unique auxiliary device IDs.
 static AUXILIARY_ID_COUNTER: Atomic<u32> = Atomic::new(0);
 

@@ -42,6 +48,10 @@
 pub(crate) struct NovaCore<'bound> {
     /// Firmware-control registration.
     _fwctl: fwctl::Registration<'bound, NovaCoreFwCtl>,
+    #[cfg(CONFIG_PCI_IOV)]
+    #[allow(clippy::type_complexity)]
+    #[pin]
+    _vf_registration: pci::VfRegistration<'bound, ForLt!(NovaCoreVfApi<'_>)>,
     #[pin]
     pub(crate) gpu: Gpu<'bound>,
     bar: pci::Bar<'bound, BAR0_SIZE>,

@@ -114,6 +124,11 @@ fn probe<'bound>(
         pin_init::pin_init_scope(move || {
             dev_dbg!(pdev, "Probe Nova Core GPU driver.\n");
 
+            #[cfg(CONFIG_PCI_IOV)]
+            if pdev.is_virtfn() {
+                return Err(ENODEV);
+            }
+
             pdev.enable_device_mem()?;
             pdev.set_master();
 

@@ -144,6 +159,18 @@ fn probe<'bound>(
                 // Run optional GPU selftests.
                 #[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
                 _: { gpu.run_selftests(pdev) },
+                #[cfg(CONFIG_PCI_IOV)]
+                _vf_registration <- {
+                    // SAFETY: `gpu` is initialized at its pinned address and
+                    // outlives the registration and its borrowed API data.
+                    let gpu = unsafe { &(*this.as_ptr()).gpu };
+                    let api = NovaCoreVfApi::new(gpu, pdev);
+                    // SAFETY: Probe has exclusive access to the registration
+                    // slot and no VFs are enabled before successful probe. The
+                    // registration keeps PF services available until VF removal
+                    // completes and is dropped before its borrowed GPU resources.
+                    unsafe { pci::VfRegistration::new(pdev, Ok(api)) }
+                },
                 _reg: auxiliary::Registration::new(
                     pdev.as_ref(),
                     c"nova-drm",

@@ -176,4 +203,24 @@ fn probe<'bound>(
             }))
         })
     }
+
+    #[cfg(CONFIG_PCI_IOV)]
+    fn sriov_enable<'bound, 'callback>(
+        _dev: &'bound pci::Device<Core<'_>>,
+        _this: Pin<&Self::Data<'bound>>,
+        token: pci::SriovEnable<'callback>,
+    ) -> Result<pci::SriovEnabled<'callback>> {
+        let num_vfs = token.num_vfs();
+        token.enable(num_vfs)
+    }
+
+    #[cfg(CONFIG_PCI_IOV)]
+    fn sriov_disable<'bound>(
+        _dev: &'bound pci::Device<Core<'_>>,
+        _this: Pin<&Self::Data<'bound>>,
+        token: pci::SriovDisable<'_>,
+    ) -> Result {
+        token.disable();
+        Ok(())
+    }
 }

index 619f32c79415..ee511bdf3544 100644
@@ -33,6 +33,8 @@
 #[cfg(CONFIG_PCI_IOV)]
 pub use vgpu::vgpu_api::{
     NovaCoreVfApi,
+    NovaCoreVfApiHandle,
+    VgpuInstance,
     VgpuTypeInfo, //
 };
 

103 lines not on this page · raw message

366 lines not on this page · raw message