rust: Add Per-CPU Variable API

index 0f1b5d115985..d56bbe6334d3 100644
@@ -29,7 +29,9 @@
 #include "page.c"
 #include "platform.c"
 #include "pci.c"
+#include "percpu.c"
 #include "pid_namespace.c"
+#include "preempt.c"
 #include "rbtree.c"
 #include "rcu.c"
 #include "refcount.c"

new file mode 100644
index 000000000000..a091389f730f
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/percpu.h>
+
+void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
+{
+	return __alloc_percpu(sz, align);
+}
+

new file mode 100644
index 000000000000..2c7529528ddd
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/preempt.h>
+
+void rust_helper_preempt_disable(void)
+{
+	preempt_disable();
+}
+
+void rust_helper_preempt_enable(void)
+{
+	preempt_enable();
+}
+

index 6b4774b2b1c3..733f9ff8b888 100644
@@ -95,6 +95,9 @@
 pub mod page;
 #[cfg(CONFIG_PCI)]
 pub mod pci;
+// Only x86_64 is supported by percpu for now
+#[cfg(CONFIG_X86_64)]
+pub mod percpu;
 pub mod pid_namespace;
 pub mod platform;
 pub mod prelude;

299 lines not on this page · raw message

new file mode 100644
index 000000000000..14c04b12e7f0
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Contains abstractions for disabling CPU preemption. See `CpuGuard`.
+
+/// A RAII guard for bindings::preempt_disable and bindings::preempt_enable. Guarantees preemption
+/// is disabled for as long as this object exists.
+pub struct CpuGuard {
+    // Don't make one without using new()
+    _phantom: (),
+}
+
+impl CpuGuard {
+    /// Create a new CpuGuard. Disables preemption for its lifetime.
+    pub fn new() -> Self {
+        // SAFETY: There are no preconditions required to call preempt_disable
+        unsafe {
+            bindings::preempt_disable();
+        }
+        CpuGuard { _phantom: () }
+    }
+}
+
+impl Default for CpuGuard {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Drop for CpuGuard {
+    fn drop(&mut self) {
+        // SAFETY: There are no preconditions required to call preempt_enable
+        unsafe {
+            bindings::preempt_enable();
+        }
+    }
+}

120 lines not on this page · raw message

index a091389f730f..0e9b2fed3ebd 100644
@@ -1,9 +1,20 @@
 // SPDX-License-Identifier: GPL-2.0
 
 #include <linux/percpu.h>
+#include <linux/smp.h>
 
 void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
 {
 	return __alloc_percpu(sz, align);
 }
 
+void rust_helper_on_each_cpu(smp_call_func_t func, void *info, int wait)
+{
+	on_each_cpu(func, info, wait);
+}
+
+int rust_helper_smp_processor_id(void)
+{
+	return smp_processor_id();
+}
+

index a9652e6ece08..114015435a85 100644
@@ -25,6 +25,26 @@
 define_per_cpu!(PERCPU: i64 = 0);
 define_per_cpu!(UPERCPU: u64 = 0);
 
+macro_rules! make_optimization_test {
+    ($ty:ty) => {
+        let mut test: DynamicPerCpu<$ty> = DynamicPerCpu::new(GFP_KERNEL).unwrap();
+        {
+            let _ = CpuGuard::new();
+            // SAFETY: No other usage of `test`
+            unsafe { test.get(CpuGuard::new()) }.with(|val: &mut $ty| *val = 10);
+            test.num().add(1);
+            // SAFETY: No other usage of `test`
+            unsafe { test.get(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 11));
+            test.num().add(10);
+            // SAFETY: No other usage of `test`
+            unsafe { test.get(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 21));
+            test.num().sub(5);
+            // SAFETY: No other usage of `test`
+            unsafe { test.get(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 16));
+        }
+    };
+}
+
 impl kernel::Module for PerCpuTestModule {
     fn init(_module: &'static ThisModule) -> Result<Self, Error> {
         pr_info!("rust percpu test start\n");

@@ -94,6 +114,22 @@ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
 
         pr_info!("rust dynamic percpu test done\n");
 
+        pr_info!("rust numeric optimizations test start\n");
+
+        make_optimization_test!(u8);
+        make_optimization_test!(u16);
+        make_optimization_test!(u32);
+        make_optimization_test!(u64);
+        make_optimization_test!(usize);
+
+        make_optimization_test!(i8);
+        make_optimization_test!(i16);
+        make_optimization_test!(i32);
+        make_optimization_test!(i64);
+        make_optimization_test!(isize);
+
+        pr_info!("rust numeric optimizations test done\n");
+
         // Return Err to unload the module
         Result::Err(EINVAL)
     }

index a912f74349e0..9659bff2d889 100644
@@ -2,6 +2,7 @@
 //! This module contains abstractions for creating and using per-CPU variables from Rust.
 //! See the define_per_cpu! macro and the PerCpu<T> type.
 pub mod cpu_guard;
+pub mod numeric;
 
 use bindings::{alloc_percpu, free_percpu};
 

117 lines not on this page · raw message

index 9659bff2d889..ffcfe02124ab 100644
@@ -23,7 +23,10 @@
 
 /// Holds a dynamically-allocated per-CPU variable.
 pub struct DynamicPerCpu<T> {
+    // INVARIANT: `ptr` is managed by `alloc` and the value of `ptr` does not change for the
+    // lifetime of `self`.
     alloc: Arc<PerCpuAllocation<T>>,
+    ptr: PerCpuPtr<T>,
 }
 
 /// Holds a statically-allocated per-CPU variable.

@@ -199,9 +202,10 @@ impl<T> DynamicPerCpu<T> {
     pub fn new(flags: Flags) -> Option<Self> {
         let alloc: PerCpuAllocation<T> = PerCpuAllocation::new()?;
 
+        let ptr = alloc.0;
         let arc = Arc::new(alloc, flags).ok()?;
 
-        Some(Self { alloc: arc })
+        Some(Self { alloc: arc, ptr })
     }
 
     /// Wraps a `PerCpuAllocation<T>` in a `PerCpu<T>`

@@ -210,8 +214,9 @@ pub fn new(flags: Flags) -> Option<Self> {
     /// * `alloc` - The allocation to use
     /// * `flags` - The flags used to allocate an `Arc` that keeps track of the `PerCpuAllocation`.
     pub fn new_from_allocation(alloc: PerCpuAllocation<T>, flags: Flags) -> Option<Self> {
+        let ptr = alloc.0;
         let arc = Arc::new(alloc, flags).ok()?;
-        Some(Self { alloc: arc })
+        Some(Self { alloc: arc, ptr })
     }
 }
 

@@ -219,7 +224,7 @@ pub fn new_from_allocation(alloc: PerCpuAllocation<T>, flags: Flags) -> Option<S
 // don't deallocate the underlying `PerCpuAllocation` until `self` is dropped.
 unsafe impl<T> PerCpu<T> for DynamicPerCpu<T> {
     unsafe fn ptr(&mut self) -> &PerCpuPtr<T> {
-        &self.alloc.0
+        &self.ptr
     }
 }
 

@@ -227,6 +232,7 @@ impl<T> Clone for DynamicPerCpu<T> {
     fn clone(&self) -> Self {
         Self {
             alloc: self.alloc.clone(),
+            ptr: self.ptr,
         }
     }
 }

index e4008f872af1..1b37cc7e5c19 100644
@@ -62,7 +62,7 @@ impl DynamicPerCpu<$ty> {
             /// Returns a `PerCpuNumeric` that can be used to manipulate the underlying per-CPU
             /// variable.
             pub fn num(&self) -> PerCpuNumeric<'_, $ty> {
-                PerCpuNumeric { ptr: &self.alloc.0 }
+                PerCpuNumeric { ptr: &self.ptr }
             }
         }
         impl StaticPerCpu<$ty> {