diff options
Diffstat (limited to 'rust/pin-init')
| -rw-r--r-- | rust/pin-init/CONTRIBUTING.md | 72 | ||||
| -rw-r--r-- | rust/pin-init/README.md | 228 | ||||
| -rw-r--r-- | rust/pin-init/examples/big_struct_in_place.rs | 39 | ||||
| -rw-r--r-- | rust/pin-init/examples/error.rs | 27 | ||||
| -rw-r--r-- | rust/pin-init/examples/linked_list.rs | 161 | ||||
| -rw-r--r-- | rust/pin-init/examples/mutex.rs | 209 | ||||
| -rw-r--r-- | rust/pin-init/examples/pthread_mutex.rs | 178 | ||||
| -rw-r--r-- | rust/pin-init/examples/static_init.rs | 122 | ||||
| -rw-r--r-- | rust/pin-init/internal/src/helpers.rs | 152 | ||||
| -rw-r--r-- | rust/pin-init/internal/src/lib.rs | 48 | ||||
| -rw-r--r-- | rust/pin-init/internal/src/pin_data.rs | 132 | ||||
| -rw-r--r-- | rust/pin-init/internal/src/pinned_drop.rs | 52 | ||||
| -rw-r--r-- | rust/pin-init/internal/src/zeroable.rs | 76 | ||||
| -rw-r--r-- | rust/pin-init/src/__internal.rs | 292 | ||||
| -rw-r--r-- | rust/pin-init/src/alloc.rs | 156 | ||||
| -rw-r--r-- | rust/pin-init/src/lib.rs | 1483 | ||||
| -rw-r--r-- | rust/pin-init/src/macros.rs | 1415 |
17 files changed, 4842 insertions, 0 deletions
diff --git a/rust/pin-init/CONTRIBUTING.md b/rust/pin-init/CONTRIBUTING.md new file mode 100644 index 000000000000..16c899a7ae0b --- /dev/null +++ b/rust/pin-init/CONTRIBUTING.md @@ -0,0 +1,72 @@ +# Contributing to `pin-init` + +Thanks for showing interest in contributing to `pin-init`! This document outlines the guidelines for +contributing to `pin-init`. + +All contributions are double-licensed under Apache 2.0 and MIT. You can find the respective licenses +in the `LICENSE-APACHE` and `LICENSE-MIT` files. + +## Non-Code Contributions + +### Bug Reports + +For any type of bug report, please submit an issue using the bug report issue template. + +If the issue is a soundness issue, please privately report it as a security vulnerability via the +GitHub web interface. + +### Feature Requests + +If you have any feature requests, please submit an issue using the feature request issue template. + +### Questions and Getting Help + +You can ask questions in the Discussions page of the GitHub repository. If you're encountering +problems or just have questions related to `pin-init` in the Linux kernel, you can also ask your +questions in the [Rust-for-Linux Zulip](https://rust-for-linux.zulipchat.com/) or see +<https://rust-for-linux.com/contact>. + +## Contributing Code + +### Linux Kernel + +`pin-init` is used by the Linux kernel and all commits are synchronized to it. For this reason, the +same requirements for commits apply to `pin-init`. See [the kernel's documentation] for details. The +rest of this document will also cover some of the rules listed there and additional ones. + +[the kernel's documentation]: https://docs.kernel.org/process/submitting-patches.html + +Contributions to `pin-init` ideally go through the [GitHub repository], because that repository runs +a CI with lots of tests not present in the kernel. However, patches are also accepted (though not +preferred). Do note that there are some files that are only present in the GitHub repository such as +tests, licenses and cargo related files. Making changes to them can only happen via GitHub. + +[GitHub repository]: https://github.com/Rust-for-Linux/pin-init + +### Commit Style + +Everything must compile without errors or warnings and all tests must pass after **every commit**. +This is important for bisection and also required by the kernel. + +Each commit should be a single, logically cohesive change. Of course it's best to keep the changes +small and digestible, but logically linked changes should be made in the same commit. For example, +when fixing typos, create a single commit that fixes all of them instead of one commit per typo. + +Commits must have a meaningful commit title. Commits with changes to files in the `internal` +directory should have a title prefixed with `internal:`. The commit message should explain the +change and its rationale. You also have to add your `Signed-off-by` tag, see [Developer's +Certificate of Origin]. This has to be done for both mailing list submissions as well as GitHub +submissions. + +[Developer's Certificate of Origin]: https://docs.kernel.org/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin + +Any changes made to public APIs must be documented not only in the commit message, but also in the +`CHANGELOG.md` file. This is especially important for breaking changes, as those warrant a major +version bump. + +If you make changes to the top-level crate documentation, you also need to update the `README.md` +via `cargo rdme`. + +Some of these rules can be ignored if the change is done solely to files that are not present in the +kernel version of this library. Those files are documented in the `sync-kernel.sh` script at the +very bottom in the `--exclude` flag given to the `git am` command. diff --git a/rust/pin-init/README.md b/rust/pin-init/README.md new file mode 100644 index 000000000000..3d04796b212b --- /dev/null +++ b/rust/pin-init/README.md @@ -0,0 +1,228 @@ +[](https://crates.io/crates/pin-init) +[](https://docs.rs/pin-init/) +[](https://deps.rs/repo/github/Rust-for-Linux/pin-init) + +[](#nightly-only) + +# `pin-init` + +<!-- cargo-rdme start --> + +Library to safely and fallibly initialize pinned `struct`s using in-place constructors. + +[Pinning][pinning] is Rust's way of ensuring data does not move. + +It also allows in-place initialization of big `struct`s that would otherwise produce a stack +overflow. + +This library's main use-case is in [Rust-for-Linux]. Although this version can be used +standalone. + +There are cases when you want to in-place initialize a struct. For example when it is very big +and moving it from the stack is not an option, because it is bigger than the stack itself. +Another reason would be that you need the address of the object to initialize it. This stands +in direct conflict with Rust's normal process of first initializing an object and then moving +it into it's final memory location. For more information, see +<https://rust-for-linux.com/the-safe-pinned-initialization-problem>. + +This library allows you to do in-place initialization safely. + +### Nightly Needed for `alloc` feature + +This library requires the [`allocator_api` unstable feature] when the `alloc` feature is +enabled and thus this feature can only be used with a nightly compiler. When enabling the +`alloc` feature, the user will be required to activate `allocator_api` as well. + +[`allocator_api` unstable feature]: https://doc.rust-lang.org/nightly/unstable-book/library-features/allocator-api.html + +The feature is enabled by default, thus by default `pin-init` will require a nightly compiler. +However, using the crate on stable compilers is possible by disabling `alloc`. In practice this +will require the `std` feature, because stable compilers have neither `Box` nor `Arc` in no-std +mode. + +## Overview + +To initialize a `struct` with an in-place constructor you will need two things: +- an in-place constructor, +- a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`], + [`Box<T>`] or any other smart pointer that supports this library). + +To get an in-place constructor there are generally three options: +- directly creating an in-place constructor using the [`pin_init!`] macro, +- a custom function/macro returning an in-place constructor provided by someone else, +- using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. + +Aside from pinned initialization, this library also supports in-place construction without +pinning, the macros/types/functions are generally named like the pinned variants without the +`pin_` prefix. + +## Examples + +Throughout the examples we will often make use of the `CMutex` type which can be found in +`../examples/mutex.rs`. It is essentially a userland rebuild of the `struct mutex` type from +the Linux kernel. It also uses a wait list and a basic spinlock. Importantly the wait list +requires it to be pinned to be locked and thus is a prime candidate for using this library. + +### Using the [`pin_init!`] macro + +If you want to use [`PinInit`], then you will have to annotate your `struct` with +`#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for +[structurally pinned fields]. After doing this, you can then create an in-place constructor via +[`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is +that you need to write `<-` instead of `:` for fields that you want to initialize in-place. + +```rust +use pin_init::{pin_data, pin_init, InPlaceInit}; + +#[pin_data] +struct Foo { + #[pin] + a: CMutex<usize>, + b: u32, +} + +let foo = pin_init!(Foo { + a <- CMutex::new(42), + b: 24, +}); +``` + +`foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like +(or just the stack) to actually initialize a `Foo`: + +```rust +let foo: Result<Pin<Box<Foo>>, AllocError> = Box::pin_init(foo); +``` + +For more information see the [`pin_init!`] macro. + +### Using a custom function/macro that returns an initializer + +Many types that use this library supply a function/macro that returns an initializer, because +the above method only works for types where you can access the fields. + +```rust +let mtx: Result<Pin<Arc<CMutex<usize>>>, _> = Arc::pin_init(CMutex::new(42)); +``` + +To declare an init macro/function you just return an [`impl PinInit<T, E>`]: + +```rust +#[pin_data] +struct DriverData { + #[pin] + status: CMutex<i32>, + buffer: Box<[u8; 1_000_000]>, +} + +impl DriverData { + fn new() -> impl PinInit<Self, Error> { + try_pin_init!(Self { + status <- CMutex::new(0), + buffer: Box::init(pin_init::zeroed())?, + }? Error) + } +} +``` + +### Manual creation of an initializer + +Often when working with primitives the previous approaches are not sufficient. That is where +[`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a +[`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure +actually does the initialization in the correct way. Here are the things to look out for +(we are calling the parameter to the closure `slot`): +- when the closure returns `Ok(())`, then it has completed the initialization successfully, so + `slot` now contains a valid bit pattern for the type `T`, +- when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so + you need to take care to clean up anything if your initialization fails mid-way, +- you may assume that `slot` will stay pinned even after the closure returns until `drop` of + `slot` gets called. + +```rust +use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; +use core::{ + ptr::addr_of_mut, + marker::PhantomPinned, + cell::UnsafeCell, + pin::Pin, + mem::MaybeUninit, +}; +mod bindings { + #[repr(C)] + pub struct foo { + /* fields from C ... */ + } + extern "C" { + pub fn init_foo(ptr: *mut foo); + pub fn destroy_foo(ptr: *mut foo); + #[must_use = "you must check the error return code"] + pub fn enable_foo(ptr: *mut foo, flags: u32) -> i32; + } +} + +/// # Invariants +/// +/// `foo` is always initialized +#[pin_data(PinnedDrop)] +pub struct RawFoo { + #[pin] + _p: PhantomPinned, + #[pin] + foo: UnsafeCell<MaybeUninit<bindings::foo>>, +} + +impl RawFoo { + pub fn new(flags: u32) -> impl PinInit<Self, i32> { + // SAFETY: + // - when the closure returns `Ok(())`, then it has successfully initialized and + // enabled `foo`, + // - when it returns `Err(e)`, then it has cleaned up before + unsafe { + pin_init_from_closure(move |slot: *mut Self| { + // `slot` contains uninit memory, avoid creating a reference. + let foo = addr_of_mut!((*slot).foo); + let foo = UnsafeCell::raw_get(foo).cast::<bindings::foo>(); + + // Initialize the `foo` + bindings::init_foo(foo); + + // Try to enable it. + let err = bindings::enable_foo(foo, flags); + if err != 0 { + // Enabling has failed, first clean up the foo and then return the error. + bindings::destroy_foo(foo); + Err(err) + } else { + // All fields of `RawFoo` have been initialized, since `_p` is a ZST. + Ok(()) + } + }) + } + } +} + +#[pinned_drop] +impl PinnedDrop for RawFoo { + fn drop(self: Pin<&mut Self>) { + // SAFETY: Since `foo` is initialized, destroying is safe. + unsafe { bindings::destroy_foo(self.foo.get().cast::<bindings::foo>()) }; + } +} +``` + +For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside +the `kernel` crate. The [`sync`] module is a good starting point. + +[`sync`]: https://rust.docs.kernel.org/kernel/sync/index.html +[pinning]: https://doc.rust-lang.org/std/pin/index.html +[structurally pinned fields]: https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field +[stack]: https://docs.rs/pin-init/latest/pin_init/macro.stack_pin_init.html +[`Arc<T>`]: https://doc.rust-lang.org/stable/alloc/sync/struct.Arc.html +[`Box<T>`]: https://doc.rust-lang.org/stable/alloc/boxed/struct.Box.html +[`impl PinInit<Foo>`]: https://docs.rs/pin-init/latest/pin_init/trait.PinInit.html +[`impl PinInit<T, E>`]: https://docs.rs/pin-init/latest/pin_init/trait.PinInit.html +[`impl Init<T, E>`]: https://docs.rs/pin-init/latest/pin_init/trait.Init.html +[Rust-for-Linux]: https://rust-for-linux.com/ + +<!-- cargo-rdme end --> diff --git a/rust/pin-init/examples/big_struct_in_place.rs b/rust/pin-init/examples/big_struct_in_place.rs new file mode 100644 index 000000000000..30d44a334ffd --- /dev/null +++ b/rust/pin-init/examples/big_struct_in_place.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use pin_init::*; + +// Struct with size over 1GiB +#[derive(Debug)] +pub struct BigStruct { + buf: [u8; 1024 * 1024 * 1024], + a: u64, + b: u64, + c: u64, + d: u64, + managed_buf: ManagedBuf, +} + +#[derive(Debug)] +pub struct ManagedBuf { + buf: [u8; 1024 * 1024], +} + +impl ManagedBuf { + pub fn new() -> impl Init<Self> { + init!(ManagedBuf { buf <- zeroed() }) + } +} + +fn main() { + // we want to initialize the struct in-place, otherwise we would get a stackoverflow + let buf: Box<BigStruct> = Box::init(init!(BigStruct { + buf <- zeroed(), + a: 7, + b: 186, + c: 7789, + d: 34, + managed_buf <- ManagedBuf::new(), + })) + .unwrap(); + println!("{}", core::mem::size_of_val(&*buf)); +} diff --git a/rust/pin-init/examples/error.rs b/rust/pin-init/examples/error.rs new file mode 100644 index 000000000000..e0cc258746ce --- /dev/null +++ b/rust/pin-init/examples/error.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::convert::Infallible; + +#[cfg(feature = "alloc")] +use std::alloc::AllocError; + +#[derive(Debug)] +pub struct Error; + +impl From<Infallible> for Error { + fn from(e: Infallible) -> Self { + match e {} + } +} + +#[cfg(feature = "alloc")] +impl From<AllocError> for Error { + fn from(_: AllocError) -> Self { + Self + } +} + +#[allow(dead_code)] +fn main() {} diff --git a/rust/pin-init/examples/linked_list.rs b/rust/pin-init/examples/linked_list.rs new file mode 100644 index 000000000000..6d7eb0a0ec0d --- /dev/null +++ b/rust/pin-init/examples/linked_list.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::{ + cell::Cell, + convert::Infallible, + marker::PhantomPinned, + pin::Pin, + ptr::{self, NonNull}, +}; + +use pin_init::*; + +#[expect(unused_attributes)] +mod error; +use error::Error; + +#[pin_data(PinnedDrop)] +#[repr(C)] +#[derive(Debug)] +pub struct ListHead { + next: Link, + prev: Link, + #[pin] + pin: PhantomPinned, +} + +impl ListHead { + #[inline] + pub fn new() -> impl PinInit<Self, Infallible> { + try_pin_init!(&this in Self { + next: unsafe { Link::new_unchecked(this) }, + prev: unsafe { Link::new_unchecked(this) }, + pin: PhantomPinned, + }? Infallible) + } + + #[inline] + pub fn insert_next(list: &ListHead) -> impl PinInit<Self, Infallible> + '_ { + try_pin_init!(&this in Self { + prev: list.next.prev().replace(unsafe { Link::new_unchecked(this)}), + next: list.next.replace(unsafe { Link::new_unchecked(this)}), + pin: PhantomPinned, + }? Infallible) + } + + #[inline] + pub fn insert_prev(list: &ListHead) -> impl PinInit<Self, Infallible> + '_ { + try_pin_init!(&this in Self { + next: list.prev.next().replace(unsafe { Link::new_unchecked(this)}), + prev: list.prev.replace(unsafe { Link::new_unchecked(this)}), + pin: PhantomPinned, + }? Infallible) + } + + #[inline] + pub fn next(&self) -> Option<NonNull<Self>> { + if ptr::eq(self.next.as_ptr(), self) { + None + } else { + Some(unsafe { NonNull::new_unchecked(self.next.as_ptr() as *mut Self) }) + } + } + + #[allow(dead_code)] + pub fn size(&self) -> usize { + let mut size = 1; + let mut cur = self.next.clone(); + while !ptr::eq(self, cur.cur()) { + cur = cur.next().clone(); + size += 1; + } + size + } +} + +#[pinned_drop] +impl PinnedDrop for ListHead { + //#[inline] + fn drop(self: Pin<&mut Self>) { + if !ptr::eq(self.next.as_ptr(), &*self) { + let next = unsafe { &*self.next.as_ptr() }; + let prev = unsafe { &*self.prev.as_ptr() }; + next.prev.set(&self.prev); + prev.next.set(&self.next); + } + } +} + +#[repr(transparent)] +#[derive(Clone, Debug)] +struct Link(Cell<NonNull<ListHead>>); + +impl Link { + /// # Safety + /// + /// The contents of the pointer should form a consistent circular + /// linked list; for example, a "next" link should be pointed back + /// by the target `ListHead`'s "prev" link and a "prev" link should be + /// pointed back by the target `ListHead`'s "next" link. + #[inline] + unsafe fn new_unchecked(ptr: NonNull<ListHead>) -> Self { + Self(Cell::new(ptr)) + } + + #[inline] + fn next(&self) -> &Link { + unsafe { &(*self.0.get().as_ptr()).next } + } + + #[inline] + fn prev(&self) -> &Link { + unsafe { &(*self.0.get().as_ptr()).prev } + } + + #[allow(dead_code)] + fn cur(&self) -> &ListHead { + unsafe { &*self.0.get().as_ptr() } + } + + #[inline] + fn replace(&self, other: Link) -> Link { + unsafe { Link::new_unchecked(self.0.replace(other.0.get())) } + } + + #[inline] + fn as_ptr(&self) -> *const ListHead { + self.0.get().as_ptr() + } + + #[inline] + fn set(&self, val: &Link) { + self.0.set(val.0.get()); + } +} + +#[allow(dead_code)] +#[cfg_attr(test, test)] +fn main() -> Result<(), Error> { + let a = Box::pin_init(ListHead::new())?; + stack_pin_init!(let b = ListHead::insert_next(&a)); + stack_pin_init!(let c = ListHead::insert_next(&a)); + stack_pin_init!(let d = ListHead::insert_next(&b)); + let e = Box::pin_init(ListHead::insert_next(&b))?; + println!("a ({a:p}): {a:?}"); + println!("b ({b:p}): {b:?}"); + println!("c ({c:p}): {c:?}"); + println!("d ({d:p}): {d:?}"); + println!("e ({e:p}): {e:?}"); + let mut inspect = &*a; + while let Some(next) = inspect.next() { + println!("({inspect:p}): {inspect:?}"); + inspect = unsafe { &*next.as_ptr() }; + if core::ptr::eq(inspect, &*a) { + break; + } + } + Ok(()) +} diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs new file mode 100644 index 000000000000..073bb79341d1 --- /dev/null +++ b/rust/pin-init/examples/mutex.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] +#![allow(clippy::missing_safety_doc)] + +use core::{ + cell::{Cell, UnsafeCell}, + marker::PhantomPinned, + ops::{Deref, DerefMut}, + pin::Pin, + sync::atomic::{AtomicBool, Ordering}, +}; +use std::{ + sync::Arc, + thread::{self, park, sleep, Builder, Thread}, + time::Duration, +}; + +use pin_init::*; +#[expect(unused_attributes)] +#[path = "./linked_list.rs"] +pub mod linked_list; +use linked_list::*; + +pub struct SpinLock { + inner: AtomicBool, +} + +impl SpinLock { + #[inline] + pub fn acquire(&self) -> SpinLockGuard<'_> { + while self + .inner + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + while self.inner.load(Ordering::Relaxed) { + thread::yield_now(); + } + } + SpinLockGuard(self) + } + + #[inline] + #[allow(clippy::new_without_default)] + pub const fn new() -> Self { + Self { + inner: AtomicBool::new(false), + } + } +} + +pub struct SpinLockGuard<'a>(&'a SpinLock); + +impl Drop for SpinLockGuard<'_> { + #[inline] + fn drop(&mut self) { + self.0.inner.store(false, Ordering::Release); + } +} + +#[pin_data] +pub struct CMutex<T> { + #[pin] + wait_list: ListHead, + spin_lock: SpinLock, + locked: Cell<bool>, + #[pin] + data: UnsafeCell<T>, +} + +impl<T> CMutex<T> { + #[inline] + pub fn new(val: impl PinInit<T>) -> impl PinInit<Self> { + pin_init!(CMutex { + wait_list <- ListHead::new(), + spin_lock: SpinLock::new(), + locked: Cell::new(false), + data <- unsafe { + pin_init_from_closure(|slot: *mut UnsafeCell<T>| { + val.__pinned_init(slot.cast::<T>()) + }) + }, + }) + } + + #[inline] + pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> { + let mut sguard = self.spin_lock.acquire(); + if self.locked.get() { + stack_pin_init!(let wait_entry = WaitEntry::insert_new(&self.wait_list)); + // println!("wait list length: {}", self.wait_list.size()); + while self.locked.get() { + drop(sguard); + park(); + sguard = self.spin_lock.acquire(); + } + // This does have an effect, as the ListHead inside wait_entry implements Drop! + #[expect(clippy::drop_non_drop)] + drop(wait_entry); + } + self.locked.set(true); + unsafe { + Pin::new_unchecked(CMutexGuard { + mtx: self, + _pin: PhantomPinned, + }) + } + } + + #[allow(dead_code)] + pub fn get_data_mut(self: Pin<&mut Self>) -> &mut T { + // SAFETY: we have an exclusive reference and thus nobody has access to data. + unsafe { &mut *self.data.get() } + } +} + +unsafe impl<T: Send> Send for CMutex<T> {} +unsafe impl<T: Send> Sync for CMutex<T> {} + +pub struct CMutexGuard<'a, T> { + mtx: &'a CMutex<T>, + _pin: PhantomPinned, +} + +impl<T> Drop for CMutexGuard<'_, T> { + #[inline] + fn drop(&mut self) { + let sguard = self.mtx.spin_lock.acquire(); + self.mtx.locked.set(false); + if let Some(list_field) = self.mtx.wait_list.next() { + let wait_entry = list_field.as_ptr().cast::<WaitEntry>(); + unsafe { (*wait_entry).thread.unpark() }; + } + drop(sguard); + } +} + +impl<T> Deref for CMutexGuard<'_, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + unsafe { &*self.mtx.data.get() } + } +} + +impl<T> DerefMut for CMutexGuard<'_, T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.mtx.data.get() } + } +} + +#[pin_data] +#[repr(C)] +struct WaitEntry { + #[pin] + wait_list: ListHead, + thread: Thread, +} + +impl WaitEntry { + #[inline] + fn insert_new(list: &ListHead) -> impl PinInit<Self> + '_ { + pin_init!(Self { + thread: thread::current(), + wait_list <- ListHead::insert_prev(list), + }) + } +} + +#[cfg(not(any(feature = "std", feature = "alloc")))] +fn main() {} + +#[allow(dead_code)] +#[cfg_attr(test, test)] +#[cfg(any(feature = "std", feature = "alloc"))] +fn main() { + let mtx: Pin<Arc<CMutex<usize>>> = Arc::pin_init(CMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = if cfg!(miri) { 100 } else { 1_000 }; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}", &*mtx.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); +} diff --git a/rust/pin-init/examples/pthread_mutex.rs b/rust/pin-init/examples/pthread_mutex.rs new file mode 100644 index 000000000000..5ac22f1880d2 --- /dev/null +++ b/rust/pin-init/examples/pthread_mutex.rs @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// inspired by <https://github.com/nbdd0121/pin-init/blob/trunk/examples/pthread_mutex.rs> +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] +#[cfg(not(windows))] +mod pthread_mtx { + #[cfg(feature = "alloc")] + use core::alloc::AllocError; + use core::{ + cell::UnsafeCell, + marker::PhantomPinned, + mem::MaybeUninit, + ops::{Deref, DerefMut}, + pin::Pin, + }; + use pin_init::*; + use std::convert::Infallible; + + #[pin_data(PinnedDrop)] + pub struct PThreadMutex<T> { + #[pin] + raw: UnsafeCell<libc::pthread_mutex_t>, + data: UnsafeCell<T>, + #[pin] + pin: PhantomPinned, + } + + unsafe impl<T: Send> Send for PThreadMutex<T> {} + unsafe impl<T: Send> Sync for PThreadMutex<T> {} + + #[pinned_drop] + impl<T> PinnedDrop for PThreadMutex<T> { + fn drop(self: Pin<&mut Self>) { + unsafe { + libc::pthread_mutex_destroy(self.raw.get()); + } + } + } + + #[derive(Debug)] + pub enum Error { + #[expect(dead_code)] + IO(std::io::Error), + Alloc, + } + + impl From<Infallible> for Error { + fn from(e: Infallible) -> Self { + match e {} + } + } + + #[cfg(feature = "alloc")] + impl From<AllocError> for Error { + fn from(_: AllocError) -> Self { + Self::Alloc + } + } + + impl<T> PThreadMutex<T> { + pub fn new(data: T) -> impl PinInit<Self, Error> { + fn init_raw() -> impl PinInit<UnsafeCell<libc::pthread_mutex_t>, Error> { + let init = |slot: *mut UnsafeCell<libc::pthread_mutex_t>| { + // we can cast, because `UnsafeCell` has the same layout as T. + let slot: *mut libc::pthread_mutex_t = slot.cast(); + let mut attr = MaybeUninit::uninit(); + let attr = attr.as_mut_ptr(); + // SAFETY: ptr is valid + let ret = unsafe { libc::pthread_mutexattr_init(attr) }; + if ret != 0 { + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + // SAFETY: attr is initialized + let ret = unsafe { + libc::pthread_mutexattr_settype(attr, libc::PTHREAD_MUTEX_NORMAL) + }; + if ret != 0 { + // SAFETY: attr is initialized + unsafe { libc::pthread_mutexattr_destroy(attr) }; + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + // SAFETY: slot is valid + unsafe { slot.write(libc::PTHREAD_MUTEX_INITIALIZER) }; + // SAFETY: attr and slot are valid ptrs and attr is initialized + let ret = unsafe { libc::pthread_mutex_init(slot, attr) }; + // SAFETY: attr was initialized + unsafe { libc::pthread_mutexattr_destroy(attr) }; + if ret != 0 { + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + Ok(()) + }; + // SAFETY: mutex has been initialized + unsafe { pin_init_from_closure(init) } + } + try_pin_init!(Self { + data: UnsafeCell::new(data), + raw <- init_raw(), + pin: PhantomPinned, + }? Error) + } + + pub fn lock(&self) -> PThreadMutexGuard<'_, T> { + // SAFETY: raw is always initialized + unsafe { libc::pthread_mutex_lock(self.raw.get()) }; + PThreadMutexGuard { mtx: self } + } + } + + pub struct PThreadMutexGuard<'a, T> { + mtx: &'a PThreadMutex<T>, + } + + impl<T> Drop for PThreadMutexGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: raw is always initialized + unsafe { libc::pthread_mutex_unlock(self.mtx.raw.get()) }; + } + } + + impl<T> Deref for PThreadMutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.mtx.data.get() } + } + } + + impl<T> DerefMut for PThreadMutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.mtx.data.get() } + } + } +} + +#[cfg_attr(test, test)] +fn main() { + #[cfg(all(any(feature = "std", feature = "alloc"), not(windows)))] + { + use core::pin::Pin; + use pin_init::*; + use pthread_mtx::*; + use std::{ + sync::Arc, + thread::{sleep, Builder}, + time::Duration, + }; + let mtx: Pin<Arc<PThreadMutex<usize>>> = Arc::try_pin_init(PThreadMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = 1_000_000; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}", &*mtx.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); + } +} diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs new file mode 100644 index 000000000000..3487d761aa26 --- /dev/null +++ b/rust/pin-init/examples/static_init.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::{ + cell::{Cell, UnsafeCell}, + mem::MaybeUninit, + ops, + pin::Pin, + time::Duration, +}; +use pin_init::*; +use std::{ + sync::Arc, + thread::{sleep, Builder}, +}; + +#[expect(unused_attributes)] +mod mutex; +use mutex::*; + +pub struct StaticInit<T, I> { + cell: UnsafeCell<MaybeUninit<T>>, + init: Cell<Option<I>>, + lock: SpinLock, + present: Cell<bool>, +} + +unsafe impl<T: Sync, I> Sync for StaticInit<T, I> {} +unsafe impl<T: Send, I> Send for StaticInit<T, I> {} + +impl<T, I: PinInit<T>> StaticInit<T, I> { + pub const fn new(init: I) -> Self { + Self { + cell: UnsafeCell::new(MaybeUninit::uninit()), + init: Cell::new(Some(init)), + lock: SpinLock::new(), + present: Cell::new(false), + } + } +} + +impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> { + type Target = T; + fn deref(&self) -> &Self::Target { + if self.present.get() { + unsafe { (*self.cell.get()).assume_init_ref() } + } else { + println!("acquire spinlock on static init"); + let _guard = self.lock.acquire(); + println!("rechecking present..."); + std::thread::sleep(std::time::Duration::from_millis(200)); + if self.present.get() { + return unsafe { (*self.cell.get()).assume_init_ref() }; + } + println!("doing init"); + let ptr = self.cell.get().cast::<T>(); + match self.init.take() { + Some(f) => unsafe { f.__pinned_init(ptr).unwrap() }, + None => unsafe { core::hint::unreachable_unchecked() }, + } + self.present.set(true); + unsafe { (*self.cell.get()).assume_init_ref() } + } + } +} + +pub struct CountInit; + +unsafe impl PinInit<CMutex<usize>> for CountInit { + unsafe fn __pinned_init( + self, + slot: *mut CMutex<usize>, + ) -> Result<(), core::convert::Infallible> { + let init = CMutex::new(0); + std::thread::sleep(std::time::Duration::from_millis(1000)); + unsafe { init.__pinned_init(slot) } + } +} + +pub static COUNT: StaticInit<CMutex<usize>, CountInit> = StaticInit::new(CountInit); + +#[cfg(not(any(feature = "std", feature = "alloc")))] +fn main() {} + +#[cfg(any(feature = "std", feature = "alloc"))] +fn main() { + let mtx: Pin<Arc<CMutex<usize>>> = Arc::pin_init(CMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = 1_000; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *COUNT.lock() += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + *mtx.lock() += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + *COUNT.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + std::thread::sleep(std::time::Duration::from_millis(10)); + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}, {:?}", &*mtx.lock(), &*COUNT.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); +} diff --git a/rust/pin-init/internal/src/helpers.rs b/rust/pin-init/internal/src/helpers.rs new file mode 100644 index 000000000000..236f989a50f2 --- /dev/null +++ b/rust/pin-init/internal/src/helpers.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(not(kernel))] +use proc_macro2 as proc_macro; + +use proc_macro::{TokenStream, TokenTree}; + +/// Parsed generics. +/// +/// See the field documentation for an explanation what each of the fields represents. +/// +/// # Examples +/// +/// ```rust,ignore +/// # let input = todo!(); +/// let (Generics { decl_generics, impl_generics, ty_generics }, rest) = parse_generics(input); +/// quote! { +/// struct Foo<$($decl_generics)*> { +/// // ... +/// } +/// +/// impl<$impl_generics> Foo<$ty_generics> { +/// fn foo() { +/// // ... +/// } +/// } +/// } +/// ``` +pub(crate) struct Generics { + /// The generics with bounds and default values (e.g. `T: Clone, const N: usize = 0`). + /// + /// Use this on type definitions e.g. `struct Foo<$decl_generics> ...` (or `union`/`enum`). + pub(crate) decl_generics: Vec<TokenTree>, + /// The generics with bounds (e.g. `T: Clone, const N: usize`). + /// + /// Use this on `impl` blocks e.g. `impl<$impl_generics> Trait for ...`. + pub(crate) impl_generics: Vec<TokenTree>, + /// The generics without bounds and without default values (e.g. `T, N`). + /// + /// Use this when you use the type that is declared with these generics e.g. + /// `Foo<$ty_generics>`. + pub(crate) ty_generics: Vec<TokenTree>, +} + +/// Parses the given `TokenStream` into `Generics` and the rest. +/// +/// The generics are not present in the rest, but a where clause might remain. +pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec<TokenTree>) { + // The generics with bounds and default values. + let mut decl_generics = vec![]; + // `impl_generics`, the declared generics with their bounds. + let mut impl_generics = vec![]; + // Only the names of the generics, without any bounds. + let mut ty_generics = vec![]; + // Tokens not related to the generics e.g. the `where` token and definition. + let mut rest = vec![]; + // The current level of `<`. + let mut nesting = 0; + let mut toks = input.into_iter(); + // If we are at the beginning of a generic parameter. + let mut at_start = true; + let mut skip_until_comma = false; + while let Some(tt) = toks.next() { + if nesting == 1 && matches!(&tt, TokenTree::Punct(p) if p.as_char() == '>') { + // Found the end of the generics. + break; + } else if nesting >= 1 { + decl_generics.push(tt.clone()); + } + match tt.clone() { + TokenTree::Punct(p) if p.as_char() == '<' => { + if nesting >= 1 && !skip_until_comma { + // This is inside of the generics and part of some bound. + impl_generics.push(tt); + } + nesting += 1; + } + TokenTree::Punct(p) if p.as_char() == '>' => { + // This is a parsing error, so we just end it here. + if nesting == 0 { + break; + } else { + nesting -= 1; + if nesting >= 1 && !skip_until_comma { + // We are still inside of the generics and part of some bound. + impl_generics.push(tt); + } + } + } + TokenTree::Punct(p) if skip_until_comma && p.as_char() == ',' => { + if nesting == 1 { + impl_generics.push(tt.clone()); + impl_generics.push(tt); + skip_until_comma = false; + } + } + _ if !skip_until_comma => { + match nesting { + // If we haven't entered the generics yet, we still want to keep these tokens. + 0 => rest.push(tt), + 1 => { + // Here depending on the token, it might be a generic variable name. + match tt.clone() { + TokenTree::Ident(i) if at_start && i.to_string() == "const" => { + let Some(name) = toks.next() else { + // Parsing error. + break; + }; + impl_generics.push(tt); + impl_generics.push(name.clone()); + ty_generics.push(name.clone()); + decl_generics.push(name); + at_start = false; + } + TokenTree::Ident(_) if at_start => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + at_start = false; + } + TokenTree::Punct(p) if p.as_char() == ',' => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + at_start = true; + } + // Lifetimes begin with `'`. + TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + } + // Generics can have default values, we skip these. + TokenTree::Punct(p) if p.as_char() == '=' => { + skip_until_comma = true; + } + _ => impl_generics.push(tt), + } + } + _ => impl_generics.push(tt), + } + } + _ => {} + } + } + rest.extend(toks); + ( + Generics { + impl_generics, + decl_generics, + ty_generics, + }, + rest, + ) +} diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs new file mode 100644 index 000000000000..babe5e878550 --- /dev/null +++ b/rust/pin-init/internal/src/lib.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT` +// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is +// touched by Kconfig when the version string from the compiler changes. + +//! `pin-init` proc macros. + +#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] +// Allow `.into()` to convert +// - `proc_macro2::TokenStream` into `proc_macro::TokenStream` in the user-space version. +// - `proc_macro::TokenStream` into `proc_macro::TokenStream` in the kernel version. +// Clippy warns on this conversion, but it's required by the user-space version. +// +// Remove once we have `proc_macro2` in the kernel. +#![allow(clippy::useless_conversion)] +// Documentation is done in the pin-init crate instead. +#![allow(missing_docs)] + +use proc_macro::TokenStream; + +#[cfg(kernel)] +#[path = "../../../macros/quote.rs"] +#[macro_use] +mod quote; +#[cfg(not(kernel))] +#[macro_use] +extern crate quote; + +mod helpers; +mod pin_data; +mod pinned_drop; +mod zeroable; + +#[proc_macro_attribute] +pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { + pin_data::pin_data(inner.into(), item.into()).into() +} + +#[proc_macro_attribute] +pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { + pinned_drop::pinned_drop(args.into(), input.into()).into() +} + +#[proc_macro_derive(Zeroable)] +pub fn derive_zeroable(input: TokenStream) -> TokenStream { + zeroable::derive(input.into()).into() +} diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs new file mode 100644 index 000000000000..87d4a7eb1d35 --- /dev/null +++ b/rust/pin-init/internal/src/pin_data.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(not(kernel))] +use proc_macro2 as proc_macro; + +use crate::helpers::{parse_generics, Generics}; +use proc_macro::{Group, Punct, Spacing, TokenStream, TokenTree}; + +pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { + // This proc-macro only does some pre-parsing and then delegates the actual parsing to + // `pin_init::__pin_data!`. + + let ( + Generics { + impl_generics, + decl_generics, + ty_generics, + }, + rest, + ) = parse_generics(input); + // The struct definition might contain the `Self` type. Since `__pin_data!` will define a new + // type with the same generics and bounds, this poses a problem, since `Self` will refer to the + // new type as opposed to this struct definition. Therefore we have to replace `Self` with the + // concrete name. + + // Errors that occur when replacing `Self` with `struct_name`. + let mut errs = TokenStream::new(); + // The name of the struct with ty_generics. + let struct_name = rest + .iter() + .skip_while(|tt| !matches!(tt, TokenTree::Ident(i) if i.to_string() == "struct")) + .nth(1) + .and_then(|tt| match tt { + TokenTree::Ident(_) => { + let tt = tt.clone(); + let mut res = vec![tt]; + if !ty_generics.is_empty() { + // We add this, so it is maximally compatible with e.g. `Self::CONST` which + // will be replaced by `StructName::<$generics>::CONST`. + res.push(TokenTree::Punct(Punct::new(':', Spacing::Joint))); + res.push(TokenTree::Punct(Punct::new(':', Spacing::Alone))); + res.push(TokenTree::Punct(Punct::new('<', Spacing::Alone))); + res.extend(ty_generics.iter().cloned()); + res.push(TokenTree::Punct(Punct::new('>', Spacing::Alone))); + } + Some(res) + } + _ => None, + }) + .unwrap_or_else(|| { + // If we did not find the name of the struct then we will use `Self` as the replacement + // and add a compile error to ensure it does not compile. + errs.extend( + "::core::compile_error!(\"Could not locate type name.\");" + .parse::<TokenStream>() + .unwrap(), + ); + "Self".parse::<TokenStream>().unwrap().into_iter().collect() + }); + let impl_generics = impl_generics + .into_iter() + .flat_map(|tt| replace_self_and_deny_type_defs(&struct_name, tt, &mut errs)) + .collect::<Vec<_>>(); + let mut rest = rest + .into_iter() + .flat_map(|tt| { + // We ignore top level `struct` tokens, since they would emit a compile error. + if matches!(&tt, TokenTree::Ident(i) if i.to_string() == "struct") { + vec![tt] + } else { + replace_self_and_deny_type_defs(&struct_name, tt, &mut errs) + } + }) + .collect::<Vec<_>>(); + // This should be the body of the struct `{...}`. + let last = rest.pop(); + let mut quoted = quote!(::pin_init::__pin_data! { + parse_input: + @args(#args), + @sig(#(#rest)*), + @impl_generics(#(#impl_generics)*), + @ty_generics(#(#ty_generics)*), + @decl_generics(#(#decl_generics)*), + @body(#last), + }); + quoted.extend(errs); + quoted +} + +/// Replaces `Self` with `struct_name` and errors on `enum`, `trait`, `struct` `union` and `impl` +/// keywords. +/// +/// The error is appended to `errs` to allow normal parsing to continue. +fn replace_self_and_deny_type_defs( + struct_name: &Vec<TokenTree>, + tt: TokenTree, + errs: &mut TokenStream, +) -> Vec<TokenTree> { + match tt { + TokenTree::Ident(ref i) + if i.to_string() == "enum" + || i.to_string() == "trait" + || i.to_string() == "struct" + || i.to_string() == "union" + || i.to_string() == "impl" => + { + errs.extend( + format!( + "::core::compile_error!(\"Cannot use `{i}` inside of struct definition with \ + `#[pin_data]`.\");" + ) + .parse::<TokenStream>() + .unwrap() + .into_iter() + .map(|mut tok| { + tok.set_span(tt.span()); + tok + }), + ); + vec![tt] + } + TokenTree::Ident(i) if i.to_string() == "Self" => struct_name.clone(), + TokenTree::Literal(_) | TokenTree::Punct(_) | TokenTree::Ident(_) => vec![tt], + TokenTree::Group(g) => vec![TokenTree::Group(Group::new( + g.delimiter(), + g.stream() + .into_iter() + .flat_map(|tt| replace_self_and_deny_type_defs(struct_name, tt, errs)) + .collect(), + ))], + } +} diff --git a/rust/pin-init/internal/src/pinned_drop.rs b/rust/pin-init/internal/src/pinned_drop.rs new file mode 100644 index 000000000000..c824dd8b436d --- /dev/null +++ b/rust/pin-init/internal/src/pinned_drop.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(not(kernel))] +use proc_macro2 as proc_macro; + +use proc_macro::{TokenStream, TokenTree}; + +pub(crate) fn pinned_drop(_args: TokenStream, input: TokenStream) -> TokenStream { + let mut toks = input.into_iter().collect::<Vec<_>>(); + assert!(!toks.is_empty()); + // Ensure that we have an `impl` item. + assert!(matches!(&toks[0], TokenTree::Ident(i) if i.to_string() == "impl")); + // Ensure that we are implementing `PinnedDrop`. + let mut nesting: usize = 0; + let mut pinned_drop_idx = None; + for (i, tt) in toks.iter().enumerate() { + match tt { + TokenTree::Punct(p) if p.as_char() == '<' => { + nesting += 1; + } + TokenTree::Punct(p) if p.as_char() == '>' => { + nesting = nesting.checked_sub(1).unwrap(); + continue; + } + _ => {} + } + if i >= 1 && nesting == 0 { + // Found the end of the generics, this should be `PinnedDrop`. + assert!( + matches!(tt, TokenTree::Ident(i) if i.to_string() == "PinnedDrop"), + "expected 'PinnedDrop', found: '{:?}'", + tt + ); + pinned_drop_idx = Some(i); + break; + } + } + let idx = pinned_drop_idx + .unwrap_or_else(|| panic!("Expected an `impl` block implementing `PinnedDrop`.")); + // Fully qualify the `PinnedDrop`, as to avoid any tampering. + toks.splice(idx..idx, quote!(::pin_init::)); + // Take the `{}` body and call the declarative macro. + if let Some(TokenTree::Group(last)) = toks.pop() { + let last = last.stream(); + quote!(::pin_init::__pinned_drop! { + @impl_sig(#(#toks)*), + @impl_body(#last), + }) + } else { + TokenStream::from_iter(toks) + } +} diff --git a/rust/pin-init/internal/src/zeroable.rs b/rust/pin-init/internal/src/zeroable.rs new file mode 100644 index 000000000000..acc94008c152 --- /dev/null +++ b/rust/pin-init/internal/src/zeroable.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 + +#[cfg(not(kernel))] +use proc_macro2 as proc_macro; + +use crate::helpers::{parse_generics, Generics}; +use proc_macro::{TokenStream, TokenTree}; + +pub(crate) fn derive(input: TokenStream) -> TokenStream { + let ( + Generics { + impl_generics, + decl_generics: _, + ty_generics, + }, + mut rest, + ) = parse_generics(input); + // This should be the body of the struct `{...}`. + let last = rest.pop(); + // Now we insert `Zeroable` as a bound for every generic parameter in `impl_generics`. + let mut new_impl_generics = Vec::with_capacity(impl_generics.len()); + // Are we inside of a generic where we want to add `Zeroable`? + let mut in_generic = !impl_generics.is_empty(); + // Have we already inserted `Zeroable`? + let mut inserted = false; + // Level of `<>` nestings. + let mut nested = 0; + for tt in impl_generics { + match &tt { + // If we find a `,`, then we have finished a generic/constant/lifetime parameter. + TokenTree::Punct(p) if nested == 0 && p.as_char() == ',' => { + if in_generic && !inserted { + new_impl_generics.extend(quote! { : ::pin_init::Zeroable }); + } + in_generic = true; + inserted = false; + new_impl_generics.push(tt); + } + // If we find `'`, then we are entering a lifetime. + TokenTree::Punct(p) if nested == 0 && p.as_char() == '\'' => { + in_generic = false; + new_impl_generics.push(tt); + } + TokenTree::Punct(p) if nested == 0 && p.as_char() == ':' => { + new_impl_generics.push(tt); + if in_generic { + new_impl_generics.extend(quote! { ::pin_init::Zeroable + }); + inserted = true; + } + } + TokenTree::Punct(p) if p.as_char() == '<' => { + nested += 1; + new_impl_generics.push(tt); + } + TokenTree::Punct(p) if p.as_char() == '>' => { + assert!(nested > 0); + nested -= 1; + new_impl_generics.push(tt); + } + _ => new_impl_generics.push(tt), + } + } + assert_eq!(nested, 0); + if in_generic && !inserted { + new_impl_generics.extend(quote! { : ::pin_init::Zeroable }); + } + quote! { + ::pin_init::__derive_zeroable!( + parse_input: + @sig(#(#rest)*), + @impl_generics(#(#new_impl_generics)*), + @ty_generics(#(#ty_generics)*), + @body(#last), + ); + } +} diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs new file mode 100644 index 000000000000..557b5948cddc --- /dev/null +++ b/rust/pin-init/src/__internal.rs @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! This module contains library internal items. +//! +//! These items must not be used outside of this crate and the pin-init-internal crate located at +//! `../internal`. + +use super::*; + +/// See the [nomicon] for what subtyping is. See also [this table]. +/// +/// The reason for not using `PhantomData<*mut T>` is that that type never implements [`Send`] and +/// [`Sync`]. Hence `fn(*mut T) -> *mut T` is used, as that type always implements them. +/// +/// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html +/// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns +pub(crate) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; + +/// Module-internal type implementing `PinInit` and `Init`. +/// +/// It is unsafe to create this type, since the closure needs to fulfill the same safety +/// requirement as the `__pinned_init`/`__init` functions. +pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>); + +// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the +// `__init` invariants. +unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T, E> +where + F: FnOnce(*mut T) -> Result<(), E>, +{ + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + (self.0)(slot) + } +} + +// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the +// `__pinned_init` invariants. +unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T, E> +where + F: FnOnce(*mut T) -> Result<(), E>, +{ + #[inline] + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + (self.0)(slot) + } +} + +/// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate +/// the pin projections within the initializers. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait HasPinData { + type PinData: PinData; + + #[expect(clippy::missing_safety_doc)] + unsafe fn __pin_data() -> Self::PinData; +} + +/// Marker trait for pinning data of structs. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait PinData: Copy { + type Datee: ?Sized + HasPinData; + + /// Type inference helper function. + fn make_closure<F, O, E>(self, f: F) -> F + where + F: FnOnce(*mut Self::Datee) -> Result<O, E>, + { + f + } +} + +/// This trait is automatically implemented for every type. It aims to provide the same type +/// inference help as `HasPinData`. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait HasInitData { + type InitData: InitData; + + #[expect(clippy::missing_safety_doc)] + unsafe fn __init_data() -> Self::InitData; +} + +/// Same function as `PinData`, but for arbitrary data. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait InitData: Copy { + type Datee: ?Sized + HasInitData; + + /// Type inference helper function. + fn make_closure<F, O, E>(self, f: F) -> F + where + F: FnOnce(*mut Self::Datee) -> Result<O, E>, + { + f + } +} + +pub struct AllData<T: ?Sized>(Invariant<T>); + +impl<T: ?Sized> Clone for AllData<T> { + fn clone(&self) -> Self { + *self + } +} + +impl<T: ?Sized> Copy for AllData<T> {} + +// SAFETY: TODO. +unsafe impl<T: ?Sized> InitData for AllData<T> { + type Datee = T; +} + +// SAFETY: TODO. +unsafe impl<T: ?Sized> HasInitData for T { + type InitData = AllData<T>; + + unsafe fn __init_data() -> Self::InitData { + AllData(PhantomData) + } +} + +/// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive. +/// +/// # Invariants +/// +/// If `self.is_init` is true, then `self.value` is initialized. +/// +/// [`stack_pin_init`]: crate::stack_pin_init +pub struct StackInit<T> { + value: MaybeUninit<T>, + is_init: bool, +} + +impl<T> Drop for StackInit<T> { + #[inline] + fn drop(&mut self) { + if self.is_init { + // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is + // true, `self.value` is initialized. + unsafe { self.value.assume_init_drop() }; + } + } +} + +impl<T> StackInit<T> { + /// Creates a new [`StackInit<T>`] that is uninitialized. Use [`stack_pin_init`] instead of this + /// primitive. + /// + /// [`stack_pin_init`]: crate::stack_pin_init + #[inline] + pub fn uninit() -> Self { + Self { + value: MaybeUninit::uninit(), + is_init: false, + } + } + + /// Initializes the contents and returns the result. + #[inline] + pub fn init<E>(self: Pin<&mut Self>, init: impl PinInit<T, E>) -> Result<Pin<&mut T>, E> { + // SAFETY: We never move out of `this`. + let this = unsafe { Pin::into_inner_unchecked(self) }; + // The value is currently initialized, so it needs to be dropped before we can reuse + // the memory (this is a safety guarantee of `Pin`). + if this.is_init { + this.is_init = false; + // SAFETY: `this.is_init` was true and therefore `this.value` is initialized. + unsafe { this.value.assume_init_drop() }; + } + // SAFETY: The memory slot is valid and this type ensures that it will stay pinned. + unsafe { init.__pinned_init(this.value.as_mut_ptr())? }; + // INVARIANT: `this.value` is initialized above. + this.is_init = true; + // SAFETY: The slot is now pinned, since we will never give access to `&mut T`. + Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) }) + } +} + +#[test] +fn stack_init_reuse() { + use ::std::{borrow::ToOwned, println, string::String}; + use core::pin::pin; + + #[derive(Debug)] + struct Foo { + a: usize, + b: String, + } + let mut slot: Pin<&mut StackInit<Foo>> = pin!(StackInit::uninit()); + let value: Result<Pin<&mut Foo>, core::convert::Infallible> = + slot.as_mut().init(crate::init!(Foo { + a: 42, + b: "Hello".to_owned(), + })); + let value = value.unwrap(); + println!("{value:?}"); + let value: Result<Pin<&mut Foo>, core::convert::Infallible> = + slot.as_mut().init(crate::init!(Foo { + a: 24, + b: "world!".to_owned(), + })); + let value = value.unwrap(); + println!("{value:?}"); +} + +/// When a value of this type is dropped, it drops a `T`. +/// +/// Can be forgotten to prevent the drop. +pub struct DropGuard<T: ?Sized> { + ptr: *mut T, +} + +impl<T: ?Sized> DropGuard<T> { + /// Creates a new [`DropGuard<T>`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer. + /// + /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`: + /// - has not been dropped, + /// - is not accessible by any other means, + /// - will not be dropped by any other means. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + Self { ptr } + } +} + +impl<T: ?Sized> Drop for DropGuard<T> { + #[inline] + fn drop(&mut self) { + // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function + // ensuring that this operation is safe. + unsafe { ptr::drop_in_place(self.ptr) } + } +} + +/// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely +/// created struct. This is needed, because the `drop` function is safe, but should not be called +/// manually. +pub struct OnlyCallFromDrop(()); + +impl OnlyCallFromDrop { + /// # Safety + /// + /// This function should only be called from the [`Drop::drop`] function and only be used to + /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type. + pub unsafe fn new() -> Self { + Self(()) + } +} + +/// Initializer that always fails. +/// +/// Used by [`assert_pinned!`]. +/// +/// [`assert_pinned!`]: crate::assert_pinned +pub struct AlwaysFail<T: ?Sized> { + _t: PhantomData<T>, +} + +impl<T: ?Sized> AlwaysFail<T> { + /// Creates a new initializer that always fails. + pub fn new() -> Self { + Self { _t: PhantomData } + } +} + +impl<T: ?Sized> Default for AlwaysFail<T> { + fn default() -> Self { + Self::new() + } +} + +// SAFETY: `__pinned_init` always fails, which is always okay. +unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> { + unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> { + Err(()) + } +} diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs new file mode 100644 index 000000000000..5017f57442d8 --- /dev/null +++ b/rust/pin-init/src/alloc.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(all(feature = "alloc", not(feature = "std")))] +use alloc::{boxed::Box, sync::Arc}; +#[cfg(feature = "alloc")] +use core::alloc::AllocError; +use core::{mem::MaybeUninit, pin::Pin}; +#[cfg(feature = "std")] +use std::sync::Arc; + +#[cfg(not(feature = "alloc"))] +type AllocError = core::convert::Infallible; + +use crate::{ + init_from_closure, pin_init_from_closure, InPlaceWrite, Init, PinInit, ZeroableOption, +}; + +pub extern crate alloc; + +// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: +// <https://doc.rust-lang.org/stable/std/option/index.html#representation>). +unsafe impl<T> ZeroableOption for Box<T> {} + +/// Smart pointer that can initialize memory in-place. +pub trait InPlaceInit<T>: Sized { + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + where + E: From<AllocError>; + + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn pin_init(init: impl PinInit<T>) -> Result<Pin<Self>, AllocError> { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + pin_init_from_closure(|slot| match init.__pinned_init(slot) { + Ok(()) => Ok(()), + Err(i) => match i {}, + }) + }; + Self::try_pin_init(init) + } + + /// Use the given initializer to in-place initialize a `T`. + fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + where + E: From<AllocError>; + + /// Use the given initializer to in-place initialize a `T`. + fn init(init: impl Init<T>) -> Result<Self, AllocError> { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + init_from_closure(|slot| match init.__init(slot) { + Ok(()) => Ok(()), + Err(i) => match i {}, + }) + }; + Self::try_init(init) + } +} + +#[cfg(feature = "alloc")] +macro_rules! try_new_uninit { + ($type:ident) => { + $type::try_new_uninit()? + }; +} +#[cfg(all(feature = "std", not(feature = "alloc")))] +macro_rules! try_new_uninit { + ($type:ident) => { + $type::new_uninit() + }; +} + +impl<T> InPlaceInit<T> for Box<T> { + #[inline] + fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + where + E: From<AllocError>, + { + try_new_uninit!(Box).write_pin_init(init) + } + + #[inline] + fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + where + E: From<AllocError>, + { + try_new_uninit!(Box).write_init(init) + } +} + +impl<T> InPlaceInit<T> for Arc<T> { + #[inline] + fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + where + E: From<AllocError>, + { + let mut this = try_new_uninit!(Arc); + let Some(slot) = Arc::get_mut(&mut this) else { + // SAFETY: the Arc has just been created and has no external references + unsafe { core::hint::unreachable_unchecked() } + }; + let slot = slot.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized and this is the only `Arc` to that data. + Ok(unsafe { Pin::new_unchecked(this.assume_init()) }) + } + + #[inline] + fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + where + E: From<AllocError>, + { + let mut this = try_new_uninit!(Arc); + let Some(slot) = Arc::get_mut(&mut this) else { + // SAFETY: the Arc has just been created and has no external references + unsafe { core::hint::unreachable_unchecked() } + }; + let slot = slot.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { this.assume_init() }) + } +} + +impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { + type Initialized = Box<T>; + + fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }) + } + + fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }.into()) + } +} diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs new file mode 100644 index 000000000000..0806c689f693 --- /dev/null +++ b/rust/pin-init/src/lib.rs @@ -0,0 +1,1483 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Library to safely and fallibly initialize pinned `struct`s using in-place constructors. +//! +//! [Pinning][pinning] is Rust's way of ensuring data does not move. +//! +//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack +//! overflow. +//! +//! This library's main use-case is in [Rust-for-Linux]. Although this version can be used +//! standalone. +//! +//! There are cases when you want to in-place initialize a struct. For example when it is very big +//! and moving it from the stack is not an option, because it is bigger than the stack itself. +//! Another reason would be that you need the address of the object to initialize it. This stands +//! in direct conflict with Rust's normal process of first initializing an object and then moving +//! it into it's final memory location. For more information, see +//! <https://rust-for-linux.com/the-safe-pinned-initialization-problem>. +//! +//! This library allows you to do in-place initialization safely. +//! +//! ## Nightly Needed for `alloc` feature +//! +//! This library requires the [`allocator_api` unstable feature] when the `alloc` feature is +//! enabled and thus this feature can only be used with a nightly compiler. When enabling the +//! `alloc` feature, the user will be required to activate `allocator_api` as well. +//! +//! [`allocator_api` unstable feature]: https://doc.rust-lang.org/nightly/unstable-book/library-features/allocator-api.html +//! +//! The feature is enabled by default, thus by default `pin-init` will require a nightly compiler. +//! However, using the crate on stable compilers is possible by disabling `alloc`. In practice this +//! will require the `std` feature, because stable compilers have neither `Box` nor `Arc` in no-std +//! mode. +//! +//! # Overview +//! +//! To initialize a `struct` with an in-place constructor you will need two things: +//! - an in-place constructor, +//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`], +//! [`Box<T>`] or any other smart pointer that supports this library). +//! +//! To get an in-place constructor there are generally three options: +//! - directly creating an in-place constructor using the [`pin_init!`] macro, +//! - a custom function/macro returning an in-place constructor provided by someone else, +//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. +//! +//! Aside from pinned initialization, this library also supports in-place construction without +//! pinning, the macros/types/functions are generally named like the pinned variants without the +//! `pin_` prefix. +//! +//! # Examples +//! +//! Throughout the examples we will often make use of the `CMutex` type which can be found in +//! `../examples/mutex.rs`. It is essentially a userland rebuild of the `struct mutex` type from +//! the Linux kernel. It also uses a wait list and a basic spinlock. Importantly the wait list +//! requires it to be pinned to be locked and thus is a prime candidate for using this library. +//! +//! ## Using the [`pin_init!`] macro +//! +//! If you want to use [`PinInit`], then you will have to annotate your `struct` with +//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for +//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via +//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is +//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. +//! +//! ```rust +//! # #![expect(clippy::disallowed_names)] +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use core::pin::Pin; +//! use pin_init::{pin_data, pin_init, InPlaceInit}; +//! +//! #[pin_data] +//! struct Foo { +//! #[pin] +//! a: CMutex<usize>, +//! b: u32, +//! } +//! +//! let foo = pin_init!(Foo { +//! a <- CMutex::new(42), +//! b: 24, +//! }); +//! # let _ = Box::pin_init(foo); +//! ``` +//! +//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like +//! (or just the stack) to actually initialize a `Foo`: +//! +//! ```rust +//! # #![expect(clippy::disallowed_names)] +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use core::{alloc::AllocError, pin::Pin}; +//! # use pin_init::*; +//! # +//! # #[pin_data] +//! # struct Foo { +//! # #[pin] +//! # a: CMutex<usize>, +//! # b: u32, +//! # } +//! # +//! # let foo = pin_init!(Foo { +//! # a <- CMutex::new(42), +//! # b: 24, +//! # }); +//! let foo: Result<Pin<Box<Foo>>, AllocError> = Box::pin_init(foo); +//! ``` +//! +//! For more information see the [`pin_init!`] macro. +//! +//! ## Using a custom function/macro that returns an initializer +//! +//! Many types that use this library supply a function/macro that returns an initializer, because +//! the above method only works for types where you can access the fields. +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use pin_init::*; +//! # use std::sync::Arc; +//! # use core::pin::Pin; +//! let mtx: Result<Pin<Arc<CMutex<usize>>>, _> = Arc::pin_init(CMutex::new(42)); +//! ``` +//! +//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # use pin_init::*; +//! # #[path = "../examples/error.rs"] mod error; use error::Error; +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! #[pin_data] +//! struct DriverData { +//! #[pin] +//! status: CMutex<i32>, +//! buffer: Box<[u8; 1_000_000]>, +//! } +//! +//! impl DriverData { +//! fn new() -> impl PinInit<Self, Error> { +//! try_pin_init!(Self { +//! status <- CMutex::new(0), +//! buffer: Box::init(pin_init::zeroed())?, +//! }? Error) +//! } +//! } +//! ``` +//! +//! ## Manual creation of an initializer +//! +//! Often when working with primitives the previous approaches are not sufficient. That is where +//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a +//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure +//! actually does the initialization in the correct way. Here are the things to look out for +//! (we are calling the parameter to the closure `slot`): +//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so +//! `slot` now contains a valid bit pattern for the type `T`, +//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so +//! you need to take care to clean up anything if your initialization fails mid-way, +//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of +//! `slot` gets called. +//! +//! ```rust +//! # #![feature(extern_types)] +//! use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; +//! use core::{ +//! ptr::addr_of_mut, +//! marker::PhantomPinned, +//! cell::UnsafeCell, +//! pin::Pin, +//! mem::MaybeUninit, +//! }; +//! mod bindings { +//! #[repr(C)] +//! pub struct foo { +//! /* fields from C ... */ +//! } +//! extern "C" { +//! pub fn init_foo(ptr: *mut foo); +//! pub fn destroy_foo(ptr: *mut foo); +//! #[must_use = "you must check the error return code"] +//! pub fn enable_foo(ptr: *mut foo, flags: u32) -> i32; +//! } +//! } +//! +//! /// # Invariants +//! /// +//! /// `foo` is always initialized +//! #[pin_data(PinnedDrop)] +//! pub struct RawFoo { +//! #[pin] +//! _p: PhantomPinned, +//! #[pin] +//! foo: UnsafeCell<MaybeUninit<bindings::foo>>, +//! } +//! +//! impl RawFoo { +//! pub fn new(flags: u32) -> impl PinInit<Self, i32> { +//! // SAFETY: +//! // - when the closure returns `Ok(())`, then it has successfully initialized and +//! // enabled `foo`, +//! // - when it returns `Err(e)`, then it has cleaned up before +//! unsafe { +//! pin_init_from_closure(move |slot: *mut Self| { +//! // `slot` contains uninit memory, avoid creating a reference. +//! let foo = addr_of_mut!((*slot).foo); +//! let foo = UnsafeCell::raw_get(foo).cast::<bindings::foo>(); +//! +//! // Initialize the `foo` +//! bindings::init_foo(foo); +//! +//! // Try to enable it. +//! let err = bindings::enable_foo(foo, flags); +//! if err != 0 { +//! // Enabling has failed, first clean up the foo and then return the error. +//! bindings::destroy_foo(foo); +//! Err(err) +//! } else { +//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. +//! Ok(()) +//! } +//! }) +//! } +//! } +//! } +//! +//! #[pinned_drop] +//! impl PinnedDrop for RawFoo { +//! fn drop(self: Pin<&mut Self>) { +//! // SAFETY: Since `foo` is initialized, destroying is safe. +//! unsafe { bindings::destroy_foo(self.foo.get().cast::<bindings::foo>()) }; +//! } +//! } +//! ``` +//! +//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside +//! the `kernel` crate. The [`sync`] module is a good starting point. +//! +//! [`sync`]: https://rust.docs.kernel.org/kernel/sync/index.html +//! [pinning]: https://doc.rust-lang.org/std/pin/index.html +//! [structurally pinned fields]: +//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field +//! [stack]: crate::stack_pin_init +#![cfg_attr( + kernel, + doc = "[`Arc<T>`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#![cfg_attr( + kernel, + doc = "[`Box<T>`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#![cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")] +#![cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] +//! [`impl PinInit<Foo>`]: crate::PinInit +//! [`impl PinInit<T, E>`]: crate::PinInit +//! [`impl Init<T, E>`]: crate::Init +//! [Rust-for-Linux]: https://rust-for-linux.com/ + +#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] +#![cfg_attr( + all( + any(feature = "alloc", feature = "std"), + not(RUSTC_NEW_UNINIT_IS_STABLE) + ), + feature(new_uninit) +)] +#![forbid(missing_docs, unsafe_op_in_unsafe_fn)] +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::{ + cell::UnsafeCell, + convert::Infallible, + marker::PhantomData, + mem::MaybeUninit, + num::*, + pin::Pin, + ptr::{self, NonNull}, +}; + +#[doc(hidden)] +pub mod __internal; +#[doc(hidden)] +pub mod macros; + +#[cfg(any(feature = "std", feature = "alloc"))] +mod alloc; +#[cfg(any(feature = "std", feature = "alloc"))] +pub use alloc::InPlaceInit; + +/// Used to specify the pinning information of the fields of a struct. +/// +/// This is somewhat similar in purpose as +/// [pin-project-lite](https://crates.io/crates/pin-project-lite). +/// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each +/// field you want to structurally pin. +/// +/// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`, +/// then `#[pin]` directs the type of initializer that is required. +/// +/// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this +/// macro, and change your `Drop` implementation to `PinnedDrop` annotated with +/// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care. +/// +/// # Examples +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// use pin_init::pin_data; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data] +/// struct DriverData { +/// #[pin] +/// queue: CMutex<Vec<Command>>, +/// buf: Box<[u8; 1024 * 1024]>, +/// } +/// ``` +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # mod bindings { pub struct info; pub unsafe fn destroy_info(_: *mut info) {} } +/// use core::pin::Pin; +/// use pin_init::{pin_data, pinned_drop, PinnedDrop}; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data(PinnedDrop)] +/// struct DriverData { +/// #[pin] +/// queue: CMutex<Vec<Command>>, +/// buf: Box<[u8; 1024 * 1024]>, +/// raw_info: *mut bindings::info, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for DriverData { +/// fn drop(self: Pin<&mut Self>) { +/// unsafe { bindings::destroy_info(self.raw_info) }; +/// } +/// } +/// ``` +pub use ::pin_init_internal::pin_data; + +/// Used to implement `PinnedDrop` safely. +/// +/// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`. +/// +/// # Examples +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # mod bindings { pub struct info; pub unsafe fn destroy_info(_: *mut info) {} } +/// use core::pin::Pin; +/// use pin_init::{pin_data, pinned_drop, PinnedDrop}; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data(PinnedDrop)] +/// struct DriverData { +/// #[pin] +/// queue: CMutex<Vec<Command>>, +/// buf: Box<[u8; 1024 * 1024]>, +/// raw_info: *mut bindings::info, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for DriverData { +/// fn drop(self: Pin<&mut Self>) { +/// unsafe { bindings::destroy_info(self.raw_info) }; +/// } +/// } +/// ``` +pub use ::pin_init_internal::pinned_drop; + +/// Derives the [`Zeroable`] trait for the given struct. +/// +/// This can only be used for structs where every field implements the [`Zeroable`] trait. +/// +/// # Examples +/// +/// ``` +/// use pin_init::Zeroable; +/// +/// #[derive(Zeroable)] +/// pub struct DriverData { +/// id: i64, +/// buf_ptr: *mut u8, +/// len: usize, +/// } +/// ``` +pub use ::pin_init_internal::Zeroable; + +/// Initialize and pin a type directly on the stack. +/// +/// # Examples +/// +/// ```rust +/// # #![expect(clippy::disallowed_names)] +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex<usize>, +/// b: Bar, +/// } +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_pin_init!(let foo = pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Bar { +/// x: 64, +/// }, +/// })); +/// let foo: Pin<&mut Foo> = foo; +/// println!("a: {}", &*foo.a.lock()); +/// ``` +/// +/// # Syntax +/// +/// A normal `let` binding with optional type annotation. The expression is expected to implement +/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error +/// type, then use [`stack_try_pin_init!`]. +#[macro_export] +macro_rules! stack_pin_init { + (let $var:ident $(: $t:ty)? = $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = match $crate::__internal::StackInit::init($var, val) { + Ok(res) => res, + Err(x) => { + let x: ::core::convert::Infallible = x; + match x {} + } + }; + }; +} + +/// Initialize and pin a type directly on the stack. +/// +/// # Examples +/// +/// ```rust +/// # #![expect(clippy::disallowed_names)] +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex<usize>, +/// b: Box<Bar>, +/// } +/// +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_try_pin_init!(let foo: Foo = try_pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Box::try_new(Bar { +/// x: 64, +/// })?, +/// }? Error)); +/// let foo = foo.unwrap(); +/// println!("a: {}", &*foo.a.lock()); +/// ``` +/// +/// ```rust +/// # #![expect(clippy::disallowed_names)] +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex<usize>, +/// b: Box<Bar>, +/// } +/// +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_try_pin_init!(let foo: Foo =? try_pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Box::try_new(Bar { +/// x: 64, +/// })?, +/// }? Error)); +/// println!("a: {}", &*foo.a.lock()); +/// # Ok::<_, Error>(()) +/// ``` +/// +/// # Syntax +/// +/// A normal `let` binding with optional type annotation. The expression is expected to implement +/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the +/// `=` will propagate this error. +#[macro_export] +macro_rules! stack_try_pin_init { + (let $var:ident $(: $t:ty)? = $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = $crate::__internal::StackInit::init($var, val); + }; + (let $var:ident $(: $t:ty)? =? $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = $crate::__internal::StackInit::init($var, val)?; + }; +} + +/// Construct an in-place, pinned initializer for `struct`s. +/// +/// This macro defaults the error to [`Infallible`]. If you need a different error, then use +/// [`try_pin_init!`]. +/// +/// The syntax is almost identical to that of a normal `struct` initializer: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Foo { +/// a: usize, +/// b: Bar, +/// } +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// # fn demo() -> impl PinInit<Foo> { +/// let a = 42; +/// +/// let initializer = pin_init!(Foo { +/// a, +/// b: Bar { +/// x: 64, +/// }, +/// }); +/// # initializer } +/// # Box::pin_init(demo()).unwrap(); +/// ``` +/// +/// Arbitrary Rust expressions can be used to set the value of a variable. +/// +/// The fields are initialized in the order that they appear in the initializer. So it is possible +/// to read already initialized fields using raw pointers. +/// +/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the +/// initializer. +/// +/// # Init-functions +/// +/// When working with this library it is often desired to let others construct your types without +/// giving access to all fields. This is where you would normally write a plain function `new` that +/// would return a new instance of your type. With this library that is also possible. However, +/// there are a few extra things to keep in mind. +/// +/// To create an initializer function, simply declare it like this: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// impl Foo { +/// fn new() -> impl PinInit<Self> { +/// pin_init!(Self { +/// a: 42, +/// b: Bar { +/// x: 64, +/// }, +/// }) +/// } +/// } +/// ``` +/// +/// Users of `Foo` can now create it like this: +/// +/// ```rust +/// # #![expect(clippy::disallowed_names)] +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// # impl Foo { +/// # fn new() -> impl PinInit<Self> { +/// # pin_init!(Self { +/// # a: 42, +/// # b: Bar { +/// # x: 64, +/// # }, +/// # }) +/// # } +/// # } +/// let foo = Box::pin_init(Foo::new()); +/// ``` +/// +/// They can also easily embed it into their own `struct`s: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// # impl Foo { +/// # fn new() -> impl PinInit<Self> { +/// # pin_init!(Self { +/// # a: 42, +/// # b: Bar { +/// # x: 64, +/// # }, +/// # }) +/// # } +/// # } +/// #[pin_data] +/// struct FooContainer { +/// #[pin] +/// foo1: Foo, +/// #[pin] +/// foo2: Foo, +/// other: u32, +/// } +/// +/// impl FooContainer { +/// fn new(other: u32) -> impl PinInit<Self> { +/// pin_init!(Self { +/// foo1 <- Foo::new(), +/// foo2 <- Foo::new(), +/// other, +/// }) +/// } +/// } +/// ``` +/// +/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. +/// This signifies that the given field is initialized in-place. As with `struct` initializers, just +/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. +/// +/// # Syntax +/// +/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with +/// the following modifications is expected: +/// - Fields that you want to initialize in-place have to use `<-` instead of `:`. +/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] +/// pointer named `this` inside of the initializer. +/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the +/// struct, this initializes every field with 0 and then runs all initializers specified in the +/// body. This can only be done if [`Zeroable`] is implemented for the struct. +/// +/// For instance: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; +/// #[pin_data] +/// #[derive(Zeroable)] +/// struct Buf { +/// // `ptr` points into `buf`. +/// ptr: *mut u8, +/// buf: [u8; 64], +/// #[pin] +/// pin: PhantomPinned, +/// } +/// +/// let init = pin_init!(&this in Buf { +/// buf: [0; 64], +/// // SAFETY: TODO. +/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, +/// pin: PhantomPinned, +/// }); +/// let init = pin_init!(Buf { +/// buf: [1; 64], +/// ..Zeroable::zeroed() +/// }); +/// ``` +/// +/// [`NonNull<Self>`]: core::ptr::NonNull +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `macros` inside of `macros.rs`. +#[macro_export] +macro_rules! pin_init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }) => { + $crate::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? { + $($fields)* + }? ::core::convert::Infallible) + }; +} + +/// Construct an in-place, fallible pinned initializer for `struct`s. +/// +/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. +/// +/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop +/// initialization and return the error. +/// +/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when +/// initialization fails, the memory can be safely deallocated without any further modifications. +/// +/// The syntax is identical to [`pin_init!`] with the following exception: you must append `? $type` +/// after the `struct` initializer to specify the error type you want to use. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// use pin_init::{pin_data, try_pin_init, PinInit, InPlaceInit, zeroed}; +/// +/// #[pin_data] +/// struct BigBuf { +/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], +/// ptr: *mut u8, +/// } +/// +/// impl BigBuf { +/// fn new() -> impl PinInit<Self, Error> { +/// try_pin_init!(Self { +/// big: Box::init(zeroed())?, +/// small: [0; 1024 * 1024], +/// ptr: core::ptr::null_mut(), +/// }? Error) +/// } +/// } +/// # let _ = Box::pin_init(BigBuf::new()); +/// ``` +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `macros` inside of `macros.rs`. +#[macro_export] +macro_rules! try_pin_init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }? $err:ty) => { + $crate::__init_internal!( + @this($($this)?), + @typ($t $(::<$($generics),*>)? ), + @fields($($fields)*), + @error($err), + @data(PinData, use_data), + @has_data(HasPinData, __pin_data), + @construct_closure(pin_init_from_closure), + @munch_fields($($fields)*), + ) + } +} + +/// Construct an in-place initializer for `struct`s. +/// +/// This macro defaults the error to [`Infallible`]. If you need a different error, then use +/// [`try_init!`]. +/// +/// The syntax is identical to [`pin_init!`] and its safety caveats also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. +/// +/// This initializer is for initializing data in-place that might later be moved. If you want to +/// pin-initialize, use [`pin_init!`]. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::InPlaceInit; +/// use pin_init::{init, Init, zeroed}; +/// +/// struct BigBuf { +/// small: [u8; 1024 * 1024], +/// } +/// +/// impl BigBuf { +/// fn new() -> impl Init<Self> { +/// init!(Self { +/// small <- zeroed(), +/// }) +/// } +/// } +/// # let _ = Box::init(BigBuf::new()); +/// ``` +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `macros` inside of `macros.rs`. +#[macro_export] +macro_rules! init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }) => { + $crate::try_init!($(&$this in)? $t $(::<$($generics),*>)? { + $($fields)* + }? ::core::convert::Infallible) + } +} + +/// Construct an in-place fallible initializer for `struct`s. +/// +/// If the initialization can complete without error (or [`Infallible`]), then use +/// [`init!`]. +/// +/// The syntax is identical to [`try_pin_init!`]. You need to specify a custom error +/// via `? $type` after the `struct` initializer. +/// The safety caveats from [`try_pin_init!`] also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # use core::alloc::AllocError; +/// # use pin_init::InPlaceInit; +/// use pin_init::{try_init, Init, zeroed}; +/// +/// struct BigBuf { +/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], +/// } +/// +/// impl BigBuf { +/// fn new() -> impl Init<Self, AllocError> { +/// try_init!(Self { +/// big: Box::init(zeroed())?, +/// small: [0; 1024 * 1024], +/// }? AllocError) +/// } +/// } +/// # let _ = Box::init(BigBuf::new()); +/// ``` +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `macros` inside of `macros.rs`. +#[macro_export] +macro_rules! try_init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }? $err:ty) => { + $crate::__init_internal!( + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error($err), + @data(InitData, /*no use_data*/), + @has_data(HasInitData, __init_data), + @construct_closure(init_from_closure), + @munch_fields($($fields)*), + ) + } +} + +/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is +/// structurally pinned. +/// +/// # Example +/// +/// This will succeed: +/// ``` +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct MyStruct { +/// #[pin] +/// some_field: u64, +/// } +/// +/// assert_pinned!(MyStruct, some_field, u64); +/// ``` +/// +/// This will fail: +/// ```compile_fail +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct MyStruct { +/// some_field: u64, +/// } +/// +/// assert_pinned!(MyStruct, some_field, u64); +/// ``` +/// +/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To +/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can +/// only be used when the macro is invoked from a function body. +/// ``` +/// # use core::pin::Pin; +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct Foo<T> { +/// #[pin] +/// elem: T, +/// } +/// +/// impl<T> Foo<T> { +/// fn project(self: Pin<&mut Self>) -> Pin<&mut T> { +/// assert_pinned!(Foo<T>, elem, T, inline); +/// +/// // SAFETY: The field is structurally pinned. +/// unsafe { self.map_unchecked_mut(|me| &mut me.elem) } +/// } +/// } +/// ``` +#[macro_export] +macro_rules! assert_pinned { + ($ty:ty, $field:ident, $field_ty:ty, inline) => { + let _ = move |ptr: *mut $field_ty| { + // SAFETY: This code is unreachable. + let data = unsafe { <$ty as $crate::__internal::HasPinData>::__pin_data() }; + let init = $crate::__internal::AlwaysFail::<$field_ty>::new(); + // SAFETY: This code is unreachable. + unsafe { data.$field(ptr, init) }.ok(); + }; + }; + + ($ty:ty, $field:ident, $field_ty:ty) => { + const _: () = { + $crate::assert_pinned!($ty, $field, $field_ty, inline); + }; + }; +} + +/// A pin-initializer for the type `T`. +/// +/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can +/// be [`Box<T>`], [`Arc<T>`] or even the stack (see [`stack_pin_init!`]). +/// +/// Also see the [module description](self). +/// +/// # Safety +/// +/// When implementing this trait you will need to take great care. Also there are probably very few +/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. +/// +/// The [`PinInit::__pinned_init`] function: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +/// +#[cfg_attr( + kernel, + doc = "[`Arc<T>`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#[cfg_attr( + kernel, + doc = "[`Box<T>`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#[cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")] +#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] +#[must_use = "An initializer must be used in order to create its value."] +pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { + /// Initializes `slot`. + /// + /// # Safety + /// + /// - `slot` is a valid pointer to uninitialized memory. + /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to + /// deallocate. + /// - `slot` will not move until it is dropped, i.e. it will be pinned. + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; + + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// # #![feature(allocator_api)] + /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; + /// # use pin_init::*; + /// let mtx_init = CMutex::new(42); + /// // Make the initializer print the value. + /// let mtx_init = mtx_init.pin_chain(|mtx| { + /// println!("{:?}", mtx.get_data_mut()); + /// Ok(()) + /// }); + /// ``` + fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> + where + F: FnOnce(Pin<&mut T>) -> Result<(), E>, + { + ChainPinInit(self, f, PhantomData) + } +} + +/// An initializer returned by [`PinInit::pin_chain`]. +pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>); + +// SAFETY: The `__pinned_init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +// - considers `slot` pinned. +unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E> +where + I: PinInit<T, E>, + F: FnOnce(Pin<&mut T>) -> Result<(), E>, +{ + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__pinned_init`. + unsafe { self.0.__pinned_init(slot)? }; + // SAFETY: The above call initialized `slot` and we still have unique access. + let val = unsafe { &mut *slot }; + // SAFETY: `slot` is considered pinned. + let val = unsafe { Pin::new_unchecked(val) }; + // SAFETY: `slot` was initialized above. + (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) }) + } +} + +/// An initializer for `T`. +/// +/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can +/// be [`Box<T>`], [`Arc<T>`] or even the stack (see [`stack_pin_init!`]). Because +/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well. +/// +/// Also see the [module description](self). +/// +/// # Safety +/// +/// When implementing this trait you will need to take great care. Also there are probably very few +/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. +/// +/// The [`Init::__init`] function: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +/// +/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same +/// code as `__init`. +/// +/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to +/// move the pointee after initialization. +/// +#[cfg_attr( + kernel, + doc = "[`Arc<T>`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#[cfg_attr( + kernel, + doc = "[`Box<T>`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#[cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")] +#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] +#[must_use = "An initializer must be used in order to create its value."] +pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { + /// Initializes `slot`. + /// + /// # Safety + /// + /// - `slot` is a valid pointer to uninitialized memory. + /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to + /// deallocate. + unsafe fn __init(self, slot: *mut T) -> Result<(), E>; + + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// # #![expect(clippy::disallowed_names)] + /// use pin_init::{init, zeroed, Init}; + /// + /// struct Foo { + /// buf: [u8; 1_000_000], + /// } + /// + /// impl Foo { + /// fn setup(&mut self) { + /// println!("Setting up foo"); + /// } + /// } + /// + /// let foo = init!(Foo { + /// buf <- zeroed() + /// }).chain(|foo| { + /// foo.setup(); + /// Ok(()) + /// }); + /// ``` + fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> + where + F: FnOnce(&mut T) -> Result<(), E>, + { + ChainInit(self, f, PhantomData) + } +} + +/// An initializer returned by [`Init::chain`]. +pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>); + +// SAFETY: The `__init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> +where + I: Init<T, E>, + F: FnOnce(&mut T) -> Result<(), E>, +{ + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + unsafe { self.0.__pinned_init(slot)? }; + // SAFETY: The above call initialized `slot` and we still have unique access. + (self.1)(unsafe { &mut *slot }).inspect_err(|_| + // SAFETY: `slot` was initialized above. + unsafe { core::ptr::drop_in_place(slot) }) + } +} + +// SAFETY: `__pinned_init` behaves exactly the same as `__init`. +unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> +where + I: Init<T, E>, + F: FnOnce(&mut T) -> Result<(), E>, +{ + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. + unsafe { self.__init(slot) } + } +} + +/// Creates a new [`PinInit<T, E>`] from the given closure. +/// +/// # Safety +/// +/// The closure: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - may assume that the `slot` does not move if `T: !Unpin`, +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +#[inline] +pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( + f: impl FnOnce(*mut T) -> Result<(), E>, +) -> impl PinInit<T, E> { + __internal::InitClosure(f, PhantomData) +} + +/// Creates a new [`Init<T, E>`] from the given closure. +/// +/// # Safety +/// +/// The closure: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - the `slot` may move after initialization. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +#[inline] +pub const unsafe fn init_from_closure<T: ?Sized, E>( + f: impl FnOnce(*mut T) -> Result<(), E>, +) -> impl Init<T, E> { + __internal::InitClosure(f, PhantomData) +} + +/// An initializer that leaves the memory uninitialized. +/// +/// The initializer is a no-op. The `slot` memory is not changed. +#[inline] +pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { + // SAFETY: The memory is allowed to be uninitialized. + unsafe { init_from_closure(|_| Ok(())) } +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// # use pin_init::*; +/// use pin_init::init_array_from_fn; +/// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +pub fn init_array_from_fn<I, const N: usize, T, E>( + mut make_init: impl FnMut(usize) -> I, +) -> impl Init<[T; N], E> +where + I: Init<T, E>, +{ + let init = move |slot: *mut [T; N]| { + let slot = slot.cast::<T>(); + for i in 0..N { + let init = make_init(i); + // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. + let ptr = unsafe { slot.add(i) }; + // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` + // requirements. + if let Err(e) = unsafe { init.__init(ptr) } { + // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return + // `Err` below, `slot` will be considered uninitialized memory. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; + return Err(e); + } + } + Ok(()) + }; + // SAFETY: The initializer above initializes every element of the array. On failure it drops + // any initialized elements and returns `Err`. + unsafe { init_from_closure(init) } +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// # use core::pin::Pin; +/// use pin_init::pin_init_array_from_fn; +/// use std::sync::Arc; +/// let array: Pin<Arc<[CMutex<usize>; 1_000]>> = +/// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +pub fn pin_init_array_from_fn<I, const N: usize, T, E>( + mut make_init: impl FnMut(usize) -> I, +) -> impl PinInit<[T; N], E> +where + I: PinInit<T, E>, +{ + let init = move |slot: *mut [T; N]| { + let slot = slot.cast::<T>(); + for i in 0..N { + let init = make_init(i); + // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. + let ptr = unsafe { slot.add(i) }; + // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` + // requirements. + if let Err(e) = unsafe { init.__pinned_init(ptr) } { + // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return + // `Err` below, `slot` will be considered uninitialized memory. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; + return Err(e); + } + } + Ok(()) + }; + // SAFETY: The initializer above initializes every element of the array. On failure it drops + // any initialized elements and returns `Err`. + unsafe { pin_init_from_closure(init) } +} + +// SAFETY: Every type can be initialized by-value. +unsafe impl<T, E> Init<T, E> for T { + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: TODO. + unsafe { slot.write(self) }; + Ok(()) + } +} + +// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`. +unsafe impl<T, E> PinInit<T, E> for T { + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: TODO. + unsafe { self.__init(slot) } + } +} + +/// Smart pointer containing uninitialized memory and that can write a value. +pub trait InPlaceWrite<T> { + /// The type `Self` turns into when the contents are initialized. + type Initialized; + + /// Use the given initializer to write a value into `self`. + /// + /// Does not drop the current value and considers it as uninitialized memory. + fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E>; + + /// Use the given pin-initializer to write a value into `self`. + /// + /// Does not drop the current value and considers it as uninitialized memory. + fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>; +} + +/// Trait facilitating pinned destruction. +/// +/// Use [`pinned_drop`] to implement this trait safely: +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// use core::pin::Pin; +/// #[pin_data(PinnedDrop)] +/// struct Foo { +/// #[pin] +/// mtx: CMutex<usize>, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for Foo { +/// fn drop(self: Pin<&mut Self>) { +/// println!("Foo is being dropped!"); +/// } +/// } +/// ``` +/// +/// # Safety +/// +/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. +pub unsafe trait PinnedDrop: __internal::HasPinData { + /// Executes the pinned destructor of this type. + /// + /// While this function is marked safe, it is actually unsafe to call it manually. For this + /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code + /// and thus prevents this function from being called where it should not. + /// + /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute + /// automatically. + fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); +} + +/// Marker trait for types that can be initialized by writing just zeroes. +/// +/// # Safety +/// +/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words, +/// this is not UB: +/// +/// ```rust,ignore +/// let val: Self = unsafe { core::mem::zeroed() }; +/// ``` +pub unsafe trait Zeroable {} + +/// Marker trait for types that allow `Option<Self>` to be set to all zeroes in order to write +/// `None` to that location. +/// +/// # Safety +/// +/// The implementer needs to ensure that `unsafe impl Zeroable for Option<Self> {}` is sound. +pub unsafe trait ZeroableOption {} + +// SAFETY: by the safety requirement of `ZeroableOption`, this is valid. +unsafe impl<T: ZeroableOption> Zeroable for Option<T> {} + +/// Create a new zeroed T. +/// +/// The returned initializer will write `0x00` to every byte of the given `slot`. +#[inline] +pub fn zeroed<T: Zeroable>() -> impl Init<T> { + // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` + // and because we write all zeroes, the memory is initialized. + unsafe { + init_from_closure(|slot: *mut T| { + slot.write_bytes(0, 1); + Ok(()) + }) + } +} + +macro_rules! impl_zeroable { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? Zeroable for $t {})* + }; +} + +impl_zeroable! { + // SAFETY: All primitives that are allowed to be zero. + bool, + char, + u8, u16, u32, u64, u128, usize, + i8, i16, i32, i64, i128, isize, + f32, f64, + + // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list; + // creating an instance of an uninhabited type is immediate undefined behavior. For more on + // uninhabited/empty types, consult The Rustonomicon: + // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference + // also has information on undefined behavior: + // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>. + // + // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists. + {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (), + + // SAFETY: Type is allowed to take any value, including all zeros. + {<T>} MaybeUninit<T>, + + // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. + {<T: ?Sized + Zeroable>} UnsafeCell<T>, + + // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>). + Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, + Option<NonZeroU128>, Option<NonZeroUsize>, + Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>, + Option<NonZeroI128>, Option<NonZeroIsize>, + {<T>} Option<NonNull<T>>, + + // SAFETY: `null` pointer is valid. + // + // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be + // null. + // + // When `Pointee` gets stabilized, we could use + // `T: ?Sized where <T as Pointee>::Metadata: Zeroable` + {<T>} *mut T, {<T>} *const T, + + // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be + // zero. + {<T>} *mut [T], {<T>} *const [T], *mut str, *const str, + + // SAFETY: `T` is `Zeroable`. + {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>, +} + +macro_rules! impl_tuple_zeroable { + ($(,)?) => {}; + ($first:ident, $($t:ident),* $(,)?) => { + // SAFETY: All elements are zeroable and padding can be zero. + unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} + impl_tuple_zeroable!($($t),* ,); + } +} + +impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); diff --git a/rust/pin-init/src/macros.rs b/rust/pin-init/src/macros.rs new file mode 100644 index 000000000000..361623324d5c --- /dev/null +++ b/rust/pin-init/src/macros.rs @@ -0,0 +1,1415 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! This module provides the macros that actually implement the proc-macros `pin_data` and +//! `pinned_drop`. It also contains `__init_internal`, the implementation of the +//! `{try_}{pin_}init!` macros. +//! +//! These macros should never be called directly, since they expect their input to be +//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in +//! safe code! Use the public facing macros instead. +//! +//! This architecture has been chosen because the kernel does not yet have access to `syn` which +//! would make matters a lot easier for implementing these as proc-macros. +//! +//! Since this library and the kernel implementation should diverge as little as possible, the same +//! approach has been taken here. +//! +//! # Macro expansion example +//! +//! This section is intended for readers trying to understand the macros in this module and the +//! `[try_][pin_]init!` macros from `lib.rs`. +//! +//! We will look at the following example: +//! +//! ```rust,ignore +//! #[pin_data] +//! #[repr(C)] +//! struct Bar<T> { +//! #[pin] +//! t: T, +//! pub x: usize, +//! } +//! +//! impl<T> Bar<T> { +//! fn new(t: T) -> impl PinInit<Self> { +//! pin_init!(Self { t, x: 0 }) +//! } +//! } +//! +//! #[pin_data(PinnedDrop)] +//! struct Foo { +//! a: usize, +//! #[pin] +//! b: Bar<u32>, +//! } +//! +//! #[pinned_drop] +//! impl PinnedDrop for Foo { +//! fn drop(self: Pin<&mut Self>) { +//! println!("{self:p} is getting dropped."); +//! } +//! } +//! +//! let a = 42; +//! let initializer = pin_init!(Foo { +//! a, +//! b <- Bar::new(36), +//! }); +//! ``` +//! +//! This example includes the most common and important features of the pin-init API. +//! +//! Below you can find individual section about the different macro invocations. Here are some +//! general things we need to take into account when designing macros: +//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()` +//! this ensures that the correct item is used, since users could define their own `mod core {}` +//! and then their own `panic!` inside to execute arbitrary code inside of our macro. +//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied +//! expressions inside of an `unsafe` block in the macro, because this would allow users to do +//! `unsafe` operations without an associated `unsafe` block. +//! +//! ## `#[pin_data]` on `Bar` +//! +//! This macro is used to specify which fields are structurally pinned and which fields are not. It +//! is placed on the struct definition and allows `#[pin]` to be placed on the fields. +//! +//! Here is the definition of `Bar` from our example: +//! +//! ```rust,ignore +//! #[pin_data] +//! #[repr(C)] +//! struct Bar<T> { +//! #[pin] +//! t: T, +//! pub x: usize, +//! } +//! ``` +//! +//! This expands to the following code: +//! +//! ```rust,ignore +//! // Firstly the normal definition of the struct, attributes are preserved: +//! #[repr(C)] +//! struct Bar<T> { +//! t: T, +//! pub x: usize, +//! } +//! // Then an anonymous constant is defined, this is because we do not want any code to access the +//! // types that we define inside: +//! const _: () = { +//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics, +//! // since we need to implement access functions for each field and thus need to know its +//! // type. +//! struct __ThePinData<T> { +//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>, +//! } +//! // We implement `Copy` for the pin-data struct, since all functions it defines will take +//! // `self` by value. +//! impl<T> ::core::clone::Clone for __ThePinData<T> { +//! fn clone(&self) -> Self { +//! *self +//! } +//! } +//! impl<T> ::core::marker::Copy for __ThePinData<T> {} +//! // For every field of `Bar`, the pin-data struct will define a function with the same name +//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the +//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field +//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`). +//! #[allow(dead_code)] +//! impl<T> __ThePinData<T> { +//! unsafe fn t<E>( +//! self, +//! slot: *mut T, +//! // Since `t` is `#[pin]`, this is `PinInit`. +//! init: impl ::pin_init::PinInit<T, E>, +//! ) -> ::core::result::Result<(), E> { +//! unsafe { ::pin_init::PinInit::__pinned_init(init, slot) } +//! } +//! pub unsafe fn x<E>( +//! self, +//! slot: *mut usize, +//! // Since `x` is not `#[pin]`, this is `Init`. +//! init: impl ::pin_init::Init<usize, E>, +//! ) -> ::core::result::Result<(), E> { +//! unsafe { ::pin_init::Init::__init(init, slot) } +//! } +//! } +//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct +//! // that we constructed above. +//! unsafe impl<T> ::pin_init::__internal::HasPinData for Bar<T> { +//! type PinData = __ThePinData<T>; +//! unsafe fn __pin_data() -> Self::PinData { +//! __ThePinData { +//! __phantom: ::core::marker::PhantomData, +//! } +//! } +//! } +//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data +//! // struct. This is important to ensure that no user can implement a rogue `__pin_data` +//! // function without using `unsafe`. +//! unsafe impl<T> ::pin_init::__internal::PinData for __ThePinData<T> { +//! type Datee = Bar<T>; +//! } +//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is +//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned +//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our +//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist +//! // for two reasons: +//! // - `__phantom`: every generic must be used, since we cannot really know which generics +//! // are used, we declare all and then use everything here once. +//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant +//! // over it. The lifetime is needed to work around the limitation that trait bounds must +//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is +//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler +//! // into accepting these bounds regardless. +//! #[allow(dead_code)] +//! struct __Unpin<'__pin, T> { +//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, +//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>, +//! // Our only `#[pin]` field is `t`. +//! t: T, +//! } +//! #[doc(hidden)] +//! impl<'__pin, T> ::core::marker::Unpin for Bar<T> +//! where +//! __Unpin<'__pin, T>: ::core::marker::Unpin, +//! {} +//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users +//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to +//! // UB with only safe code, so we disallow this by giving a trait implementation error using +//! // a direct impl and a blanket implementation. +//! trait MustNotImplDrop {} +//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do +//! // (normally people want to know if a type has any kind of drop glue at all, here we want +//! // to know if it has any kind of custom drop glue, which is exactly what this bound does). +//! #[expect(drop_bounds)] +//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {} +//! impl<T> MustNotImplDrop for Bar<T> {} +//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to +//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed +//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`. +//! #[expect(non_camel_case_types)] +//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} +//! impl< +//! T: ::pin_init::PinnedDrop, +//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} +//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {} +//! }; +//! ``` +//! +//! ## `pin_init!` in `impl Bar` +//! +//! This macro creates an pin-initializer for the given struct. It requires that the struct is +//! annotated by `#[pin_data]`. +//! +//! Here is the impl on `Bar` defining the new function: +//! +//! ```rust,ignore +//! impl<T> Bar<T> { +//! fn new(t: T) -> impl PinInit<Self> { +//! pin_init!(Self { t, x: 0 }) +//! } +//! } +//! ``` +//! +//! This expands to the following code: +//! +//! ```rust,ignore +//! impl<T> Bar<T> { +//! fn new(t: T) -> impl PinInit<Self> { +//! { +//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok` +//! // return type and shadow it later when we insert the arbitrary user code. That way +//! // there will be no possibility of returning without `unsafe`. +//! struct __InitOk; +//! // Get the data about fields from the supplied type. +//! // - the function is unsafe, hence the unsafe block +//! // - we `use` the `HasPinData` trait in the block, it is only available in that +//! // scope. +//! let data = unsafe { +//! use ::pin_init::__internal::HasPinData; +//! Self::__pin_data() +//! }; +//! // Ensure that `data` really is of type `PinData` and help with type inference: +//! let init = ::pin_init::__internal::PinData::make_closure::< +//! _, +//! __InitOk, +//! ::core::convert::Infallible, +//! >(data, move |slot| { +//! { +//! // Shadow the structure so it cannot be used to return early. If a user +//! // tries to write `return Ok(__InitOk)`, then they get a type error, +//! // since that will refer to this struct instead of the one defined +//! // above. +//! struct __InitOk; +//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`. +//! { +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) }; +//! } +//! // Since initialization could fail later (not in this case, since the +//! // error type is `Infallible`) we will need to drop this field if there +//! // is an error later. This `DropGuard` will drop the field when it gets +//! // dropped and has not yet been forgotten. +//! let __t_guard = unsafe { +//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t)) +//! }; +//! // Expansion of `x: 0,`: +//! // Since this can be an arbitrary expression we cannot place it inside +//! // of the `unsafe` block, so we bind it here. +//! { +//! let x = 0; +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) }; +//! } +//! // We again create a `DropGuard`. +//! let __x_guard = unsafe { +//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x)) +//! }; +//! // Since initialization has successfully completed, we can now forget +//! // the guards. This is not `mem::forget`, since we only have +//! // `&DropGuard`. +//! ::core::mem::forget(__x_guard); +//! ::core::mem::forget(__t_guard); +//! // Here we use the type checker to ensure that every field has been +//! // initialized exactly once, since this is `if false` it will never get +//! // executed, but still type-checked. +//! // Additionally we abuse `slot` to automatically infer the correct type +//! // for the struct. This is also another check that every field is +//! // accessible from this scope. +//! #[allow(unreachable_code, clippy::diverging_sub_expression)] +//! let _ = || { +//! unsafe { +//! ::core::ptr::write( +//! slot, +//! Self { +//! // We only care about typecheck finding every field +//! // here, the expression does not matter, just conjure +//! // one using `panic!()`: +//! t: ::core::panic!(), +//! x: ::core::panic!(), +//! }, +//! ); +//! }; +//! }; +//! } +//! // We leave the scope above and gain access to the previously shadowed +//! // `__InitOk` that we need to return. +//! Ok(__InitOk) +//! }); +//! // Change the return type from `__InitOk` to `()`. +//! let init = move | +//! slot, +//! | -> ::core::result::Result<(), ::core::convert::Infallible> { +//! init(slot).map(|__InitOk| ()) +//! }; +//! // Construct the initializer. +//! let init = unsafe { +//! ::pin_init::pin_init_from_closure::< +//! _, +//! ::core::convert::Infallible, +//! >(init) +//! }; +//! init +//! } +//! } +//! } +//! ``` +//! +//! ## `#[pin_data]` on `Foo` +//! +//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the +//! differences/new things in the expansion of the `Foo` definition: +//! +//! ```rust,ignore +//! #[pin_data(PinnedDrop)] +//! struct Foo { +//! a: usize, +//! #[pin] +//! b: Bar<u32>, +//! } +//! ``` +//! +//! This expands to the following code: +//! +//! ```rust,ignore +//! struct Foo { +//! a: usize, +//! b: Bar<u32>, +//! } +//! const _: () = { +//! struct __ThePinData { +//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>, +//! } +//! impl ::core::clone::Clone for __ThePinData { +//! fn clone(&self) -> Self { +//! *self +//! } +//! } +//! impl ::core::marker::Copy for __ThePinData {} +//! #[allow(dead_code)] +//! impl __ThePinData { +//! unsafe fn b<E>( +//! self, +//! slot: *mut Bar<u32>, +//! init: impl ::pin_init::PinInit<Bar<u32>, E>, +//! ) -> ::core::result::Result<(), E> { +//! unsafe { ::pin_init::PinInit::__pinned_init(init, slot) } +//! } +//! unsafe fn a<E>( +//! self, +//! slot: *mut usize, +//! init: impl ::pin_init::Init<usize, E>, +//! ) -> ::core::result::Result<(), E> { +//! unsafe { ::pin_init::Init::__init(init, slot) } +//! } +//! } +//! unsafe impl ::pin_init::__internal::HasPinData for Foo { +//! type PinData = __ThePinData; +//! unsafe fn __pin_data() -> Self::PinData { +//! __ThePinData { +//! __phantom: ::core::marker::PhantomData, +//! } +//! } +//! } +//! unsafe impl ::pin_init::__internal::PinData for __ThePinData { +//! type Datee = Foo; +//! } +//! #[allow(dead_code)] +//! struct __Unpin<'__pin> { +//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, +//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>, +//! b: Bar<u32>, +//! } +//! #[doc(hidden)] +//! impl<'__pin> ::core::marker::Unpin for Foo +//! where +//! __Unpin<'__pin>: ::core::marker::Unpin, +//! {} +//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to +//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like +//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`. +//! impl ::core::ops::Drop for Foo { +//! fn drop(&mut self) { +//! // Since we are getting dropped, no one else has a reference to `self` and thus we +//! // can assume that we never move. +//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; +//! // Create the unsafe token that proves that we are inside of a destructor, this +//! // type is only allowed to be created in a destructor. +//! let token = unsafe { ::pin_init::__internal::OnlyCallFromDrop::new() }; +//! ::pin_init::PinnedDrop::drop(pinned, token); +//! } +//! } +//! }; +//! ``` +//! +//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo` +//! +//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an +//! extra parameter that should not be used at all. The macro hides that parameter. +//! +//! Here is the `PinnedDrop` impl for `Foo`: +//! +//! ```rust,ignore +//! #[pinned_drop] +//! impl PinnedDrop for Foo { +//! fn drop(self: Pin<&mut Self>) { +//! println!("{self:p} is getting dropped."); +//! } +//! } +//! ``` +//! +//! This expands to the following code: +//! +//! ```rust,ignore +//! // `unsafe`, full path and the token parameter are added, everything else stays the same. +//! unsafe impl ::pin_init::PinnedDrop for Foo { +//! fn drop(self: Pin<&mut Self>, _: ::pin_init::__internal::OnlyCallFromDrop) { +//! println!("{self:p} is getting dropped."); +//! } +//! } +//! ``` +//! +//! ## `pin_init!` on `Foo` +//! +//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion +//! of `pin_init!` on `Foo`: +//! +//! ```rust,ignore +//! let a = 42; +//! let initializer = pin_init!(Foo { +//! a, +//! b <- Bar::new(36), +//! }); +//! ``` +//! +//! This expands to the following code: +//! +//! ```rust,ignore +//! let a = 42; +//! let initializer = { +//! struct __InitOk; +//! let data = unsafe { +//! use ::pin_init::__internal::HasPinData; +//! Foo::__pin_data() +//! }; +//! let init = ::pin_init::__internal::PinData::make_closure::< +//! _, +//! __InitOk, +//! ::core::convert::Infallible, +//! >(data, move |slot| { +//! { +//! struct __InitOk; +//! { +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; +//! } +//! let __a_guard = unsafe { +//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a)) +//! }; +//! let init = Bar::new(36); +//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? }; +//! let __b_guard = unsafe { +//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b)) +//! }; +//! ::core::mem::forget(__b_guard); +//! ::core::mem::forget(__a_guard); +//! #[allow(unreachable_code, clippy::diverging_sub_expression)] +//! let _ = || { +//! unsafe { +//! ::core::ptr::write( +//! slot, +//! Foo { +//! a: ::core::panic!(), +//! b: ::core::panic!(), +//! }, +//! ); +//! }; +//! }; +//! } +//! Ok(__InitOk) +//! }); +//! let init = move | +//! slot, +//! | -> ::core::result::Result<(), ::core::convert::Infallible> { +//! init(slot).map(|__InitOk| ()) +//! }; +//! let init = unsafe { +//! ::pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) +//! }; +//! init +//! }; +//! ``` + +#[cfg(kernel)] +pub use ::macros::paste; +#[cfg(not(kernel))] +pub use ::paste::paste; + +/// Creates a `unsafe impl<...> PinnedDrop for $type` block. +/// +/// See [`PinnedDrop`] for more information. +#[doc(hidden)] +#[macro_export] +macro_rules! __pinned_drop { + ( + @impl_sig($($impl_sig:tt)*), + @impl_body( + $(#[$($attr:tt)*])* + fn drop($($sig:tt)*) { + $($inner:tt)* + } + ), + ) => { + // SAFETY: TODO. + unsafe $($impl_sig)* { + // Inherit all attributes and the type/ident tokens for the signature. + $(#[$($attr)*])* + fn drop($($sig)*, _: $crate::__internal::OnlyCallFromDrop) { + $($inner)* + } + } + } +} + +/// This macro first parses the struct definition such that it separates pinned and not pinned +/// fields. Afterwards it declares the struct and implement the `PinData` trait safely. +#[doc(hidden)] +#[macro_export] +macro_rules! __pin_data { + // Proc-macro entry point, this is supplied by the proc-macro pre-parsing. + (parse_input: + @args($($pinned_drop:ident)?), + @sig( + $(#[$($struct_attr:tt)*])* + $vis:vis struct $name:ident + $(where $($whr:tt)*)? + ), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), + @body({ $($fields:tt)* }), + ) => { + // We now use token munching to iterate through all of the fields. While doing this we + // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user + // wants these to be structurally pinned. The rest of the fields are the + // 'not pinned fields'. Additionally we collect all fields, since we need them in the right + // order to declare the struct. + // + // In this call we also put some explaining comments for the parameters. + $crate::__pin_data!(find_pinned_fields: + // Attributes on the struct itself, these will just be propagated to be put onto the + // struct definition. + @struct_attrs($(#[$($struct_attr)*])*), + // The visibility of the struct. + @vis($vis), + // The name of the struct. + @name($name), + // The 'impl generics', the generics that will need to be specified on the struct inside + // of an `impl<$ty_generics>` block. + @impl_generics($($impl_generics)*), + // The 'ty generics', the generics that will need to be specified on the impl blocks. + @ty_generics($($ty_generics)*), + // The 'decl generics', the generics that need to be specified on the struct + // definition. + @decl_generics($($decl_generics)*), + // The where clause of any impl block and the declaration. + @where($($($whr)*)?), + // The remaining fields tokens that need to be processed. + // We add a `,` at the end to ensure correct parsing. + @fields_munch($($fields)* ,), + // The pinned fields. + @pinned(), + // The not pinned fields. + @not_pinned(), + // All fields. + @fields(), + // The accumulator containing all attributes already parsed. + @accum(), + // Contains `yes` or `` to indicate if `#[pin]` was found on the current field. + @is_pinned(), + // The proc-macro argument, this should be `PinnedDrop` or ``. + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), + @where($($whr:tt)*), + // We found a PhantomPinned field, this should generally be pinned! + @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + // This field is not pinned. + @is_pinned(), + @pinned_drop($($pinned_drop:ident)?), + ) => { + ::core::compile_error!(concat!( + "The field `", + stringify!($field), + "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.", + )); + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), + @where($($whr)*), + @fields_munch($($rest)*), + @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,), + @not_pinned($($not_pinned)*), + @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,), + @accum(), + @is_pinned(), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), + @where($($whr:tt)*), + // We reached the field declaration. + @fields_munch($field:ident : $type:ty, $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + // This field is pinned. + @is_pinned(yes), + @pinned_drop($($pinned_drop:ident)?), + ) => { + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), + @where($($whr)*), + @fields_munch($($rest)*), + @pinned($($pinned)* $($accum)* $field: $type,), + @not_pinned($($not_pinned)*), + @fields($($fields)* $($accum)* $field: $type,), + @accum(), + @is_pinned(), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), + @where($($whr:tt)*), + // We reached the field declaration. + @fields_munch($field:ident : $type:ty, $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + // This field is not pinned. + @is_pinned(), + @pinned_drop($($pinned_drop:ident)?), + ) => { + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), + @where($($whr)*), + @fields_munch($($rest)*), + @pinned($($pinned)*), + @not_pinned($($not_pinned)* $($accum)* $field: $type,), + @fields($($fields)* $($accum)* $field: $type,), + @accum(), + @is_pinned(), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), + @where($($whr:tt)*), + // We found the `#[pin]` attr. + @fields_munch(#[pin] $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + @is_pinned($($is_pinned:ident)?), + @pinned_drop($($pinned_drop:ident)?), + ) => { + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), + @where($($whr)*), + @fields_munch($($rest)*), + // We do not include `#[pin]` in the list of attributes, since it is not actually an + // attribute that is defined somewhere. + @pinned($($pinned)*), + @not_pinned($($not_pinned)*), + @fields($($fields)*), + @accum($($accum)*), + // Set this to `yes`. + @is_pinned(yes), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), + @where($($whr:tt)*), + // We reached the field declaration with visibility, for simplicity we only munch the + // visibility and put it into `$accum`. + @fields_munch($fvis:vis $field:ident $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + @is_pinned($($is_pinned:ident)?), + @pinned_drop($($pinned_drop:ident)?), + ) => { + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), + @where($($whr)*), + @fields_munch($field $($rest)*), + @pinned($($pinned)*), + @not_pinned($($not_pinned)*), + @fields($($fields)*), + @accum($($accum)* $fvis), + @is_pinned($($is_pinned)?), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), + @where($($whr:tt)*), + // Some other attribute, just put it into `$accum`. + @fields_munch(#[$($attr:tt)*] $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + @is_pinned($($is_pinned:ident)?), + @pinned_drop($($pinned_drop:ident)?), + ) => { + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), + @where($($whr)*), + @fields_munch($($rest)*), + @pinned($($pinned)*), + @not_pinned($($not_pinned)*), + @fields($($fields)*), + @accum($($accum)* #[$($attr)*]), + @is_pinned($($is_pinned)?), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), + @where($($whr:tt)*), + // We reached the end of the fields, plus an optional additional comma, since we added one + // before and the user is also allowed to put a trailing comma. + @fields_munch($(,)?), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum(), + @is_pinned(), + @pinned_drop($($pinned_drop:ident)?), + ) => { + // Declare the struct with all fields in the correct order. + $($struct_attrs)* + $vis struct $name <$($decl_generics)*> + where $($whr)* + { + $($fields)* + } + + // We put the rest into this const item, because it then will not be accessible to anything + // outside. + const _: () = { + // We declare this struct which will host all of the projection function for our type. + // it will be invariant over all generic parameters which are inherited from the + // struct. + $vis struct __ThePinData<$($impl_generics)*> + where $($whr)* + { + __phantom: ::core::marker::PhantomData< + fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> + >, + } + + impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*> + where $($whr)* + { + fn clone(&self) -> Self { *self } + } + + impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*> + where $($whr)* + {} + + // Make all projection functions. + $crate::__pin_data!(make_pin_data: + @pin_data(__ThePinData), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @where($($whr)*), + @pinned($($pinned)*), + @not_pinned($($not_pinned)*), + ); + + // SAFETY: We have added the correct projection functions above to `__ThePinData` and + // we also use the least restrictive generics possible. + unsafe impl<$($impl_generics)*> + $crate::__internal::HasPinData for $name<$($ty_generics)*> + where $($whr)* + { + type PinData = __ThePinData<$($ty_generics)*>; + + unsafe fn __pin_data() -> Self::PinData { + __ThePinData { __phantom: ::core::marker::PhantomData } + } + } + + // SAFETY: TODO. + unsafe impl<$($impl_generics)*> + $crate::__internal::PinData for __ThePinData<$($ty_generics)*> + where $($whr)* + { + type Datee = $name<$($ty_generics)*>; + } + + // This struct will be used for the unpin analysis. Since only structurally pinned + // fields are relevant whether the struct should implement `Unpin`. + #[allow(dead_code)] + struct __Unpin <'__pin, $($impl_generics)*> + where $($whr)* + { + __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, + __phantom: ::core::marker::PhantomData< + fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> + >, + // Only the pinned fields. + $($pinned)* + } + + #[doc(hidden)] + impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*> + where + __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin, + $($whr)* + {} + + // We need to disallow normal `Drop` implementation, the exact behavior depends on + // whether `PinnedDrop` was specified as the parameter. + $crate::__pin_data!(drop_prevention: + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @where($($whr)*), + @pinned_drop($($pinned_drop)?), + ); + }; + }; + // When no `PinnedDrop` was specified, then we have to prevent implementing drop. + (drop_prevention: + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + @pinned_drop(), + ) => { + // We prevent this by creating a trait that will be implemented for all types implementing + // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, + // if it also implements `Drop` + trait MustNotImplDrop {} + #[expect(drop_bounds)] + impl<T: ::core::ops::Drop> MustNotImplDrop for T {} + impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*> + where $($whr)* {} + // We also take care to prevent users from writing a useless `PinnedDrop` implementation. + // They might implement `PinnedDrop` correctly for the struct, but forget to give + // `PinnedDrop` as the parameter to `#[pin_data]`. + #[expect(non_camel_case_types)] + trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} + impl<T: $crate::PinnedDrop> + UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} + impl<$($impl_generics)*> + UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*> + where $($whr)* {} + }; + // When `PinnedDrop` was specified we just implement `Drop` and delegate. + (drop_prevention: + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + @pinned_drop(PinnedDrop), + ) => { + impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*> + where $($whr)* + { + fn drop(&mut self) { + // SAFETY: Since this is a destructor, `self` will not move after this function + // terminates, since it is inaccessible. + let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; + // SAFETY: Since this is a drop function, we can create this token to call the + // pinned destructor of this type. + let token = unsafe { $crate::__internal::OnlyCallFromDrop::new() }; + $crate::PinnedDrop::drop(pinned, token); + } + } + }; + // If some other parameter was specified, we emit a readable error. + (drop_prevention: + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + @pinned_drop($($rest:tt)*), + ) => { + compile_error!( + "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.", + stringify!($($rest)*), + ); + }; + (make_pin_data: + @pin_data($pin_data:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?), + @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?), + ) => { + // For every field, we create a projection function according to its projection type. If a + // field is structurally pinned, then it must be initialized via `PinInit`, if it is not + // structurally pinned, then it can be initialized via `Init`. + // + // The functions are `unsafe` to prevent accidentally calling them. + #[allow(dead_code)] + #[expect(clippy::missing_safety_doc)] + impl<$($impl_generics)*> $pin_data<$($ty_generics)*> + where $($whr)* + { + $( + $(#[$($p_attr)*])* + $pvis unsafe fn $p_field<E>( + self, + slot: *mut $p_type, + init: impl $crate::PinInit<$p_type, E>, + ) -> ::core::result::Result<(), E> { + // SAFETY: TODO. + unsafe { $crate::PinInit::__pinned_init(init, slot) } + } + )* + $( + $(#[$($attr)*])* + $fvis unsafe fn $field<E>( + self, + slot: *mut $type, + init: impl $crate::Init<$type, E>, + ) -> ::core::result::Result<(), E> { + // SAFETY: TODO. + unsafe { $crate::Init::__init(init, slot) } + } + )* + } + }; +} + +/// The internal init macro. Do not call manually! +/// +/// This is called by the `{try_}{pin_}init!` macros with various inputs. +/// +/// This macro has multiple internal call configurations, these are always the very first ident: +/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros. +/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled. +/// - `init_slot`: recursively creates the code that initializes all fields in `slot`. +/// - `make_initializer`: recursively create the struct initializer that guarantees that every +/// field has been initialized exactly once. +#[doc(hidden)] +#[macro_export] +macro_rules! __init_internal { + ( + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @munch_fields(), + ) => { + $crate::__init_internal!(with_update_parsed: + @this($($this)?), + @typ($t), + @fields($($fields)*), + @error($err), + @data($data, $($use_data)?), + @has_data($has_data, $get_data), + @construct_closure($construct_closure), + @zeroed(), // Nothing means default behavior. + ) + }; + ( + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @munch_fields(..Zeroable::zeroed()), + ) => { + $crate::__init_internal!(with_update_parsed: + @this($($this)?), + @typ($t), + @fields($($fields)*), + @error($err), + @data($data, $($use_data)?), + @has_data($has_data, $get_data), + @construct_closure($construct_closure), + @zeroed(()), // `()` means zero all fields not mentioned. + ) + }; + ( + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @munch_fields($ignore:tt $($rest:tt)*), + ) => { + $crate::__init_internal!( + @this($($this)?), + @typ($t), + @fields($($fields)*), + @error($err), + @data($data, $($use_data)?), + @has_data($has_data, $get_data), + @construct_closure($construct_closure), + @munch_fields($($rest)*), + ) + }; + (with_update_parsed: + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @zeroed($($init_zeroed:expr)?), + ) => {{ + // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return + // type and shadow it later when we insert the arbitrary user code. That way there will be + // no possibility of returning without `unsafe`. + struct __InitOk; + // Get the data about fields from the supplied type. + // + // SAFETY: TODO. + let data = unsafe { + use $crate::__internal::$has_data; + // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal + // information that is associated to already parsed fragments, so a path fragment + // cannot be used in this position. Doing the retokenization results in valid rust + // code. + $crate::macros::paste!($t::$get_data()) + }; + // Ensure that `data` really is of type `$data` and help with type inference: + let init = $crate::__internal::$data::make_closure::<_, __InitOk, $err>( + data, + move |slot| { + { + // Shadow the structure so it cannot be used to return early. + struct __InitOk; + // If `$init_zeroed` is present we should zero the slot now and not emit an + // error when fields are missing (since they will be zeroed). We also have to + // check that the type actually implements `Zeroable`. + $({ + fn assert_zeroable<T: $crate::Zeroable>(_: *mut T) {} + // Ensure that the struct is indeed `Zeroable`. + assert_zeroable(slot); + // SAFETY: The type implements `Zeroable` by the check above. + unsafe { ::core::ptr::write_bytes(slot, 0, 1) }; + $init_zeroed // This will be `()` if set. + })? + // Create the `this` so it can be referenced by the user inside of the + // expressions creating the individual fields. + $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? + // Initialize every field. + $crate::__init_internal!(init_slot($($use_data)?): + @data(data), + @slot(slot), + @guards(), + @munch_fields($($fields)*,), + ); + // We use unreachable code to ensure that all fields have been mentioned exactly + // once, this struct initializer will still be type-checked and complain with a + // very natural error message if a field is forgotten/mentioned more than once. + #[allow(unreachable_code, clippy::diverging_sub_expression)] + let _ = || { + $crate::__init_internal!(make_initializer: + @slot(slot), + @type_name($t), + @munch_fields($($fields)*,), + @acc(), + ); + }; + } + Ok(__InitOk) + } + ); + let init = move |slot| -> ::core::result::Result<(), $err> { + init(slot).map(|__InitOk| ()) + }; + // SAFETY: TODO. + let init = unsafe { $crate::$construct_closure::<_, $err>(init) }; + init + }}; + (init_slot($($use_data:ident)?): + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + @munch_fields($(..Zeroable::zeroed())? $(,)?), + ) => { + // Endpoint of munching, no fields are left. If execution reaches this point, all fields + // have been initialized. Therefore we can now dismiss the guards by forgetting them. + $(::core::mem::forget($guards);)* + }; + (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields. + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + // In-place initialization syntax. + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + ) => { + let init = $val; + // Call the initializer. + // + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we + // return when an error/panic occurs. + // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. + unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? }; + // Create the drop guard: + // + // We rely on macro hygiene to make it impossible for users to access this local variable. + // We use `paste!` to create new hygiene for `$field`. + $crate::macros::paste! { + // SAFETY: We forget the guard later when initialization has succeeded. + let [< __ $field _guard >] = unsafe { + $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::__init_internal!(init_slot($use_data): + @data($data), + @slot($slot), + @guards([< __ $field _guard >], $($guards,)*), + @munch_fields($($rest)*), + ); + } + }; + (init_slot(): // No `use_data`, so we use `Init::__init` directly. + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + // In-place initialization syntax. + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + ) => { + let init = $val; + // Call the initializer. + // + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we + // return when an error/panic occurs. + unsafe { $crate::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? }; + // Create the drop guard: + // + // We rely on macro hygiene to make it impossible for users to access this local variable. + // We use `paste!` to create new hygiene for `$field`. + $crate::macros::paste! { + // SAFETY: We forget the guard later when initialization has succeeded. + let [< __ $field _guard >] = unsafe { + $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::__init_internal!(init_slot(): + @data($data), + @slot($slot), + @guards([< __ $field _guard >], $($guards,)*), + @munch_fields($($rest)*), + ); + } + }; + (init_slot($($use_data:ident)?): + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + // Init by-value. + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + ) => { + { + $(let $field = $val;)? + // Initialize the field. + // + // SAFETY: The memory at `slot` is uninitialized. + unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; + } + // Create the drop guard: + // + // We rely on macro hygiene to make it impossible for users to access this local variable. + // We use `paste!` to create new hygiene for `$field`. + $crate::macros::paste! { + // SAFETY: We forget the guard later when initialization has succeeded. + let [< __ $field _guard >] = unsafe { + $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::__init_internal!(init_slot($($use_data)?): + @data($data), + @slot($slot), + @guards([< __ $field _guard >], $($guards,)*), + @munch_fields($($rest)*), + ); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields(..Zeroable::zeroed() $(,)?), + @acc($($acc:tt)*), + ) => { + // Endpoint, nothing more to munch, create the initializer. Since the users specified + // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have + // not been overwritten are thus zero and initialized. We still check that all fields are + // actually accessible by using the struct update syntax ourselves. + // We are inside of a closure that is never executed and thus we can abuse `slot` to + // get the correct type inference here: + #[allow(unused_assignments)] + unsafe { + let mut zeroed = ::core::mem::zeroed(); + // We have to use type inference here to make zeroed have the correct type. This does + // not get executed, so it has no effect. + ::core::ptr::write($slot, zeroed); + zeroed = ::core::mem::zeroed(); + // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal + // information that is associated to already parsed fragments, so a path fragment + // cannot be used in this position. Doing the retokenization results in valid rust + // code. + $crate::macros::paste!( + ::core::ptr::write($slot, $t { + $($acc)* + ..zeroed + }); + ); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields($(,)?), + @acc($($acc:tt)*), + ) => { + // Endpoint, nothing more to munch, create the initializer. + // Since we are in the closure that is never called, this will never get executed. + // We abuse `slot` to get the correct type inference here: + // + // SAFETY: TODO. + unsafe { + // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal + // information that is associated to already parsed fragments, so a path fragment + // cannot be used in this position. Doing the retokenization results in valid rust + // code. + $crate::macros::paste!( + ::core::ptr::write($slot, $t { + $($acc)* + }); + ); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + @acc($($acc:tt)*), + ) => { + $crate::__init_internal!(make_initializer: + @slot($slot), + @type_name($t), + @munch_fields($($rest)*), + @acc($($acc)* $field: ::core::panic!(),), + ); + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + @acc($($acc:tt)*), + ) => { + $crate::__init_internal!(make_initializer: + @slot($slot), + @type_name($t), + @munch_fields($($rest)*), + @acc($($acc)* $field: ::core::panic!(),), + ); + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __derive_zeroable { + (parse_input: + @sig( + $(#[$($struct_attr:tt)*])* + $vis:vis struct $name:ident + $(where $($whr:tt)*)? + ), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @body({ + $( + $(#[$($field_attr:tt)*])* + $field:ident : $field_ty:ty + ),* $(,)? + }), + ) => { + // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. + #[automatically_derived] + unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*> + where + $($($whr)*)? + {} + const _: () = { + fn assert_zeroable<T: ?::core::marker::Sized + $crate::Zeroable>() {} + fn ensure_zeroable<$($impl_generics)*>() + where $($($whr)*)? + { + $(assert_zeroable::<$field_ty>();)* + } + }; + }; +} |