From 8abba4b0ea920d23799c2d7b6985617e6392d176 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Sat, 31 Aug 2024 22:44:59 +0900 Subject: [PATCH] riscv: Support zaamo target feature https://github.com/taiki-e/portable-atomic/issues/123#issuecomment-2322844699 --- .github/.cspell/project-dictionary.txt | 1 + build.rs | 9 ++- src/imp/interrupt/mod.rs | 42 ++++++++++-- src/imp/interrupt/riscv.rs | 6 +- src/imp/riscv.rs | 91 +++++++++++++++++++++----- tools/build.sh | 6 ++ tools/no-std.sh | 6 ++ 7 files changed, 135 insertions(+), 26 deletions(-) diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt index 67ace4a2..5dd64632 100644 --- a/.github/.cspell/project-dictionary.txt +++ b/.github/.cspell/project-dictionary.txt @@ -173,4 +173,5 @@ xilf xmmword xsave xsub +zaamo Zhaoxin diff --git a/build.rs b/build.rs index adf79226..923a91f4 100644 --- a/build.rs +++ b/build.rs @@ -47,7 +47,7 @@ fn main() { if version.minor >= 80 { println!( - r#"cargo:rustc-check-cfg=cfg(target_feature,values("quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"# + r#"cargo:rustc-check-cfg=cfg(target_feature,values("zaamo","quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"# ); // Custom cfgs set by build script. Not public API. @@ -58,7 +58,7 @@ fn main() { // TODO: handle multi-line target_feature_fallback // grep -E 'target_feature_fallback\("' build.rs | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' println!( - r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6"))"# + r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo"))"# ); } @@ -309,6 +309,11 @@ fn main() { target_feature_fallback("mclass", mclass); } } + "riscv32" | "riscv64" => { + // As of rustc 1.80, target_feature "zaamo" is not available on rustc side: + // https://github.com/rust-lang/rust/blob/1.80.0/compiler/rustc_target/src/target_features.rs#L273 + target_feature_fallback("zaamo", false); // amo*.{w,d} + } "powerpc64" => { // For Miri and ThreadSanitizer. if version.nightly && version.llvm >= 15 { diff --git a/src/imp/interrupt/mod.rs b/src/imp/interrupt/mod.rs index cccd9ef2..0a6ea6cd 100644 --- a/src/imp/interrupt/mod.rs +++ b/src/imp/interrupt/mod.rs @@ -172,11 +172,25 @@ impl AtomicPtr { #[inline] pub(crate) fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T { let _ = order; - #[cfg(portable_atomic_force_amo)] + #[cfg(all( + any(target_arch = "riscv32", target_arch = "riscv64"), + any( + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", + ), + ))] { self.as_native().swap(ptr, order) } - #[cfg(not(portable_atomic_force_amo))] + #[cfg(not(all( + any(target_arch = "riscv32", target_arch = "riscv64"), + any( + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", + ), + )))] // SAFETY: any data races are prevented by disabling interrupts (see // module-level comments) and the raw pointer is valid because we got it // from a reference. @@ -279,10 +293,24 @@ macro_rules! atomic_int { }; (load_store_atomic $([$kind:ident])?, $atomic_type:ident, $int_type:ident, $align:literal) => { atomic_int!(base, $atomic_type, $int_type, $align); - #[cfg(not(portable_atomic_force_amo))] - atomic_int!(cas[emulate], $atomic_type, $int_type); - #[cfg(portable_atomic_force_amo)] + #[cfg(all( + any(target_arch = "riscv32", target_arch = "riscv64"), + any( + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", + ), + ))] atomic_int!(cas $([$kind])?, $atomic_type, $int_type); + #[cfg(not(all( + any(target_arch = "riscv32", target_arch = "riscv64"), + any( + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", + ), + )))] + atomic_int!(cas[emulate], $atomic_type, $int_type); impl $atomic_type { #[inline] #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] @@ -568,7 +596,7 @@ macro_rules! atomic_int { } } }; - // cfg(portable_atomic_force_amo) 32-bit(RV32)/{32,64}-bit(RV64) RMW + // 32-bit(RV32)/{32,64}-bit(RV64) RMW with Zaamo extension (cas, $atomic_type:ident, $int_type:ident) => { impl $atomic_type { #[inline] @@ -675,7 +703,7 @@ macro_rules! atomic_int { } } }; - // cfg(portable_atomic_force_amo) {8,16}-bit RMW + // {8,16}-bit RMW with Zaamo extension (cas[sub_word], $atomic_type:ident, $int_type:ident) => { impl $atomic_type { #[inline] diff --git a/src/imp/interrupt/riscv.rs b/src/imp/interrupt/riscv.rs index d14f4d37..8437ffb2 100644 --- a/src/imp/interrupt/riscv.rs +++ b/src/imp/interrupt/riscv.rs @@ -1,8 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT // Refs: -// - https://five-embeddev.com/riscv-isa-manual/latest/machine.html#machine-status-registers-mstatus-and-mstatush -// - https://five-embeddev.com/riscv-isa-manual/latest/supervisor.html#sstatus +// - https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/machine.adoc#machine-status-mstatus-and-mstatush-registers +// - https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/supervisor.adoc#supervisor-status-sstatus-register +// +// See also src/imp/riscv.rs. // // Generated asm: // - riscv64gc https://godbolt.org/z/zTrzT1Ee7 diff --git a/src/imp/riscv.rs b/src/imp/riscv.rs index c1947193..1a60fe50 100644 --- a/src/imp/riscv.rs +++ b/src/imp/riscv.rs @@ -11,9 +11,11 @@ // - RISC-V Instruction Set Manual Volume I: Unprivileged ISA // https://riscv.org/wp-content/uploads/2019/12/riscv-spec-20191213.pdf // - RISC-V Atomics ABI Specification -// https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/HEAD/riscv-atomic.adoc +// https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20240829-13bfa9f54634cb60d86b9b333e109f077805b4b3/riscv-atomic.adoc // - "Mappings from C/C++ primitives to RISC-V primitives." table in RISC-V Instruction Set Manual: -// https://five-embeddev.com/riscv-isa-manual/latest/memory.html#sec:memory:porting +// https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/mm-eplan.adoc#code-porting-and-mapping-guidelines +// - ""Zaamo" Extension for Atomic Memory Operations" in RISC-V Instruction Set Manual: +// https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/a-st-ext.adoc#zaamo-extension-for-atomic-memory-operations // - atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit // // Generated asm: @@ -24,14 +26,24 @@ use core::arch::asm; use core::{cell::UnsafeCell, sync::atomic::Ordering}; -#[cfg(any(test, portable_atomic_force_amo))] +#[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", +))] #[cfg(target_arch = "riscv32")] macro_rules! w { () => { "" }; } -#[cfg(any(test, portable_atomic_force_amo))] +#[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", +))] #[cfg(target_arch = "riscv64")] macro_rules! w { () => { @@ -39,7 +51,12 @@ macro_rules! w { }; } -#[cfg(any(test, portable_atomic_force_amo))] +#[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", +))] macro_rules! atomic_rmw_amo_order { ($op:ident, $order:ident) => { match $order { @@ -52,14 +69,19 @@ macro_rules! atomic_rmw_amo_order { } }; } -#[cfg(any(test, portable_atomic_force_amo))] +#[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", +))] macro_rules! atomic_rmw_amo { ($op:ident, $dst:ident, $val:ident, $order:ident, $asm_suffix:tt) => {{ let out; macro_rules! op { ($asm_order:tt) => { // SAFETY: The user guaranteed that the AMO instruction is available in this - // system by setting the portable_atomic_force_amo and + // system by setting the portable_atomic_force_amo/target_feature and // portable_atomic_unsafe_assume_single_core. // The caller of this macro must guarantee the validity of the pointer. asm!( @@ -80,7 +102,12 @@ macro_rules! atomic_rmw_amo { }}; } // 32-bit val.wrapping_shl(shift) but no extra `& (u32::BITS - 1)` -#[cfg(any(test, portable_atomic_force_amo))] +#[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", +))] #[inline] fn sllw(val: u32, shift: u32) -> u32 { // SAFETY: Calling sll{,w} is safe. @@ -97,7 +124,12 @@ fn sllw(val: u32, shift: u32) -> u32 { } } // 32-bit val.wrapping_shr(shift) but no extra `& (u32::BITS - 1)` -#[cfg(any(test, portable_atomic_force_amo))] +#[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", +))] macro_rules! srlw { ($val:expr, $shift:expr) => { // SAFETY: Calling srl{,w} is safe. @@ -240,7 +272,11 @@ macro_rules! atomic_load_store { macro_rules! atomic_ptr { ($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $asm_suffix:tt) => { atomic_load_store!($([$($generics)*])? $atomic_type, $value_type, $asm_suffix); - #[cfg(portable_atomic_force_amo)] + #[cfg(any( + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", + ))] impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { #[inline] pub(crate) fn swap(&self, val: $value_type, order: Ordering) -> $value_type { @@ -257,7 +293,12 @@ macro_rules! atomic { ($atomic_type:ident, $value_type:ty, $asm_suffix:tt, $max:tt, $min:tt) => { atomic_load_store!($atomic_type, $value_type, $asm_suffix); // There is no amo{sub,nand,neg}. - #[cfg(any(test, portable_atomic_force_amo))] + #[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", + ))] impl $atomic_type { #[inline] pub(crate) fn swap(&self, val: $value_type, order: Ordering) -> $value_type { @@ -328,21 +369,36 @@ macro_rules! atomic { }; } -#[cfg(any(test, portable_atomic_force_amo))] +#[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", +))] trait ZeroExtend: Copy { /// Zero-extends `self` to `u32` if it is smaller than 32-bit. fn zero_extend(self) -> u32; } macro_rules! zero_extend { ($int:ident, $uint:ident) => { - #[cfg(any(test, portable_atomic_force_amo))] + #[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", + ))] impl ZeroExtend for $uint { #[inline] fn zero_extend(self) -> u32 { self as u32 } } - #[cfg(any(test, portable_atomic_force_amo))] + #[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", + ))] impl ZeroExtend for $int { #[allow(clippy::cast_sign_loss)] #[inline] @@ -358,7 +414,12 @@ zero_extend!(i16, u16); macro_rules! atomic_sub_word { ($atomic_type:ident, $value_type:ty, $unsigned_type:ty, $asm_suffix:tt) => { atomic_load_store!($atomic_type, $value_type, $asm_suffix); - #[cfg(any(test, portable_atomic_force_amo))] + #[cfg(any( + test, + portable_atomic_force_amo, + target_feature = "zaamo", + portable_atomic_target_feature = "zaamo", + ))] impl $atomic_type { #[inline] pub(crate) fn fetch_and(&self, val: $value_type, order: Ordering) -> $value_type { diff --git a/tools/build.sh b/tools/build.sh index 31ad3f90..dfd86a37 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -408,6 +408,9 @@ build() { CARGO_TARGET_DIR="${target_dir}/api-test-assume-single-core-force-amo" \ RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core --cfg portable_atomic_force_amo" \ x_cargo "${args[@]}" --feature-powerset --manifest-path tests/api-test/Cargo.toml "$@" + CARGO_TARGET_DIR="${target_dir}/api-test-assume-single-core-zaamo" \ + RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core -C target-feature=+zaamo" \ + x_cargo "${args[@]}" --feature-powerset --manifest-path tests/api-test/Cargo.toml "$@" fi ;; esac @@ -454,6 +457,9 @@ build() { CARGO_TARGET_DIR="${target_dir}/assume-single-core-force-amo" \ RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core --cfg portable_atomic_force_amo" \ x_cargo "${args[@]}" --exclude-features "critical-section" "$@" + CARGO_TARGET_DIR="${target_dir}/assume-single-core-zaamo" \ + RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core -C target-feature=+zaamo" \ + x_cargo "${args[@]}" --exclude-features "critical-section" "$@" fi ;; esac diff --git a/tools/no-std.sh b/tools/no-std.sh index 633b3356..2ef9e657 100755 --- a/tools/no-std.sh +++ b/tools/no-std.sh @@ -209,6 +209,12 @@ run() { CARGO_TARGET_DIR="${target_dir}/no-std-test-force-amo" \ RUSTFLAGS="${target_rustflags} --cfg portable_atomic_force_amo" \ x_cargo "${args[@]}" --release "$@" + CARGO_TARGET_DIR="${target_dir}/no-std-test-zaamo" \ + RUSTFLAGS="${target_rustflags} -C target-feature=+zaamo" \ + x_cargo "${args[@]}" "$@" + CARGO_TARGET_DIR="${target_dir}/no-std-test-zaamo" \ + RUSTFLAGS="${target_rustflags} -C target-feature=+zaamo" \ + x_cargo "${args[@]}" --release "$@" fi ;; esac