riscv: Support zaamo target feature

#123 (comment)
taiki-e · Aug 31, 2024 · 8abba4b · 8abba4b
1 parent 10d47de
commit 8abba4b
Show file tree

Hide file tree

Showing 7 changed files with 135 additions and 26 deletions.
diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt
@@ -173,4 +173,5 @@ xilf
 xmmword
 xsave
 xsub
+zaamo
 Zhaoxin
diff --git a/build.rs b/build.rs
@@ -47,7 +47,7 @@ fn main() {
 
     if version.minor >= 80 {
         println!(
-            r#"cargo:rustc-check-cfg=cfg(target_feature,values("quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"#
+            r#"cargo:rustc-check-cfg=cfg(target_feature,values("zaamo","quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"#
         );
 
         // Custom cfgs set by build script. Not public API.
@@ -58,7 +58,7 @@ fn main() {
         // TODO: handle multi-line target_feature_fallback
         // grep -E 'target_feature_fallback\("' build.rs | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ','
         println!(
-            r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6"))"#
+            r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo"))"#
         );
     }
 
@@ -309,6 +309,11 @@ fn main() {
                 target_feature_fallback("mclass", mclass);
             }
         }
+        "riscv32" | "riscv64" => {
+            // As of rustc 1.80, target_feature "zaamo" is not available on rustc side:
+            // https://github.com/rust-lang/rust/blob/1.80.0/compiler/rustc_target/src/target_features.rs#L273
+            target_feature_fallback("zaamo", false); // amo*.{w,d}
+        }
         "powerpc64" => {
             // For Miri and ThreadSanitizer.
             if version.nightly && version.llvm >= 15 {

diff --git a/src/imp/interrupt/mod.rs b/src/imp/interrupt/mod.rs
@@ -172,11 +172,25 @@ impl<T> AtomicPtr<T> {
     #[inline]
     pub(crate) fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T {
         let _ = order;
-        #[cfg(portable_atomic_force_amo)]
+        #[cfg(all(
+            any(target_arch = "riscv32", target_arch = "riscv64"),
+            any(
+                portable_atomic_force_amo,
+                target_feature = "zaamo",
+                portable_atomic_target_feature = "zaamo",
+            ),
+        ))]
         {
             self.as_native().swap(ptr, order)
         }
-        #[cfg(not(portable_atomic_force_amo))]
+        #[cfg(not(all(
+            any(target_arch = "riscv32", target_arch = "riscv64"),
+            any(
+                portable_atomic_force_amo,
+                target_feature = "zaamo",
+                portable_atomic_target_feature = "zaamo",
+            ),
+        )))]
         // SAFETY: any data races are prevented by disabling interrupts (see
         // module-level comments) and the raw pointer is valid because we got it
         // from a reference.
@@ -279,10 +293,24 @@ macro_rules! atomic_int {
     };
     (load_store_atomic $([$kind:ident])?, $atomic_type:ident, $int_type:ident, $align:literal) => {
         atomic_int!(base, $atomic_type, $int_type, $align);
-        #[cfg(not(portable_atomic_force_amo))]
-        atomic_int!(cas[emulate], $atomic_type, $int_type);
-        #[cfg(portable_atomic_force_amo)]
+        #[cfg(all(
+            any(target_arch = "riscv32", target_arch = "riscv64"),
+            any(
+                portable_atomic_force_amo,
+                target_feature = "zaamo",
+                portable_atomic_target_feature = "zaamo",
+            ),
+        ))]
         atomic_int!(cas $([$kind])?, $atomic_type, $int_type);
+        #[cfg(not(all(
+            any(target_arch = "riscv32", target_arch = "riscv64"),
+            any(
+                portable_atomic_force_amo,
+                target_feature = "zaamo",
+                portable_atomic_target_feature = "zaamo",
+            ),
+        )))]
+        atomic_int!(cas[emulate], $atomic_type, $int_type);
         impl $atomic_type {
             #[inline]
             #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
@@ -568,7 +596,7 @@ macro_rules! atomic_int {
             }
         }
     };
-    // cfg(portable_atomic_force_amo) 32-bit(RV32)/{32,64}-bit(RV64) RMW
+    // 32-bit(RV32)/{32,64}-bit(RV64) RMW with Zaamo extension
     (cas, $atomic_type:ident, $int_type:ident) => {
         impl $atomic_type {
             #[inline]
@@ -675,7 +703,7 @@ macro_rules! atomic_int {
             }
         }
     };
-    // cfg(portable_atomic_force_amo) {8,16}-bit RMW
+    // {8,16}-bit RMW with Zaamo extension
     (cas[sub_word], $atomic_type:ident, $int_type:ident) => {
         impl $atomic_type {
             #[inline]

diff --git a/src/imp/interrupt/riscv.rs b/src/imp/interrupt/riscv.rs
@@ -1,8 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
 // Refs:
-// - https://five-embeddev.com/riscv-isa-manual/latest/machine.html#machine-status-registers-mstatus-and-mstatush
-// - https://five-embeddev.com/riscv-isa-manual/latest/supervisor.html#sstatus
+// - https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/machine.adoc#machine-status-mstatus-and-mstatush-registers
+// - https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/supervisor.adoc#supervisor-status-sstatus-register
+//
+// See also src/imp/riscv.rs.
 //
 // Generated asm:
 // - riscv64gc https://godbolt.org/z/zTrzT1Ee7

diff --git a/src/imp/riscv.rs b/src/imp/riscv.rs
@@ -11,9 +11,11 @@
 // - RISC-V Instruction Set Manual Volume I: Unprivileged ISA
 //   https://riscv.org/wp-content/uploads/2019/12/riscv-spec-20191213.pdf
 // - RISC-V Atomics ABI Specification
-//   https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/HEAD/riscv-atomic.adoc
+//   https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20240829-13bfa9f54634cb60d86b9b333e109f077805b4b3/riscv-atomic.adoc
 // - "Mappings from C/C++ primitives to RISC-V primitives." table in RISC-V Instruction Set Manual:
-//   https://five-embeddev.com/riscv-isa-manual/latest/memory.html#sec:memory:porting
+//   https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/mm-eplan.adoc#code-porting-and-mapping-guidelines
+// - ""Zaamo" Extension for Atomic Memory Operations" in RISC-V Instruction Set Manual:
+//   https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/a-st-ext.adoc#zaamo-extension-for-atomic-memory-operations
 // - atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit
 //
 // Generated asm:
@@ -24,22 +26,37 @@
 use core::arch::asm;
 use core::{cell::UnsafeCell, sync::atomic::Ordering};
 
-#[cfg(any(test, portable_atomic_force_amo))]
+#[cfg(any(
+    test,
+    portable_atomic_force_amo,
+    target_feature = "zaamo",
+    portable_atomic_target_feature = "zaamo",
+))]
 #[cfg(target_arch = "riscv32")]
 macro_rules! w {
     () => {
         ""
     };
 }
-#[cfg(any(test, portable_atomic_force_amo))]
+#[cfg(any(
+    test,
+    portable_atomic_force_amo,
+    target_feature = "zaamo",
+    portable_atomic_target_feature = "zaamo",
+))]
 #[cfg(target_arch = "riscv64")]
 macro_rules! w {
     () => {
         "w"
     };
 }
 
-#[cfg(any(test, portable_atomic_force_amo))]
+#[cfg(any(
+    test,
+    portable_atomic_force_amo,
+    target_feature = "zaamo",
+    portable_atomic_target_feature = "zaamo",
+))]
 macro_rules! atomic_rmw_amo_order {
     ($op:ident, $order:ident) => {
         match $order {
@@ -52,14 +69,19 @@ macro_rules! atomic_rmw_amo_order {
         }
     };
 }
-#[cfg(any(test, portable_atomic_force_amo))]
+#[cfg(any(
+    test,
+    portable_atomic_force_amo,
+    target_feature = "zaamo",
+    portable_atomic_target_feature = "zaamo",
+))]
 macro_rules! atomic_rmw_amo {
     ($op:ident, $dst:ident, $val:ident, $order:ident, $asm_suffix:tt) => {{
         let out;
         macro_rules! op {
             ($asm_order:tt) => {
                 // SAFETY: The user guaranteed that the AMO instruction is available in this
-                // system by setting the portable_atomic_force_amo and
+                // system by setting the portable_atomic_force_amo/target_feature and
                 // portable_atomic_unsafe_assume_single_core.
                 // The caller of this macro must guarantee the validity of the pointer.
                 asm!(
@@ -80,7 +102,12 @@ macro_rules! atomic_rmw_amo {
     }};
 }
 // 32-bit val.wrapping_shl(shift) but no extra `& (u32::BITS - 1)`
-#[cfg(any(test, portable_atomic_force_amo))]
+#[cfg(any(
+    test,
+    portable_atomic_force_amo,
+    target_feature = "zaamo",
+    portable_atomic_target_feature = "zaamo",
+))]
 #[inline]
 fn sllw(val: u32, shift: u32) -> u32 {
     // SAFETY: Calling sll{,w} is safe.
@@ -97,7 +124,12 @@ fn sllw(val: u32, shift: u32) -> u32 {
     }
 }
 // 32-bit val.wrapping_shr(shift) but no extra `& (u32::BITS - 1)`
-#[cfg(any(test, portable_atomic_force_amo))]
+#[cfg(any(
+    test,
+    portable_atomic_force_amo,
+    target_feature = "zaamo",
+    portable_atomic_target_feature = "zaamo",
+))]
 macro_rules! srlw {
     ($val:expr, $shift:expr) => {
         // SAFETY: Calling srl{,w} is safe.
@@ -240,7 +272,11 @@ macro_rules! atomic_load_store {
 macro_rules! atomic_ptr {
     ($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $asm_suffix:tt) => {
         atomic_load_store!($([$($generics)*])? $atomic_type, $value_type, $asm_suffix);
-        #[cfg(portable_atomic_force_amo)]
+        #[cfg(any(
+            portable_atomic_force_amo,
+            target_feature = "zaamo",
+            portable_atomic_target_feature = "zaamo",
+        ))]
         impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
             #[inline]
             pub(crate) fn swap(&self, val: $value_type, order: Ordering) -> $value_type {
@@ -257,7 +293,12 @@ macro_rules! atomic {
     ($atomic_type:ident, $value_type:ty, $asm_suffix:tt, $max:tt, $min:tt) => {
         atomic_load_store!($atomic_type, $value_type, $asm_suffix);
         // There is no amo{sub,nand,neg}.
-        #[cfg(any(test, portable_atomic_force_amo))]
+        #[cfg(any(
+            test,
+            portable_atomic_force_amo,
+            target_feature = "zaamo",
+            portable_atomic_target_feature = "zaamo",
+        ))]
         impl $atomic_type {
             #[inline]
             pub(crate) fn swap(&self, val: $value_type, order: Ordering) -> $value_type {
@@ -328,21 +369,36 @@ macro_rules! atomic {
     };
 }
 
-#[cfg(any(test, portable_atomic_force_amo))]
+#[cfg(any(
+    test,
+    portable_atomic_force_amo,
+    target_feature = "zaamo",
+    portable_atomic_target_feature = "zaamo",
+))]
 trait ZeroExtend: Copy {
     /// Zero-extends `self` to `u32` if it is smaller than 32-bit.
     fn zero_extend(self) -> u32;
 }
 macro_rules! zero_extend {
     ($int:ident, $uint:ident) => {
-        #[cfg(any(test, portable_atomic_force_amo))]
+        #[cfg(any(
+            test,
+            portable_atomic_force_amo,
+            target_feature = "zaamo",
+            portable_atomic_target_feature = "zaamo",
+        ))]
         impl ZeroExtend for $uint {
             #[inline]
             fn zero_extend(self) -> u32 {
                 self as u32
             }
         }
-        #[cfg(any(test, portable_atomic_force_amo))]
+        #[cfg(any(
+            test,
+            portable_atomic_force_amo,
+            target_feature = "zaamo",
+            portable_atomic_target_feature = "zaamo",
+        ))]
         impl ZeroExtend for $int {
             #[allow(clippy::cast_sign_loss)]
             #[inline]
@@ -358,7 +414,12 @@ zero_extend!(i16, u16);
 macro_rules! atomic_sub_word {
     ($atomic_type:ident, $value_type:ty, $unsigned_type:ty, $asm_suffix:tt) => {
         atomic_load_store!($atomic_type, $value_type, $asm_suffix);
-        #[cfg(any(test, portable_atomic_force_amo))]
+        #[cfg(any(
+            test,
+            portable_atomic_force_amo,
+            target_feature = "zaamo",
+            portable_atomic_target_feature = "zaamo",
+        ))]
         impl $atomic_type {
             #[inline]
             pub(crate) fn fetch_and(&self, val: $value_type, order: Ordering) -> $value_type {

diff --git a/tools/build.sh b/tools/build.sh
@@ -408,6 +408,9 @@ build() {
                                         CARGO_TARGET_DIR="${target_dir}/api-test-assume-single-core-force-amo" \
                                             RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core --cfg portable_atomic_force_amo" \
                                             x_cargo "${args[@]}" --feature-powerset --manifest-path tests/api-test/Cargo.toml "$@"
+                                        CARGO_TARGET_DIR="${target_dir}/api-test-assume-single-core-zaamo" \
+                                            RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core -C target-feature=+zaamo" \
+                                            x_cargo "${args[@]}" --feature-powerset --manifest-path tests/api-test/Cargo.toml "$@"
                                     fi
                                     ;;
                             esac
@@ -454,6 +457,9 @@ build() {
                                         CARGO_TARGET_DIR="${target_dir}/assume-single-core-force-amo" \
                                             RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core --cfg portable_atomic_force_amo" \
                                             x_cargo "${args[@]}" --exclude-features "critical-section" "$@"
+                                        CARGO_TARGET_DIR="${target_dir}/assume-single-core-zaamo" \
+                                            RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core -C target-feature=+zaamo" \
+                                            x_cargo "${args[@]}" --exclude-features "critical-section" "$@"
                                     fi
                                     ;;
                             esac

diff --git a/tools/no-std.sh b/tools/no-std.sh
@@ -209,6 +209,12 @@ run() {
                     CARGO_TARGET_DIR="${target_dir}/no-std-test-force-amo" \
                         RUSTFLAGS="${target_rustflags} --cfg portable_atomic_force_amo" \
                         x_cargo "${args[@]}" --release "$@"
+                    CARGO_TARGET_DIR="${target_dir}/no-std-test-zaamo" \
+                        RUSTFLAGS="${target_rustflags} -C target-feature=+zaamo" \
+                        x_cargo "${args[@]}" "$@"
+                    CARGO_TARGET_DIR="${target_dir}/no-std-test-zaamo" \
+                        RUSTFLAGS="${target_rustflags} -C target-feature=+zaamo" \
+                        x_cargo "${args[@]}" --release "$@"
                 fi
                 ;;
         esac
-Original file line number
+Diff line change
@@ Expand Up / @@ -173,4 +173,5 @@ xilf @@
     xmmword
     xsave
     xsub
+    zaamo
     Zhaoxin