Enable outline-atomics by default on more AArch64 platforms

compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs

@@ -21,7 +21,7 @@ pub(crate) fn target() -> Target {
             max_atomic_width: Some(128),
             // As documented in https://developer.android.com/ndk/guides/cpu-features.html
             // the neon (ASIMD) and FP must exist on all android aarch64 targets.
-            features: "+v8a,+neon".into(),
+            features: "+v8a,+neon,+outline-atomics".into(),
             // the AAPCS64 expects use of non-leaf frame pointers per
             // https://github.com/ARM-software/abi-aa/blob/4492d1570eb70c8fd146623e0db65b2d241f12e7/aapcs64/aapcs64.rst#the-frame-pointer
             // and we tend to encounter interesting bugs in AArch64 unwinding code if we do not

compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs

@@ -3,7 +3,7 @@ use crate::spec::{Arch, Cc, FramePointer, LinkerFlavor, Lld, Target, TargetMetad
 pub(crate) fn target() -> Target {
     let mut base = base::windows_gnullvm::opts();
     base.max_atomic_width = Some(128);
-    base.features = "+v8a,+neon".into();
+    base.features = "+v8a,+neon,+outline-atomics".into();
     base.linker = Some("aarch64-w64-mingw32-clang".into());
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::No, Lld::No), &["-m", "arm64pe"]);
 

compiler/rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs

@@ -3,7 +3,7 @@ use crate::spec::{Arch, FramePointer, Target, TargetMetadata, base};
 pub(crate) fn target() -> Target {
     let mut base = base::windows_msvc::opts();
     base.max_atomic_width = Some(128);
-    base.features = "+v8a,+neon".into();
+    base.features = "+v8a,+neon,+outline-atomics".into();
 
     // Microsoft recommends enabling frame pointers on Arm64 Windows.
     // From https://learn.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=msvc-170#integer-registers

compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs

@@ -5,7 +5,7 @@ use crate::spec::{
 pub(crate) fn target() -> Target {
     let mut base = base::fuchsia::opts();
     base.cpu = "generic".into();
-    base.features = "+v8a,+crc,+aes,+sha2,+neon".into();
+    base.features = "+v8a,+crc,+aes,+sha2,+neon,+outline-atomics".into();
     base.max_atomic_width = Some(128);
     base.stack_probes = StackProbeType::Inline;
     base.supported_sanitizers = SanitizerSet::ADDRESS

compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs

@@ -13,7 +13,7 @@ pub(crate) fn target() -> Target {
         data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(),
         arch: Arch::AArch64,
         options: TargetOptions {
-            features: "+v8a".into(),
+            features: "+v8a,+outline-atomics".into(),
             max_atomic_width: Some(128),
             stack_probes: StackProbeType::Inline,
             ..base::openbsd::opts()

library/compiler-builtins/compiler-builtins/src/aarch64_linux.rs → library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs

@@ -196,7 +196,7 @@ macro_rules! compare_and_swap {
                     "cbnz   w17, 0b",
                     "1:",
                     "ret",
-                    have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
+                    have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
                 }
             }
         }
@@ -228,7 +228,7 @@ macro_rules! compare_and_swap_i128 {
                     "cbnz   w15, 0b",
                     "1:",
                     "ret",
-                    have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
+                    have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
                 }
             }
         }
@@ -256,7 +256,7 @@ macro_rules! swap {
                     concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"),
                     "cbnz   w17, 0b",
                     "ret",
-                    have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
+                    have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
                 }
             }
         }
@@ -286,7 +286,7 @@ macro_rules! fetch_op {
                     concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"),
                     "cbnz  w15, 0b",
                     "ret",
-                    have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
+                    have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
                 }
             }
         }

library/compiler-builtins/compiler-builtins/src/lib.rs

@@ -55,8 +55,8 @@ pub mod arm;
 #[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))]
 pub mod aarch64;
 
-#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
-pub mod aarch64_linux;
+#[cfg(all(target_arch = "aarch64", target_feature = "outline-atomics"))]
+pub mod aarch64_outline_atomics;
 
 #[cfg(all(
     kernel_user_helpers,

library/std/src/sys/configure_builtins.rs

@@ -1,13 +1,49 @@
-/// Hook into .init_array to enable LSE atomic operations at startup, if
-/// supported.
-#[cfg(all(target_arch = "aarch64", target_os = "linux", not(feature = "compiler-builtins-c")))]
+/// Enable LSE atomic operations at startup, if supported.
+///
+/// Linker sections are based on what [`ctor`] does, with priorities to run slightly before user
+/// code:
+///
+/// - Apple uses the section `__mod_init_func`, `mod_init_funcs` is needed to set
+///   `S_MOD_INIT_FUNC_POINTERS`. There doesn't seem to be a way to indicate priorities.
+/// - Windows uses `.CRT$XCT`, which is run before user constructors (these should use `.CRT$XCU`).
+/// - ELF uses `.init_array` with a priority of 90, which runs before our `ARGV_INIT_ARRAY`
+///   initializer (priority 99). Both are within the 0-100 implementation-reserved range, per docs
+///   for the [`prio-ctor-dtor`] warning, and this matches compiler-rt's `CONSTRUCTOR_PRIORITY`.
+///
+/// To save startup time, the initializer is only run if outline atomic routines from
+/// compiler-builtins may be used. If LSE is known to be available then the calls are never
+/// emitted, and if we build the C intrinsics then it has its own initializer using the symbol
+/// `__aarch64_have_lse_atomics`.
+///
+/// Initialization is done in a global constructor to so we get the same behavior regardless of
+/// whether Rust's `init` is used, or if we are in a `dylib` or `no_main` situation (as opposed
+/// to doing it as part of pre-main startup). This also matches C implementations.
+///
+/// Ideally `core` would have something similar, but detecting the CPU features requires the
+/// auxiliary vector from the OS. We do the initialization in `std` rather than as part of
+/// `compiler-builtins` because a builtins->std dependency isn't possible, and inlining parts of
+/// `std-detect` would be much messier.
+///
+/// [`ctor`]: https://github.com/mmastrac/rust-ctor/blob/63382b833ddcbfb8b064f4e86bfa1ed4026ff356/shared/src/macros/mod.rs#L522-L534
+/// [`prio-ctor-dtor`]: https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html
+#[cfg(all(
+    target_arch = "aarch64",
+    target_feature = "outline-atomics",
+    not(target_feature = "lse"),
+    not(feature = "compiler-builtins-c"),
+))]
 #[used]
-#[unsafe(link_section = ".init_array.90")]
+#[cfg_attr(target_vendor = "apple", unsafe(link_section = "__DATA,__mod_init_func,mod_init_funcs"))]
+#[cfg_attr(target_os = "windows", unsafe(link_section = ".CRT$XCT"))]
+#[cfg_attr(
+    not(any(target_vendor = "apple", target_os = "windows")),
+    unsafe(link_section = ".init_array.90")
+)]
 static RUST_LSE_INIT: extern "C" fn() = {
     extern "C" fn init_lse() {
         use crate::arch;
 
-        // This is provided by compiler-builtins::aarch64_linux.
+        // This is provided by compiler-builtins::aarch64_outline_atomics.
         unsafe extern "C" {
             fn __rust_enable_lse();
         }