From 230e91d2f6a6cb495bff8db9bbfc1530c97294d5 Mon Sep 17 00:00:00 2001 From: Urgau Date: Sun, 14 Dec 2025 13:29:59 +0100 Subject: [PATCH 01/11] Prevent double encoding when the filename wasn't remapped at all This is done by making the `local` part of `RealFileName` none. This works because `maybe_remapped` is equal to `local` when no remapping happened. --- compiler/rustc_span/src/lib.rs | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index 49b2e0c1ff1a7..63c51164761d9 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -370,7 +370,11 @@ impl RealFileName { /// /// May not exists if the filename was imported from another crate. pub fn local_path(&self) -> Option<&Path> { - self.local.as_ref().map(|lp| lp.name.as_ref()) + if self.was_not_remapped() { + Some(&self.maybe_remapped.name) + } else { + self.local.as_ref().map(|lp| lp.name.as_ref()) + } } /// Returns the path suitable for reading from the file system on the local host, @@ -378,7 +382,11 @@ impl RealFileName { /// /// May not exists if the filename was imported from another crate. pub fn into_local_path(self) -> Option { - self.local.map(|lp| lp.name) + if self.was_not_remapped() { + Some(self.maybe_remapped.name) + } else { + self.local.map(|lp| lp.name) + } } /// Returns whenever the filename was remapped. @@ -386,6 +394,18 @@ impl RealFileName { !self.scopes.is_empty() } + /// Returns whenever the filename was fully remapped. + #[inline] + fn was_fully_remapped(&self) -> bool { + self.scopes.is_all() + } + + /// Returns whenever the filename was not remapped. + #[inline] + fn was_not_remapped(&self) -> bool { + self.scopes.is_empty() + } + /// Returns an empty `RealFileName` /// /// Useful as the working directory input to `SourceMap::to_real_filename`. @@ -420,9 +440,14 @@ impl RealFileName { /// Update the filename for encoding in the crate metadata. /// /// Currently it's about removing the local part when the filename - /// is fully remapped. + /// is either fully remapped or not remapped at all. + #[inline] pub fn update_for_crate_metadata(&mut self) { - if self.scopes.is_all() { + if self.was_fully_remapped() || self.was_not_remapped() { + // NOTE: This works because when the filename is fully + // remapped, we don't care about the `local` part, + // and when the filename is not remapped at all, + // `maybe_remapped` and `local` are equal. self.local = None; } } From 7227bfed0847703f65e0628f60a860f5cad09594 Mon Sep 17 00:00:00 2001 From: Urgau Date: Sun, 14 Dec 2025 13:30:25 +0100 Subject: [PATCH 02/11] Prefer using `was_fully_remapped` helper function --- compiler/rustc_span/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index 63c51164761d9..53363fae933ee 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -313,7 +313,7 @@ impl Hash for RealFileName { // remapped path if that exists. This is because remapped paths to // sysroot crates (/rust/$hash or /rust/$version) remain stable even // if the corresponding local path changes. - if !self.scopes.is_all() { + if !self.was_fully_remapped() { self.local.hash(state); } self.maybe_remapped.hash(state); From c877536864fc011dfcbcbde60f5d21698c6eee71 Mon Sep 17 00:00:00 2001 From: Urgau Date: Sun, 14 Dec 2025 13:32:16 +0100 Subject: [PATCH 03/11] Add `#[inline]` to a bunch of filename related functions --- compiler/rustc_span/src/lib.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index 53363fae933ee..630656f9ec5c6 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -241,12 +241,14 @@ bitflags::bitflags! { } impl Encodable for RemapPathScopeComponents { + #[inline] fn encode(&self, s: &mut E) { s.emit_u8(self.bits()); } } impl Decodable for RemapPathScopeComponents { + #[inline] fn decode(s: &mut D) -> RemapPathScopeComponents { RemapPathScopeComponents::from_bits(s.read_u8()) .expect("invalid bits for RemapPathScopeComponents") @@ -308,6 +310,7 @@ struct InnerRealFileName { } impl Hash for RealFileName { + #[inline] fn hash(&self, state: &mut H) { // To prevent #70924 from happening again we should only hash the // remapped path if that exists. This is because remapped paths to @@ -327,6 +330,7 @@ impl RealFileName { /// ## Panic /// /// Only one scope components can be given to this function. + #[inline] pub fn path(&self, scope: RemapPathScopeComponents) -> &Path { assert!( scope.bits().count_ones() == 1, @@ -351,6 +355,7 @@ impl RealFileName { /// ## Panic /// /// Only one scope components can be given to this function. + #[inline] pub fn embeddable_name(&self, scope: RemapPathScopeComponents) -> (&Path, &Path) { assert!( scope.bits().count_ones() == 1, @@ -369,6 +374,7 @@ impl RealFileName { /// if this information exists. /// /// May not exists if the filename was imported from another crate. + #[inline] pub fn local_path(&self) -> Option<&Path> { if self.was_not_remapped() { Some(&self.maybe_remapped.name) @@ -381,6 +387,7 @@ impl RealFileName { /// if this information exists. /// /// May not exists if the filename was imported from another crate. + #[inline] pub fn into_local_path(self) -> Option { if self.was_not_remapped() { Some(self.maybe_remapped.name) @@ -390,6 +397,7 @@ impl RealFileName { } /// Returns whenever the filename was remapped. + #[inline] pub(crate) fn was_remapped(&self) -> bool { !self.scopes.is_empty() } @@ -409,6 +417,7 @@ impl RealFileName { /// Returns an empty `RealFileName` /// /// Useful as the working directory input to `SourceMap::to_real_filename`. + #[inline] pub fn empty() -> RealFileName { RealFileName { local: Some(InnerRealFileName { @@ -554,6 +563,7 @@ impl FileName { /// if this information exists. /// /// Avoid embedding this in build artifacts. Prefer using the `display` method. + #[inline] pub fn prefer_remapped_unconditionally(&self) -> FileNameDisplay<'_> { FileNameDisplay { inner: self, display_pref: FileNameDisplayPreference::Remapped } } @@ -562,16 +572,19 @@ impl FileName { /// if this information exists. /// /// Avoid embedding this in build artifacts. Prefer using the `display` method. + #[inline] pub fn prefer_local_unconditionally(&self) -> FileNameDisplay<'_> { FileNameDisplay { inner: self, display_pref: FileNameDisplayPreference::Local } } /// Returns a short (either the filename or an empty string). + #[inline] pub fn short(&self) -> FileNameDisplay<'_> { FileNameDisplay { inner: self, display_pref: FileNameDisplayPreference::Short } } /// Returns a `Display`-able path for the given scope. + #[inline] pub fn display(&self, scope: RemapPathScopeComponents) -> FileNameDisplay<'_> { FileNameDisplay { inner: self, display_pref: FileNameDisplayPreference::Scope(scope) } } From 075f4cd57f3c2b175510c25638fc23bc87d7633b Mon Sep 17 00:00:00 2001 From: Urgau Date: Sun, 14 Dec 2025 13:44:14 +0100 Subject: [PATCH 04/11] Restore embedding warning and simply `{,into_}local_path` methods --- compiler/rustc_span/src/lib.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index 630656f9ec5c6..3d641905d3251 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -374,12 +374,16 @@ impl RealFileName { /// if this information exists. /// /// May not exists if the filename was imported from another crate. + /// + /// Avoid embedding this in build artifacts; prefer `path()` or `embeddable_name()`. #[inline] pub fn local_path(&self) -> Option<&Path> { if self.was_not_remapped() { Some(&self.maybe_remapped.name) + } else if let Some(local) = &self.local { + Some(&local.name) } else { - self.local.as_ref().map(|lp| lp.name.as_ref()) + None } } @@ -387,12 +391,16 @@ impl RealFileName { /// if this information exists. /// /// May not exists if the filename was imported from another crate. + /// + /// Avoid embedding this in build artifacts; prefer `path()` or `embeddable_name()`. #[inline] pub fn into_local_path(self) -> Option { if self.was_not_remapped() { Some(self.maybe_remapped.name) + } else if let Some(local) = self.local { + Some(local.name) } else { - self.local.map(|lp| lp.name) + None } } From a07bd236bdb47d165c9c7f62e35e2a9a3d773ad6 Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Thu, 13 Nov 2025 23:51:01 +0800 Subject: [PATCH 05/11] rustc_target: Add `efiapi` ABI support for LoongArch This commit adds basic `efiapi` ABI support for LoongArch by recognizing `extern "efiapi"` in the ABI map and inline asm clobber handling, and mapping it to the C calling convention. This change is intentionally submitted ahead of the full LoongArch UEFI target support. While UEFI binaries are ultimately produced as PE images, LoongArch UEFI applications can already be developed by building ELF objects, applying relocation fixups, and converting them to PE in a later step. For such workflows, having `efiapi` properly recognized by the compiler is a prerequisite, even without a dedicated UEFI target. Landing this ABI support early helps unblock LoongArch UEFI application and driver development, and allows the remaining UEFI-specific pieces to be introduced incrementally in follow-up patches. MCP: https://github.com/rust-lang/compiler-team/issues/953 --- compiler/rustc_target/src/asm/mod.rs | 4 ++-- compiler/rustc_target/src/spec/abi_map.rs | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs index 05b24d7109481..0078866ab9503 100644 --- a/compiler/rustc_target/src/asm/mod.rs +++ b/compiler/rustc_target/src/asm/mod.rs @@ -997,8 +997,8 @@ impl InlineAsmClobberAbi { _ => Err(&["C", "system"]), }, InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => match name { - "C" | "system" => Ok(InlineAsmClobberAbi::LoongArch), - _ => Err(&["C", "system"]), + "C" | "system" | "efiapi" => Ok(InlineAsmClobberAbi::LoongArch), + _ => Err(&["C", "system", "efiapi"]), }, InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => match name { "C" | "system" => Ok(if target.abi == Abi::Spe { diff --git a/compiler/rustc_target/src/spec/abi_map.rs b/compiler/rustc_target/src/spec/abi_map.rs index 8126227d80e48..d7fc18cd3761e 100644 --- a/compiler/rustc_target/src/spec/abi_map.rs +++ b/compiler/rustc_target/src/spec/abi_map.rs @@ -56,6 +56,7 @@ impl AbiMap { ArmVer::Other }), Arch::Avr => ArchKind::Avr, + Arch::LoongArch32 | Arch::LoongArch64 => ArchKind::LoongArch, Arch::Msp430 => ArchKind::Msp430, Arch::Nvptx64 => ArchKind::Nvptx, Arch::RiscV32 | Arch::RiscV64 => ArchKind::Riscv, @@ -108,7 +109,10 @@ impl AbiMap { (ExternAbi::EfiApi, ArchKind::Arm(..)) => CanonAbi::Arm(ArmCall::Aapcs), (ExternAbi::EfiApi, ArchKind::X86_64) => CanonAbi::X86(X86Call::Win64), - (ExternAbi::EfiApi, ArchKind::Aarch64 | ArchKind::Riscv | ArchKind::X86) => CanonAbi::C, + ( + ExternAbi::EfiApi, + ArchKind::Aarch64 | ArchKind::LoongArch | ArchKind::Riscv | ArchKind::X86, + ) => CanonAbi::C, (ExternAbi::EfiApi, _) => return AbiMapping::Invalid, /* arm */ @@ -196,6 +200,7 @@ enum ArchKind { Amdgpu, Arm(ArmVer), Avr, + LoongArch, Msp430, Nvptx, Riscv, From 383053e01602a1a754a170d7c201cc43fff87739 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 5 Sep 2025 15:27:17 -0400 Subject: [PATCH 06/11] Enable `outline-atomics` by default on AArch64 FreeBSD Many aarch64 targets without LSE in the baseline enable the `outline-atomics` feature, which uses runtime detection of LSE for its faster atomic ops. This provides nontrivial performance improvements on most hardware from the past decade, at a small cost to anything pre-LSE. This matches what Clang does [1]. [1]: https://github.com/llvm/llvm-project/commit/e24f90190c772b6fdd915cd0a2e55cbd468c3024 --- .../rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs index 69a65e6b0f024..47775f9684008 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs @@ -15,7 +15,7 @@ pub(crate) fn target() -> Target { data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: Arch::AArch64, options: TargetOptions { - features: "+v8a".into(), + features: "+v8a,+outline-atomics".into(), max_atomic_width: Some(128), stack_probes: StackProbeType::Inline, supported_sanitizers: SanitizerSet::ADDRESS From b31ee3af9c67c777143c0b6707a4439c78391b69 Mon Sep 17 00:00:00 2001 From: Moulins Date: Thu, 18 Dec 2025 00:27:02 +0100 Subject: [PATCH 07/11] layout: Store inverse memory index in `FieldsShape::Arbitrary` All usages of `memory_index` start by calling `invert_bijective_mapping`, so storing the inverted mapping directly saves some work and simplifies the code. --- compiler/rustc_abi/src/layout.rs | 46 ++++++++---------- compiler/rustc_abi/src/layout/coroutine.rs | 46 +++++++++--------- compiler/rustc_abi/src/layout/simple.rs | 6 +-- compiler/rustc_abi/src/lib.rs | 47 ++----------------- .../src/pretty_clif.rs | 4 +- .../rustc_const_eval/src/interpret/visitor.rs | 18 +++---- compiler/rustc_index/src/slice.rs | 3 -- compiler/rustc_transmute/src/layout/tree.rs | 5 +- compiler/rustc_ty_utils/src/layout.rs | 4 +- src/tools/miri/src/helpers.rs | 7 ++- tests/ui/abi/c-zst.aarch64-darwin.stderr | 4 +- tests/ui/abi/c-zst.powerpc-linux.stderr | 4 +- tests/ui/abi/c-zst.s390x-linux.stderr | 4 +- tests/ui/abi/c-zst.sparc64-linux.stderr | 4 +- tests/ui/abi/c-zst.x86_64-linux.stderr | 4 +- .../ui/abi/c-zst.x86_64-pc-windows-gnu.stderr | 4 +- tests/ui/abi/debug.generic.stderr | 20 ++++---- tests/ui/abi/debug.loongarch64.stderr | 20 ++++---- tests/ui/abi/debug.riscv64.stderr | 20 ++++---- tests/ui/abi/pass-indirectly-attr.stderr | 8 ++-- tests/ui/abi/sysv64-zst.stderr | 4 +- .../pass-by-value-abi.aarch64.stderr | 4 +- tests/ui/c-variadic/pass-by-value-abi.rs | 2 +- .../c-variadic/pass-by-value-abi.win.stderr | 4 +- .../pass-by-value-abi.x86_64.stderr | 12 ++--- .../enum-discriminant/wrapping_niche.stderr | 16 +++---- tests/ui/layout/debug.stderr | 14 +++--- tests/ui/layout/hexagon-enum.stderr | 20 ++++---- ...-scalarpair-payload-might-be-uninit.stderr | 34 +++++++------- .../issue-96185-overaligned-enum.stderr | 12 ++--- tests/ui/layout/thumb-enum.stderr | 20 ++++---- .../layout/zero-sized-array-enum-niche.stderr | 26 +++++----- ...-variants.aarch64-unknown-linux-gnu.stderr | 16 +++---- ...-c-dead-variants.armebv7r-none-eabi.stderr | 16 +++---- ...-dead-variants.i686-pc-windows-msvc.stderr | 16 +++---- ...d-variants.x86_64-unknown-linux-gnu.stderr | 16 +++---- tests/ui/repr/repr-c-int-dead-variants.stderr | 16 +++---- tests/ui/type/pattern_types/non_null.stderr | 10 ++-- .../ui/type/pattern_types/or_patterns.stderr | 4 +- .../type/pattern_types/range_patterns.stderr | 22 ++++----- 40 files changed, 254 insertions(+), 308 deletions(-) diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index b852d71b80082..4f1594d02a82e 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -714,7 +714,7 @@ impl LayoutCalculator { }, fields: FieldsShape::Arbitrary { offsets: [niche_offset].into(), - memory_index: [0].into(), + in_memory_order: [FieldIdx::new(0)].into(), }, backend_repr: abi, largest_niche, @@ -1008,8 +1008,8 @@ impl LayoutCalculator { let pair = LayoutData::::scalar_pair(&self.cx, tag, prim_scalar); let pair_offsets = match pair.fields { - FieldsShape::Arbitrary { ref offsets, ref memory_index } => { - assert_eq!(memory_index.raw, [0, 1]); + FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => { + assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]); offsets } _ => panic!("encountered a non-arbitrary layout during enum layout"), @@ -1061,7 +1061,7 @@ impl LayoutCalculator { }, fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), - memory_index: [0].into(), + in_memory_order: [FieldIdx::new(0)].into(), }, largest_niche, uninhabited, @@ -1110,10 +1110,10 @@ impl LayoutCalculator { let pack = repr.pack; let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align }; let mut max_repr_align = repr.align; - let mut inverse_memory_index: IndexVec = fields.indices().collect(); + let mut in_memory_order: IndexVec = fields.indices().collect(); let optimize_field_order = !repr.inhibit_struct_field_reordering(); let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() }; - let optimizing = &mut inverse_memory_index.raw[..end]; + let optimizing = &mut in_memory_order.raw[..end]; let fields_excluding_tail = &fields.raw[..end]; // unsizable tail fields are excluded so that we use the same seed for the sized and unsized layouts. let field_seed = fields_excluding_tail @@ -1248,12 +1248,10 @@ impl LayoutCalculator { // regardless of the status of `-Z randomize-layout` } } - // inverse_memory_index holds field indices by increasing memory offset. - // That is, if field 5 has offset 0, the first element of inverse_memory_index is 5. + // in_memory_order holds field indices by increasing memory offset. + // That is, if field 5 has offset 0, the first element of in_memory_order is 5. // We now write field offsets to the corresponding offset slot; // field 5 with offset 0 puts 0 in offsets[5]. - // At the bottom of this function, we invert `inverse_memory_index` to - // produce `memory_index` (see `invert_mapping`). let mut unsized_field = None::<&F>; let mut offsets = IndexVec::from_elem(Size::ZERO, fields); let mut offset = Size::ZERO; @@ -1265,7 +1263,7 @@ impl LayoutCalculator { align = align.max(prefix_align); offset = prefix_size.align_to(prefix_align); } - for &i in &inverse_memory_index { + for &i in &in_memory_order { let field = &fields[i]; if let Some(unsized_field) = unsized_field { return Err(LayoutCalculatorError::UnexpectedUnsized(*unsized_field)); @@ -1322,18 +1320,6 @@ impl LayoutCalculator { debug!("univariant min_size: {:?}", offset); let min_size = offset; - // As stated above, inverse_memory_index holds field indices by increasing offset. - // This makes it an already-sorted view of the offsets vec. - // To invert it, consider: - // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0. - // Field 5 would be the first element, so memory_index is i: - // Note: if we didn't optimize, it's already right. - let memory_index = if optimize_field_order { - inverse_memory_index.invert_bijective_mapping() - } else { - debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices())); - inverse_memory_index.into_iter().map(|it| it.index() as u32).collect() - }; let size = min_size.align_to(align); // FIXME(oli-obk): deduplicate and harden these checks if size.bytes() >= dl.obj_size_bound() { @@ -1389,8 +1375,11 @@ impl LayoutCalculator { let pair = LayoutData::::scalar_pair(&self.cx, a, b); let pair_offsets = match pair.fields { - FieldsShape::Arbitrary { ref offsets, ref memory_index } => { - assert_eq!(memory_index.raw, [0, 1]); + FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => { + assert_eq!( + in_memory_order.raw, + [FieldIdx::new(0), FieldIdx::new(1)] + ); offsets } FieldsShape::Primitive @@ -1434,7 +1423,7 @@ impl LayoutCalculator { Ok(LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, - fields: FieldsShape::Arbitrary { offsets, memory_index }, + fields: FieldsShape::Arbitrary { offsets, in_memory_order }, backend_repr: abi, largest_niche, uninhabited, @@ -1530,7 +1519,10 @@ where Ok(LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, - fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), memory_index: [0].into() }, + fields: FieldsShape::Arbitrary { + offsets: [Size::ZERO].into(), + in_memory_order: [FieldIdx::new(0)].into(), + }, backend_repr: repr, largest_niche: elt.largest_niche, uninhabited: false, diff --git a/compiler/rustc_abi/src/layout/coroutine.rs b/compiler/rustc_abi/src/layout/coroutine.rs index 2b22276d4aed7..815cf1e28a08c 100644 --- a/compiler/rustc_abi/src/layout/coroutine.rs +++ b/compiler/rustc_abi/src/layout/coroutine.rs @@ -182,33 +182,29 @@ pub(super) fn layout< // CoroutineLayout. debug!("prefix = {:#?}", prefix); let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields { - FieldsShape::Arbitrary { mut offsets, memory_index } => { - let mut inverse_memory_index = memory_index.invert_bijective_mapping(); - + FieldsShape::Arbitrary { mut offsets, in_memory_order } => { // "a" (`0..b_start`) and "b" (`b_start..`) correspond to // "outer" and "promoted" fields respectively. let b_start = tag_index.plus(1); let offsets_b = IndexVec::from_raw(offsets.raw.split_off(b_start.index())); let offsets_a = offsets; - // Disentangle the "a" and "b" components of `inverse_memory_index` + // Disentangle the "a" and "b" components of `in_memory_order` // by preserving the order but keeping only one disjoint "half" each. // FIXME(eddyb) build a better abstraction for permutations, if possible. - let inverse_memory_index_b: IndexVec = inverse_memory_index - .iter() - .filter_map(|&i| i.index().checked_sub(b_start.index()).map(FieldIdx::new)) - .collect(); - inverse_memory_index.raw.retain(|&i| i.index() < b_start.index()); - let inverse_memory_index_a = inverse_memory_index; - - // Since `inverse_memory_index_{a,b}` each only refer to their - // respective fields, they can be safely inverted - let memory_index_a = inverse_memory_index_a.invert_bijective_mapping(); - let memory_index_b = inverse_memory_index_b.invert_bijective_mapping(); + let mut in_memory_order_a = IndexVec::::new(); + let mut in_memory_order_b = IndexVec::::new(); + for i in in_memory_order { + if let Some(j) = i.index().checked_sub(b_start.index()) { + in_memory_order_b.push(FieldIdx::new(j)); + } else { + in_memory_order_a.push(i); + } + } let outer_fields = - FieldsShape::Arbitrary { offsets: offsets_a, memory_index: memory_index_a }; - (outer_fields, offsets_b, memory_index_b) + FieldsShape::Arbitrary { offsets: offsets_a, in_memory_order: in_memory_order_a }; + (outer_fields, offsets_b, in_memory_order_b.invert_bijective_mapping()) } _ => unreachable!(), }; @@ -236,7 +232,7 @@ pub(super) fn layout< )?; variant.variants = Variants::Single { index }; - let FieldsShape::Arbitrary { offsets, memory_index } = variant.fields else { + let FieldsShape::Arbitrary { offsets, in_memory_order } = variant.fields else { unreachable!(); }; @@ -249,8 +245,9 @@ pub(super) fn layout< // promoted fields were being used, but leave the elements not in the // subset as `invalid_field_idx`, which we can filter out later to // obtain a valid (bijective) mapping. + let memory_index = in_memory_order.invert_bijective_mapping(); let invalid_field_idx = promoted_memory_index.len() + memory_index.len(); - let mut combined_inverse_memory_index = + let mut combined_in_memory_order = IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx); let mut offsets_and_memory_index = iter::zip(offsets, memory_index); @@ -268,19 +265,18 @@ pub(super) fn layout< (promoted_offsets[field_idx], promoted_memory_index[field_idx]) } }; - combined_inverse_memory_index[memory_index] = i; + combined_in_memory_order[memory_index] = i; offset }) .collect(); - // Remove the unused slots and invert the mapping to obtain the - // combined `memory_index` (also see previous comment). - combined_inverse_memory_index.raw.retain(|&i| i.index() != invalid_field_idx); - let combined_memory_index = combined_inverse_memory_index.invert_bijective_mapping(); + // Remove the unused slots to obtain the combined `in_memory_order` + // (also see previous comment). + combined_in_memory_order.raw.retain(|&i| i.index() != invalid_field_idx); variant.fields = FieldsShape::Arbitrary { offsets: combined_offsets, - memory_index: combined_memory_index, + in_memory_order: combined_in_memory_order, }; size = size.max(variant.size); diff --git a/compiler/rustc_abi/src/layout/simple.rs b/compiler/rustc_abi/src/layout/simple.rs index b3807c8727396..3784611b157be 100644 --- a/compiler/rustc_abi/src/layout/simple.rs +++ b/compiler/rustc_abi/src/layout/simple.rs @@ -16,7 +16,7 @@ impl LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Arbitrary { offsets: IndexVec::new(), - memory_index: IndexVec::new(), + in_memory_order: IndexVec::new(), }, backend_repr: BackendRepr::Memory { sized }, largest_niche: None, @@ -108,7 +108,7 @@ impl LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Arbitrary { offsets: [Size::ZERO, b_offset].into(), - memory_index: [0, 1].into(), + in_memory_order: [FieldIdx::new(0), FieldIdx::new(1)].into(), }, backend_repr: BackendRepr::ScalarPair(a, b), largest_niche, @@ -133,7 +133,7 @@ impl LayoutData { Some(fields) => FieldsShape::Union(fields), None => FieldsShape::Arbitrary { offsets: IndexVec::new(), - memory_index: IndexVec::new(), + in_memory_order: IndexVec::new(), }, }, backend_repr: BackendRepr::Memory { sized: true }, diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 53b2c3f36dfc9..061ad8617893c 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -1636,19 +1636,14 @@ pub enum FieldsShape { // FIXME(eddyb) use small vector optimization for the common case. offsets: IndexVec, - /// Maps source order field indices to memory order indices, + /// Maps memory order field indices to source order indices, /// depending on how the fields were reordered (if at all). /// This is a permutation, with both the source order and the /// memory order using the same (0..n) index ranges. /// - /// Note that during computation of `memory_index`, sometimes - /// it is easier to operate on the inverse mapping (that is, - /// from memory order to source order), and that is usually - /// named `inverse_memory_index`. - /// // FIXME(eddyb) build a better abstraction for permutations, if possible. // FIXME(camlorn) also consider small vector optimization here. - memory_index: IndexVec, + in_memory_order: IndexVec, }, } @@ -1682,51 +1677,17 @@ impl FieldsShape { } } - #[inline] - pub fn memory_index(&self, i: usize) -> usize { - match *self { - FieldsShape::Primitive => { - unreachable!("FieldsShape::memory_index: `Primitive`s have no fields") - } - FieldsShape::Union(_) | FieldsShape::Array { .. } => i, - FieldsShape::Arbitrary { ref memory_index, .. } => { - memory_index[FieldIdx::new(i)].try_into().unwrap() - } - } - } - /// Gets source indices of the fields by increasing offsets. #[inline] pub fn index_by_increasing_offset(&self) -> impl ExactSizeIterator { - let mut inverse_small = [0u8; 64]; - let mut inverse_big = IndexVec::new(); - let use_small = self.count() <= inverse_small.len(); - - // We have to write this logic twice in order to keep the array small. - if let FieldsShape::Arbitrary { ref memory_index, .. } = *self { - if use_small { - for (field_idx, &mem_idx) in memory_index.iter_enumerated() { - inverse_small[mem_idx as usize] = field_idx.index() as u8; - } - } else { - inverse_big = memory_index.invert_bijective_mapping(); - } - } - // Primitives don't really have fields in the way that structs do, // but having this return an empty iterator for them is unhelpful // since that makes them look kinda like ZSTs, which they're not. let pseudofield_count = if let FieldsShape::Primitive = self { 1 } else { self.count() }; - (0..pseudofield_count).map(move |i| match *self { + (0..pseudofield_count).map(move |i| match self { FieldsShape::Primitive | FieldsShape::Union(_) | FieldsShape::Array { .. } => i, - FieldsShape::Arbitrary { .. } => { - if use_small { - inverse_small[i] as usize - } else { - inverse_big[i as u32].index() - } - } + FieldsShape::Arbitrary { in_memory_order, .. } => in_memory_order[i as u32].index(), }) } } diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs index 2878fa7aa298a..65779b38ad1c0 100644 --- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs +++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs @@ -10,7 +10,7 @@ //! function u0:22(i64) -> i8, i8 system_v { //! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd361e9f5c3d1c4deE //! ; instance Instance { def: Item(DefId(0:42 ~ example[3895]::{impl#0}::call_once)), args: ['{erased}, '{erased}] } -//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } +//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], in_memory_order: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], in_memory_order: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } //! //! ; kind loc.idx param pass mode ty //! ; ssa _0 (u8, u8) 2b 1 var=(0, 1) @@ -41,7 +41,7 @@ //! ; //! ; _0 = >::call_mut(move _3, copy _2) //! v2 = stack_load.i64 ss0 -//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } +//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], in_memory_order: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } //! v3, v4 = call fn0(v1, v2) ; v1 = 1 //! v5 -> v3 //! v6 -> v4 diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs index b5de10c7dcd11..95106c7c143b3 100644 --- a/compiler/rustc_const_eval/src/interpret/visitor.rs +++ b/compiler/rustc_const_eval/src/interpret/visitor.rs @@ -4,7 +4,7 @@ use std::num::NonZero; use rustc_abi::{FieldIdx, FieldsShape, VariantIdx, Variants}; -use rustc_index::IndexVec; +use rustc_index::{Idx as _, IndexVec}; use rustc_middle::mir::interpret::InterpResult; use rustc_middle::ty::{self, Ty}; use tracing::trace; @@ -27,13 +27,15 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { /// This function provides the chance to reorder the order in which fields are visited for /// `FieldsShape::Aggregate`. /// - /// The default means we iterate in source declaration order; alternatively this can do some - /// work with `memory_index` to iterate in memory order. + /// The default means we iterate in source declaration order; alternatively this can use + /// `in_memory_order` to iterate in memory order. #[inline(always)] fn aggregate_field_iter( - memory_index: &IndexVec, - ) -> impl Iterator + 'static { - memory_index.indices() + in_memory_order: &IndexVec, + ) -> impl Iterator { + // Allow the optimizer to elide the bounds checking when creating each index. + let _ = FieldIdx::new(in_memory_order.len()); + (0..in_memory_order.len()).map(FieldIdx::new) } // Recursive actions, ready to be overloaded. @@ -168,8 +170,8 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { &FieldsShape::Union(fields) => { self.visit_union(v, fields)?; } - FieldsShape::Arbitrary { memory_index, .. } => { - for idx in Self::aggregate_field_iter(memory_index) { + FieldsShape::Arbitrary { in_memory_order, .. } => { + for idx in Self::aggregate_field_iter(in_memory_order) { let field = self.ecx().project_field(v, idx)?; self.visit_field(v, idx.as_usize(), &field)?; } diff --git a/compiler/rustc_index/src/slice.rs b/compiler/rustc_index/src/slice.rs index d2702bdb0571d..415fe370b702c 100644 --- a/compiler/rustc_index/src/slice.rs +++ b/compiler/rustc_index/src/slice.rs @@ -181,9 +181,6 @@ impl IndexSlice { /// Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`, /// assuming the values in `self` are a permutation of `0..self.len()`. /// - /// This is used to go between `memory_index` (source field order to memory order) - /// and `inverse_memory_index` (memory order to source field order). - /// See also `FieldsShape::Arbitrary::memory_index` for more details. // FIXME(eddyb) build a better abstraction for permutations, if possible. pub fn invert_bijective_mapping(&self) -> IndexVec { debug_assert_eq!( diff --git a/compiler/rustc_transmute/src/layout/tree.rs b/compiler/rustc_transmute/src/layout/tree.rs index d9ebdadf3b79c..1202ed2384315 100644 --- a/compiler/rustc_transmute/src/layout/tree.rs +++ b/compiler/rustc_transmute/src/layout/tree.rs @@ -491,7 +491,7 @@ pub(crate) mod rustc { ) -> Result { // This constructor does not support non-`FieldsShape::Arbitrary` // layouts. - let FieldsShape::Arbitrary { offsets, memory_index } = layout.fields() else { + let FieldsShape::Arbitrary { offsets, in_memory_order } = layout.fields() else { return Err(Err::NotYetSupported); }; @@ -519,8 +519,7 @@ pub(crate) mod rustc { } // Append the fields, in memory order, to the layout. - let inverse_memory_index = memory_index.invert_bijective_mapping(); - for &field_idx in inverse_memory_index.iter() { + for &field_idx in in_memory_order.iter() { // Add interfield padding. let padding_needed = offsets[field_idx] - size; let padding = Self::padding(padding_needed.bytes_usize()); diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 6962eeddf1370..62f3667ad7f4f 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -9,7 +9,7 @@ use rustc_abi::{ use rustc_hashes::Hash64; use rustc_hir::attrs::AttributeKind; use rustc_hir::find_attr; -use rustc_index::IndexVec; +use rustc_index::{Idx as _, IndexVec}; use rustc_middle::bug; use rustc_middle::query::Providers; use rustc_middle::traits::ObligationCause; @@ -374,7 +374,7 @@ fn layout_of_uncached<'tcx>( // specifically care about pattern types will have to handle it. layout.fields = FieldsShape::Arbitrary { offsets: [Size::ZERO].into_iter().collect(), - memory_index: [0].into_iter().collect(), + in_memory_order: [FieldIdx::new(0)].into_iter().collect(), }; tcx.mk_layout(layout) } diff --git a/src/tools/miri/src/helpers.rs b/src/tools/miri/src/helpers.rs index f11860f4a1adf..218e4ffe5e2f7 100644 --- a/src/tools/miri/src/helpers.rs +++ b/src/tools/miri/src/helpers.rs @@ -584,10 +584,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } fn aggregate_field_iter( - memory_index: &IndexVec, - ) -> impl Iterator + 'static { - let inverse_memory_index = memory_index.invert_bijective_mapping(); - inverse_memory_index.into_iter() + in_memory_order: &IndexVec, + ) -> impl Iterator { + in_memory_order.iter().copied() } // Hook to detect `UnsafeCell`. diff --git a/tests/ui/abi/c-zst.aarch64-darwin.stderr b/tests/ui/abi/c-zst.aarch64-darwin.stderr index d050bfcbaa173..6d2ac90c0c975 100644 --- a/tests/ui/abi/c-zst.aarch64-darwin.stderr +++ b/tests/ui/abi/c-zst.aarch64-darwin.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -41,7 +41,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/c-zst.powerpc-linux.stderr b/tests/ui/abi/c-zst.powerpc-linux.stderr index 816addd795761..f297aa984dd2e 100644 --- a/tests/ui/abi/c-zst.powerpc-linux.stderr +++ b/tests/ui/abi/c-zst.powerpc-linux.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/c-zst.s390x-linux.stderr b/tests/ui/abi/c-zst.s390x-linux.stderr index 816addd795761..f297aa984dd2e 100644 --- a/tests/ui/abi/c-zst.s390x-linux.stderr +++ b/tests/ui/abi/c-zst.s390x-linux.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/c-zst.sparc64-linux.stderr b/tests/ui/abi/c-zst.sparc64-linux.stderr index 816addd795761..f297aa984dd2e 100644 --- a/tests/ui/abi/c-zst.sparc64-linux.stderr +++ b/tests/ui/abi/c-zst.sparc64-linux.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/c-zst.x86_64-linux.stderr b/tests/ui/abi/c-zst.x86_64-linux.stderr index d050bfcbaa173..6d2ac90c0c975 100644 --- a/tests/ui/abi/c-zst.x86_64-linux.stderr +++ b/tests/ui/abi/c-zst.x86_64-linux.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -41,7 +41,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr index 816addd795761..f297aa984dd2e 100644 --- a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr +++ b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/debug.generic.stderr b/tests/ui/abi/debug.generic.stderr index 04d6f50872a39..8a031b79780a8 100644 --- a/tests/ui/abi/debug.generic.stderr +++ b/tests/ui/abi/debug.generic.stderr @@ -244,7 +244,7 @@ error: fn_abi_of(test_generic) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -328,7 +328,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -400,7 +400,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -479,7 +479,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -552,7 +552,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -629,7 +629,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -701,7 +701,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -779,7 +779,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -851,7 +851,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -962,7 +962,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/debug.loongarch64.stderr b/tests/ui/abi/debug.loongarch64.stderr index 85c888c4fae08..00bd3febde4ec 100644 --- a/tests/ui/abi/debug.loongarch64.stderr +++ b/tests/ui/abi/debug.loongarch64.stderr @@ -244,7 +244,7 @@ error: fn_abi_of(test_generic) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -328,7 +328,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -400,7 +400,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -479,7 +479,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -552,7 +552,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -629,7 +629,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -701,7 +701,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -779,7 +779,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -851,7 +851,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -962,7 +962,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/debug.riscv64.stderr b/tests/ui/abi/debug.riscv64.stderr index 85c888c4fae08..00bd3febde4ec 100644 --- a/tests/ui/abi/debug.riscv64.stderr +++ b/tests/ui/abi/debug.riscv64.stderr @@ -244,7 +244,7 @@ error: fn_abi_of(test_generic) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -328,7 +328,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -400,7 +400,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -479,7 +479,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -552,7 +552,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -629,7 +629,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -701,7 +701,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -779,7 +779,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -851,7 +851,7 @@ error: ABIs are not compatible }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -962,7 +962,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/pass-indirectly-attr.stderr b/tests/ui/abi/pass-indirectly-attr.stderr index a93982dacfa1b..03fe9ea46a4b7 100644 --- a/tests/ui/abi/pass-indirectly-attr.stderr +++ b/tests/ui/abi/pass-indirectly-attr.stderr @@ -24,7 +24,7 @@ error: fn_abi_of(extern_c) = FnAbi { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -65,7 +65,7 @@ error: fn_abi_of(extern_c) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -106,7 +106,7 @@ error: fn_abi_of(extern_rust) = FnAbi { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -165,7 +165,7 @@ error: fn_abi_of(extern_rust) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/abi/sysv64-zst.stderr b/tests/ui/abi/sysv64-zst.stderr index 2233e8e4f623e..82d3793c35328 100644 --- a/tests/ui/abi/sysv64-zst.stderr +++ b/tests/ui/abi/sysv64-zst.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -41,7 +41,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr index a86b28d98ecc6..584416f58f861 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr +++ b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(take_va_list) = FnAbi { }, fields: Arbitrary { offsets: $OFFSETS, - memory_index: $MEMORY_INDEX, + in_memory_order: $MEMORY_INDEX, }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(take_va_list) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/c-variadic/pass-by-value-abi.rs b/tests/ui/c-variadic/pass-by-value-abi.rs index 1e3935067cc27..a7cf37295a74a 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.rs +++ b/tests/ui/c-variadic/pass-by-value-abi.rs @@ -1,7 +1,7 @@ //@ check-fail //@ normalize-stderr: "randomization_seed: \d+" -> "randomization_seed: $$SEED" //@ normalize-stderr: "valid_range: 0\.\.=\d+" -> "valid_range: 0..=$$MAX" -//@ normalize-stderr: "memory_index: \[[^\]]+\]" -> "memory_index: $$MEMORY_INDEX" +//@ normalize-stderr: "in_memory_order: \[[^\]]+\]" -> "in_memory_order: $$MEMORY_INDEX" //@ normalize-stderr: "offsets: \[[^\]]+\]" -> "offsets: $$OFFSETS" //@ revisions: x86_64 aarch64 win //@ compile-flags: -O diff --git a/tests/ui/c-variadic/pass-by-value-abi.win.stderr b/tests/ui/c-variadic/pass-by-value-abi.win.stderr index 8ff93961f6678..b8e3f699b30e8 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.win.stderr +++ b/tests/ui/c-variadic/pass-by-value-abi.win.stderr @@ -20,7 +20,7 @@ error: fn_abi_of(take_va_list) = FnAbi { ), fields: Arbitrary { offsets: $OFFSETS, - memory_index: $MEMORY_INDEX, + in_memory_order: $MEMORY_INDEX, }, largest_niche: None, uninhabited: false, @@ -55,7 +55,7 @@ error: fn_abi_of(take_va_list) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr index aafe89d59e333..b5e0e8589af16 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr +++ b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr @@ -13,7 +13,7 @@ error: fn_abi_of(take_va_list) = FnAbi { }, fields: Arbitrary { offsets: $OFFSETS, - memory_index: $MEMORY_INDEX, + in_memory_order: $MEMORY_INDEX, }, largest_niche: None, uninhabited: false, @@ -52,7 +52,7 @@ error: fn_abi_of(take_va_list) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -91,7 +91,7 @@ error: fn_abi_of(take_va_list_sysv64) = FnAbi { }, fields: Arbitrary { offsets: $OFFSETS, - memory_index: $MEMORY_INDEX, + in_memory_order: $MEMORY_INDEX, }, largest_niche: None, uninhabited: false, @@ -130,7 +130,7 @@ error: fn_abi_of(take_va_list_sysv64) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -171,7 +171,7 @@ error: fn_abi_of(take_va_list_win64) = FnAbi { }, fields: Arbitrary { offsets: $OFFSETS, - memory_index: $MEMORY_INDEX, + in_memory_order: $MEMORY_INDEX, }, largest_niche: None, uninhabited: false, @@ -210,7 +210,7 @@ error: fn_abi_of(take_va_list_win64) = FnAbi { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/enum-discriminant/wrapping_niche.stderr b/tests/ui/enum-discriminant/wrapping_niche.stderr index e3e1755e14dd4..9b97ad4aeac7e 100644 --- a/tests/ui/enum-discriminant/wrapping_niche.stderr +++ b/tests/ui/enum-discriminant/wrapping_niche.stderr @@ -16,7 +16,7 @@ error: layout_of(UnsignedAroundZero) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -52,7 +52,7 @@ error: layout_of(UnsignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -73,7 +73,7 @@ error: layout_of(UnsignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -94,7 +94,7 @@ error: layout_of(UnsignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -134,7 +134,7 @@ error: layout_of(SignedAroundZero) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -170,7 +170,7 @@ error: layout_of(SignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -191,7 +191,7 @@ error: layout_of(SignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -212,7 +212,7 @@ error: layout_of(SignedAroundZero) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/layout/debug.stderr b/tests/ui/layout/debug.stderr index b2ce6385ab654..79ce21eb532b0 100644 --- a/tests/ui/layout/debug.stderr +++ b/tests/ui/layout/debug.stderr @@ -16,7 +16,7 @@ error: layout_of(E) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -52,7 +52,7 @@ error: layout_of(E) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -77,7 +77,7 @@ error: layout_of(E) = Layout { Size(4 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, 2, @@ -130,7 +130,7 @@ error: layout_of(S) = Layout { Size(8 bytes), Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, 2, 1, @@ -200,7 +200,7 @@ error: layout_of(Result) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -251,7 +251,7 @@ error: layout_of(Result) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -289,7 +289,7 @@ error: layout_of(Result) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/layout/hexagon-enum.stderr b/tests/ui/layout/hexagon-enum.stderr index 659013c8a6f1a..20e0a8642a66f 100644 --- a/tests/ui/layout/hexagon-enum.stderr +++ b/tests/ui/layout/hexagon-enum.stderr @@ -16,7 +16,7 @@ error: layout_of(A) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -52,7 +52,7 @@ error: layout_of(A) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -92,7 +92,7 @@ error: layout_of(B) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -128,7 +128,7 @@ error: layout_of(B) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -168,7 +168,7 @@ error: layout_of(C) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -204,7 +204,7 @@ error: layout_of(C) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -244,7 +244,7 @@ error: layout_of(P) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -280,7 +280,7 @@ error: layout_of(P) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -320,7 +320,7 @@ error: layout_of(T) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -356,7 +356,7 @@ error: layout_of(T) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr b/tests/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr index 2087fedeb19bc..61cfcbdc07f75 100644 --- a/tests/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr +++ b/tests/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr @@ -22,7 +22,7 @@ error: layout_of(MissingPayloadField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -72,7 +72,7 @@ error: layout_of(MissingPayloadField) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -95,7 +95,7 @@ error: layout_of(MissingPayloadField) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -142,7 +142,7 @@ error: layout_of(CommonPayloadField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -193,7 +193,7 @@ error: layout_of(CommonPayloadField) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(CommonPayloadField) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -279,7 +279,7 @@ error: layout_of(CommonPayloadFieldIsMaybeUninit) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -329,7 +329,7 @@ error: layout_of(CommonPayloadFieldIsMaybeUninit) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -366,7 +366,7 @@ error: layout_of(CommonPayloadFieldIsMaybeUninit) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -414,7 +414,7 @@ error: layout_of(NicheFirst) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -470,7 +470,7 @@ error: layout_of(NicheFirst) = Layout { Size(0 bytes), Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -503,7 +503,7 @@ error: layout_of(NicheFirst) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -524,7 +524,7 @@ error: layout_of(NicheFirst) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -570,7 +570,7 @@ error: layout_of(NicheSecond) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -626,7 +626,7 @@ error: layout_of(NicheSecond) = Layout { Size(1 bytes), Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 1, 0, ], @@ -659,7 +659,7 @@ error: layout_of(NicheSecond) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -680,7 +680,7 @@ error: layout_of(NicheSecond) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/layout/issue-96185-overaligned-enum.stderr b/tests/ui/layout/issue-96185-overaligned-enum.stderr index 6bcc5b4906b50..64e2f42c042f1 100644 --- a/tests/ui/layout/issue-96185-overaligned-enum.stderr +++ b/tests/ui/layout/issue-96185-overaligned-enum.stderr @@ -10,7 +10,7 @@ error: layout_of(Aligned1) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -46,7 +46,7 @@ error: layout_of(Aligned1) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -69,7 +69,7 @@ error: layout_of(Aligned1) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -113,7 +113,7 @@ error: layout_of(Aligned2) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -149,7 +149,7 @@ error: layout_of(Aligned2) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -172,7 +172,7 @@ error: layout_of(Aligned2) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/layout/thumb-enum.stderr b/tests/ui/layout/thumb-enum.stderr index 1ef22daf2a9d6..a6e603652123b 100644 --- a/tests/ui/layout/thumb-enum.stderr +++ b/tests/ui/layout/thumb-enum.stderr @@ -16,7 +16,7 @@ error: layout_of(A) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -52,7 +52,7 @@ error: layout_of(A) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -92,7 +92,7 @@ error: layout_of(B) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -128,7 +128,7 @@ error: layout_of(B) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -168,7 +168,7 @@ error: layout_of(C) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -204,7 +204,7 @@ error: layout_of(C) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -244,7 +244,7 @@ error: layout_of(P) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -280,7 +280,7 @@ error: layout_of(P) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -320,7 +320,7 @@ error: layout_of(T) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -356,7 +356,7 @@ error: layout_of(T) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, diff --git a/tests/ui/layout/zero-sized-array-enum-niche.stderr b/tests/ui/layout/zero-sized-array-enum-niche.stderr index 1707b8aff81cf..23f9092778085 100644 --- a/tests/ui/layout/zero-sized-array-enum-niche.stderr +++ b/tests/ui/layout/zero-sized-array-enum-niche.stderr @@ -10,7 +10,7 @@ error: layout_of(Result<[u32; 0], bool>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -48,7 +48,7 @@ error: layout_of(Result<[u32; 0], bool>) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -73,7 +73,7 @@ error: layout_of(Result<[u32; 0], bool>) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -118,7 +118,7 @@ error: layout_of(MultipleAlignments) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -156,7 +156,7 @@ error: layout_of(MultipleAlignments) = Layout { offsets: [ Size(2 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -181,7 +181,7 @@ error: layout_of(MultipleAlignments) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -206,7 +206,7 @@ error: layout_of(MultipleAlignments) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -251,7 +251,7 @@ error: layout_of(Result<[u32; 0], Packed>>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -289,7 +289,7 @@ error: layout_of(Result<[u32; 0], Packed>>) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -314,7 +314,7 @@ error: layout_of(Result<[u32; 0], Packed>>) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -359,7 +359,7 @@ error: layout_of(Result<[u32; 0], Packed>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -401,7 +401,7 @@ error: layout_of(Result<[u32; 0], Packed>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -426,7 +426,7 @@ error: layout_of(Result<[u32; 0], Packed>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/repr/repr-c-dead-variants.aarch64-unknown-linux-gnu.stderr b/tests/ui/repr/repr-c-dead-variants.aarch64-unknown-linux-gnu.stderr index 1a1929c530ff4..62f6ec92d493c 100644 --- a/tests/ui/repr/repr-c-dead-variants.aarch64-unknown-linux-gnu.stderr +++ b/tests/ui/repr/repr-c-dead-variants.aarch64-unknown-linux-gnu.stderr @@ -16,7 +16,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -60,7 +60,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -108,7 +108,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -158,7 +158,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -195,7 +195,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -270,7 +270,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { Size(8 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -298,7 +298,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/repr/repr-c-dead-variants.armebv7r-none-eabi.stderr b/tests/ui/repr/repr-c-dead-variants.armebv7r-none-eabi.stderr index 67729e9fcef9c..3e0efad974cd2 100644 --- a/tests/ui/repr/repr-c-dead-variants.armebv7r-none-eabi.stderr +++ b/tests/ui/repr/repr-c-dead-variants.armebv7r-none-eabi.stderr @@ -16,7 +16,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -60,7 +60,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -108,7 +108,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -158,7 +158,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -195,7 +195,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -270,7 +270,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { Size(8 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -298,7 +298,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/repr/repr-c-dead-variants.i686-pc-windows-msvc.stderr b/tests/ui/repr/repr-c-dead-variants.i686-pc-windows-msvc.stderr index 1a1929c530ff4..62f6ec92d493c 100644 --- a/tests/ui/repr/repr-c-dead-variants.i686-pc-windows-msvc.stderr +++ b/tests/ui/repr/repr-c-dead-variants.i686-pc-windows-msvc.stderr @@ -16,7 +16,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -60,7 +60,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -108,7 +108,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -158,7 +158,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -195,7 +195,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -270,7 +270,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { Size(8 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -298,7 +298,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/repr/repr-c-dead-variants.x86_64-unknown-linux-gnu.stderr b/tests/ui/repr/repr-c-dead-variants.x86_64-unknown-linux-gnu.stderr index 1a1929c530ff4..62f6ec92d493c 100644 --- a/tests/ui/repr/repr-c-dead-variants.x86_64-unknown-linux-gnu.stderr +++ b/tests/ui/repr/repr-c-dead-variants.x86_64-unknown-linux-gnu.stderr @@ -16,7 +16,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -60,7 +60,7 @@ error: layout_of(Univariant) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -108,7 +108,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -158,7 +158,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -195,7 +195,7 @@ error: layout_of(TwoVariants) = Layout { offsets: [ Size(4 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -270,7 +270,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { Size(8 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -298,7 +298,7 @@ error: layout_of(DeadBranchHasOtherField) = Layout { offsets: [ Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/repr/repr-c-int-dead-variants.stderr b/tests/ui/repr/repr-c-int-dead-variants.stderr index d88a842f88482..b25e4a9b7b6f5 100644 --- a/tests/ui/repr/repr-c-int-dead-variants.stderr +++ b/tests/ui/repr/repr-c-int-dead-variants.stderr @@ -16,7 +16,7 @@ error: layout_of(UnivariantU8) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -60,7 +60,7 @@ error: layout_of(UnivariantU8) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -108,7 +108,7 @@ error: layout_of(TwoVariantsU8) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -158,7 +158,7 @@ error: layout_of(TwoVariantsU8) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -195,7 +195,7 @@ error: layout_of(TwoVariantsU8) = Layout { offsets: [ Size(1 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -231,7 +231,7 @@ error: layout_of(DeadBranchHasOtherFieldU8) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -270,7 +270,7 @@ error: layout_of(DeadBranchHasOtherFieldU8) = Layout { Size(8 bytes), Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, 1, ], @@ -298,7 +298,7 @@ error: layout_of(DeadBranchHasOtherFieldU8) = Layout { offsets: [ Size(8 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/type/pattern_types/non_null.stderr b/tests/ui/type/pattern_types/non_null.stderr index ad61e9a591473..0847af2d086fa 100644 --- a/tests/ui/type/pattern_types/non_null.stderr +++ b/tests/ui/type/pattern_types/non_null.stderr @@ -17,7 +17,7 @@ error: layout_of((*const T) is !null) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -64,7 +64,7 @@ error: layout_of(Option<(*const ()) is !null>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -96,7 +96,7 @@ error: layout_of(Option<(*const ()) is !null>) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -126,7 +126,7 @@ error: layout_of(Option<(*const ()) is !null>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -186,7 +186,7 @@ error: layout_of((*const [u8]) is !null) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/type/pattern_types/or_patterns.stderr b/tests/ui/type/pattern_types/or_patterns.stderr index adc7320829b95..7206c570187b4 100644 --- a/tests/ui/type/pattern_types/or_patterns.stderr +++ b/tests/ui/type/pattern_types/or_patterns.stderr @@ -57,7 +57,7 @@ error: layout_of((i8) is (i8::MIN..=-1 | 1..)) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -102,7 +102,7 @@ error: layout_of((i8) is (i8::MIN..=-2 | 0..)) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, diff --git a/tests/ui/type/pattern_types/range_patterns.stderr b/tests/ui/type/pattern_types/range_patterns.stderr index c9a846e22f581..6c9acac7eb7b8 100644 --- a/tests/ui/type/pattern_types/range_patterns.stderr +++ b/tests/ui/type/pattern_types/range_patterns.stderr @@ -16,7 +16,7 @@ error: layout_of(NonZero) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -61,7 +61,7 @@ error: layout_of((u32) is 1..) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -106,7 +106,7 @@ error: layout_of(Option<(u32) is 1..>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -137,7 +137,7 @@ error: layout_of(Option<(u32) is 1..>) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -166,7 +166,7 @@ error: layout_of(Option<(u32) is 1..>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -217,7 +217,7 @@ error: layout_of(Option>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -248,7 +248,7 @@ error: layout_of(Option>) = Layout { }, fields: Arbitrary { offsets: [], - memory_index: [], + in_memory_order: [], }, largest_niche: None, uninhabited: false, @@ -277,7 +277,7 @@ error: layout_of(Option>) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -328,7 +328,7 @@ error: layout_of(NonZeroU32New) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -401,7 +401,7 @@ error: layout_of((i8) is -10..=10) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, @@ -446,7 +446,7 @@ error: layout_of((i8) is i8::MIN..=0) = Layout { offsets: [ Size(0 bytes), ], - memory_index: [ + in_memory_order: [ 0, ], }, From 3e4944d573af5bae3def57572fb0ed16e152ca88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Dom=C3=ADnguez?= Date: Mon, 8 Dec 2025 20:47:53 +0100 Subject: [PATCH 08/11] Split runtime global logic and cache kernel specific one --- compiler/rustc_codegen_llvm/src/base.rs | 16 +- .../src/builder/gpu_offload.rs | 186 ++++++++++++------ compiler/rustc_codegen_llvm/src/context.rs | 9 + compiler/rustc_codegen_llvm/src/intrinsic.rs | 25 ++- compiler/rustc_feature/src/builtin_attrs.rs | 2 +- tests/codegen-llvm/gpu_offload/gpu_host.rs | 18 +- 6 files changed, 171 insertions(+), 85 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 6cbddfec46318..16455b4c79cd6 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -23,13 +23,14 @@ use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrs, SanitizerFnAttrs}; use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::TyCtxt; -use rustc_session::config::DebugInfo; +use rustc_session::config::{DebugInfo, Offload}; use rustc_span::Symbol; use rustc_target::spec::SanitizerSet; use super::ModuleLlvm; use crate::attributes; use crate::builder::Builder; +use crate::builder::gpu_offload::OffloadGlobals; use crate::context::CodegenCx; use crate::llvm::{self, Value}; @@ -85,6 +86,19 @@ pub(crate) fn compile_codegen_unit( let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str()); { let mut cx = CodegenCx::new(tcx, cgu, &llvm_module); + + // Declare and store globals shared by all offload kernels + // + // These globals are left in the LLVM-IR host module so all kernels can access them. + // They are necessary for correct offload execution. We do this here to simplify the + // `offload` intrinsic, avoiding the need for tracking whether it's the first + // intrinsic call or not. + if cx.sess().opts.unstable_opts.offload.contains(&Offload::Enable) + && !cx.sess().target.is_like_gpu + { + cx.offload_globals.replace(Some(OffloadGlobals::declare(&cx))); + } + let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); for &(mono_item, data) in &mono_items { mono_item.predefine::>( diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index 5d1ddd057d88a..f28037fb9cff6 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -2,17 +2,76 @@ use std::ffi::CString; use llvm::Linkage::*; use rustc_abi::Align; -use rustc_codegen_ssa::traits::BaseTypeCodegenMethods; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; use rustc_middle::ty::offload_meta::OffloadMetadata; -use crate::builder::SBuilder; +use crate::builder::Builder; +use crate::common::CodegenCx; use crate::llvm::AttributePlace::Function; -use crate::llvm::{self, BasicBlock, Linkage, Type, Value}; +use crate::llvm::{self, Linkage, Type, Value}; use crate::{SimpleCx, attributes}; +// LLVM kernel-independent globals required for offloading +pub(crate) struct OffloadGlobals<'ll> { + pub launcher_fn: &'ll llvm::Value, + pub launcher_ty: &'ll llvm::Type, + + pub bin_desc: &'ll llvm::Type, + + pub kernel_args_ty: &'ll llvm::Type, + + pub offload_entry_ty: &'ll llvm::Type, + + pub begin_mapper: &'ll llvm::Value, + pub end_mapper: &'ll llvm::Value, + pub mapper_fn_ty: &'ll llvm::Type, + + pub ident_t_global: &'ll llvm::Value, + + pub register_lib: &'ll llvm::Value, + pub unregister_lib: &'ll llvm::Value, + pub init_rtls: &'ll llvm::Value, +} + +impl<'ll> OffloadGlobals<'ll> { + pub(crate) fn declare(cx: &CodegenCx<'ll, '_>) -> Self { + let (launcher_fn, launcher_ty) = generate_launcher(cx); + let kernel_args_ty = KernelArgsTy::new_decl(cx); + let offload_entry_ty = TgtOffloadEntry::new_decl(cx); + let (begin_mapper, _, end_mapper, mapper_fn_ty) = gen_tgt_data_mappers(cx); + let ident_t_global = generate_at_one(cx); + + let tptr = cx.type_ptr(); + let ti32 = cx.type_i32(); + let tgt_bin_desc_ty = vec![ti32, tptr, tptr, tptr]; + let bin_desc = cx.type_named_struct("struct.__tgt_bin_desc"); + cx.set_struct_body(bin_desc, &tgt_bin_desc_ty, false); + + let register_lib = declare_offload_fn(&cx, "__tgt_register_lib", mapper_fn_ty); + let unregister_lib = declare_offload_fn(&cx, "__tgt_unregister_lib", mapper_fn_ty); + let init_ty = cx.type_func(&[], cx.type_void()); + let init_rtls = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty); + + OffloadGlobals { + launcher_fn, + launcher_ty, + bin_desc, + kernel_args_ty, + offload_entry_ty, + begin_mapper, + end_mapper, + mapper_fn_ty, + ident_t_global, + register_lib, + unregister_lib, + init_rtls, + } + } +} + // ; Function Attrs: nounwind // declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr) #2 -fn generate_launcher<'ll>(cx: &'ll SimpleCx<'_>) -> (&'ll llvm::Value, &'ll llvm::Type) { +fn generate_launcher<'ll>(cx: &CodegenCx<'ll, '_>) -> (&'ll llvm::Value, &'ll llvm::Type) { let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); let ti32 = cx.type_i32(); @@ -30,7 +89,7 @@ fn generate_launcher<'ll>(cx: &'ll SimpleCx<'_>) -> (&'ll llvm::Value, &'ll llvm // @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @0 }, align 8 // FIXME(offload): @0 should include the file name (e.g. lib.rs) in which the function to be // offloaded was defined. -fn generate_at_one<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Value { +pub(crate) fn generate_at_one<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll llvm::Value { let unknown_txt = ";unknown;unknown;0;0;;"; let c_entry_name = CString::new(unknown_txt).unwrap(); let c_val = c_entry_name.as_bytes_with_nul(); @@ -68,7 +127,7 @@ pub(crate) struct TgtOffloadEntry { } impl TgtOffloadEntry { - pub(crate) fn new_decl<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Type { + pub(crate) fn new_decl<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll llvm::Type { let offload_entry_ty = cx.type_named_struct("struct.__tgt_offload_entry"); let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); @@ -82,7 +141,7 @@ impl TgtOffloadEntry { } fn new<'ll>( - cx: &'ll SimpleCx<'_>, + cx: &CodegenCx<'ll, '_>, region_id: &'ll Value, llglobal: &'ll Value, ) -> [&'ll Value; 9] { @@ -126,7 +185,7 @@ impl KernelArgsTy { const OFFLOAD_VERSION: u64 = 3; const FLAGS: u64 = 0; const TRIPCOUNT: u64 = 0; - fn new_decl<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll Type { + fn new_decl<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Type { let kernel_arguments_ty = cx.type_named_struct("struct.__tgt_kernel_arguments"); let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); @@ -140,8 +199,8 @@ impl KernelArgsTy { kernel_arguments_ty } - fn new<'ll>( - cx: &'ll SimpleCx<'_>, + fn new<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, num_args: u64, memtransfer_types: &'ll Value, geps: [&'ll Value; 3], @@ -171,7 +230,8 @@ impl KernelArgsTy { } // Contains LLVM values needed to manage offloading for a single kernel. -pub(crate) struct OffloadKernelData<'ll> { +#[derive(Copy, Clone)] +pub(crate) struct OffloadKernelGlobals<'ll> { pub offload_sizes: &'ll llvm::Value, pub memtransfer_types: &'ll llvm::Value, pub region_id: &'ll llvm::Value, @@ -179,7 +239,7 @@ pub(crate) struct OffloadKernelData<'ll> { } fn gen_tgt_data_mappers<'ll>( - cx: &'ll SimpleCx<'_>, + cx: &CodegenCx<'ll, '_>, ) -> (&'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Type) { let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); @@ -241,12 +301,18 @@ pub(crate) fn add_global<'ll>( // mapped to/from the gpu. It also returns a region_id with the name of this kernel, to be // concatenated into the list of region_ids. pub(crate) fn gen_define_handling<'ll>( - cx: &SimpleCx<'ll>, - offload_entry_ty: &'ll llvm::Type, + cx: &CodegenCx<'ll, '_>, metadata: &[OffloadMetadata], - types: &[&Type], - symbol: &str, -) -> OffloadKernelData<'ll> { + types: &[&'ll Type], + symbol: String, + offload_globals: &OffloadGlobals<'ll>, +) -> OffloadKernelGlobals<'ll> { + if let Some(entry) = cx.offload_kernel_cache.borrow().get(&symbol) { + return *entry; + } + + let offload_entry_ty = offload_globals.offload_entry_ty; + // It seems like non-pointer values are automatically mapped. So here, we focus on pointer (or // reference) types. let ptr_meta = types.iter().zip(metadata).filter_map(|(&x, meta)| match cx.type_kind(x) { @@ -274,7 +340,7 @@ pub(crate) fn gen_define_handling<'ll>( let initializer = cx.get_const_i8(0); let region_id = add_unnamed_global(&cx, &name, initializer, WeakAnyLinkage); - let c_entry_name = CString::new(symbol).unwrap(); + let c_entry_name = CString::new(symbol.clone()).unwrap(); let c_val = c_entry_name.as_bytes_with_nul(); let offload_entry_name = format!(".offloading.entry_name.{symbol}"); @@ -298,11 +364,16 @@ pub(crate) fn gen_define_handling<'ll>( let c_section_name = CString::new("llvm_offload_entries").unwrap(); llvm::set_section(offload_entry, &c_section_name); - OffloadKernelData { offload_sizes, memtransfer_types, region_id, offload_entry } + let result = + OffloadKernelGlobals { offload_sizes, memtransfer_types, region_id, offload_entry }; + + cx.offload_kernel_cache.borrow_mut().insert(symbol, result); + + result } fn declare_offload_fn<'ll>( - cx: &'ll SimpleCx<'_>, + cx: &CodegenCx<'ll, '_>, name: &str, ty: &'ll llvm::Type, ) -> &'ll llvm::Value { @@ -335,28 +406,28 @@ fn declare_offload_fn<'ll>( // 4. set insert point after kernel call. // 5. generate all the GEPS and stores, to be used in 6) // 6. generate __tgt_target_data_end calls to move data from the GPU -pub(crate) fn gen_call_handling<'ll>( - cx: &SimpleCx<'ll>, - bb: &BasicBlock, - offload_data: &OffloadKernelData<'ll>, +pub(crate) fn gen_call_handling<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, + offload_data: &OffloadKernelGlobals<'ll>, args: &[&'ll Value], types: &[&Type], metadata: &[OffloadMetadata], + offload_globals: &OffloadGlobals<'ll>, ) { - let OffloadKernelData { offload_sizes, offload_entry, memtransfer_types, region_id } = + let cx = builder.cx; + let OffloadKernelGlobals { offload_sizes, offload_entry, memtransfer_types, region_id } = offload_data; - let (tgt_decl, tgt_target_kernel_ty) = generate_launcher(&cx); - // %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } - let tptr = cx.type_ptr(); - let ti32 = cx.type_i32(); - let tgt_bin_desc_ty = vec![ti32, tptr, tptr, tptr]; - let tgt_bin_desc = cx.type_named_struct("struct.__tgt_bin_desc"); - cx.set_struct_body(tgt_bin_desc, &tgt_bin_desc_ty, false); - let tgt_kernel_decl = KernelArgsTy::new_decl(&cx); - let (begin_mapper_decl, _, end_mapper_decl, fn_ty) = gen_tgt_data_mappers(&cx); + let tgt_decl = offload_globals.launcher_fn; + let tgt_target_kernel_ty = offload_globals.launcher_ty; - let mut builder = SBuilder::build(cx, bb); + // %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } + let tgt_bin_desc = offload_globals.bin_desc; + + let tgt_kernel_decl = offload_globals.kernel_args_ty; + let begin_mapper_decl = offload_globals.begin_mapper; + let end_mapper_decl = offload_globals.end_mapper; + let fn_ty = offload_globals.mapper_fn_ty; let num_args = types.len() as u64; let ip = unsafe { llvm::LLVMRustGetInsertPoint(&builder.llbuilder) }; @@ -378,9 +449,8 @@ pub(crate) fn gen_call_handling<'ll>( // Step 0) // %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } // %6 = alloca %struct.__tgt_bin_desc, align 8 - let llfn = unsafe { llvm::LLVMGetBasicBlockParent(bb) }; unsafe { - llvm::LLVMRustPositionBuilderPastAllocas(&builder.llbuilder, llfn); + llvm::LLVMRustPositionBuilderPastAllocas(&builder.llbuilder, builder.llfn()); } let tgt_bin_desc_alloca = builder.direct_alloca(tgt_bin_desc, Align::EIGHT, "EmptyDesc"); @@ -413,16 +483,16 @@ pub(crate) fn gen_call_handling<'ll>( } let mapper_fn_ty = cx.type_func(&[cx.type_ptr()], cx.type_void()); - let register_lib_decl = declare_offload_fn(&cx, "__tgt_register_lib", mapper_fn_ty); - let unregister_lib_decl = declare_offload_fn(&cx, "__tgt_unregister_lib", mapper_fn_ty); + let register_lib_decl = offload_globals.register_lib; + let unregister_lib_decl = offload_globals.unregister_lib; let init_ty = cx.type_func(&[], cx.type_void()); - let init_rtls_decl = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty); + let init_rtls_decl = offload_globals.init_rtls; // FIXME(offload): Later we want to add them to the wrapper code, rather than our main function. // call void @__tgt_register_lib(ptr noundef %6) - builder.call(mapper_fn_ty, register_lib_decl, &[tgt_bin_desc_alloca], None); + builder.call(mapper_fn_ty, None, None, register_lib_decl, &[tgt_bin_desc_alloca], None, None); // call void @__tgt_init_all_rtls() - builder.call(init_ty, init_rtls_decl, &[], None); + builder.call(init_ty, None, None, init_rtls_decl, &[], None, None); for i in 0..num_args { let idx = cx.get_const_i32(i); @@ -437,15 +507,15 @@ pub(crate) fn gen_call_handling<'ll>( // For now we have a very simplistic indexing scheme into our // offload_{baseptrs,ptrs,sizes}. We will probably improve this along with our gpu frontend pr. - fn get_geps<'a, 'll>( - builder: &mut SBuilder<'a, 'll>, - cx: &'ll SimpleCx<'ll>, + fn get_geps<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, ty: &'ll Type, ty2: &'ll Type, a1: &'ll Value, a2: &'ll Value, a4: &'ll Value, ) -> [&'ll Value; 3] { + let cx = builder.cx; let i32_0 = cx.get_const_i32(0); let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, i32_0]); @@ -454,9 +524,8 @@ pub(crate) fn gen_call_handling<'ll>( [gep1, gep2, gep3] } - fn generate_mapper_call<'a, 'll>( - builder: &mut SBuilder<'a, 'll>, - cx: &'ll SimpleCx<'ll>, + fn generate_mapper_call<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, geps: [&'ll Value; 3], o_type: &'ll Value, fn_to_call: &'ll Value, @@ -464,20 +533,20 @@ pub(crate) fn gen_call_handling<'ll>( num_args: u64, s_ident_t: &'ll Value, ) { + let cx = builder.cx; let nullptr = cx.const_null(cx.type_ptr()); let i64_max = cx.get_const_i64(u64::MAX); let num_args = cx.get_const_i32(num_args); let args = vec![s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type, nullptr, nullptr]; - builder.call(fn_ty, fn_to_call, &args, None); + builder.call(fn_ty, None, None, fn_to_call, &args, None, None); } // Step 2) - let s_ident_t = generate_at_one(&cx); - let geps = get_geps(&mut builder, &cx, ty, ty2, a1, a2, a4); + let s_ident_t = offload_globals.ident_t_global; + let geps = get_geps(builder, ty, ty2, a1, a2, a4); generate_mapper_call( - &mut builder, - &cx, + builder, geps, memtransfer_types, begin_mapper_decl, @@ -504,14 +573,13 @@ pub(crate) fn gen_call_handling<'ll>( region_id, a5, ]; - builder.call(tgt_target_kernel_ty, tgt_decl, &args, None); + builder.call(tgt_target_kernel_ty, None, None, tgt_decl, &args, None, None); // %41 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args) // Step 4) - let geps = get_geps(&mut builder, &cx, ty, ty2, a1, a2, a4); + let geps = get_geps(builder, ty, ty2, a1, a2, a4); generate_mapper_call( - &mut builder, - &cx, + builder, geps, memtransfer_types, end_mapper_decl, @@ -520,7 +588,5 @@ pub(crate) fn gen_call_handling<'ll>( s_ident_t, ); - builder.call(mapper_fn_ty, unregister_lib_decl, &[tgt_bin_desc_alloca], None); - - drop(builder); + builder.call(mapper_fn_ty, None, None, unregister_lib_decl, &[tgt_bin_desc_alloca], None, None); } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 6caf60e3cc41e..03da70d48f7d6 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -35,6 +35,7 @@ use smallvec::SmallVec; use crate::abi::to_llvm_calling_convention; use crate::back::write::to_llvm_code_model; +use crate::builder::gpu_offload::{OffloadGlobals, OffloadKernelGlobals}; use crate::callee::get_fn; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; use crate::llvm::{self, Metadata, MetadataKindId, Module, Type, Value}; @@ -156,6 +157,12 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// Cache of Objective-C selector references pub objc_selrefs: RefCell>, + + /// Globals shared by the offloading runtime + pub offload_globals: RefCell>>, + + /// Cache of kernel-specific globals + pub offload_kernel_cache: RefCell>>, } fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { @@ -639,6 +646,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { objc_class_t: Cell::new(None), objc_classrefs: Default::default(), objc_selrefs: Default::default(), + offload_globals: Default::default(), + offload_kernel_cache: Default::default(), }, PhantomData, ) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 33541f7b695f8..46a7c55a7baac 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -26,7 +26,7 @@ use tracing::debug; use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; -use crate::builder::gpu_offload::TgtOffloadEntry; +use crate::builder::gpu_offload::{gen_call_handling, gen_define_handling}; use crate::context::CodegenCx; use crate::errors::{ AutoDiffWithoutEnable, AutoDiffWithoutLto, OffloadWithoutEnable, OffloadWithoutFatLTO, @@ -1287,8 +1287,6 @@ fn codegen_offload<'ll, 'tcx>( let args = get_args_from_tuple(bx, args[1], fn_target); let target_symbol = symbol_name_for_instance_in_crate(tcx, fn_target, LOCAL_CRATE); - let offload_entry_ty = TgtOffloadEntry::new_decl(&cx); - let sig = tcx.fn_sig(fn_target.def_id()).skip_binder().skip_binder(); let inputs = sig.inputs(); @@ -1296,17 +1294,16 @@ fn codegen_offload<'ll, 'tcx>( let types = inputs.iter().map(|ty| cx.layout_of(*ty).llvm_type(cx)).collect::>(); - let offload_data = crate::builder::gpu_offload::gen_define_handling( - cx, - offload_entry_ty, - &metadata, - &types, - &target_symbol, - ); - - // FIXME(Sa4dUs): pass the original builder once we separate kernel launch logic from globals - let bb = unsafe { llvm::LLVMGetInsertBlock(bx.llbuilder) }; - crate::builder::gpu_offload::gen_call_handling(cx, bb, &offload_data, &args, &types, &metadata); + let offload_globals_ref = cx.offload_globals.borrow(); + let offload_globals = match offload_globals_ref.as_ref() { + Some(globals) => globals, + None => { + // Offload is not initialized, cannot continue + return; + } + }; + let offload_data = gen_define_handling(&cx, &metadata, &types, target_symbol, offload_globals); + gen_call_handling(bx, &offload_data, &args, &types, &metadata, offload_globals); } fn get_args_from_tuple<'ll, 'tcx>( diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index 6eefb2f48d127..c8245b0e43bd5 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -1118,7 +1118,7 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ template!(Word, List: &[r#""...""#]), DuplicatesOk, EncodeCrossCrate::Yes, ), - rustc_attr!( + rustc_attr!( rustc_offload_kernel, Normal, template!(Word), DuplicatesOk, EncodeCrossCrate::Yes, diff --git a/tests/codegen-llvm/gpu_offload/gpu_host.rs b/tests/codegen-llvm/gpu_offload/gpu_host.rs index b0f83c825705f..06bb2d92d8ba2 100644 --- a/tests/codegen-llvm/gpu_offload/gpu_host.rs +++ b/tests/codegen-llvm/gpu_offload/gpu_host.rs @@ -11,6 +11,7 @@ // when inside of a function called main. This, too, is a temporary workaround for not having a // frontend. +#![feature(rustc_attrs)] #![feature(core_intrinsics)] #![no_main] @@ -21,29 +22,31 @@ fn main() { core::hint::black_box(&x); } -// CHECK: %struct.__tgt_offload_entry = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr } // CHECK: %struct.ident_t = type { i32, i32, i32, i32, ptr } +// CHECK: %struct.__tgt_offload_entry = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr } // CHECK: %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } // CHECK: %struct.__tgt_kernel_arguments = type { i32, i32, ptr, ptr, ptr, ptr, ptr, ptr, i64, i64, [3 x i32], [3 x i32], i32 } +// CHECK: @anon.{{.*}}.0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1 +// CHECK: @anon.{{.*}}.1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @anon.{{.*}}.0 }, align 8 + // CHECK: @.offload_sizes._kernel_1 = private unnamed_addr constant [1 x i64] [i64 1024] // CHECK: @.offload_maptypes._kernel_1 = private unnamed_addr constant [1 x i64] [i64 35] // CHECK: @._kernel_1.region_id = internal unnamed_addr constant i8 0 // CHECK: @.offloading.entry_name._kernel_1 = internal unnamed_addr constant [10 x i8] c"_kernel_1\00", section ".llvm.rodata.offloading", align 1 // CHECK: @.offloading.entry._kernel_1 = internal constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @._kernel_1.region_id, ptr @.offloading.entry_name._kernel_1, i64 0, i64 0, ptr null }, section "llvm_offload_entries", align 8 -// CHECK: @anon.{{.*}}.0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1 -// CHECK: @anon.{{.*}}.1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @anon.{{.*}}.0 }, align 8 +// CHECK: Function Attrs: nounwind +// CHECK: declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr) -// CHECK: Function Attrs: -// CHECK-NEXT: define{{( dso_local)?}} void @main() +// CHECK: define{{( dso_local)?}} void @main() // CHECK-NEXT: start: // CHECK-NEXT: %0 = alloca [8 x i8], align 8 // CHECK-NEXT: %x = alloca [1024 x i8], align 16 // CHECK: call void @kernel_1(ptr noalias noundef nonnull align 4 dereferenceable(1024) %x) // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %0) // CHECK-NEXT: store ptr %x, ptr %0, align 8 -// CHECK-NEXT: call void asm sideeffect "", "r,~{memory}"(ptr nonnull %0) #4, !srcloc !4 +// CHECK-NEXT: call void asm sideeffect "", "r,~{memory}"(ptr nonnull %0) // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %0) // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %x) // CHECK-NEXT: ret void @@ -92,9 +95,6 @@ fn main() { // CHECK-NEXT: ret void // CHECK-NEXT: } -// CHECK: Function Attrs: nounwind -// CHECK: declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr) - #[unsafe(no_mangle)] #[inline(never)] pub fn kernel_1(x: &mut [f32; 256]) { From 8bafb632026871b4e6849f80d63b3f4d8a5fb517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Dom=C3=ADnguez?= Date: Mon, 8 Dec 2025 22:47:05 +0100 Subject: [PATCH 09/11] Remove outdated comment --- compiler/rustc_codegen_llvm/src/back/write.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index c0c01b80372f3..c6a1440c86dac 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -710,8 +710,7 @@ pub(crate) unsafe fn llvm_optimize( if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Enable) { let cx = SimpleCx::new(module.module_llvm.llmod(), module.module_llvm.llcx, cgcx.pointer_size); - // For now we only support up to 10 kernels named kernel_0 ... kernel_9, a follow-up PR is - // introducing a proper offload intrinsic to solve this limitation. + for func in cx.get_functions() { let offload_kernel = "offload-kernel"; if attributes::has_string_attr(func, offload_kernel) { From 04c2d2be1379cd67521d04d7142d3edd680d6b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Dom=C3=ADnguez?= Date: Thu, 18 Dec 2025 13:14:41 +0100 Subject: [PATCH 10/11] Remove `region_id` unnamed attr --- compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs | 2 +- tests/codegen-llvm/gpu_offload/gpu_host.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index f28037fb9cff6..046501d08c482 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -338,7 +338,7 @@ pub(crate) fn gen_define_handling<'ll>( let name = format!(".{symbol}.region_id"); let initializer = cx.get_const_i8(0); - let region_id = add_unnamed_global(&cx, &name, initializer, WeakAnyLinkage); + let region_id = add_global(&cx, &name, initializer, WeakAnyLinkage); let c_entry_name = CString::new(symbol.clone()).unwrap(); let c_val = c_entry_name.as_bytes_with_nul(); diff --git a/tests/codegen-llvm/gpu_offload/gpu_host.rs b/tests/codegen-llvm/gpu_offload/gpu_host.rs index 06bb2d92d8ba2..ac179a65828d7 100644 --- a/tests/codegen-llvm/gpu_offload/gpu_host.rs +++ b/tests/codegen-llvm/gpu_offload/gpu_host.rs @@ -32,7 +32,7 @@ fn main() { // CHECK: @.offload_sizes._kernel_1 = private unnamed_addr constant [1 x i64] [i64 1024] // CHECK: @.offload_maptypes._kernel_1 = private unnamed_addr constant [1 x i64] [i64 35] -// CHECK: @._kernel_1.region_id = internal unnamed_addr constant i8 0 +// CHECK: @._kernel_1.region_id = internal constant i8 0 // CHECK: @.offloading.entry_name._kernel_1 = internal unnamed_addr constant [10 x i8] c"_kernel_1\00", section ".llvm.rodata.offloading", align 1 // CHECK: @.offloading.entry._kernel_1 = internal constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @._kernel_1.region_id, ptr @.offloading.entry_name._kernel_1, i64 0, i64 0, ptr null }, section "llvm_offload_entries", align 8 From 6608f6ace7a0267cff3920216f1b697cabc3ca67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Fri, 19 Dec 2025 13:57:34 +0100 Subject: [PATCH 11/11] split up expansion code of eii macro into functions --- compiler/rustc_builtin_macros/src/eii.rs | 404 +++++++++++++---------- 1 file changed, 238 insertions(+), 166 deletions(-) diff --git a/compiler/rustc_builtin_macros/src/eii.rs b/compiler/rustc_builtin_macros/src/eii.rs index b97cb1daec534..e29b29fc6ccb0 100644 --- a/compiler/rustc_builtin_macros/src/eii.rs +++ b/compiler/rustc_builtin_macros/src/eii.rs @@ -1,11 +1,12 @@ use rustc_ast::token::{Delimiter, TokenKind}; use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_ast::{ - DUMMY_NODE_ID, EiiExternTarget, EiiImpl, ItemKind, Stmt, StmtKind, ast, token, tokenstream, + Attribute, DUMMY_NODE_ID, EiiExternTarget, EiiImpl, ItemKind, MetaItem, Path, Stmt, StmtKind, + Visibility, ast, }; use rustc_ast_pretty::pprust::path_to_string; use rustc_expand::base::{Annotatable, ExtCtxt}; -use rustc_span::{Ident, Span, kw, sym}; +use rustc_span::{ErrorGuaranteed, Ident, Span, kw, sym}; use thin_vec::{ThinVec, thin_vec}; use crate::errors::{ @@ -52,12 +53,12 @@ pub(crate) fn unsafe_eii( fn eii_( ecx: &mut ExtCtxt<'_>, - span: Span, + eii_attr_span: Span, meta_item: &ast::MetaItem, item: Annotatable, impl_unsafe: bool, ) -> Vec { - let span = ecx.with_def_site_ctxt(span); + let eii_attr_span = ecx.with_def_site_ctxt(eii_attr_span); let (item, stmt) = if let Annotatable::Item(item) = item { (item, false) @@ -67,7 +68,7 @@ fn eii_( (item.clone(), true) } else { ecx.dcx().emit_err(EiiSharedMacroExpectedFunction { - span, + span: eii_attr_span, name: path_to_string(&meta_item.path), }); return vec![item]; @@ -77,113 +78,218 @@ fn eii_( let item = *item; - let ast::Item { attrs, id: _, span: item_span, vis, kind: ItemKind::Fn(mut func), tokens: _ } = - item - else { + let ast::Item { attrs, id: _, span: _, vis, kind: ItemKind::Fn(func), tokens: _ } = item else { ecx.dcx().emit_err(EiiSharedMacroExpectedFunction { - span, + span: eii_attr_span, name: path_to_string(&meta_item.path), }); return vec![Annotatable::Item(Box::new(item))]; }; - // Detect when this is the *second* eii attribute on an item. - let mut new_attrs = ThinVec::new(); - for i in attrs { - if i.has_name(sym::eii) { - ecx.dcx().emit_err(EiiOnlyOnce { - span: i.span, - first_span: span, - name: path_to_string(&meta_item.path), - }); - } else { - new_attrs.push(i); - } + let attrs_from_decl = + filter_attrs_for_multiple_eii_attr(ecx, attrs, eii_attr_span, &meta_item.path); + + let Ok(macro_name) = name_for_impl_macro(ecx, &func, &meta_item) else { + return vec![Annotatable::Item(orig_item)]; + }; + + // span of the declaring item without attributes + let item_span = func.sig.span; + // span of the eii attribute and the item below it, i.e. the full declaration + let decl_span = eii_attr_span.to(item_span); + let foreign_item_name = func.ident; + + let mut return_items = Vec::new(); + + if func.body.is_some() { + return_items.push(Box::new(generate_default_impl( + &func, + impl_unsafe, + macro_name, + eii_attr_span, + item_span, + ))) + } + + return_items.push(Box::new(generate_foreign_item( + ecx, + eii_attr_span, + item_span, + *func, + vis, + &attrs_from_decl, + ))); + return_items.push(Box::new(generate_attribute_macro_to_implement( + ecx, + eii_attr_span, + macro_name, + foreign_item_name, + impl_unsafe, + decl_span, + ))); + + if stmt { + return_items + .into_iter() + .map(|i| { + Annotatable::Stmt(Box::new(Stmt { + id: DUMMY_NODE_ID, + kind: StmtKind::Item(i), + span: eii_attr_span, + })) + }) + .collect() + } else { + return_items.into_iter().map(|i| Annotatable::Item(i)).collect() } - let attrs = new_attrs; +} - let macro_name = if meta_item.is_word() { - func.ident +/// Decide on the name of the macro that can be used to implement the EII. +/// This is either an explicitly given name, or the name of the item in the +/// declaration of the EII. +fn name_for_impl_macro( + ecx: &mut ExtCtxt<'_>, + func: &ast::Fn, + meta_item: &MetaItem, +) -> Result { + if meta_item.is_word() { + Ok(func.ident) } else if let Some([first]) = meta_item.meta_item_list() && let Some(m) = first.meta_item() && m.path.segments.len() == 1 { - m.path.segments[0].ident + Ok(m.path.segments[0].ident) } else { - ecx.dcx().emit_err(EiiMacroExpectedMaxOneArgument { + Err(ecx.dcx().emit_err(EiiMacroExpectedMaxOneArgument { span: meta_item.span, name: path_to_string(&meta_item.path), - }); - return vec![Annotatable::Item(orig_item)]; - }; + })) + } +} - let mut return_items = Vec::new(); +/// Ensure that in the list of attrs, there's only a single `eii` attribute. +fn filter_attrs_for_multiple_eii_attr( + ecx: &mut ExtCtxt<'_>, + attrs: ThinVec, + eii_attr_span: Span, + eii_attr_path: &Path, +) -> ThinVec { + attrs + .into_iter() + .filter(|i| { + if i.has_name(sym::eii) { + ecx.dcx().emit_err(EiiOnlyOnce { + span: i.span, + first_span: eii_attr_span, + name: path_to_string(eii_attr_path), + }); + false + } else { + true + } + }) + .collect() +} - if func.body.is_some() { - let mut default_func = func.clone(); - func.body = None; - default_func.eii_impls.push(ast::EiiImpl { - node_id: DUMMY_NODE_ID, - eii_macro_path: ast::Path::from_ident(macro_name), - impl_safety: if impl_unsafe { ast::Safety::Unsafe(span) } else { ast::Safety::Default }, - span, - inner_span: macro_name.span, - is_default: true, // important! - }); +fn generate_default_impl( + func: &ast::Fn, + impl_unsafe: bool, + macro_name: Ident, + eii_attr_span: Span, + item_span: Span, +) -> ast::Item { + // FIXME: re-add some original attrs + let attrs = ThinVec::new(); + + let mut default_func = func.clone(); + default_func.eii_impls.push(EiiImpl { + node_id: DUMMY_NODE_ID, + inner_span: macro_name.span, + eii_macro_path: ast::Path::from_ident(macro_name), + impl_safety: if impl_unsafe { + ast::Safety::Unsafe(eii_attr_span) + } else { + ast::Safety::Default + }, + span: eii_attr_span, + is_default: true, + }); - return_items.push(Box::new(ast::Item { - attrs: ThinVec::new(), - id: ast::DUMMY_NODE_ID, - span, - vis: ast::Visibility { span, kind: ast::VisibilityKind::Inherited, tokens: None }, - kind: ast::ItemKind::Const(Box::new(ast::ConstItem { - ident: Ident { name: kw::Underscore, span }, - defaultness: ast::Defaultness::Final, - generics: ast::Generics::default(), - ty: Box::new(ast::Ty { - id: DUMMY_NODE_ID, - kind: ast::TyKind::Tup(ThinVec::new()), - span, - tokens: None, - }), - rhs: Some(ast::ConstItemRhs::Body(Box::new(ast::Expr { - id: DUMMY_NODE_ID, - kind: ast::ExprKind::Block( - Box::new(ast::Block { - stmts: thin_vec![ast::Stmt { - id: DUMMY_NODE_ID, - kind: ast::StmtKind::Item(Box::new(ast::Item { - attrs: thin_vec![], // FIXME: re-add some original attrs - id: DUMMY_NODE_ID, - span: item_span, - vis: ast::Visibility { - span, - kind: ast::VisibilityKind::Inherited, - tokens: None - }, - kind: ItemKind::Fn(default_func), - tokens: None, - })), - span - }], - id: DUMMY_NODE_ID, - rules: ast::BlockCheckMode::Default, - span, - tokens: None, - }), - None, - ), - span, - attrs: ThinVec::new(), - tokens: None, - }))), - define_opaque: None, - })), + ast::Item { + attrs: ThinVec::new(), + id: ast::DUMMY_NODE_ID, + span: eii_attr_span, + vis: ast::Visibility { + span: eii_attr_span, + kind: ast::VisibilityKind::Inherited, tokens: None, - })) + }, + kind: ast::ItemKind::Const(Box::new(ast::ConstItem { + ident: Ident { name: kw::Underscore, span: eii_attr_span }, + defaultness: ast::Defaultness::Final, + generics: ast::Generics::default(), + ty: Box::new(ast::Ty { + id: DUMMY_NODE_ID, + kind: ast::TyKind::Tup(ThinVec::new()), + span: eii_attr_span, + tokens: None, + }), + rhs: Some(ast::ConstItemRhs::Body(Box::new(ast::Expr { + id: DUMMY_NODE_ID, + kind: ast::ExprKind::Block( + Box::new(ast::Block { + stmts: thin_vec![ast::Stmt { + id: DUMMY_NODE_ID, + kind: ast::StmtKind::Item(Box::new(ast::Item { + attrs, + id: DUMMY_NODE_ID, + span: item_span, + vis: ast::Visibility { + span: eii_attr_span, + kind: ast::VisibilityKind::Inherited, + tokens: None + }, + kind: ItemKind::Fn(Box::new(default_func)), + tokens: None, + })), + span: eii_attr_span + }], + id: DUMMY_NODE_ID, + rules: ast::BlockCheckMode::Default, + span: eii_attr_span, + tokens: None, + }), + None, + ), + span: eii_attr_span, + attrs: ThinVec::new(), + tokens: None, + }))), + define_opaque: None, + })), + tokens: None, } +} - let decl_span = span.to(func.sig.span); +/// Generates a foreign item, like +/// +/// ```rust, ignore +/// extern "…" { safe fn item(); } +/// ``` +fn generate_foreign_item( + ecx: &mut ExtCtxt<'_>, + eii_attr_span: Span, + item_span: Span, + mut func: ast::Fn, + vis: Visibility, + attrs_from_decl: &[Attribute], +) -> ast::Item { + let mut foreign_item_attrs = ThinVec::new(); + foreign_item_attrs.extend_from_slice(attrs_from_decl); + + // Add the rustc_eii_extern_item on the foreign item. Usually, foreign items are mangled. + // This attribute makes sure that we later know that this foreign item's symbol should not be. + foreign_item_attrs.push(ecx.attr_word(sym::rustc_eii_extern_item, eii_attr_span)); let abi = match func.sig.header.ext { // extern "X" fn => extern "X" {} @@ -196,85 +302,69 @@ fn eii_( suffix: None, symbol_unescaped: sym::Rust, style: ast::StrStyle::Cooked, - span, + span: eii_attr_span, }), }; // ABI has been moved to the extern {} block, so we remove it from the fn item. func.sig.header.ext = ast::Extern::None; + func.body = None; // And mark safe functions explicitly as `safe fn`. if func.sig.header.safety == ast::Safety::Default { func.sig.header.safety = ast::Safety::Safe(func.sig.span); } - // extern "…" { safe fn item(); } - let mut extern_item_attrs = attrs.clone(); - extern_item_attrs.push(ast::Attribute { - kind: ast::AttrKind::Normal(Box::new(ast::NormalAttr { - item: ast::AttrItem { - unsafety: ast::Safety::Default, - // Add the rustc_eii_extern_item on the foreign item. Usually, foreign items are mangled. - // This attribute makes sure that we later know that this foreign item's symbol should not be. - path: ast::Path::from_ident(Ident::new(sym::rustc_eii_extern_item, span)), - args: ast::AttrArgs::Empty, - tokens: None, - }, - tokens: None, - })), - id: ecx.sess.psess.attr_id_generator.mk_attr_id(), - style: ast::AttrStyle::Outer, - span, - }); - - let extern_block = Box::new(ast::Item { + ast::Item { attrs: ast::AttrVec::default(), id: ast::DUMMY_NODE_ID, - span, - vis: ast::Visibility { span, kind: ast::VisibilityKind::Inherited, tokens: None }, + span: eii_attr_span, + vis: ast::Visibility { + span: eii_attr_span, + kind: ast::VisibilityKind::Inherited, + tokens: None, + }, kind: ast::ItemKind::ForeignMod(ast::ForeignMod { - extern_span: span, - safety: ast::Safety::Unsafe(span), + extern_span: eii_attr_span, + safety: ast::Safety::Unsafe(eii_attr_span), abi, items: From::from([Box::new(ast::ForeignItem { - attrs: extern_item_attrs, + attrs: foreign_item_attrs, id: ast::DUMMY_NODE_ID, span: item_span, vis, - kind: ast::ForeignItemKind::Fn(func.clone()), + kind: ast::ForeignItemKind::Fn(Box::new(func.clone())), tokens: None, })]), }), tokens: None, - }); + } +} - let mut macro_attrs = attrs.clone(); - macro_attrs.push( - // #[builtin_macro(eii_shared_macro)] - ast::Attribute { - kind: ast::AttrKind::Normal(Box::new(ast::NormalAttr { - item: ast::AttrItem { - unsafety: ast::Safety::Default, - path: ast::Path::from_ident(Ident::new(sym::rustc_builtin_macro, span)), - args: ast::AttrArgs::Delimited(ast::DelimArgs { - dspan: DelimSpan::from_single(span), - delim: Delimiter::Parenthesis, - tokens: TokenStream::new(vec![tokenstream::TokenTree::token_alone( - token::TokenKind::Ident(sym::eii_shared_macro, token::IdentIsRaw::No), - span, - )]), - }), - tokens: None, - }, - tokens: None, - })), - id: ecx.sess.psess.attr_id_generator.mk_attr_id(), - style: ast::AttrStyle::Outer, - span, - }, - ); +/// Generate a stub macro (a bit like in core) that will roughly look like: +/// +/// ```rust, ignore, example +/// // Since this a stub macro, the actual code that expands it lives in the compiler. +/// // This attribute tells the compiler that +/// #[builtin_macro(eii_shared_macro)] +/// // the metadata to link this macro to the generated foreign item. +/// #[eii_extern_target()] +/// macro macro_name { () => {} } +/// ``` +fn generate_attribute_macro_to_implement( + ecx: &mut ExtCtxt<'_>, + span: Span, + macro_name: Ident, + foreign_item_name: Ident, + impl_unsafe: bool, + decl_span: Span, +) -> ast::Item { + let mut macro_attrs = ThinVec::new(); + + // #[builtin_macro(eii_shared_macro)] + macro_attrs.push(ecx.attr_nested_word(sym::rustc_builtin_macro, sym::eii_shared_macro, span)); - let macro_def = Box::new(ast::Item { + ast::Item { attrs: macro_attrs, id: ast::DUMMY_NODE_ID, span, @@ -305,33 +395,15 @@ fn eii_( ]), }), macro_rules: false, - // #[eii_extern_target(func.ident)] + // #[eii_extern_target(foreign_item_ident)] eii_extern_target: Some(ast::EiiExternTarget { - extern_item_path: ast::Path::from_ident(func.ident), + extern_item_path: ast::Path::from_ident(foreign_item_name), impl_unsafe, span: decl_span, }), }, ), tokens: None, - }); - - return_items.push(extern_block); - return_items.push(macro_def); - - if stmt { - return_items - .into_iter() - .map(|i| { - Annotatable::Stmt(Box::new(Stmt { - id: DUMMY_NODE_ID, - kind: StmtKind::Item(i), - span, - })) - }) - .collect() - } else { - return_items.into_iter().map(|i| Annotatable::Item(i)).collect() } } @@ -436,10 +508,10 @@ pub(crate) fn eii_shared_macro( f.eii_impls.push(EiiImpl { node_id: DUMMY_NODE_ID, + inner_span: meta_item.path.span, eii_macro_path: meta_item.path.clone(), impl_safety: meta_item.unsafety, span, - inner_span: meta_item.path.span, is_default, });