Rollup of 13 pull requests

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -26,7 +26,7 @@ use rustc_middle::mir::{
 };
 use rustc_middle::ty::print::PrintTraitRefExt as _;
 use rustc_middle::ty::{
-    self, PredicateKind, Ty, TyCtxt, TypeSuperVisitable, TypeVisitor, Upcast,
+    self, CallerBoundsIterator, PredicateKind, Ty, TyCtxt, TypeSuperVisitable, TypeVisitor, Upcast,
     suggest_constraining_type_params,
 };
 use rustc_mir_dataflow::move_paths::{Init, InitKind, InitLocation, MoveOutIndex, MovePathIndex};
@@ -285,7 +285,7 @@ impl<'diag, 'tcx> MirBorrowckCtxt<'_, 'diag, 'tcx> {
             let place = &self.move_data.move_paths[mpi].place;
             let ty = place.ty(self.body, self.infcx.tcx).ty;
 
-            if self.infcx.param_env.caller_bounds().any(|c| {
+            if self.infcx.param_env.caller_bounds().all_clauses(self.infcx.tcx).iter().any(|c| {
                 c.as_trait_clause().is_some_and(|pred| {
                     pred.skip_binder().self_ty() == ty && self.infcx.tcx.is_fn_trait(pred.def_id())
                 })

compiler/rustc_borrowck/src/type_check/free_region_relations.rs

@@ -11,7 +11,7 @@ use rustc_infer::traits::query::MirBorrowckImpliedOutlivesBounds;
 use rustc_infer::traits::query::type_op::Normalize;
 use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::traits::query::OutlivesBound;
-use rustc_middle::ty::{self, RegionVid, Ty, TypeVisitableExt};
+use rustc_middle::ty::{self, CallerBoundsIterator, RegionVid, Ty, TypeVisitableExt};
 use rustc_span::{ErrorGuaranteed, Span};
 use rustc_trait_selection::solve::NoSolution;
 use rustc_trait_selection::traits::query::type_op;
@@ -211,7 +211,7 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> {
         // Normalize the assumptions we use to borrowck the program.
         let mut constraints = vec![];
         let mut known_type_outlives_obligations = vec![];
-        for bound in param_env.caller_bounds() {
+        for bound in param_env.caller_bounds().non_trait_clauses().iter() {
             if let Some(outlives) = bound.as_type_outlives_clause() {
                 known_type_outlives_obligations.push(outlives);
             };
@@ -371,7 +371,7 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> {
         // Normalize the assumptions we use to borrowck the program.
         let mut constraints = vec![];
         let mut known_type_outlives_obligations = vec![];
-        for bound in param_env.caller_bounds() {
+        for bound in param_env.caller_bounds().non_trait_clauses().iter() {
             if let Some(outlives) = bound.as_type_outlives_clause() {
                 known_type_outlives_obligations.push(outlives);
             };

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1204,17 +1204,13 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         // FIXME(antoyo)
     }
 
-    fn store(&mut self, val: RValue<'gcc>, ptr: RValue<'gcc>, align: Align) -> RValue<'gcc> {
-        self.store_with_flags(val, ptr, align, MemFlags::empty())
-    }
-
     fn store_with_flags(
         &mut self,
         val: RValue<'gcc>,
         ptr: RValue<'gcc>,
         align: Align,
         flags: MemFlags,
-    ) -> RValue<'gcc> {
+    ) {
         let ptr = self.check_store(val, ptr);
         let destination = ptr.dereference(self.location);
         // NOTE: libgccjit does not support specifying the alignment on the assignment, so we cast
@@ -1231,10 +1227,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         let modified_destination = modified_ptr.dereference(self.location);
         self.llbb().add_assignment(self.location, modified_destination, val);
         // FIXME(antoyo): handle `MemFlags::NONTEMPORAL`.
-        // NOTE: dummy value here since it's never used. FIXME(antoyo): API should not return a value here?
         // When adding support for NONTEMPORAL, make sure to not just emit MOVNT on x86; see the
         // LLVM backend for details.
-        self.cx.context.new_rvalue_zero(self.type_i32())
     }
 
     fn atomic_store(

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -657,22 +657,17 @@ pub(crate) unsafe fn llvm_optimize<'ll>(
         None
     };
 
-    fn handle_offload<'ll>(cx: &SimpleCx<'ll>, old_fn: &'ll llvm::Value) {
+    // Wraps `old_fn` into a new kernel that takes the implicit `dyn_ptr` as its first argument.
+    // Returns the new function, or `None` if nothing was wrapped
+    fn handle_offload<'ll>(
+        cx: &SimpleCx<'ll>,
+        old_fn: &'ll llvm::Value,
+    ) -> Option<&'ll llvm::Value> {
         let old_fn_ty = cx.get_type_of_global(old_fn);
         let old_param_types = cx.func_params_types(old_fn_ty);
         let old_param_count = old_param_types.len();
         if old_param_count == 0 {
-            return;
-        }
-
-        let first_param = llvm::get_param(old_fn, 0);
-        let c_name = llvm::get_value_name(first_param);
-        let first_arg_name = str::from_utf8(&c_name).unwrap();
-        // We might call llvm_optimize (and thus this code) multiple times on the same IR,
-        // but we shouldn't add this helper ptr multiple times.
-        // FIXME(offload): This could break if the user calls his first argument `dyn_ptr`.
-        if first_arg_name == "dyn_ptr" {
-            return;
+            return None;
         }
 
         // Create the new parameter list, with ptr as the first argument
@@ -747,6 +742,7 @@ pub(crate) unsafe fn llvm_optimize<'ll>(
         }
         // Now we can re-use the old name, without name collision.
         llvm::set_value_name(new_fn, &name);
+        Some(new_fn)
     }
 
     if cgcx.target_is_like_gpu
@@ -755,10 +751,13 @@ pub(crate) unsafe fn llvm_optimize<'ll>(
         let cx = SimpleCx::new(llmod, llcx, cgcx.pointer_size);
         for func in cx.get_functions() {
             let offload_kernel = "offload-kernel";
-            if attributes::has_string_attr(func, offload_kernel) {
-                handle_offload(&cx, func);
+            if !attributes::has_string_attr(func, offload_kernel) {
+                continue;
+            }
+            match handle_offload(&cx, func) {
+                Some(new_fn) => attributes::remove_string_attr_from_llfn(new_fn, offload_kernel),
+                None => attributes::remove_string_attr_from_llfn(func, offload_kernel),
             }
-            attributes::remove_string_attr_from_llfn(func, offload_kernel);
         }
     }
 
@@ -822,8 +821,9 @@ pub(crate) unsafe fn llvm_optimize<'ll>(
     };
 
     // This assumes that we previously compiled our kernels for a gpu target, which created a
-    // `<crate>.bc` artifact. The user is supposed to provide us with a path to this artifact, we
-    // don't need any other artifacts from the previous run.
+    // `<crate>.ptx` (NVPTX) or `<crate>.elf` (AMDGPU) artifact.
+    // The user is supposed to provide us with a path to this artifact,
+    // we don't need any other artifacts from the previous run.
     if !cgcx.target_is_like_gpu && is_final_stage {
         if let Some(device_path) = config
             .offload
@@ -833,15 +833,18 @@ pub(crate) unsafe fn llvm_optimize<'ll>(
             let device_pathbuf = PathBuf::from(device_path);
             if device_pathbuf.is_relative() {
                 dcx.emit_err(crate::diagnostics::OffloadWithoutAbsPath);
-            } else if device_pathbuf.extension().and_then(|e| e.to_str()) != Some("bc") {
+            } else if !matches!(
+                device_pathbuf.extension().and_then(|e| e.to_str()),
+                Some("ptx" | "elf")
+            ) {
                 dcx.emit_err(crate::diagnostics::OffloadWrongFileName);
             } else if !device_pathbuf.exists() {
                 dcx.emit_err(crate::diagnostics::OffloadNonexistingPath);
             }
-            let device_bitcode_c = path_to_c_string(device_pathbuf.as_path());
+            let device_image_c = path_to_c_string(device_pathbuf.as_path());
             let ok = unsafe {
                 llvm::RustOffloadWrapper::get_instance()
-                    .llvm_rust_offload_wrap_images(llmod, device_bitcode_c.as_c_str())
+                    .llvm_rust_offload_wrap_images(llmod, device_image_c.as_c_str())
             };
             if !ok {
                 dcx.emit_err(crate::diagnostics::OffloadWrapImagesFailed);

compiler/rustc_codegen_llvm/src/builder.rs

@@ -905,17 +905,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         self.set_metadata_node(load, llvm::MD_nonnull, &[]);
     }
 
-    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
-        self.store_with_flags(val, ptr, align, MemFlags::empty())
-    }
-
     fn store_with_flags(
         &mut self,
         val: &'ll Value,
         ptr: &'ll Value,
         align: Align,
         flags: MemFlags,
-    ) -> &'ll Value {
+    ) {
         debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
         assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
         unsafe {
@@ -966,7 +962,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
                 let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, args.as_ptr(), args.len());
                 self.set_metadata(store, id, md);
             }
-            store
         }
     }
 

compiler/rustc_codegen_llvm/src/diagnostics.rs

@@ -73,7 +73,7 @@ pub(crate) struct RustOffloadComponentMissing {
 
 #[derive(Diagnostic)]
 #[diag(
-    "using the offload feature requires -Z offload=<Device or Host=/absolute/path/to/<crate>.bc>"
+    "using the offload feature requires -Z offload=<Device or Host=/absolute/path/to/<crate>.ptx or <crate>.elf>"
 )]
 pub(crate) struct OffloadWithoutEnable;
 
@@ -82,18 +82,20 @@ pub(crate) struct OffloadWithoutEnable;
 pub(crate) struct OffloadWithoutFatLTO;
 
 #[derive(Diagnostic)]
-#[diag("using the `-Z offload=Host=/absolute/path/to/<crate>.bc` flag requires an absolute path")]
+#[diag(
+    "using the `-Z offload=Host=/absolute/path/to/<crate>.ptx` or `<crate>.elf` flag requires an absolute path"
+)]
 pub(crate) struct OffloadWithoutAbsPath;
 
 #[derive(Diagnostic)]
 #[diag(
-    "using the `-Z offload=Host=/absolute/path/to/<crate>.bc` flag must point to a .bc file produced by the device compilation (--emit=llvm-bc)"
+    "using the `-Z offload=Host=/absolute/path/to/<crate>.ptx` or `<crate>.elf` flag must point to a .ptx or .elf file produced by the device compilation"
 )]
 pub(crate) struct OffloadWrongFileName;
 
 #[derive(Diagnostic)]
 #[diag(
-    "the given path/file to `<crate>.bc` does not exist. Did you forget to run the device compilation first?"
+    "the given path/file to `<crate>.ptx` or `<crate>.elf` does not exist. Did you forget to run the device compilation first?"
 )]
 pub(crate) struct OffloadNonexistingPath;
 

compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs

@@ -6,9 +6,8 @@ use super::ffi::{Module, Value};
 
 type LLVMRustOffloadMapperFn = unsafe extern "C" fn(&Value, &Value, *const &Value);
 type LLVMRustOffloadWrapImagesFn =
-    for<'ll> unsafe extern "C" fn(Module<'ll>, *const c_char, *const c_char) -> bool;
+    for<'ll> unsafe extern "C" fn(Module<'ll>, *const c_char) -> bool;
 
-use rustc_fs_util::path_to_c_string;
 use rustc_session::config::host_tuple;
 use rustc_session::filesearch;
 
@@ -17,7 +16,6 @@ use crate::llvm;
 pub(crate) struct RustOffloadWrapper {
     LLVMRustOffloadMapper: LLVMRustOffloadMapperFn,
     LLVMRustOffloadWrapImages: LLVMRustOffloadWrapImagesFn,
-    lld_path: Option<PathBuf>,
     // Keep the dynamic library loaded while the function pointers are used.
     _lib: libloading::Library,
 }
@@ -59,18 +57,15 @@ impl RustOffloadWrapper {
     pub(crate) unsafe fn llvm_rust_offload_wrap_images<'ll>(
         &self,
         host_m: Module<'ll>,
-        device_bitcode_path: &CStr,
+        device_image_path: &CStr,
     ) -> bool {
-        let lld_c = self.lld_path.as_deref().map(path_to_c_string).unwrap_or_default();
-        unsafe {
-            (self.LLVMRustOffloadWrapImages)(host_m, lld_c.as_ptr(), device_bitcode_path.as_ptr())
-        }
+        unsafe { (self.LLVMRustOffloadWrapImages)(host_m, device_image_path.as_ptr()) }
     }
 
     fn call_dynamic(
         sysroot: &rustc_session::config::Sysroot,
     ) -> Result<Self, RustOffloadLibraryError> {
-        let (rust_offload_path, lld_path) = Self::get_offload_and_lld_paths(sysroot)?;
+        let rust_offload_path = Self::get_offload_path(sysroot)?;
         let lib = unsafe { libloading::Library::new(rust_offload_path)? };
 
         let llvm_rust_offload_wrapper =
@@ -81,14 +76,13 @@ impl RustOffloadWrapper {
         Ok(Self {
             LLVMRustOffloadMapper: llvm_rust_offload_wrapper,
             LLVMRustOffloadWrapImages: llvm_rust_offload_wrap_images,
-            lld_path,
             _lib: lib,
         })
     }
 
-    fn get_offload_and_lld_paths(
+    fn get_offload_path(
         sysroot: &rustc_session::config::Sysroot,
-    ) -> Result<(PathBuf, Option<PathBuf>), RustOffloadLibraryError> {
+    ) -> Result<PathBuf, RustOffloadLibraryError> {
         let llvm_version_major = llvm::LLVMRustVersionMajor();
         let mut searched = Vec::new();
 
@@ -102,11 +96,7 @@ impl RustOffloadWrapper {
                 continue;
             }
 
-            let lld_path = filesearch::make_target_bin_path(root, host_tuple())
-                .join(format!("rust-lld{}", std::env::consts::EXE_SUFFIX));
-            let lld_path = lld_path.is_file().then_some(lld_path);
-
-            return Ok((rust_offload_path, lld_path));
+            return Ok(rust_offload_path);
         }
 
         Err(RustOffloadLibraryError::NotFound {

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -96,8 +96,19 @@ fn process_builtin_attrs(
             AttributeKind::LinkSection { name } => codegen_fn_attrs.link_section = Some(*name),
             AttributeKind::NoMangle(attr_span) => {
                 interesting_spans.no_mangle = Some(*attr_span);
-                if tcx.opt_item_name(did.to_def_id()).is_some() {
-                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_MANGLE;
+                if let Some(name) = tcx.opt_item_name(did.to_def_id()) {
+                    // Don't override #[export_name].
+
+                    // Also don't override #[link_name]. All places where #[link_name] is allowed
+                    // shouldn't allow #[no_mangle], so #[link_name] shouldn't be a concern here,
+                    // however currently #[no_mangle] is currently merely a warning on foreign
+                    // items rather than a hard error, so we still need to take #[no_mangle] +
+                    // #[link_name] into account.
+                    // FIXME remove this comment once #[no_mangle] on foreign items is a hard error.
+
+                    if codegen_fn_attrs.symbol_name.is_none() {
+                        codegen_fn_attrs.symbol_name = Some(name);
+                    }
                 } else {
                     tcx.dcx()
                         .span_delayed_bug(*attr_span, "no_mangle should be on a named function");
@@ -429,6 +440,8 @@ fn apply_overrides(tcx: TyCtxt<'_>, did: LocalDefId, codegen_fn_attrs: &mut Code
     }
 
     // Foreign items by default use no mangling for their symbol name.
+    // Usually, `symbol_name` is controlled by `#[link_name]`/`#[export_name]`. Here we additionally
+    // set `symbol_name` on most foreign items so that their their symbol name does not get mangled.
     if tcx.is_foreign_item(did) {
         codegen_fn_attrs.flags |= CodegenFnAttrFlags::FOREIGN_ITEM;
 
@@ -444,16 +457,21 @@ fn apply_overrides(tcx: TyCtxt<'_>, did: LocalDefId, codegen_fn_attrs: &mut Code
             //   Implementing an EII does the appropriate name resolution to make sure the implementations
             //   get the same symbol name as the *mangled* foreign item they refer to so that's all good.
         } else if codegen_fn_attrs.symbol_name.is_some() {
-            // * This can be overridden with the `#[link_name]` attribute
+            // * If the name is already set by `#[link_name]`, we don't overwrite it.
+        } else if codegen_fn_attrs.link_ordinal.is_some() {
+            // * `#[link_ordinal]` and `#[link_name]` are incompatible with each other, so
+            //   disable the implicit `#[link_name]` for foreign items below to avoid an
+            //   error.
         } else {
+            // This effectively adds an implicit `#[link_name = "<item name>"]`.
             // NOTE: there's one more exception that we cannot apply here. On wasm,
             // some items cannot be `no_mangle`.
             // However, we don't have enough information here to determine that.
             // As such, no_mangle foreign items on wasm that have the same defid as some
             // import will *still* be mangled despite this.
             //
-            // if none of the exceptions apply; apply no_mangle
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_MANGLE;
+            // if none of the exceptions apply; do not mangle the symbol name
+            codegen_fn_attrs.symbol_name = Some(tcx.item_name(did));
         }
     }
 }
@@ -552,22 +570,24 @@ fn handle_lang_items(
     // strippable by the linker.
     //
     // Additionally weak lang items have predetermined symbol names.
-    if let Some(lang_item) = lang_item
+    let link_name_override = if let Some(lang_item) = lang_item
         && let Some(link_name) = lang_item.link_name()
     {
         codegen_fn_attrs.flags |= CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL;
-        codegen_fn_attrs.symbol_name = Some(link_name);
-    }
+        Some(link_name)
+    } else {
+        None
+    };
 
-    // error when using no_mangle on a lang item item
+    // error when using no_mangle, or export_name on a lang item item
     if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL)
-        && codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_MANGLE)
+        && codegen_fn_attrs.symbol_name.is_some()
     {
         let mut err = tcx
             .dcx()
             .struct_span_err(
                 interesting_spans.no_mangle.unwrap_or_default(),
-                "`#[no_mangle]` cannot be used on internal language items",
+                "`#[no_mangle]` and `#[export_name]` cannot be used on internal language items",
             )
             .with_note("Rustc requires this item to have a specific mangled name.")
             .with_span_label(tcx.def_span(did), "should be the internal language item");
@@ -583,6 +603,10 @@ fn handle_lang_items(
         }
         err.emit();
     }
+
+    if let Some(link_name_override) = link_name_override {
+        codegen_fn_attrs.symbol_name = Some(link_name_override);
+    }
 }
 
 /// Generate the [`CodegenFnAttrs`] for an item (identified by the [`LocalDefId`]).