diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f5cc409e36..a394748abd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,10 @@ jobs: - name: Install the libraries needed to build librsvg if: ${{ contains(matrix.commands, '--projects') }} - run: sudo apt-get install libcairo2-dev libpango1.0-dev libfontconfig1-dev libfreetype-dev libharfbuzz-dev libxml2-dev libglib2.0-dev + # The runner image's package index can be stale. + run: | + sudo apt-get update + sudo apt-get install libcairo2-dev libpango1.0-dev libfontconfig1-dev libfreetype-dev libharfbuzz-dev libxml2-dev libglib2.0-dev - name: Download artifact run: curl -LO https://github.com/rust-lang/gcc/releases/latest/download/${{ matrix.libgccjit_version.gcc }} diff --git a/build_system/asm-tester/src/main.rs b/build_system/asm-tester/src/main.rs index 00ee4ac9365..fc6df1f5930 100644 --- a/build_system/asm-tester/src/main.rs +++ b/build_system/asm-tester/src/main.rs @@ -24,9 +24,15 @@ impl Config { } } "--" => { - config.rustc_flags.extend(&mut args); - // Nothing else to be read but the `break` makes it more clear. - break; + // compiletest passes its own `--out-dir` to auxiliary builds, and rustc + // rejects a second one. + while let Some(arg) = args.next() { + if arg == "--out-dir" { + args.next(); + } else { + config.rustc_flags.push(arg); + } + } } arg => return Err(format!("Unknown argument {arg:?}")), } diff --git a/example/mini_core.rs b/example/mini_core.rs index 2d5a29ceb81..2827d003fe7 100644 --- a/example/mini_core.rs +++ b/example/mini_core.rs @@ -145,6 +145,15 @@ unsafe impl Freeze for *mut T {} unsafe impl Freeze for &T {} unsafe impl Freeze for &mut T {} +#[lang = "unsafe_unpin"] +pub unsafe auto trait UnsafeUnpin {} + +unsafe impl UnsafeUnpin for PhantomData {} +unsafe impl UnsafeUnpin for *const T {} +unsafe impl UnsafeUnpin for *mut T {} +unsafe impl UnsafeUnpin for &T {} +unsafe impl UnsafeUnpin for &mut T {} + #[lang = "structural_peq"] pub trait StructuralPartialEq {} diff --git a/src/attributes.rs b/src/attributes.rs index 41db5e83bdc..9a6868ca80e 100644 --- a/src/attributes.rs +++ b/src/attributes.rs @@ -1,16 +1,39 @@ +#[cfg(feature = "master")] +use std::cell::{OnceCell, RefCell}; + #[cfg(feature = "master")] use gccjit::FnAttribute; use gccjit::Function; #[cfg(feature = "master")] use rustc_abi::{CanonAbi, InterruptKind}; #[cfg(feature = "master")] +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +#[cfg(feature = "master")] +use rustc_data_structures::graph::scc::Sccs; +#[cfg(feature = "master")] +use rustc_data_structures::graph::vec_graph::VecGraph; +#[cfg(feature = "master")] use rustc_hir::attrs::InlineAttr; use rustc_hir::attrs::InstructionSetAttr; #[cfg(feature = "master")] -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; +#[cfg(feature = "master")] +use rustc_middle::mir::interpret::{AllocId, GlobalAlloc, Scalar}; +#[cfg(feature = "master")] +use rustc_middle::mir::visit::Visitor; +#[cfg(feature = "master")] +use rustc_middle::mir::{self, Location, TerminatorKind, traversal}; #[cfg(feature = "master")] -use rustc_middle::mir::TerminatorKind; +use rustc_middle::mono::{CollectionMode, MonoItem}; use rustc_middle::ty; +#[cfg(feature = "master")] +use rustc_middle::ty::adjustment::PointerCoercion; +#[cfg(feature = "master")] +use rustc_middle::ty::layout::FnAbiOf; +#[cfg(feature = "master")] +use rustc_session::config::OptLevel; +#[cfg(feature = "master")] +use rustc_span::def_id::DefId; use rustc_target::callconv::FnAbi; #[cfg(feature = "master")] use rustc_target::spec::Arch; @@ -20,34 +43,435 @@ use crate::base; use crate::context::CodegenCx; use crate::gcc_util::to_gcc_features; -/// Checks if the function `instance` is recursively inline. -/// Returns `false` if a functions is guaranteed to be non-recursive, and `true` if it *might* be recursive. +/// Whether the GCC function being annotated gets a body in this codegen unit. +#[derive(Clone, Copy)] +pub enum FnBody { + Defined, + Declared, +} + +/// What we need to know, per codegen unit, to decide where `always_inline` is safe. #[cfg(feature = "master")] -fn recursively_inline<'gcc, 'tcx>( - cx: &CodegenCx<'gcc, 'tcx>, +#[derive(Default)] +pub struct InlineAnalysis<'tcx> { + /// Whether GCC could end up inlining the instance into itself. + in_cycle: RefCell, bool>>, + interrupt_callees: OnceCell>>, + definitions: OnceCell>>, +} + +/// How a function reaches something GCC could inline into it. +#[cfg(feature = "master")] +#[derive(Clone, Copy, PartialEq, Eq)] +enum Edge { + Call, + /// Only its address is taken; GCC may still turn that into a direct call. + Address, +} + +/// The inlining we ask GCC for, before checking whether it can honor `always_inline`. +#[cfg(feature = "master")] +fn requested_inline<'tcx>( + tcx: ty::TyCtxt<'tcx>, instance: ty::Instance<'tcx>, + attrs: &CodegenFnAttrs, +) -> InlineAttr { + let inline = if attrs.flags.contains(CodegenFnAttrFlags::NAKED) { + InlineAttr::Never + } else if attrs.inline == InlineAttr::None && instance.def.requires_inline(tcx) { + InlineAttr::Hint + } else { + attrs.inline + }; + // GCC drops `weak` from a function that is also `inline`, leaving the symbol strong, and + // the linkage is what has to survive. `inline(never)` does not conflict. + match inline { + InlineAttr::Always | InlineAttr::Hint | InlineAttr::Force { .. } + if attrs.linkage.is_some_and(base::linkage_needs_weak_attribute) => + { + InlineAttr::None + } + inline => inline, + } +} + +#[cfg(feature = "master")] +fn requests_forced_inline<'tcx>(tcx: ty::TyCtxt<'tcx>, instance: ty::Instance<'tcx>) -> bool { + matches!( + requested_inline(tcx, instance, &tcx.codegen_instance_attrs(instance.def)), + InlineAttr::Always | InlineAttr::Force { .. } + ) +} + +/// Whether GCC may inline `instance` into a caller. At `-O0` it only inlines `always_inline` +/// functions; above that, anything not marked `noinline`. +#[cfg(feature = "master")] +fn gcc_may_inline<'tcx>( + tcx: ty::TyCtxt<'tcx>, + instance: ty::Instance<'tcx>, + optimize: OptLevel, ) -> bool { - // No body, so we can't check if this is recursively inline, so we assume it is. - if !cx.tcx.is_mir_available(instance.def_id()) { - return true; + match requested_inline(tcx, instance, &tcx.codegen_instance_attrs(instance.def)) { + InlineAttr::Always | InlineAttr::Force { .. } => true, + InlineAttr::Never => false, + InlineAttr::Hint | InlineAttr::None => optimize != OptLevel::No, + } +} + +/// Functions defined in this unit, under every symbol GCC knows them by. A foreign declaration +/// naming one of these symbols is, as far as GCC is concerned, that function. +#[cfg(feature = "master")] +fn definitions<'a, 'gcc, 'tcx>( + cx: &'a CodegenCx<'gcc, 'tcx>, +) -> &'a FxHashMap<&'tcx str, ty::Instance<'tcx>> { + cx.inline_analysis.definitions.get_or_init(|| { + let tcx = cx.tcx; + let mut definitions = FxHashMap::default(); + for item in cx.codegen_unit.items().keys() { + let MonoItem::Fn(instance) = *item else { continue }; + definitions.insert(tcx.symbol_name(instance).name, instance); + // An EII implementation also gets a wrapper under the declaration's symbol, and + // `add_function_aliases` annotates it with this same instance. + let attrs = tcx.codegen_instance_attrs(instance.def); + for &(alias, ..) in &attrs.foreign_item_symbol_aliases { + definitions.insert(tcx.symbol_name(ty::Instance::mono(tcx, alias)).name, instance); + } + } + definitions + }) +} + +/// Functions whose code GCC could pull into `instance`: what it calls, and what it takes the +/// address of (fn pointers, vtables, statics), since GCC turns calls through a known address into +/// direct calls. +#[cfg(feature = "master")] +fn inline_edges<'gcc, 'tcx>( + cx: &CodegenCx<'gcc, 'tcx>, + instance: ty::Instance<'tcx>, +) -> Vec<(ty::Instance<'tcx>, Edge)> { + let tcx = cx.tcx; + // Nothing to inline without a body. Monomorphizations we import from another crate count as + // bodyless: querying them would also check them against the wrong target features. + let has_body = match instance.def { + ty::InstanceKind::Item(def_id) => tcx.is_mir_available(def_id), + ty::InstanceKind::Intrinsic(_) + | ty::InstanceKind::LlvmIntrinsic(_) + | ty::InstanceKind::Virtual(..) => false, + ty::InstanceKind::Shim(_) => true, + }; + if !has_body || !tcx.should_codegen_locally(instance) { + return Vec::new(); + } + // The collector already resolved all of this, and reported its errors. + let Ok((used, _)) = tcx.items_of_instance((instance, CollectionMode::UsedItems)) else { + return Vec::new(); + }; + // The collector doesn't say which uses are calls, and skips foreign declarations. + let body = tcx.instance_mir(instance.def); + let mut refs = BodyRefs { + tcx, + instance, + body, + definitions: definitions(cx), + calls: FxHashSet::default(), + addresses: FxHashSet::default(), + makes_vtables: false, + }; + for (block, data) in traversal::mono_reachable(body, tcx, instance) { + refs.visit_basic_block_data(block, data); } - // `expect_local` ought to never fail: we should be checking a function within this codegen unit. - let body = cx.tcx.optimized_mir(instance.def_id()); - for block in body.basic_blocks.iter() { - let Some(ref terminator) = block.terminator else { continue }; - // I assume that the recursive-inline issue applies only to functions, and not to drops. - // In principle, a recursive, `#[inline(always)]` drop could(?) exist, but I don't think it does. - let TerminatorKind::Call { ref func, .. } = terminator.kind else { continue }; - let Some((def, _args)) = func.const_fn_def() else { continue }; - // Check if the called function is recursively inline. - if matches!( - cx.tcx.codegen_fn_attrs(def).inline, - InlineAttr::Always | InlineAttr::Force { .. } - ) { - return true; + for item in used { + match item.node { + MonoItem::Fn(callee) => { + if !refs.calls.contains(&callee) { + refs.addresses.insert(callee); + } + } + MonoItem::Static(def_id) => { + let mut pending = Vec::new(); + static_initializer(tcx, def_id, &mut pending); + for function in allocated_fns(tcx, pending) { + refs.address_of(function); + } + } + MonoItem::GlobalAsm(_) => {} } } - false + // We don't work out which methods a new vtable holds, so with one around, anything might be + // in it. + if refs.makes_vtables { + refs.addresses.extend(refs.calls.drain()); + } + + // A function that is both called and has its address taken counts as the latter. + let calls = refs.calls.iter().filter(|&callee| !refs.addresses.contains(callee)); + let edges = calls + .map(|&callee| (callee, Edge::Call)) + .chain(refs.addresses.iter().map(|&callee| (callee, Edge::Address))); + let optimize = cx.sess().opts.optimize; + edges.filter(|&(callee, _)| gcc_may_inline(tcx, callee, optimize)).collect() +} + +/// What a body does that the collector doesn't tell us: which functions it calls directly and +/// which it takes the address of. Foreign declarations are mapped to what they name. +#[cfg(feature = "master")] +struct BodyRefs<'a, 'tcx> { + tcx: ty::TyCtxt<'tcx>, + instance: ty::Instance<'tcx>, + body: &'tcx mir::Body<'tcx>, + definitions: &'a FxHashMap<&'tcx str, ty::Instance<'tcx>>, + calls: FxHashSet>, + addresses: FxHashSet>, + /// Whether the body coerces something to `dyn Trait`, creating a vtable. + makes_vtables: bool, +} + +#[cfg(feature = "master")] +impl<'a, 'tcx> BodyRefs<'a, 'tcx> { + fn monomorphize>>(&self, value: T) -> T { + self.instance.instantiate_mir_and_normalize_erasing_regions( + self.tcx, + ty::TypingEnv::fully_monomorphized(), + ty::EarlyBinder::bind(self.tcx, value), + ) + } + + fn resolve(&self, fn_ty: ty::Ty<'tcx>) -> Option> { + let ty::FnDef(def_id, args) = *self.monomorphize(fn_ty).kind() else { return None }; + let typing_env = ty::TypingEnv::fully_monomorphized(); + ty::Instance::try_resolve(self.tcx, typing_env, def_id, args.no_bound_vars()?).ok()? + } + + /// The function GCC will see for `function`: the definition a foreign declaration names, or + /// nothing if that isn't in this unit or we don't generate the function here. + fn in_this_unit(&self, function: ty::Instance<'tcx>) -> Option> { + match function.def { + ty::InstanceKind::Item(def_id) if self.tcx.is_foreign_item(def_id) => { + let symbol = self.tcx.symbol_name(ty::Instance::mono(self.tcx, def_id)).name; + self.definitions.get(symbol).copied() + } + _ => self.tcx.should_codegen_locally(function).then_some(function), + } + } + + fn address_of(&mut self, function: ty::Instance<'tcx>) { + if let Some(function) = self.in_this_unit(function) { + self.addresses.insert(function); + } + } +} + +#[cfg(feature = "master")] +impl<'a, 'tcx> Visitor<'tcx> for BodyRefs<'a, 'tcx> { + fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, location: Location) { + if let TerminatorKind::Call { ref func, .. } | TerminatorKind::TailCall { ref func, .. } = + terminator.kind + && let Some(callee) = self.resolve(func.ty(self.body, self.tcx)) + && let Some(callee) = self.in_this_unit(callee) + { + self.calls.insert(callee); + } + self.super_terminator(terminator, location); + } + + fn visit_rvalue(&mut self, rvalue: &mir::Rvalue<'tcx>, location: Location) { + let typing_env = ty::TypingEnv::fully_monomorphized(); + if let mir::Rvalue::Cast(mir::CastKind::PointerCoercion(coercion, _), ref operand, target) = + *rvalue + { + let source = self.monomorphize(operand.ty(self.body, self.tcx)); + match (coercion, *source.kind()) { + (PointerCoercion::ReifyFnPointer(_), ty::FnDef(def_id, args)) => { + if let Some(args) = args.no_bound_vars() + && let Some(function) = + ty::Instance::resolve_for_fn_ptr(self.tcx, typing_env, def_id, args) + { + self.address_of(function); + } + } + (PointerCoercion::ClosureFnPointer(_), ty::Closure(def_id, args)) => { + let kind = ty::ClosureKind::FnOnce; + self.address_of(ty::Instance::resolve_closure(self.tcx, def_id, args, kind)); + } + (PointerCoercion::Unsize, _) => { + let target = self.monomorphize(target); + self.makes_vtables |= target.walk().any(|arg| { + arg.as_type().is_some_and(|ty| matches!(ty.kind(), ty::Dynamic(..))) + }); + } + _ => {} + } + } + self.super_rvalue(rvalue, location); + } + + fn visit_const_operand(&mut self, constant: &mir::ConstOperand<'tcx>, _: Location) { + let typing_env = ty::TypingEnv::fully_monomorphized(); + let Ok(value) = + self.monomorphize(constant.const_).eval(self.tcx, typing_env, constant.span) + else { + return; + }; + let root = match value { + mir::ConstValue::Scalar(Scalar::Ptr(ptr, _)) => ptr.provenance.alloc_id(), + mir::ConstValue::Indirect { alloc_id, .. } + | mir::ConstValue::Slice { alloc_id, .. } => alloc_id, + mir::ConstValue::Scalar(Scalar::Int(_)) | mir::ConstValue::ZeroSized => return, + }; + for function in allocated_fns(self.tcx, vec![root]) { + self.address_of(function); + } + } +} + +#[cfg(feature = "master")] +fn static_initializer(tcx: ty::TyCtxt<'_>, def_id: DefId, pending: &mut Vec) { + if tcx.is_foreign_item(def_id) || !tcx.should_codegen_locally(ty::Instance::mono(tcx, def_id)) { + return; + } + if let Ok(alloc) = tcx.eval_static_initializer(def_id) { + pending.extend(alloc.inner().provenance().ptrs().values().map(|prov| prov.alloc_id())); + } +} + +/// Functions whose addresses are stored in these allocations, or in anything they point to. +/// GCC can fold loads from read-only data, so a call through a static fn table can become a +/// direct call. +#[cfg(feature = "master")] +fn allocated_fns<'tcx>( + tcx: ty::TyCtxt<'tcx>, + mut pending: Vec, +) -> Vec> { + let mut functions = Vec::new(); + let mut seen = FxHashSet::default(); + while let Some(alloc_id) = pending.pop() { + if !seen.insert(alloc_id) { + continue; + } + match tcx.global_alloc(alloc_id) { + GlobalAlloc::Function { instance, .. } => functions.push(instance), + GlobalAlloc::Memory(alloc) => pending + .extend(alloc.inner().provenance().ptrs().values().map(|prov| prov.alloc_id())), + GlobalAlloc::VTable(ty, dyn_ty) => pending.push( + tcx.vtable_allocation(( + ty, + dyn_ty + .principal() + .map(|principal| tcx.instantiate_bound_regions_with_erased(principal)), + )), + ), + GlobalAlloc::Static(nested) => static_initializer(tcx, nested, &mut pending), + GlobalAlloc::TypeId { .. } => {} + } + } + functions +} + +/// Whether GCC could end up inlining `root` into itself. +/// +/// Classifies everything reachable from `root` at once. Only functions on a cycle are affected: +/// once they lose `always_inline`, callers that merely reach the cycle can keep it. +#[cfg(feature = "master")] +fn in_inline_cycle<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, root: ty::Instance<'tcx>) -> bool { + if let Some(&in_cycle) = cx.inline_analysis.in_cycle.borrow().get(&root) { + return in_cycle; + } + + // Collect the part of the graph reachable from `root` that hasn't been classified yet. + // Classified functions can be left out: their SCC is complete, so nothing new can join it. + let mut nodes = vec![root]; + let mut index = FxHashMap::from_iter([(root, 0)]); + let mut edges = Vec::new(); + { + let classified = cx.inline_analysis.in_cycle.borrow(); + let mut caller = 0; + while let Some(&instance) = nodes.get(caller) { + for (callee, edge) in inline_edges(cx, instance) { + if classified.contains_key(&callee) { + continue; + } + let callee = *index.entry(callee).or_insert_with(|| { + nodes.push(callee); + nodes.len() - 1 + }); + edges.push((caller, callee, edge)); + } + caller += 1; + } + } + let forced: Vec = + nodes.iter().map(|&node| requests_forced_inline(cx.tcx, node)).collect(); + + // GCC rejects any cycle made only of `always_inline` functions. A plain function in the middle + // breaks it up as long as everything is a direct call: GCC inlines those early and never + // tries to inline the plain function into itself. + let forced_edges = edges + .iter() + .filter(|&&(from, to, _)| forced[from] && forced[to]) + .map(|&(from, to, _)| (from, to)); + let mut in_cycle = in_nontrivial_scc(nodes.len(), forced_edges.collect()); + + // When optimizing, GCC also inlines plain functions, and can then resolve an address taken in + // one of them into a direct call. If that call leads back to a function it was inlined into, + // GCC rejects it, so the function whose address is taken loses `always_inline`. + if cx.sess().opts.optimize != OptLevel::No { + let all = Sccs::::new(&VecGraph::::new( + nodes.len(), + edges.iter().map(|&(from, to, _)| (from, to)).collect(), + )); + for &(from, to, edge) in &edges { + if edge == Edge::Address && forced[to] && all.scc(from) == all.scc(to) { + in_cycle[to] = true; + } + } + } + + let mut classified = cx.inline_analysis.in_cycle.borrow_mut(); + classified.extend(nodes.iter().copied().zip(in_cycle.iter().copied())); + in_cycle[0] +} + +/// Which nodes lie on a cycle: a strongly connected component with more than one node, or a +/// node with an edge to itself. +#[cfg(feature = "master")] +fn in_nontrivial_scc(num_nodes: usize, edges: Vec<(usize, usize)>) -> Vec { + let mut in_cycle = vec![false; num_nodes]; + for &(from, to) in &edges { + in_cycle[from] |= from == to; + } + let sccs = Sccs::::new(&VecGraph::::new(num_nodes, edges)); + let mut sizes = vec![0usize; sccs.num_sccs()]; + for node in 0..num_nodes { + sizes[sccs.scc(node)] += 1; + } + for (node, in_cycle) in in_cycle.iter_mut().enumerate() { + *in_cycle |= sizes[sccs.scc(node)] > 1; + } + in_cycle +} + +/// `always_inline` functions that an `x86-interrupt` handler in this unit could inline. Handlers +/// are built with `general-regs-only`, and GCC refuses to inline normally-built code into them. +#[cfg(feature = "master")] +fn interrupt_callees<'a, 'gcc, 'tcx>( + cx: &'a CodegenCx<'gcc, 'tcx>, +) -> &'a FxHashSet> { + cx.inline_analysis.interrupt_callees.get_or_init(|| { + cx.codegen_unit + .items() + .keys() + .filter_map(|item| match *item { + MonoItem::Fn(instance) => Some(instance), + MonoItem::Static(_) | MonoItem::GlobalAsm(_) => None, + }) + .filter(|&instance| { + is_x86_interrupt(Some(cx.fn_abi_of_instance(instance, ty::List::empty()))) + }) + .flat_map(|handler| inline_edges(cx, handler)) + .map(|(callee, _)| callee) + .filter(|&callee| requests_forced_inline(cx.tcx, callee)) + .collect() + }) } /// Get GCC attribute for the provided inline heuristic, attached to `instance`. @@ -57,25 +481,21 @@ fn inline_attr<'gcc, 'tcx>( cx: &CodegenCx<'gcc, 'tcx>, inline: InlineAttr, instance: ty::Instance<'tcx>, + body: FnBody, + fn_abi: Option<&FnAbi<'tcx, ty::Ty<'tcx>>>, ) -> Option> { match inline { - InlineAttr::Always => { - // We can't simply always return `always_inline` unconditionally. - // It is *NOT A HINT* and does not work for recursive functions. - // - // So, it can only be applied *if*: - // The current function does not call any functions marked `#[inline(always)]`. - // - // That prevents issues steming from recursive `#[inline(always)]` at a *relatively* small cost. - // We *only* need to check all the terminators of a function marked with this attribute. - if recursively_inline(cx, instance) { - Some(FnAttribute::Inline) - } else { - Some(FnAttribute::AlwaysInline) - } + InlineAttr::Always | InlineAttr::Force { .. } => { + // `always_inline` is *not* a hint: GCC fails the build when it can't inline, so only + // ask for it when it can. + let can_inline = matches!(body, FnBody::Defined) + // GCC never inlines functions that use va_arg. + && !fn_abi.is_some_and(|fn_abi| fn_abi.c_variadic) + && !interrupt_callees(cx).contains(&instance) + && !in_inline_cycle(cx, instance); + Some(if can_inline { FnAttribute::AlwaysInline } else { FnAttribute::Inline }) } InlineAttr::Hint => Some(FnAttribute::Inline), - InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline), InlineAttr::Never => { if cx.sess().target.arch != Arch::AmdGpu { Some(FnAttribute::NoInline) @@ -101,6 +521,7 @@ pub fn from_fn_attrs<'gcc, 'tcx>( cx: &CodegenCx<'gcc, 'tcx>, #[cfg_attr(not(feature = "master"), expect(unused_variables))] func: Function<'gcc>, instance: ty::Instance<'tcx>, + #[cfg_attr(not(feature = "master"), expect(unused_variables))] body: FnBody, #[cfg_attr(not(feature = "master"), expect(unused_variables))] fn_abi: Option< &FnAbi<'tcx, ty::Ty<'tcx>>, >, @@ -109,26 +530,8 @@ pub fn from_fn_attrs<'gcc, 'tcx>( #[cfg(feature = "master")] { - let inline = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { - InlineAttr::Never - } else if codegen_fn_attrs.inline == InlineAttr::None - && instance.def.requires_inline(cx.tcx) - { - InlineAttr::Hint - } else { - codegen_fn_attrs.inline - }; - // GCC drops `weak` from a function that is also `inline`, leaving the symbol strong, and - // the linkage is what has to survive. `inline(never)` does not conflict. - let inline = match inline { - InlineAttr::Always | InlineAttr::Hint | InlineAttr::Force { .. } - if codegen_fn_attrs.linkage.is_some_and(base::linkage_needs_weak_attribute) => - { - InlineAttr::None - } - inline => inline, - }; - if let Some(attr) = inline_attr(cx, inline, instance) { + let inline = requested_inline(cx.tcx, instance, &codegen_fn_attrs); + if let Some(attr) = inline_attr(cx, inline, instance, body, fn_abi) { if let FnAttribute::AlwaysInline = attr { func.add_attribute(FnAttribute::Inline); } diff --git a/src/callee.rs b/src/callee.rs index d3f412180da..b2bd28069d4 100644 --- a/src/callee.rs +++ b/src/callee.rs @@ -4,7 +4,7 @@ use gccjit::{Function, FunctionType}; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; -use crate::attributes; +use crate::attributes::{self, FnBody}; use crate::context::CodegenCx; /// Codegens a reference to a fn/method item, monomorphizing and @@ -70,7 +70,7 @@ pub fn get_fn<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, instance: Instance<'tcx>) cx.linkage.set(FunctionType::Extern); let func = cx.declare_fn(sym, fn_abi); - attributes::from_fn_attrs(cx, func, instance, Some(fn_abi)); + attributes::from_fn_attrs(cx, func, instance, FnBody::Declared, Some(fn_abi)); #[cfg(feature = "master")] { diff --git a/src/context.rs b/src/context.rs index 5f342e8b2dc..0f01df84f25 100644 --- a/src/context.rs +++ b/src/context.rs @@ -25,6 +25,8 @@ use rustc_target::spec::{HasTargetSpec, HasX86AbiOpt, Target, TlsModel, X86Abi}; #[cfg(feature = "master")] use crate::abi::conv_to_fn_attribute; +#[cfg(feature = "master")] +use crate::attributes::InlineAnalysis; use crate::callee::get_fn; use crate::common::SignType; use crate::type_::StructTypeKey; @@ -49,6 +51,8 @@ pub struct CodegenCx<'gcc, 'tcx> { pub functions: RefCell>>, pub intrinsics: RefCell>>, + #[cfg(feature = "master")] + pub inline_analysis: InlineAnalysis<'tcx>, pub tls_model: gccjit::TlsModel, @@ -265,6 +269,8 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { function_address_names: Default::default(), functions: RefCell::new(functions), intrinsics: RefCell::new(FxHashMap::default()), + #[cfg(feature = "master")] + inline_analysis: Default::default(), tls_model, diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs index 4d2590ac81e..c50e5e9b16b 100644 --- a/src/intrinsic/mod.rs +++ b/src/intrinsic/mod.rs @@ -609,7 +609,13 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc self.on_stack_function_params.borrow_mut().insert(func, FxHashSet::default()); - crate::attributes::from_fn_attrs(self, func, instance, None); + crate::attributes::from_fn_attrs( + self, + func, + instance, + crate::attributes::FnBody::Declared, + None, + ); func }; diff --git a/src/mono_item.rs b/src/mono_item.rs index d8170fbb085..96077657c52 100644 --- a/src/mono_item.rs +++ b/src/mono_item.rs @@ -11,6 +11,7 @@ use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use rustc_span::bug; +use crate::attributes::FnBody; use crate::consts::const_alloc_type; use crate::context::CodegenCx; use crate::type_of::LayoutGccExt; @@ -188,7 +189,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { self.linkage.set(base::linkage_to_gcc(linkage)); let fn_decl = self.declare_fn(symbol_name, fn_abi); - attributes::from_fn_attrs(self, fn_decl, instance, Some(fn_abi)); + attributes::from_fn_attrs(self, fn_decl, instance, FnBody::Defined, Some(fn_abi)); #[cfg(feature = "master")] if base::linkage_needs_weak_attribute(linkage) { diff --git a/tests/asm/always_inline.rs b/tests/asm/always_inline.rs new file mode 100644 index 00000000000..5352ffc657d --- /dev/null +++ b/tests/asm/always_inline.rs @@ -0,0 +1,31 @@ +//@ assembly-output: emit-asm +//@ only-x86_64 +//@ compile-flags: -Copt-level=0 + +#![crate_type = "lib"] +#![no_std] + +// At -O0 a plain `inline` hint does nothing, so every function in this chain has to keep +// always_inline. +#[inline(always)] +fn leaf(x: u32) -> u32 { + x ^ 0xa5a5 +} + +#[inline(always)] +fn mid(x: u32) -> u32 { + leaf(x).wrapping_mul(3) +} + +#[inline(always)] +fn top(x: u32) -> u32 { + mid(x).wrapping_add(7) +} + +// CHECK-LABEL: {{^"?_?}}entry{{"?}}: +// CHECK-NOT: call +// CHECK: ret +#[no_mangle] +pub fn entry(x: u32) -> u32 { + top(x) +} diff --git a/tests/asm/always_inline_address_cycles.rs b/tests/asm/always_inline_address_cycles.rs new file mode 100644 index 00000000000..4a1297c0e40 --- /dev/null +++ b/tests/asm/always_inline_address_cycles.rs @@ -0,0 +1,53 @@ +//@ assembly-output: emit-asm +//@ only-x86_64 +//@ compile-flags: -Copt-level=1 + +#![crate_type = "lib"] +#![no_std] + +// From -O1, GCC folds loads from read-only data, so calling through a static table or a vtable +// becomes a direct call, which closes the cycle. + +static TABLE: [fn(u32) -> u32; 1] = [via_table]; + +#[inline(always)] +fn via_table(n: u32) -> u32 { + table_caller(n) +} + +#[inline(always)] +fn table_caller(n: u32) -> u32 { + if n == 0 { + return 0; + } + TABLE[0](n - 1) + 1 +} + +trait Step { + fn step(&self, n: u32) -> u32; +} + +struct S; + +impl Step for S { + #[inline(always)] + fn step(&self, n: u32) -> u32 { + dyn_caller(n) + } +} + +#[inline(always)] +fn dyn_caller(n: u32) -> u32 { + if n == 0 { + return 0; + } + let step: &dyn Step = &S; + step.step(n - 1) + 1 +} + +// CHECK-LABEL: {{^"?_?}}entry{{"?}}: +// CHECK: ret +#[no_mangle] +pub fn entry(n: u32) -> u32 { + via_table(n) + dyn_caller(n) +} diff --git a/tests/asm/always_inline_call_and_address.rs b/tests/asm/always_inline_call_and_address.rs new file mode 100644 index 00000000000..9957b8c71bc --- /dev/null +++ b/tests/asm/always_inline_call_and_address.rs @@ -0,0 +1,31 @@ +//@ assembly-output: emit-asm +//@ only-x86_64 +//@ compile-flags: -Copt-level=2 + +#![crate_type = "lib"] +#![no_std] + +// `helper` both calls `forced` and returns its address. Once GCC inlines `helper`, that address +// turns into direct calls back into `forced`, so `forced` must lose always_inline. + +#[inline(always)] +fn forced(n: u32) -> u32 { + if n == 0 { + return 0; + } + let f = helper(); + f(n - 1).wrapping_add(f(n / 2)).wrapping_add(n) +} + +#[inline] +fn helper() -> fn(u32) -> u32 { + let _ = forced(0); + forced +} + +// CHECK-LABEL: {{^"?_?}}entry{{"?}}: +// CHECK: .size +#[no_mangle] +pub fn entry(n: u32) -> u32 { + forced(n) +} diff --git a/tests/asm/always_inline_cycle_caller.rs b/tests/asm/always_inline_cycle_caller.rs new file mode 100644 index 00000000000..1bd26e842e4 --- /dev/null +++ b/tests/asm/always_inline_cycle_caller.rs @@ -0,0 +1,27 @@ +//@ assembly-output: emit-asm +//@ only-x86_64 +//@ compile-flags: -Copt-level=0 + +#![crate_type = "lib"] +#![no_std] + +#[inline(always)] +fn countdown(n: u32) -> u32 { + if n == 0 { 0 } else { countdown(n - 1) } +} + +// Reaching a cycle is not being on one: only `countdown` loses always_inline. +#[inline(always)] +fn wrapper(n: u32) -> u32 { + countdown(n) +} + +// CHECK-LABEL: {{^"?_?}}entry{{"?}}: +// CHECK-NOT: {{call.*wrapper}} +// CHECK: {{call.*countdown}} +// CHECK-NOT: {{call.*wrapper}} +// CHECK: ret +#[no_mangle] +pub fn entry(n: u32) -> u32 { + wrapper(n) +} diff --git a/tests/asm/always_inline_foreign_decls.rs b/tests/asm/always_inline_foreign_decls.rs new file mode 100644 index 00000000000..5ef1cb94669 --- /dev/null +++ b/tests/asm/always_inline_foreign_decls.rs @@ -0,0 +1,43 @@ +//@ assembly-output: emit-asm +//@ only-x86_64 +//@ compile-flags: -Copt-level=0 + +#![crate_type = "lib"] +#![no_std] +#![feature(extern_item_impls)] +#![allow(incomplete_features, unused_attributes)] + +// Being reachable from a foreign declaration only matters when it closes a cycle. + +#[eii] +fn hook(n: u32) -> u32; + +#[hook] +#[inline(always)] +fn hook_impl(n: u32) -> u32 { + n.wrapping_mul(3) +} + +unsafe extern "C" { + #[link_name = "exported"] + fn exported_alias(n: u32) -> u32; +} + +#[no_mangle] +#[inline(always)] +pub extern "C" fn exported(n: u32) -> u32 { + n.wrapping_add(5) +} + +#[no_mangle] +pub fn through_decls(n: u32) -> u32 { + hook(n).wrapping_add(unsafe { exported_alias(n) }) +} + +// CHECK-LABEL: {{^"?_?}}entry{{"?}}: +// CHECK-NOT: call +// CHECK: ret +#[no_mangle] +pub fn entry(n: u32) -> u32 { + hook_impl(n).wrapping_add(exported(n)) +} diff --git a/tests/asm/always_inline_helper_recursion.rs b/tests/asm/always_inline_helper_recursion.rs new file mode 100644 index 00000000000..91cdbe42bfd --- /dev/null +++ b/tests/asm/always_inline_helper_recursion.rs @@ -0,0 +1,34 @@ +//@ assembly-output: emit-asm +//@ only-x86_64 +//@ compile-flags: -Copt-level=2 + +#![crate_type = "lib"] +#![no_std] + +// Recursing through a plain helper is fine for GCC as long as every step is a direct call, so +// `step` keeps always_inline. +#[inline(always)] +fn step(n: u32) -> u32 { + if n == 0 { + return 0; + } + let acc = n.wrapping_mul(0x9e3779b1).rotate_left(5); + helper(n - 1).wrapping_add(helper(n / 2)).wrapping_add(acc) +} + +fn helper(n: u32) -> u32 { + step(n) +} + +#[no_mangle] +pub fn other(n: u32) -> u32 { + step(n) +} + +// CHECK-LABEL: {{^"?_?}}entry{{"?}}: +// CHECK-NOT: {{(call|jmp).*step}} +// CHECK: .size +#[no_mangle] +pub fn entry(n: u32) -> u32 { + step(n.wrapping_add(1)) +} diff --git a/tests/asm/always_inline_imported_mono.rs b/tests/asm/always_inline_imported_mono.rs new file mode 100644 index 00000000000..dbbfdc3de9e --- /dev/null +++ b/tests/asm/always_inline_imported_mono.rs @@ -0,0 +1,25 @@ +//@ assembly-output: emit-asm +//@ only-x86_64 +//@ compile-flags: -Copt-level=1 +//@ aux-build: avx_generic.rs + +#![crate_type = "lib"] +#![no_std] + +// `imported::<()>` comes from the upstream crate, built with avx. We only declare it here, so its +// body must not be checked against our target features. +extern crate avx_generic; + +static TABLE: [fn(); 1] = [avx_generic::imported::<()>]; + +#[inline(always)] +fn wrapper() { + TABLE[0]() +} + +// CHECK-LABEL: {{^"?_?}}entry{{"?}}: +// CHECK: ret +#[no_mangle] +pub fn entry() { + wrapper() +} diff --git a/tests/asm/auxiliary/avx_generic.rs b/tests/asm/auxiliary/avx_generic.rs new file mode 100644 index 00000000000..7bdd841ef45 --- /dev/null +++ b/tests/asm/auxiliary/avx_generic.rs @@ -0,0 +1,21 @@ +//@ no-prefer-dynamic +//@ compile-flags: -Copt-level=1 -Ctarget-feature=+avx + +#![crate_type = "rlib"] +#![no_std] +#![feature(simd_ffi)] +#![allow(improper_ctypes)] + +use core::arch::x86_64::__m256; + +unsafe extern "C" { + fn takes_avx(value: __m256); +} + +pub fn imported() { + unsafe { takes_avx(core::mem::zeroed()) } +} + +pub fn instantiate() { + imported::<()>(); +} diff --git a/tests/compile/always_inline_eii.rs b/tests/compile/always_inline_eii.rs new file mode 100644 index 00000000000..3300fd6c4c0 --- /dev/null +++ b/tests/compile/always_inline_eii.rs @@ -0,0 +1,29 @@ +// Compiler: + +// To GCC, calling an EII declaration from its own implementation is a plain self-call. + +#![crate_type = "lib"] +#![feature(extern_item_impls)] +#![allow(incomplete_features, unused_attributes)] + +#[eii] +fn callback(n: u32) -> u32; + +#[inline(always)] +fn leaf(n: u32) -> u32 { + n - 1 +} + +#[callback] +#[inline(always)] +fn implementation(n: u32) -> u32 { + if n == 0 { + return 0; + } + callback(leaf(n)) + 1 +} + +#[unsafe(no_mangle)] +pub fn entry(n: u32) -> u32 { + implementation(n) +} diff --git a/tests/compile/always_inline_extern_decl.rs b/tests/compile/always_inline_extern_decl.rs new file mode 100644 index 00000000000..c22bea97beb --- /dev/null +++ b/tests/compile/always_inline_extern_decl.rs @@ -0,0 +1,15 @@ +// Compiler: + +// A declaration has no body to inline, so it must not get always_inline. + +#![crate_type = "lib"] +#![allow(unused_attributes)] + +unsafe extern "C" { + #[inline(always)] + fn abs(x: i32) -> i32; +} + +pub fn entry(x: i32) -> i32 { + unsafe { abs(x) } +} diff --git a/tests/compile/always_inline_symbol_alias.rs b/tests/compile/always_inline_symbol_alias.rs new file mode 100644 index 00000000000..ac402c3035e --- /dev/null +++ b/tests/compile/always_inline_symbol_alias.rs @@ -0,0 +1,29 @@ +// Compiler: + +// `alias` is `recurse` under another name, so the Rust call graph can't see this recursion. + +#![crate_type = "lib"] +#![allow(unused_attributes)] + +unsafe extern "C" { + #[link_name = "recurse"] + fn alias(n: u32) -> u32; +} + +#[inline(always)] +fn leaf(n: u32) -> u32 { + n - 1 +} + +#[unsafe(no_mangle)] +#[inline(always)] +pub extern "C" fn recurse(n: u32) -> u32 { + if n == 0 { + return 0; + } + unsafe { alias(leaf(n)) + 1 } +} + +pub fn entry(n: u32) -> u32 { + recurse(n) +} diff --git a/tests/compile/always_inline_variadic.rs b/tests/compile/always_inline_variadic.rs new file mode 100644 index 00000000000..7c262e24e91 --- /dev/null +++ b/tests/compile/always_inline_variadic.rs @@ -0,0 +1,19 @@ +// Compiler: + +// GCC won't inline a function that uses va_arg, cycle or not. + +#![crate_type = "lib"] + +#[inline(always)] +fn helper(n: u32) -> u32 { + n + 1 +} + +#[inline(always)] +pub unsafe extern "C" fn variadic(n: u32, mut args: ...) -> u32 { + helper(n) + unsafe { args.next_arg::() } +} + +pub fn entry(n: u32) -> u32 { + unsafe { variadic(n, n) } +} diff --git a/tests/compile/always_inline_x86_interrupt.rs b/tests/compile/always_inline_x86_interrupt.rs new file mode 100644 index 00000000000..d3f3ead5006 --- /dev/null +++ b/tests/compile/always_inline_x86_interrupt.rs @@ -0,0 +1,27 @@ +// Compiler: + +// Interrupt handlers are built with general-regs-only, and GCC won't inline normally-built code +// into them. + +#![feature(abi_x86_interrupt)] +#![crate_type = "lib"] + +static mut STATE: u64 = 0; + +#[inline(always)] +fn leaf(x: u64) -> u64 { + x.wrapping_add(1) +} + +#[inline(always)] +fn wrapper(x: u64) -> u64 { + leaf(x) +} + +pub extern "x86-interrupt" fn via_wrapper(_frame: u64) { + unsafe { STATE = wrapper(STATE) } +} + +pub extern "x86-interrupt" fn direct(_frame: u64) { + unsafe { STATE = leaf(STATE) } +} diff --git a/tests/lang_tests.rs b/tests/lang_tests.rs index 7ec0ab877b0..c149518c12d 100644 --- a/tests/lang_tests.rs +++ b/tests/lang_tests.rs @@ -286,6 +286,9 @@ fn compile_tests(tempdir: PathBuf, c_objects_dir: PathBuf, current_dir: String) "global_asm_nul_byte.rs", "naked_asm_nul_byte.rs", "x86_interrupt_first_arg_byval.rs", + "always_inline_x86_interrupt.rs", + // C-variadic definitions are still unstable on m68k. + "always_inline_variadic.rs", ], ); } diff --git a/tests/run/always_inline.rs b/tests/run/always_inline.rs index ebd741ee090..22de6b25c43 100644 --- a/tests/run/always_inline.rs +++ b/tests/run/always_inline.rs @@ -3,7 +3,7 @@ // Run-time: // status: 0 -#![feature(no_core)] +#![feature(no_core, stmt_expr_attributes)] #![no_std] #![no_core] #![no_main] @@ -44,10 +44,74 @@ fn fib_a(n: u8) -> u8 { fib_b(n - 1) + fib_b(n - 2) } +// These cycles only show up after monomorphization: the generic callers call +// `::step` and `::call_mut`, not the impls. +trait Step { + fn step(n: u8) -> u8; +} + +struct S; + +impl Step for S { + #[inline(always)] + fn step(n: u8) -> u8 { + if n == 0 { + return 0; + } + drive::(n - 1) + 1 + } +} + +#[inline(always)] +fn drive(n: u8) -> u8 { + T::step(n) +} + +#[inline(always)] +fn apply u8>(mut f: F, n: u8) -> u8 { + f(n) +} + +#[inline(always)] +fn countdown(n: u8) -> u8 { + if n == 0 { + return 0; + } + apply( + #[inline(always)] + |m| countdown(m), + n - 1, + ) + 1 +} + +// GCC turns a call through a known fn pointer into a direct call, which closes the cycle. +#[inline(always)] +fn via_pointer(n: u8) -> u8 { + if n == 0 { + return 0; + } + let f: fn(u8) -> u8 = via_direct; + f(n - 1) + 1 +} + +#[inline(always)] +fn via_direct(n: u8) -> u8 { + via_pointer(n) +} + #[no_mangle] extern "C" fn main(argc: i32, _argv: *const *const u8) -> i32 { if fib(2) != fib_a(2) { intrinsics::abort(); } + if drive::(3) != 3 { + intrinsics::abort(); + } + if countdown(3) != 3 { + intrinsics::abort(); + } + if via_direct(3) != 3 { + intrinsics::abort(); + } 0 }