Merge to tag jdk-25.0.5+3

src/hotspot/cpu/aarch64/aarch64.ad

@@ -3339,36 +3339,36 @@ encode %{
   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
   // CompareAndSwap sequence to serve as a barrier on acquiring a
   // lock.
-  enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
+  enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegL oldval, iRegL newval) %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
                Assembler::xword, /*acquire*/ true, /*release*/ true,
                /*weak*/ false, noreg);
   %}
 
-  enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
+  enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegI oldval, iRegI newval) %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
                Assembler::word, /*acquire*/ true, /*release*/ true,
                /*weak*/ false, noreg);
   %}
 
-  enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
+  enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegI oldval, iRegI newval) %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
                Assembler::halfword, /*acquire*/ true, /*release*/ true,
                /*weak*/ false, noreg);
   %}
 
-  enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
+  enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegI oldval, iRegI newval) %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
                Assembler::byte, /*acquire*/ true, /*release*/ true,
                /*weak*/ false, noreg);
   %}
 
   // auxiliary used for CompareAndSwapX to set result register
-  enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
+  enc_class aarch64_enc_cset_eq(iRegI res) %{
     Register res_reg = as_Register($res$$reg);
     __ cset(res_reg, Assembler::EQ);
   %}
@@ -8267,7 +8267,7 @@ instruct castVVMask(pRegGov dst)
 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 // can't match them
 
-instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
+instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 
   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
@@ -8285,7 +8285,7 @@ instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoS
   ins_pipe(pipe_slow);
 %}
 
-instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
+instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 
   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
@@ -8303,7 +8303,7 @@ instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoS
   ins_pipe(pipe_slow);
 %}
 
-instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
+instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 
   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
@@ -8321,7 +8321,7 @@ instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoS
   ins_pipe(pipe_slow);
 %}
 
-instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
+instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 
   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
@@ -8358,7 +8358,7 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval
   ins_pipe(pipe_slow);
 %}
 
-instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
+instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 
   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
   predicate(n->as_LoadStore()->barrier_data() == 0);
@@ -8379,7 +8379,7 @@ instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoS
 
 // alternative CompareAndSwapX when we are eliding barriers
 
-instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
+instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 
   predicate(needs_acquiring_load_exclusive(n));
   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
@@ -8398,7 +8398,7 @@ instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegI
   ins_pipe(pipe_slow);
 %}
 
-instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
+instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 
   predicate(needs_acquiring_load_exclusive(n));
   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
@@ -8417,7 +8417,7 @@ instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegI
   ins_pipe(pipe_slow);
 %}
 
-instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
+instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 
   predicate(needs_acquiring_load_exclusive(n));
   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
@@ -8436,7 +8436,7 @@ instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegI
   ins_pipe(pipe_slow);
 %}
 
-instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
+instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 
   predicate(needs_acquiring_load_exclusive(n));
   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
@@ -8474,7 +8474,7 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP new
   ins_pipe(pipe_slow);
 %}
 
-instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
+instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 
   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
   match(Set res (CompareAndSwapN mem (Binary oldval newval)));

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

@@ -56,6 +56,8 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
       __ cbz(count, done);
 
       // Is GC active?
+      assert(!saved_regs.contains(rscratch1), "Sanity: about to clobber rscratch1");
+      assert(!saved_regs.contains(rscratch2), "Sanity: about to clobber rscratch2");
       Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
       __ ldrb(rscratch1, gc_state);
       if (ShenandoahSATBBarrier && dest_uninitialized) {
@@ -66,13 +68,26 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
         __ br(Assembler::EQ, done);
       }
 
-      __ push(saved_regs, sp);
+      __ push_call_clobbered_registers();
+      // If arguments are not in proper places, shuffle them.
+      // Doing this via the stack is the most straight-forward way to avoid
+      // accidentally smashing any register.
+      if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
+        __ push(RegSet::of(src), sp);
+        __ push(RegSet::of(dst), sp);
+        __ push(RegSet::of(count), sp);
+        __ pop(RegSet::of(c_rarg2), sp);
+        __ pop(RegSet::of(c_rarg1), sp);
+        __ pop(RegSet::of(c_rarg0), sp);
+      }
+      address target = nullptr;
       if (UseCompressedOops) {
-        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop), src, dst, count);
+        target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
       } else {
-        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count);
+        target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
       }
-      __ pop(saved_regs, sp);
+      __ call_VM_leaf(target, 3);
+      __ pop_call_clobbered_registers();
       __ bind(done);
     }
   }
@@ -90,15 +105,8 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
                                                  Register pre_val,
                                                  Register thread,
                                                  Register tmp1,
-                                                 Register tmp2,
-                                                 bool tosca_live,
-                                                 bool expand_call) {
+                                                 Register tmp2) {
   assert(ShenandoahSATBBarrier, "Should be checked by caller");
-
-  // If expand_call is true then we expand the call_VM_leaf macro
-  // directly to skip generating the check by
-  // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
-
   assert(thread == rthread, "must be");
 
   Label done;
@@ -117,7 +125,12 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
 
   // Do we need to load the previous value?
   if (obj != noreg) {
-    __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+    if (UseCompressedOops) {
+      __ ldrw(pre_val, Address(obj, 0));
+      __ decode_heap_oop(pre_val);
+    } else {
+      __ ldr(pre_val, Address(obj, 0));
+    }
   }
 
   // Is the previous value null?
@@ -141,33 +154,17 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
   __ b(done);
 
   __ bind(runtime);
-  // save the live input values
-  RegSet saved = RegSet::of(pre_val);
-  if (tosca_live) saved += RegSet::of(r0);
-  if (obj != noreg) saved += RegSet::of(obj);
 
-  __ push(saved, sp);
-
-  // Calling the runtime using the regular call_VM_leaf mechanism generates
-  // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
-  // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
-  //
-  // If we care generating the pre-barrier without a frame (e.g. in the
-  // intrinsified Reference.get() routine) then rfp might be pointing to
-  // the caller frame and so this check will most likely fail at runtime.
-  //
-  // Expanding the call directly bypasses the generation of the check.
-  // So when we do not have have a full interpreter frame on the stack
-  // expand_call should be passed true.
-
-  if (expand_call) {
-    assert(pre_val != c_rarg1, "smashed arg");
-    __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
-  } else {
-    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
+  // Slow-path call
+  __ enter(/* strip_ret_addr = */ true);
+  __ push_call_clobbered_registers();
+  if (c_rarg0 != pre_val) {
+    __ mov(c_rarg0, pre_val);
   }
-
-  __ pop(saved, sp);
+  // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
+  __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
+  __ pop_call_clobbered_registers();
+  __ leave();
 
   __ bind(done);
 }
@@ -231,7 +228,6 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
   bool is_narrow  = UseCompressedOops && !is_native;
 
   Label heap_stable, not_cset;
-  __ enter(/*strip_ret_addr*/true);
   Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
   __ ldrb(rscratch2, gc_state);
 
@@ -266,37 +262,40 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
     __ tbz(rscratch2, 0, not_cset);
   }
 
+  // Slow-path call
+  __ enter(/* strip_ret_addr = */ true);
   __ push_call_clobbered_registers();
+  address target = nullptr;
   if (is_strong) {
     if (is_narrow) {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
+      target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
     } else {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
+      target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
     }
   } else if (is_weak) {
     if (is_narrow) {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
+      target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
     } else {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
+      target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
     }
   } else {
     assert(is_phantom, "only remaining strength");
     assert(!is_narrow, "phantom access cannot be narrow");
-    // AOT saved adapters need relocation for this call.
-    __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)));
+    target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
   }
-  __ blr(lr);
+  // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
+  __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
   __ mov(rscratch1, r0);
   __ pop_call_clobbered_registers();
   __ mov(r0, rscratch1);
+  __ leave();
 
   __ bind(not_cset);
 
   __ mov(result_dst, r0);
   __ pop(to_save, sp);
 
   __ bind(heap_stable);
-  __ leave();
 }
 
 //
@@ -346,18 +345,12 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
 
   // 3: apply keep-alive barrier if needed
   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
-    __ enter(/*strip_ret_addr*/true);
-    __ push_call_clobbered_registers();
     satb_barrier(masm /* masm */,
                  noreg /* obj */,
                  dst /* pre_val */,
                  rthread /* thread */,
                  tmp1 /* tmp1 */,
-                 tmp2 /* tmp2 */,
-                 true /* tosca_live */,
-                 true /* expand_call */);
-    __ pop_call_clobbered_registers();
-    __ leave();
+                 tmp2 /* tmp2 */);
   }
 }
 
@@ -399,24 +392,21 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
     __ lea(tmp3, dst);
   }
 
-  bool storing_non_null = (val != noreg);
-
   // 2: pre-barrier: SATB needs the previous value
   if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
     satb_barrier(masm,
                  tmp3 /* obj */,
                  tmp2 /* pre_val */,
                  rthread /* thread */,
                  tmp1 /* tmp */,
-                 rscratch1 /* tmp2 */,
-                 storing_non_null /* tosca_live */,
-                 false /* expand_call */);
+                 rscratch1 /* tmp2 */);
   }
 
   // Store!
   BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
 
   // 3: post-barrier: card barrier needs store address
+  bool storing_non_null = (val != noreg);
   if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
     card_barrier(masm, tmp3);
   }

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp

@@ -45,9 +45,7 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
                     Register pre_val,
                     Register thread,
                     Register tmp1,
-                    Register tmp2,
-                    bool tosca_live,
-                    bool expand_call);
+                    Register tmp2);
 
   void card_barrier(MacroAssembler* masm, Register obj);