Inline GC handle write barrier

src/coreclr/gc/gc.h

@@ -16,8 +16,6 @@ Module Name:
 #include "gcinterface.h"
 #include "env/gcenv.os.h"
 
-#include "gchandletableimpl.h"
-
 #ifdef BUILD_AS_STANDALONE
 #include "gcenv.ee.standalone.inl"
 

src/coreclr/gc/gcinternal.h

@@ -12,7 +12,7 @@
 #include "gcdesc.h"
 #include "softwarewritewatch.h"
 #include "handletable.h"
-#include "handletable.inl"
+#include "gchandletableimpl.h"
 #include "gcenv.inl"
 #include "gceventstatus.h"
 #include <minipal/memorybarrierprocesswide.h>

src/coreclr/gc/handletable.cpp

@@ -24,14 +24,6 @@
 DWORD g_dwHandles = 0;
 #endif // ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
 
-#ifndef DACCESS_COMPILE
-int GetConvertedGeneration(_UNCHECKED_OBJECTREF obj)
-{
-    int generation = g_theGCHeap->WhichGeneration(obj);
-    return generation == INT_MAX ? max_generation : generation;
-}
-#endif //DACCESS_COMPILE
-
 /****************************************************************************
  *
  * FORWARD DECLARATIONS
@@ -548,75 +540,6 @@ void HndLogSetEvent(OBJECTHANDLE handle, _UNCHECKED_OBJECTREF value)
 #endif
 }
 
-#ifndef DACCESS_COMPILE
-/*
- * HndWriteBarrierWorker
- *
- * Resets the generation number for the handle's clump to zero.
- *
- */
-void HndWriteBarrierWorker(OBJECTHANDLE handle, _UNCHECKED_OBJECTREF value)
-{
-    _ASSERTE (value != NULL);
-
-    // find the write barrier for this handle
-    uint8_t *barrier = (uint8_t *)((uintptr_t)handle & HANDLE_SEGMENT_ALIGN_MASK);
-
-    // sanity
-    _ASSERTE(barrier);
-
-    // find the offset of this handle into the segment
-    uintptr_t offset = (uintptr_t)handle & HANDLE_SEGMENT_CONTENT_MASK;
-
-    // make sure it is in the handle area and not the header
-    _ASSERTE(offset >= HANDLE_HEADER_SIZE);
-
-    // compute the clump index for this handle
-    offset = (offset - HANDLE_HEADER_SIZE) / (HANDLE_SIZE * HANDLE_HANDLES_PER_CLUMP);
-
-    // Be careful to read and write the age byte via volatile operations. Otherwise the compiler has been
-    // observed to translate the read + conditional write sequence below into an unconditional read/write
-    // (utilizing a conditional register move to determine whether the write is an update or simply writes
-    // back what was read). This is a legal transformation for non-volatile accesses but obviously leads to a
-    // race condition where we can lose an update (see the comment below for the race condition).
-    volatile uint8_t * pClumpAge = barrier + offset;
-
-    // if this age is smaller than age of the clump, update the clump age
-    if (*pClumpAge != 0) // Perf optimization: if clumpAge is 0, nothing more to do
-    {
-        // find out generation
-        int generation = GetConvertedGeneration(value);
-        uint32_t uType = HandleFetchType(handle);
-
-#ifdef FEATURE_ASYNC_PINNED_HANDLES
-        //OverlappedData need special treatment: because all user data pointed by it needs to be reported by this handle,
-        //its age is consider to be min age of the user data, to be simple, we just make it 0
-        if (uType == HNDTYPE_ASYNCPINNED)
-        {
-            generation = 0;
-        }
-#endif
-
-        if (uType == HNDTYPE_DEPENDENT)
-        {
-            generation = 0;
-        }
-
-        if (*pClumpAge > (uint8_t) generation)
-        {
-            // We have to be careful here. HndWriteBarrier is not under any synchronization
-            // Consider the scenario where 2 threads are hitting the line below at the same
-            // time. Only one will win. If the winner has an older age than the loser, we
-            // just created a potential GC hole  (The clump will not be reporting the
-            // youngest handle in the clump, thus GC may skip the clump). To fix this
-            // we just set the clump age to 0, which means that whoever wins the race
-            // results are the same, as GC will always look at the clump
-            *pClumpAge = (uint8_t)0;
-        }
-    }
-}
-#endif // DACCESS_COMPILE
-
 /*
  * HndEnumHandles
  *
@@ -1157,5 +1080,3 @@ void DEBUG_LogScanningStatistics(HandleTable *pTable, uint32_t level)
 
 
 /*--------------------------------------------------------------------------*/
-
-

src/coreclr/gc/handletable.h

@@ -89,7 +89,6 @@ HHANDLETABLE    HndGetHandleTable(OBJECTHANDLE handle);
 /*
  * write barrier
  */
-void            HndWriteBarrierWorker(OBJECTHANDLE handle, _UNCHECKED_OBJECTREF value);
 void            HndWriteBarrier(OBJECTHANDLE handle, OBJECTREF value);
 
 /*
@@ -224,4 +223,3 @@ FORCEINLINE BOOL HndIsNullOrDestroyedHandle(_UNCHECKED_OBJECTREF value)
 #include "handletable.inl"
 
 #endif //_HANDLETABLE_H
-

src/coreclr/gc/handletable.inl

@@ -7,6 +7,91 @@
 #ifndef _HANDLETABLE_INL
 #define _HANDLETABLE_INL
 
+#ifndef DACCESS_COMPILE
+#include "gc.h"
+#include "handletableconstants.h"
+
+FORCEINLINE int GetConvertedGeneration(_UNCHECKED_OBJECTREF obj)
+{
+    int generation = g_theGCHeap->WhichGeneration(obj);
+    return generation == INT_MAX ? max_generation : generation;
+}
+
+FORCEINLINE uint32_t HandleFetchType(OBJECTHANDLE handle)
+{
+    WRAPPER_NO_CONTRACT;
+
+    uint8_t* segment = reinterpret_cast<uint8_t*>(reinterpret_cast<uintptr_t>(handle) & HANDLE_SEGMENT_ALIGN_MASK);
+    _ASSERTE(segment);
+
+    uintptr_t offset = reinterpret_cast<uintptr_t>(handle) & HANDLE_SEGMENT_CONTENT_MASK;
+    _ASSERTE(offset >= HANDLE_HEADER_SIZE);
+
+    uint32_t uHandle = static_cast<uint32_t>((offset - HANDLE_HEADER_SIZE) / HANDLE_SIZE);
+    uint32_t uBlock = uHandle / HANDLE_HANDLES_PER_BLOCK;
+
+    return segment[HANDLE_SEGMENT_BLOCK_TYPE_OFFSET + uBlock];
+}
+
+FORCEINLINE void HndWriteBarrierWorker(OBJECTHANDLE handle, _UNCHECKED_OBJECTREF value)
+{
+    _ASSERTE(value != NULL);
+
+    uint8_t* barrier = (uint8_t*)((uintptr_t)handle & HANDLE_SEGMENT_ALIGN_MASK);
+    _ASSERTE(barrier);
+
+    uintptr_t offset = (uintptr_t)handle & HANDLE_SEGMENT_CONTENT_MASK;
+    _ASSERTE(offset >= HANDLE_HEADER_SIZE);
+
+    offset = (offset - HANDLE_HEADER_SIZE) / (HANDLE_SIZE * HANDLE_HANDLES_PER_CLUMP);
+
+    // Be careful to read and write the age byte via volatile operations. Otherwise the compiler has been
+    // observed to translate the read + conditional write sequence below into an unconditional read/write
+    // (utilizing a conditional register move to determine whether the write is an update or simply writes
+    // back what was read). This is a legal transformation for non-volatile accesses but obviously leads to a
+    // race condition where we can lose an update (see the comment below for the race condition).
+    volatile uint8_t* pClumpAge = barrier + offset;
+
+    if (*pClumpAge != 0)
+    {
+        int generation = GetConvertedGeneration(value);
+
+        if (generation == 0)
+        {
+            *pClumpAge = 0;
+            return;
+        }
+
+        uint32_t uType = HandleFetchType(handle);
+
+#ifdef FEATURE_ASYNC_PINNED_HANDLES
+        //OverlappedData need special treatment: because all user data pointed by it needs to be reported by this handle,
+        //its age is consider to be min age of the user data, to be simple, we just make it 0
+        if (uType == HNDTYPE_ASYNCPINNED)
+        {
+            generation = 0;
+        }
+#endif
+
+        if (uType == HNDTYPE_DEPENDENT)
+        {
+            generation = 0;
+        }
+
+        if (*pClumpAge > (uint8_t)generation)
+        {
+            // We have to be careful here. HndWriteBarrier is not under any synchronization
+            // Consider the scenario where 2 threads are hitting the line below at the same
+            // time. Only one will win. If the winner has an older age than the loser, we
+            // just created a potential GC hole  (The clump will not be reporting the
+            // youngest handle in the clump, thus GC may skip the clump). To fix this
+            // we just set the clump age to 0, which means that whoever wins the race
+            // results are the same, as GC will always look at the clump
+            *pClumpAge = (uint8_t)0;
+        }
+    }
+}
+
 inline void HndWriteBarrier(OBJECTHANDLE handle, OBJECTREF objref)
 {
     STATIC_CONTRACT_NOTHROW;
@@ -41,7 +126,7 @@ inline void HndAssignHandle(OBJECTHANDLE handle, OBJECTREF objref)
 
     // if we are doing a non-NULL pointer store then invoke the write-barrier
     if (value)
-        HndWriteBarrier(handle, objref);
+        HndWriteBarrierWorker(handle, value);
 
     // Store the pointer with release semantics so object field writes are visible
     // before the handle can publish the object to another thread.
@@ -134,5 +219,6 @@ inline BOOL HndFirstAssignHandle(OBJECTHANDLE handle, OBJECTREF objref)
     // return our result
     return success;
 }
+#endif // DACCESS_COMPILE
 
 #endif // _HANDLETABLE_INL

src/coreclr/gc/handletableconstants.h

@@ -93,6 +93,9 @@
 #define HANDLE_MASKS_PER_SEGMENT        (HANDLE_HANDLES_PER_SEGMENT / HANDLE_HANDLES_PER_MASK)
 #define HANDLE_MASKS_PER_BLOCK          (HANDLE_HANDLES_PER_BLOCK / HANDLE_HANDLES_PER_MASK)
 #define HANDLE_CLUMPS_PER_MASK          (HANDLE_HANDLES_PER_MASK / HANDLE_HANDLES_PER_CLUMP)
+#define HANDLE_SEGMENT_BLOCK_TYPE_OFFSET ((HANDLE_BLOCKS_PER_SEGMENT * sizeof(uint32_t)) + \
+                                          HANDLE_BLOCKS_PER_SEGMENT + \
+                                          (HANDLE_MASKS_PER_SEGMENT * sizeof(uint32_t)))
 
 // We use this relation to check for free mask per block.
 static_assert (HANDLE_HANDLES_PER_MASK * 2 == HANDLE_HANDLES_PER_BLOCK);

src/coreclr/gc/handletablecore.cpp

@@ -442,35 +442,6 @@ void HandleQuickSetUserData(OBJECTHANDLE handle, uintptr_t lUserData)
 
 #endif // !DACCESS_COMPILE
 
-/*
- * HandleFetchType
- *
- * Computes the type index for a given handle.
- *
- */
-uint32_t HandleFetchType(OBJECTHANDLE handle)
-{
-    WRAPPER_NO_CONTRACT;
-
-    // get the segment for this handle
-    PTR__TableSegmentHeader pSegment = HandleFetchSegmentPointer(handle);
-
-    // find the offset of this handle into the segment
-    uintptr_t offset = (uintptr_t)handle & HANDLE_SEGMENT_CONTENT_MASK;
-
-    // make sure it is in the handle area and not the header
-    _ASSERTE(offset >= HANDLE_HEADER_SIZE);
-
-    // convert the offset to a handle index
-    uint32_t uHandle = (uint32_t)((offset - HANDLE_HEADER_SIZE) / HANDLE_SIZE);
-
-    // compute the block this handle resides in
-    uint32_t uBlock = uHandle / HANDLE_HANDLES_PER_BLOCK;
-
-    // return the block's type
-    return pSegment->rgBlockType[uBlock];
-}
-
 /*
  * HandleFetchHandleTable
  *
@@ -2213,4 +2184,3 @@ void TableFreeBulkUnpreparedHandles(HandleTable *pTable, uint32_t uType, const O
 
 /*--------------------------------------------------------------------------*/
 
-

src/coreclr/gc/handletablepriv.h

@@ -168,6 +168,8 @@ struct _TableSegmentHeader
     uint8_t bSequence;
 };
 
+static_assert(offsetof(_TableSegmentHeader, rgBlockType) == HANDLE_SEGMENT_BLOCK_TYPE_OFFSET);
+
 typedef DPTR(struct _TableSegmentHeader) PTR__TableSegmentHeader;
 typedef DPTR(uintptr_t) PTR_uintptr_t;
 
@@ -609,15 +611,6 @@ PTR_uintptr_t HandleQuickFetchUserDataPointer(OBJECTHANDLE handle);
 void HandleQuickSetUserData(OBJECTHANDLE handle, uintptr_t lUserData);
 
 
-/*
- * HandleFetchType
- *
- * Computes the type index for a given handle.
- *
- */
-uint32_t HandleFetchType(OBJECTHANDLE handle);
-
-
 /*
  * HandleFetchHandleTable
  *
@@ -942,15 +935,3 @@ void CALLBACK BlockVerifyAgeMapForBlocks(PTR_TableSegment pSegment, uint32_t uBl
 PTR_TableSegment CALLBACK xxxAsyncSegmentIterator(PTR_HandleTable pTable, TableSegment *pPrevSegment, CrstHolderWithState *pCrstHolder);
 
 /*--------------------------------------------------------------------------*/
-
-#ifndef DACCESS_COMPILE
-
-/*
- * GetConvertedGeneration
- *
- * Get the generation of an object, where a frozen object is regarded as max_generation
- *
- */
-int GetConvertedGeneration(_UNCHECKED_OBJECTREF obj);
-
-#endif //DACCESS_COMPILE

src/coreclr/gc/handletablescan.cpp

@@ -1878,4 +1878,3 @@ uint32_t TableSegment::DacSize(TADDR addr)
 }
 #endif
 /*--------------------------------------------------------------------------*/
-