diff --git a/include/my_cpu.h b/include/my_cpu.h index 028dc05bd084a..66205de99a408 100644 --- a/include/my_cpu.h +++ b/include/my_cpu.h @@ -110,6 +110,25 @@ static inline void MY_RELAX_CPU(void) } +/* + Hint the CPU to start loading a cache line for reading. + + Like YieldProcessor() above, PreFetchCacheLine() is architecture-independent: + winnt.h picks the instruction per target, unlike _mm_prefetch(). + clang-cl defines __clang__ but not __GNUC__, so it must take the first branch. +*/ +static inline void my_prefetch(const void *addr) +{ +#if defined(__GNUC__) || defined(__clang__) + __builtin_prefetch(addr, 0, 3); +#elif defined(_WIN32) + PreFetchCacheLine(PF_TEMPORAL_LEVEL_1, addr); +#else + (void) addr; +#endif +} + + #ifdef HAVE_PAUSE_INSTRUCTION # ifdef __cplusplus extern "C" { diff --git a/sql/vector_mhnsw.cc b/sql/vector_mhnsw.cc index c480c36c7e7ad..d42461cae6620 100644 --- a/sql/vector_mhnsw.cc +++ b/sql/vector_mhnsw.cc @@ -22,6 +22,7 @@ #include "vector_mhnsw.h" #include #include +#include // my_prefetch() #include "bloom_filters.h" // distance can be a little bit < 0 because of fast math @@ -1347,6 +1348,19 @@ static int search_layer(MHNSW_param *p, const FVector *target, float threshold, if (res == 0xff) continue; + // A node and its vector share one allocation. Prefetch unseen nodes before + // computing distances so later loads can overlap with work on earlier ones. + for (size_t i= 0; i < 8; i++) + { + FVectorNode *link= links[i]; + if (!(res & (1 << i)) && link) + { + my_prefetch(link); + my_prefetch(reinterpret_cast(link) + + CPU_LEVEL1_DCACHE_LINESIZE); + } + } + for (size_t i= 0; i < 8; i++) { if (res & (1 << i))