From 6cbc43da8ac7a2d6cf6fec4eaf7be15ff9ce9581 Mon Sep 17 00:00:00 2001 From: Aljo Joby Date: Sat, 19 Sep 2026 21:42:14 +0530 Subject: [PATCH 1/2] uvm: coalesce adjacent TLB batch invalidate ranges Sequential PTE updates burn the 4-slot TLB batch and fall back to invalidate-all. Merge the new interval into the last queued range when they overlap or touch and neither start+size wraps. Do not bump count or total_ranges on merge. Guard ranges[] so count outside 1..MAX does not OOB. --- kernel-open/nvidia-uvm/uvm_page_tree_test.c | 220 ++++++++++++++++++++ kernel-open/nvidia-uvm/uvm_tlb_batch.c | 53 ++++- 2 files changed, 270 insertions(+), 3 deletions(-) diff --git a/kernel-open/nvidia-uvm/uvm_page_tree_test.c b/kernel-open/nvidia-uvm/uvm_page_tree_test.c index e2b81e0d64..7ad9bf18e0 100644 --- a/kernel-open/nvidia-uvm/uvm_page_tree_test.c +++ b/kernel-open/nvidia-uvm/uvm_page_tree_test.c @@ -1410,6 +1410,223 @@ static NV_STATUS test_tlb_batch_invalidates(uvm_gpu_t *gpu, const NvU64 *page_si return status; } +static NV_STATUS test_tlb_batch_coalesce(uvm_gpu_t *gpu) +{ + NV_STATUS status = NV_OK; + uvm_page_tree_t tree; + uvm_push_t push; + uvm_tlb_batch_t batch; + NvU32 depth_2m; + NvU32 depth_64k; + NvU64 wrap_size; + int i; + + MEM_NV_CHECK_RET(test_page_tree_init(gpu, &tree), NV_OK); + MEM_NV_CHECK_RET(uvm_push_begin_fake(gpu, &push), NV_OK); + + depth_2m = tree.hal->page_table_depth(UVM_PAGE_SIZE_2M); + depth_64k = tree.hal->page_table_depth(UVM_PAGE_SIZE_64K); + + fake_tlb_invals_enable(); + + // 1. Adjacent same page size + uvm_tlb_batch_begin(&tree, &batch); + for (i = 0; i < 16; ++i) { + uvm_tlb_batch_invalidate(&batch, + (NvU64)i * UVM_PAGE_SIZE_4K, + UVM_PAGE_SIZE_4K, + UVM_PAGE_SIZE_4K, + UVM_MEMBAR_NONE); + } + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 1, done); + TEST_CHECK_GOTO(g_last_fake_inval->base == 0, done); + TEST_CHECK_GOTO(g_last_fake_inval->size == 16 * UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->page_size == UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->size != (NvU64)-1, done); + fake_tlb_invals_reset(); + + // 2. Overlapping + uvm_tlb_batch_begin(&tree, &batch); + uvm_tlb_batch_invalidate(&batch, 0, 2 * UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, UVM_PAGE_SIZE_4K, 2 * UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 1, done); + TEST_CHECK_GOTO(g_last_fake_inval->base == 0, done); + TEST_CHECK_GOTO(g_last_fake_inval->size == 3 * UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->page_size == UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->size != (NvU64)-1, done); + fake_tlb_invals_reset(); + + // 3. Reverse adjacent + uvm_tlb_batch_begin(&tree, &batch); + uvm_tlb_batch_invalidate(&batch, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, 0, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 1, done); + TEST_CHECK_GOTO(g_last_fake_inval->base == 0, done); + TEST_CHECK_GOTO(g_last_fake_inval->size == 2 * UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->page_size == UVM_PAGE_SIZE_4K, done); + fake_tlb_invals_reset(); + + // 4. Gap + uvm_tlb_batch_begin(&tree, &batch); + uvm_tlb_batch_invalidate(&batch, 0, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, 2 * UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 2, done); + TEST_CHECK_GOTO(g_fake_invals[0].base == 0, done); + TEST_CHECK_GOTO(g_fake_invals[0].size == UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_fake_invals[1].base == 2 * UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_fake_invals[1].size == UVM_PAGE_SIZE_4K, done); + fake_tlb_invals_reset(); + + // 5. Mixed 2M+4K adjacent + uvm_tlb_batch_begin(&tree, &batch); + uvm_tlb_batch_invalidate(&batch, 0, UVM_PAGE_SIZE_2M, UVM_PAGE_SIZE_2M, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, UVM_PAGE_SIZE_2M, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 1, done); + TEST_CHECK_GOTO(g_last_fake_inval->base == 0, done); + TEST_CHECK_GOTO(g_last_fake_inval->size == UVM_PAGE_SIZE_2M + UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->page_size == UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->depth == depth_2m, done); + TEST_CHECK_GOTO(g_last_fake_inval->size != (NvU64)-1, done); + fake_tlb_invals_reset(); + + // 6. Mixed 64K+4K adjacent + uvm_tlb_batch_begin(&tree, &batch); + uvm_tlb_batch_invalidate(&batch, 0, UVM_PAGE_SIZE_64K, UVM_PAGE_SIZE_64K, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, UVM_PAGE_SIZE_64K, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 1, done); + TEST_CHECK_GOTO(g_last_fake_inval->base == 0, done); + TEST_CHECK_GOTO(g_last_fake_inval->size == UVM_PAGE_SIZE_64K + UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->page_size == UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->depth == depth_64k, done); + TEST_CHECK_GOTO(g_last_fake_inval->size != (NvU64)-1, done); + fake_tlb_invals_reset(); + + // 7. Contained mix + uvm_tlb_batch_begin(&tree, &batch); + uvm_tlb_batch_invalidate(&batch, 0, UVM_PAGE_SIZE_2M, UVM_PAGE_SIZE_2M, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 1, done); + TEST_CHECK_GOTO(g_last_fake_inval->base == 0, done); + TEST_CHECK_GOTO(g_last_fake_inval->size == UVM_PAGE_SIZE_2M, done); + TEST_CHECK_GOTO(g_last_fake_inval->page_size == UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->depth == depth_2m, done); + fake_tlb_invals_reset(); + + // 8. Merge then disjoint + uvm_tlb_batch_begin(&tree, &batch); + uvm_tlb_batch_invalidate(&batch, 0, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, 2 * UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, 4 * UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 2, done); + TEST_CHECK_GOTO(g_fake_invals[0].base == 0, done); + TEST_CHECK_GOTO(g_fake_invals[0].size == 3 * UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_fake_invals[1].base == 4 * UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_fake_invals[1].size == UVM_PAGE_SIZE_4K, done); + fake_tlb_invals_reset(); + + // 9. Membar max + uvm_tlb_batch_begin(&tree, &batch); + uvm_tlb_batch_invalidate(&batch, 0, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_SYS); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 1, done); + TEST_CHECK_GOTO(g_last_fake_inval->membar == UVM_MEMBAR_SYS, done); + fake_tlb_invals_reset(); + + // 10. Cap preserved + uvm_tlb_batch_begin(&tree, &batch); + for (i = 0; i < 5; ++i) { + uvm_tlb_batch_invalidate(&batch, + (NvU64)i * 2 * UVM_PAGE_SIZE_4K, + UVM_PAGE_SIZE_4K, + UVM_PAGE_SIZE_4K, + UVM_MEMBAR_NONE); + } + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 1, done); + TEST_CHECK_GOTO(g_last_fake_inval->base == 0, done); + TEST_CHECK_GOTO(g_last_fake_inval->size == (NvU64)-1, done); + fake_tlb_invals_reset(); + + // 11. count==4 touch + uvm_tlb_batch_begin(&tree, &batch); + for (i = 0; i < 4; ++i) { + uvm_tlb_batch_invalidate(&batch, + (NvU64)i * 2 * UVM_PAGE_SIZE_4K, + UVM_PAGE_SIZE_4K, + UVM_PAGE_SIZE_4K, + UVM_MEMBAR_NONE); + } + uvm_tlb_batch_invalidate(&batch, 7 * UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 4, done); + TEST_CHECK_GOTO(g_last_fake_inval->base == 6 * UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->size == 2 * UVM_PAGE_SIZE_4K, done); + TEST_CHECK_GOTO(g_last_fake_inval->size != (NvU64)-1, done); + fake_tlb_invals_reset(); + + // 12. Already-all + touch + uvm_tlb_batch_begin(&tree, &batch); + for (i = 0; i < 5; ++i) { + uvm_tlb_batch_invalidate(&batch, + (NvU64)i * 2 * UVM_PAGE_SIZE_4K, + UVM_PAGE_SIZE_4K, + UVM_PAGE_SIZE_4K, + UVM_MEMBAR_NONE); + } + uvm_tlb_batch_invalidate(&batch, 7 * UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(g_fake_invals_count == 1, done); + TEST_CHECK_GOTO(g_last_fake_inval->base == 0, done); + TEST_CHECK_GOTO(g_last_fake_inval->size == (NvU64)-1, done); + fake_tlb_invals_reset(); + + // 13. Wrap new + wrap_size = 0ull - UVM_PAGE_SIZE_4K; + uvm_tlb_batch_begin(&tree, &batch); + uvm_tlb_batch_invalidate(&batch, 0, 2 * UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, UVM_PAGE_SIZE_4K, wrap_size, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(!(g_fake_invals_count == 1 && + g_last_fake_inval->base == 0 && + g_last_fake_inval->size == 2 * UVM_PAGE_SIZE_4K), done); + TEST_CHECK_GOTO(g_fake_invals_count == 2 || + (g_fake_invals_count == 1 && + g_last_fake_inval->base == 0 && + g_last_fake_inval->size == (NvU64)-1), done); + fake_tlb_invals_reset(); + + // 14. Wrap last + wrap_size = 0ull - UVM_PAGE_SIZE_4K; + uvm_tlb_batch_begin(&tree, &batch); + uvm_tlb_batch_invalidate(&batch, UVM_PAGE_SIZE_4K, wrap_size, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_invalidate(&batch, 0, 2 * UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K, UVM_MEMBAR_NONE); + uvm_tlb_batch_end(&batch, &push, UVM_MEMBAR_NONE); + TEST_CHECK_GOTO(!(g_fake_invals_count == 1 && + g_last_fake_inval->base == 0 && + g_last_fake_inval->size == 2 * UVM_PAGE_SIZE_4K), done); + TEST_CHECK_GOTO(g_fake_invals_count == 2 || + (g_fake_invals_count == 1 && + g_last_fake_inval->base == 0 && + g_last_fake_inval->size == (NvU64)-1), done); + +done: + fake_tlb_invals_disable(); + uvm_push_end_fake(&push); + uvm_page_tree_deinit(&tree); + + return status; +} + typedef struct { NvU64 count; @@ -2189,6 +2406,7 @@ static NV_STATUS turing_test_page_tree(uvm_gpu_t *turing) MEM_NV_CHECK_RET(fast_split_double_backoff(turing), NV_OK); MEM_NV_CHECK_RET(test_tlb_invalidates_gmmu_v2(turing), NV_OK); MEM_NV_CHECK_RET(test_tlb_batch_invalidates(turing, page_sizes, num_page_sizes), NV_OK); + MEM_NV_CHECK_RET(test_tlb_batch_coalesce(turing), NV_OK); // Run the test again with a bigger limit on max pages tlb_batch_saved_max_pages = turing->parent->tlb_batch.max_pages; @@ -2233,6 +2451,7 @@ static NV_STATUS ampere_test_page_tree(uvm_gpu_t *ampere) // TLB batch invalidate MEM_NV_CHECK_RET(test_tlb_batch_invalidates(ampere, page_sizes, num_page_sizes), NV_OK); + MEM_NV_CHECK_RET(test_tlb_batch_coalesce(ampere), NV_OK); // Run the test again with a bigger limit on max pages tlb_batch_saved_max_pages = ampere->parent->tlb_batch.max_pages; @@ -2301,6 +2520,7 @@ static NV_STATUS blackwell_test_page_tree(uvm_gpu_t *blackwell) // TLB batch invalidate MEM_NV_CHECK_RET(test_tlb_batch_invalidates(blackwell, page_sizes, num_page_sizes), NV_OK); + MEM_NV_CHECK_RET(test_tlb_batch_coalesce(blackwell), NV_OK); // Run the test again with a bigger limit on max pages tlb_batch_saved_max_pages = blackwell->parent->tlb_batch.max_pages; diff --git a/kernel-open/nvidia-uvm/uvm_tlb_batch.c b/kernel-open/nvidia-uvm/uvm_tlb_batch.c index badc89f1f8..d902970e48 100644 --- a/kernel-open/nvidia-uvm/uvm_tlb_batch.c +++ b/kernel-open/nvidia-uvm/uvm_tlb_batch.c @@ -94,6 +94,49 @@ static bool tlb_batch_should_invalidate_all(uvm_tlb_batch_t *batch) return batch->total_ranges > batch->tree->gpu->parent->tlb_batch.max_ranges; } +static bool tlb_batch_try_coalesce(uvm_tlb_batch_t *batch, + NvU64 start, + NvU64 size, + NvU64 page_sizes) +{ + uvm_tlb_batch_range_t *last; + NvU64 last_end, new_end; + NvU64 old_start, old_size, old_page_sizes; + + if (batch->count == 0 || batch->count > UVM_TLB_BATCH_MAX_ENTRIES) + return false; + + if (start + size < start) + return false; + + last = &batch->ranges[batch->count - 1]; + + if (last->start + last->size < last->start) + return false; + + last_end = last->start + last->size; + new_end = start + size; + + if (last->start > new_end || start > last_end) + return false; + + old_start = last->start; + old_size = last->size; + old_page_sizes = last->page_sizes; + + last->start = min(old_start, start); + last->size = max(last_end, new_end) - last->start; + last->page_sizes = old_page_sizes | page_sizes; + + UVM_ASSERT(last->start <= old_start); + UVM_ASSERT(last->start + last->size >= old_start + old_size); + UVM_ASSERT(last->start <= start); + UVM_ASSERT(last->start + last->size >= start + size); + UVM_ASSERT(last->page_sizes == (old_page_sizes | page_sizes)); + + return true; +} + void uvm_tlb_batch_end(uvm_tlb_batch_t *batch, uvm_push_t *push, uvm_membar_t tlb_membar) { if (batch->count == 0) @@ -111,14 +154,18 @@ void uvm_tlb_batch_invalidate(uvm_tlb_batch_t *batch, NvU64 start, NvU64 size, N { uvm_tlb_batch_range_t *new_entry; + UVM_ASSERT(size > 0); + UVM_ASSERT(page_sizes != 0); + batch->membar = uvm_membar_max(tlb_membar, batch->membar); + batch->biggest_page_size = max(batch->biggest_page_size, biggest_page_size(page_sizes)); - ++batch->count; + if (tlb_batch_try_coalesce(batch, start, size, page_sizes)) + return; + ++batch->count; batch->total_ranges++; - batch->biggest_page_size = max(batch->biggest_page_size, biggest_page_size(page_sizes)); - if (tlb_batch_should_invalidate_all(batch)) return; From 2ddb268344700204efdf0f8ed3789714aeb9c004 Mon Sep 17 00:00:00 2001 From: Aljo Joby Date: Sat, 19 Sep 2026 21:59:23 +0530 Subject: [PATCH 2/2] uvm: pick least-busy channel on reserve fast path Channel 0 of each pool is always claimed first while it still has a free GPFIFO slot. Scan the pool under one lock, take the channel with the most free entries, and rotate next_hint on ties. Wait path stays first-fit. Same pick on the Confidential Computing first scan. --- kernel-open/nvidia-uvm/uvm_channel.c | 71 +++++++++++---- kernel-open/nvidia-uvm/uvm_channel.h | 4 + kernel-open/nvidia-uvm/uvm_channel_test.c | 103 ++++++++++++++++++++++ 3 files changed, 159 insertions(+), 19 deletions(-) diff --git a/kernel-open/nvidia-uvm/uvm_channel.c b/kernel-open/nvidia-uvm/uvm_channel.c index 2d0d3034e9..bc343edacd 100644 --- a/kernel-open/nvidia-uvm/uvm_channel.c +++ b/kernel-open/nvidia-uvm/uvm_channel.c @@ -401,6 +401,43 @@ static bool test_claim_and_lock_channel(uvm_channel_t *channel, return false; } +// Returns a channel that can satisfy the reserve, or NULL. Does not claim. +static uvm_channel_t *channel_pick_least_busy_locked(uvm_channel_pool_t *pool, + NvU32 num_gpfifo_entries, + uvm_channel_reserve_type_t reserve_type, + bool skip_locked_for_push) +{ + NvU32 n; + NvU32 best_available = 0; + uvm_channel_t *best = NULL; + + uvm_channel_pool_assert_locked(pool); + UVM_ASSERT(!skip_locked_for_push || g_uvm_global.conf_computing_enabled); + + for (n = 0; n < pool->num_channels; n++) { + NvU32 index = (pool->next_hint + n) % pool->num_channels; + uvm_channel_t *channel = &pool->channels[index]; + NvU32 available; + + if (skip_locked_for_push && uvm_channel_is_locked_for_push(channel)) + continue; + + if (reserve_type == UVM_CHANNEL_RESERVE_WITH_P2P && channel->suspended_p2p) + continue; + + available = channel_get_available_gpfifo_entries(channel); + if (available < num_gpfifo_entries) + continue; + + if (best == NULL || available > best_available) { + best = channel; + best_available = available; + } + } + + return best; +} + // Reserve, or release, all channels in the given pool. // // One scenario where reservation of the entire pool is useful is key rotation, @@ -512,7 +549,6 @@ static NV_STATUS channel_reserve_and_lock_in_pool(uvm_channel_pool_t *pool, { uvm_channel_t *channel; uvm_spin_loop_t spin; - NvU32 index; NV_STATUS status; UVM_ASSERT(pool); @@ -529,20 +565,15 @@ static NV_STATUS channel_reserve_and_lock_in_pool(uvm_channel_pool_t *pool, // uvm_channel_end_push() routine. uvm_down(&pool->conf_computing.push_sem); - // At least one channel is unlocked. We check if any unlocked channel is - // available, i.e., if it has free GPFIFO entries. - + // At least one channel is unlocked. Check if any unlocked channel has a + // free GPFIFO entry. channel_pool_lock(pool); - - for_each_clear_bit(index, pool->conf_computing.push_locks, pool->num_channels) { - channel = &pool->channels[index]; - - if (try_claim_channel_locked(channel, 1, reserve_type)) { - lock_channel_for_push(channel); - goto done; - } + channel = channel_pick_least_busy_locked(pool, 1, reserve_type, true); + if (channel != NULL && try_claim_channel_locked(channel, 1, reserve_type)) { + lock_channel_for_push(channel); + pool->next_hint = (uvm_channel_index_in_pool(channel) + 1) % pool->num_channels; + goto done; } - channel_pool_unlock(pool); // No channels are available. Update and check errors on all channels until @@ -593,13 +624,15 @@ static NV_STATUS channel_reserve_in_pool(uvm_channel_pool_t *pool, if (g_uvm_global.conf_computing_enabled) return channel_reserve_and_lock_in_pool(pool, reserve_type, channel_out); - uvm_for_each_channel_in_pool(channel, pool) { - // TODO: Bug 1764953: Prefer idle/less busy channels - if (try_claim_channel(channel, 1, reserve_type)) { - *channel_out = channel; - return NV_OK; - } + channel_pool_lock(pool); + channel = channel_pick_least_busy_locked(pool, 1, reserve_type, false); + if (channel != NULL && try_claim_channel_locked(channel, 1, reserve_type)) { + pool->next_hint = (uvm_channel_index_in_pool(channel) + 1) % pool->num_channels; + channel_pool_unlock(pool); + *channel_out = channel; + return NV_OK; } + channel_pool_unlock(pool); uvm_spin_loop_init(&spin); while (1) { diff --git a/kernel-open/nvidia-uvm/uvm_channel.h b/kernel-open/nvidia-uvm/uvm_channel.h index 36e0e35061..d130012f24 100644 --- a/kernel-open/nvidia-uvm/uvm_channel.h +++ b/kernel-open/nvidia-uvm/uvm_channel.h @@ -211,6 +211,10 @@ typedef struct // Number of elements in the channel array NvU32 num_channels; + // Next channel to consider when selecting a least-busy channel. + // Protected by the pool lock. + NvU32 next_hint; + // Index of the engine associated with the pool (index is an offset from the // first engine of the same engine type.) unsigned engine_index; diff --git a/kernel-open/nvidia-uvm/uvm_channel_test.c b/kernel-open/nvidia-uvm/uvm_channel_test.c index 04e30ac5de..e23fb00361 100644 --- a/kernel-open/nvidia-uvm/uvm_channel_test.c +++ b/kernel-open/nvidia-uvm/uvm_channel_test.c @@ -977,6 +977,105 @@ static NV_STATUS test_conf_computing_channel_selection(uvm_va_space_t *va_space) return status; } +// Concurrent uvm_push_begin calls on an idle multi-channel pool should reserve +// distinct channels. +static NV_STATUS test_least_busy_channel_reserve(uvm_va_space_t *va_space) +{ + NV_STATUS status = NV_OK; + uvm_push_t *pushes = NULL; + uvm_gpu_t *gpu = NULL; + NvU32 num_begun = 0; + NvU32 num_ended = 0; + + uvm_thread_context_lock_disable_tracking(); + + for_each_va_space_gpu(gpu, va_space) { + uvm_channel_type_t channel_type; + + // Nested begins are illegal with key rotation. Disable it for the + // duration of this GPU's concurrent pushes; a no-op when CC is off. + uvm_conf_computing_disable_key_rotation(gpu); + + for (channel_type = 0; channel_type < UVM_CHANNEL_TYPE_COUNT; channel_type++) { + NvU32 i; + NvU32 num_pushes; + uvm_channel_t *channel; + uvm_channel_pool_t *pool = gpu->channel_manager->pool_to_use.default_for_type[channel_type]; + + // SEC2/WLC/LCIC defaults are NULL when Confidential Computing is off. + if (pool == NULL) + continue; + + // Skip LCIC channels as those can't accept any pushes + if (uvm_channel_pool_is_lcic(pool)) + continue; + + if (pool->num_channels < 2) + continue; + + num_pushes = pool->num_channels; + + // Reclaim completed GPFIFO so available counts match an idle pool. + uvm_for_each_channel_in_pool(channel, pool) + TEST_NV_CHECK_GOTO(uvm_channel_wait(channel), error); + + pushes = uvm_kvmalloc_zero(sizeof(*pushes) * num_pushes); + TEST_CHECK_GOTO(pushes != NULL, error); + + num_begun = 0; + num_ended = 0; + + for (i = 0; i < num_pushes; i++) { + uvm_push_t *push = &pushes[i]; + status = uvm_push_begin(gpu->channel_manager, channel_type, push, "least-busy push %u", i); + TEST_NV_CHECK_GOTO(status, error); + num_begun++; + } + + for (i = 0; i < num_pushes; i++) { + NvU32 j; + + for (j = 0; j < i; j++) + TEST_CHECK_GOTO(pushes[i].channel != pushes[j].channel, error); + } + + for (i = 0; i < num_pushes; i++) { + uvm_push_t *push = &pushes[i]; + status = uvm_push_end_and_wait(push); + num_ended++; + TEST_NV_CHECK_GOTO(status, error); + } + + uvm_kvfree(pushes); + pushes = NULL; + num_begun = 0; + num_ended = 0; + } + + uvm_conf_computing_enable_key_rotation(gpu); + } + + uvm_thread_context_lock_enable_tracking(); + + return status; + +error: + if (pushes != NULL) { + NvU32 i; + + for (i = num_ended; i < num_begun; i++) + uvm_push_end(&pushes[i]); + } + + if (gpu != NULL) + uvm_conf_computing_enable_key_rotation(gpu); + + uvm_thread_context_lock_enable_tracking(); + uvm_kvfree(pushes); + + return status; +} + static NV_STATUS test_channel_iv_rotation(uvm_va_space_t *va_space) { uvm_gpu_t *gpu; @@ -1640,6 +1739,10 @@ NV_STATUS uvm_test_channel_sanity(UVM_TEST_CHANNEL_SANITY_PARAMS *params, struct if (status != NV_OK) goto done; + status = test_least_busy_channel_reserve(va_space); + if (status != NV_OK) + goto done; + status = test_channel_iv_rotation(va_space); if (status != NV_OK) goto done;