From 8e27a9c215560ad700036ad4b4d32676d61ef4b1 Mon Sep 17 00:00:00 2001 From: Song Zhixin Date: Wed, 17 Jun 2026 01:53:27 +0800 Subject: [PATCH] [PERF] Fuse multi-group block table staged writes (#44944) Signed-off-by: jesse Signed-off-by: Nick Hill Co-authored-by: Nick Hill --- tests/v1/worker/test_gpu_block_table.py | 132 ++++++++++++++++++++++++ vllm/v1/worker/gpu/block_table.py | 33 +++--- vllm/v1/worker/gpu/buffer_utils.py | 94 ++++++++++++++++- 3 files changed, 242 insertions(+), 17 deletions(-) create mode 100644 tests/v1/worker/test_gpu_block_table.py diff --git a/tests/v1/worker/test_gpu_block_table.py b/tests/v1/worker/test_gpu_block_table.py new file mode 100644 index 0000000000..31acd475ad --- /dev/null +++ b/tests/v1/worker/test_gpu_block_table.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm.platforms import current_platform +from vllm.v1.worker.gpu.block_table import BlockTables + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda(), + reason="requires CUDA", +) + + +def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch): + device = torch.device("cuda") + block_tables = BlockTables( + block_sizes=[16, 32, 8], + max_num_reqs=4, + max_num_batched_tokens=64, + max_num_blocks_per_group=[8, 8, 8], + device=device, + kernel_block_sizes=[16, 16, 8], + ) + + def fail_if_apply_write_called(): + pytest.fail("multi-group writes should use the fused apply kernel") + + for block_table in block_tables.block_tables: + monkeypatch.setattr(block_table, "apply_write", fail_if_apply_write_called) + + block_tables.append_block_ids( + req_index=0, + new_block_ids=([1, 2], [10, 11], []), + overwrite=True, + ) + block_tables.append_block_ids( + req_index=1, + new_block_ids=([3], [12], [5, 6]), + overwrite=True, + ) + block_tables.apply_staged_writes() + torch.accelerator.synchronize() + + assert torch.equal( + block_tables.block_tables[0].gpu[0, :2], + torch.tensor([1, 2], dtype=torch.int32, device=device), + ) + # Group 1 has blocks_per_kv_block == 2, so each KV block expands to two + # kernel block IDs. + assert torch.equal( + block_tables.block_tables[1].gpu[0, :4], + torch.tensor([20, 21, 22, 23], dtype=torch.int32, device=device), + ) + assert torch.equal( + block_tables.block_tables[0].gpu[1, :1], + torch.tensor([3], dtype=torch.int32, device=device), + ) + assert torch.equal( + block_tables.block_tables[1].gpu[1, :2], + torch.tensor([24, 25], dtype=torch.int32, device=device), + ) + assert torch.equal( + block_tables.block_tables[2].gpu[1, :2], + torch.tensor([5, 6], dtype=torch.int32, device=device), + ) + assert block_tables.num_blocks.np[0, 0] == 2 + assert block_tables.num_blocks.np[1, 0] == 4 + assert block_tables.num_blocks.np[2, 0] == 0 + assert block_tables.num_blocks.np[0, 1] == 1 + assert block_tables.num_blocks.np[1, 1] == 2 + assert block_tables.num_blocks.np[2, 1] == 2 + assert torch.equal( + block_tables.num_blocks.gpu[:, :2], + torch.tensor([[2, 1], [4, 2], [0, 2]], dtype=torch.int32, device=device), + ) + + for block_table in block_tables.block_tables: + assert not block_table._staged_write_indices + assert not block_table._staged_write_starts + assert not block_table._staged_write_contents + assert not block_table._staged_write_cu_lens + + block_tables.append_block_ids( + req_index=0, + new_block_ids=([7], [13], [8]), + overwrite=False, + ) + block_tables.apply_staged_writes() + torch.accelerator.synchronize() + + assert torch.equal( + block_tables.block_tables[0].gpu[0, :3], + torch.tensor([1, 2, 7], dtype=torch.int32, device=device), + ) + assert torch.equal( + block_tables.block_tables[1].gpu[0, :6], + torch.tensor([20, 21, 22, 23, 26, 27], dtype=torch.int32, device=device), + ) + assert torch.equal( + block_tables.block_tables[2].gpu[0, :1], + torch.tensor([8], dtype=torch.int32, device=device), + ) + assert block_tables.num_blocks.np[0, 0] == 3 + assert block_tables.num_blocks.np[1, 0] == 6 + assert block_tables.num_blocks.np[2, 0] == 1 + + +def test_block_tables_apply_staged_writes_single_group(): + device = torch.device("cuda") + block_tables = BlockTables( + block_sizes=[16], + max_num_reqs=2, + max_num_batched_tokens=16, + max_num_blocks_per_group=[4], + device=device, + kernel_block_sizes=[16], + ) + + block_tables.append_block_ids( + req_index=0, + new_block_ids=([1, 2],), + overwrite=True, + ) + block_tables.apply_staged_writes() + torch.accelerator.synchronize() + + assert torch.equal( + block_tables.block_tables[0].gpu[0, :2], + torch.tensor([1, 2], dtype=torch.int32, device=device), + ) diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 41692f58e7..8d41ba5a36 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -6,7 +6,12 @@ import torch from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.utils import PAD_SLOT_ID -from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor +from vllm.v1.worker.gpu.buffer_utils import ( + FusedStagedWriter, + StagedWriteTensor, + UvaBackedTensor, + _load_ptr, +) class BlockTables: @@ -52,6 +57,12 @@ class BlockTables: (self.num_kv_cache_groups, self.max_num_reqs), dtype=torch.int32, ) + self.fused_writer: FusedStagedWriter | None = None + if self.num_kv_cache_groups > 1: + # Only the multi-group path uses the fused writer. + self.fused_writer = FusedStagedWriter( + self.device, self.num_kv_cache_groups * self.max_num_reqs + ) # Block tables used for model's forward pass. # num_kv_cache_groups x [max_num_reqs, max_num_blocks] @@ -109,10 +120,15 @@ class BlockTables: self.num_blocks.np[i, req_index] = start + len(block_ids) def apply_staged_writes(self) -> None: - # TODO(woosuk): This can be inefficient since it launches one kernel per - # block table. Implement a kernel to handle all block tables at once. - for block_table in self.block_tables: - block_table.apply_write() + if self.num_kv_cache_groups == 1: + # Single group: write directly, skipping the per-write group lookup. + self.block_tables[0].apply_write() + else: + # Multiple groups: apply all block tables with one fused kernel. + assert self.fused_writer is not None + self.fused_writer.apply( + self.block_tables, self.block_table_ptrs, self.block_table_strides + ) self.num_blocks.copy_to_uva() def gather_block_tables( @@ -283,10 +299,3 @@ def _compute_slot_mappings_kernel( slot_ids = tl.where(is_local, slot_ids, PAD_ID) tl.store(slot_mapping_ptr + offset, slot_ids, mask=offset < end_idx) - - -@triton.jit -def _load_ptr(ptr_to_ptr, elem_dtype): - ptr = tl.load(ptr_to_ptr) - ptr = tl.cast(ptr, tl.pointer_type(elem_dtype)) - return tl.multiple_of(ptr, 16) diff --git a/vllm/v1/worker/gpu/buffer_utils.py b/vllm/v1/worker/gpu/buffer_utils.py index e4497de43a..cf5b2c1a2d 100644 --- a/vllm/v1/worker/gpu/buffer_utils.py +++ b/vllm/v1/worker/gpu/buffer_utils.py @@ -194,7 +194,9 @@ class StagedWriteTensor: starts_uva, write_contents, cu_lens_uva, + None, BLOCK_SIZE=1024, + MULTI_GROUP=False, ) # Clear the staged writes self.clear_staged_writes() @@ -206,15 +208,81 @@ class StagedWriteTensor: self._staged_write_cu_lens.clear() +class FusedStagedWriter: + """Applies the staged writes of several `StagedWriteTensor`s at once.""" + + def __init__( + self, device: torch.device, max_writes: int, max_concurrency: int | None = None + ): + new_pool = partial( + UvaBufferPool, dtype=torch.int32, max_concurrency=max_concurrency + ) + self.group_ids = new_pool(max_writes) + self.indices = new_pool(max_writes) + self.starts = new_pool(max_writes) + self.cu_lens = new_pool(max_writes) + self.device = device + + def apply( + self, + tensors: Sequence[StagedWriteTensor], + output_ptrs: torch.Tensor, + output_strides: torch.Tensor, + ) -> None: + """Apply and clear the staged writes of `tensors` with one kernel.""" + group_ids: list[int] = [] + indices: list[int] = [] + starts: list[int] = [] + contents: list[int | float] = [] + cu_lens: list[int] = [] + + for group_id, t in enumerate(tensors): + n = len(t._staged_write_indices) + if n == 0: + continue + + group_ids.extend([group_id] * n) + indices.extend(t._staged_write_indices) + starts.extend(t._staged_write_starts) + content_base = len(contents) + contents.extend(t._staged_write_contents) + cu_lens.extend(content_base + cu_len for cu_len in t._staged_write_cu_lens) + + if not group_ids: + return + + group_ids_uva = self.group_ids.copy_to_uva(group_ids) + indices_uva = self.indices.copy_to_uva(indices) + starts_uva = self.starts.copy_to_uva(starts) + cu_lens_uva = self.cu_lens.copy_to_uva(cu_lens) + contents_gpu = async_tensor_h2d(contents, torch.int32, self.device) + + _apply_write_kernel[(len(group_ids),)]( + output_ptrs, + output_strides, + indices_uva, + starts_uva, + contents_gpu, + cu_lens_uva, + group_ids_uva, + BLOCK_SIZE=1024, + MULTI_GROUP=True, + ) + for t in tensors: + t.clear_staged_writes() + + @triton.jit def _apply_write_kernel( - output_ptr, - output_stride, + output_ptr, # MULTI_GROUP: ptr-to-ptrs [num_groups]; else: data ptr + output_stride, # MULTI_GROUP: ptr-to-strides [num_groups]; else: row stride write_indices_ptr, write_starts_ptr, write_contents_ptr, write_cu_lens_ptr, + write_group_ids_ptr, # [num_writes], used only when MULTI_GROUP BLOCK_SIZE: tl.constexpr, + MULTI_GROUP: tl.constexpr, ): pid = tl.program_id(0) row_idx = tl.load(write_indices_ptr + pid) @@ -224,10 +292,26 @@ def _apply_write_kernel( cu_end = tl.load(write_cu_lens_ptr + pid) content_len = cu_end - cu_start + if MULTI_GROUP: + # Each write targets a different output tensor (KV cache group); + # resolve its base pointer and row stride per write. + group_id = tl.load(write_group_ids_ptr + pid) + row_ptr = _load_ptr(output_ptr + group_id, tl.int32) + row_stride = tl.load(output_stride + group_id) + else: + row_ptr = output_ptr + row_stride = output_stride + row_ptr += row_idx * row_stride + start_idx + for i in range(0, content_len, BLOCK_SIZE): block = i + tl.arange(0, BLOCK_SIZE) mask = block < content_len content = tl.load(write_contents_ptr + cu_start + block, mask=mask) - tl.store( - output_ptr + row_idx * output_stride + start_idx + block, content, mask=mask - ) + tl.store(row_ptr + block, content, mask=mask) + + +@triton.jit +def _load_ptr(ptr_to_ptr, elem_dtype): + ptr = tl.load(ptr_to_ptr) + ptr = tl.cast(ptr, tl.pointer_type(elem_dtype)) + return tl.multiple_of(ptr, 16)