[PERF] Fuse multi-group block table staged writes (#44944)

Signed-off-by: jesse <szxfml@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Song Zhixin
2026-06-16 10:53:27 -07:00
committed by GitHub
co-authored by Nick Hill
parent 7d567172fc
commit 8e27a9c215
3 changed files with 242 additions and 17 deletions
+132
View File
@@ -0,0 +1,132 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.block_table import BlockTables
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(),
reason="requires CUDA",
)
def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
device = torch.device("cuda")
block_tables = BlockTables(
block_sizes=[16, 32, 8],
max_num_reqs=4,
max_num_batched_tokens=64,
max_num_blocks_per_group=[8, 8, 8],
device=device,
kernel_block_sizes=[16, 16, 8],
)
def fail_if_apply_write_called():
pytest.fail("multi-group writes should use the fused apply kernel")
for block_table in block_tables.block_tables:
monkeypatch.setattr(block_table, "apply_write", fail_if_apply_write_called)
block_tables.append_block_ids(
req_index=0,
new_block_ids=([1, 2], [10, 11], []),
overwrite=True,
)
block_tables.append_block_ids(
req_index=1,
new_block_ids=([3], [12], [5, 6]),
overwrite=True,
)
block_tables.apply_staged_writes()
torch.accelerator.synchronize()
assert torch.equal(
block_tables.block_tables[0].gpu[0, :2],
torch.tensor([1, 2], dtype=torch.int32, device=device),
)
# Group 1 has blocks_per_kv_block == 2, so each KV block expands to two
# kernel block IDs.
assert torch.equal(
block_tables.block_tables[1].gpu[0, :4],
torch.tensor([20, 21, 22, 23], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[0].gpu[1, :1],
torch.tensor([3], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[1].gpu[1, :2],
torch.tensor([24, 25], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[2].gpu[1, :2],
torch.tensor([5, 6], dtype=torch.int32, device=device),
)
assert block_tables.num_blocks.np[0, 0] == 2
assert block_tables.num_blocks.np[1, 0] == 4
assert block_tables.num_blocks.np[2, 0] == 0
assert block_tables.num_blocks.np[0, 1] == 1
assert block_tables.num_blocks.np[1, 1] == 2
assert block_tables.num_blocks.np[2, 1] == 2
assert torch.equal(
block_tables.num_blocks.gpu[:, :2],
torch.tensor([[2, 1], [4, 2], [0, 2]], dtype=torch.int32, device=device),
)
for block_table in block_tables.block_tables:
assert not block_table._staged_write_indices
assert not block_table._staged_write_starts
assert not block_table._staged_write_contents
assert not block_table._staged_write_cu_lens
block_tables.append_block_ids(
req_index=0,
new_block_ids=([7], [13], [8]),
overwrite=False,
)
block_tables.apply_staged_writes()
torch.accelerator.synchronize()
assert torch.equal(
block_tables.block_tables[0].gpu[0, :3],
torch.tensor([1, 2, 7], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[1].gpu[0, :6],
torch.tensor([20, 21, 22, 23, 26, 27], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[2].gpu[0, :1],
torch.tensor([8], dtype=torch.int32, device=device),
)
assert block_tables.num_blocks.np[0, 0] == 3
assert block_tables.num_blocks.np[1, 0] == 6
assert block_tables.num_blocks.np[2, 0] == 1
def test_block_tables_apply_staged_writes_single_group():
device = torch.device("cuda")
block_tables = BlockTables(
block_sizes=[16],
max_num_reqs=2,
max_num_batched_tokens=16,
max_num_blocks_per_group=[4],
device=device,
kernel_block_sizes=[16],
)
block_tables.append_block_ids(
req_index=0,
new_block_ids=([1, 2],),
overwrite=True,
)
block_tables.apply_staged_writes()
torch.accelerator.synchronize()
assert torch.equal(
block_tables.block_tables[0].gpu[0, :2],
torch.tensor([1, 2], dtype=torch.int32, device=device),
)
+21 -12
View File
@@ -6,7 +6,12 @@ import torch
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
from vllm.v1.worker.gpu.buffer_utils import (
FusedStagedWriter,
StagedWriteTensor,
UvaBackedTensor,
_load_ptr,
)
class BlockTables:
@@ -52,6 +57,12 @@ class BlockTables:
(self.num_kv_cache_groups, self.max_num_reqs),
dtype=torch.int32,
)
self.fused_writer: FusedStagedWriter | None = None
if self.num_kv_cache_groups > 1:
# Only the multi-group path uses the fused writer.
self.fused_writer = FusedStagedWriter(
self.device, self.num_kv_cache_groups * self.max_num_reqs
)
# Block tables used for model's forward pass.
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
@@ -109,10 +120,15 @@ class BlockTables:
self.num_blocks.np[i, req_index] = start + len(block_ids)
def apply_staged_writes(self) -> None:
# TODO(woosuk): This can be inefficient since it launches one kernel per
# block table. Implement a kernel to handle all block tables at once.
for block_table in self.block_tables:
block_table.apply_write()
if self.num_kv_cache_groups == 1:
# Single group: write directly, skipping the per-write group lookup.
self.block_tables[0].apply_write()
else:
# Multiple groups: apply all block tables with one fused kernel.
assert self.fused_writer is not None
self.fused_writer.apply(
self.block_tables, self.block_table_ptrs, self.block_table_strides
)
self.num_blocks.copy_to_uva()
def gather_block_tables(
@@ -283,10 +299,3 @@ def _compute_slot_mappings_kernel(
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
tl.store(slot_mapping_ptr + offset, slot_ids, mask=offset < end_idx)
@triton.jit
def _load_ptr(ptr_to_ptr, elem_dtype):
ptr = tl.load(ptr_to_ptr)
ptr = tl.cast(ptr, tl.pointer_type(elem_dtype))
return tl.multiple_of(ptr, 16)
+89 -5
View File
@@ -194,7 +194,9 @@ class StagedWriteTensor:
starts_uva,
write_contents,
cu_lens_uva,
None,
BLOCK_SIZE=1024,
MULTI_GROUP=False,
)
# Clear the staged writes
self.clear_staged_writes()
@@ -206,15 +208,81 @@ class StagedWriteTensor:
self._staged_write_cu_lens.clear()
class FusedStagedWriter:
"""Applies the staged writes of several `StagedWriteTensor`s at once."""
def __init__(
self, device: torch.device, max_writes: int, max_concurrency: int | None = None
):
new_pool = partial(
UvaBufferPool, dtype=torch.int32, max_concurrency=max_concurrency
)
self.group_ids = new_pool(max_writes)
self.indices = new_pool(max_writes)
self.starts = new_pool(max_writes)
self.cu_lens = new_pool(max_writes)
self.device = device
def apply(
self,
tensors: Sequence[StagedWriteTensor],
output_ptrs: torch.Tensor,
output_strides: torch.Tensor,
) -> None:
"""Apply and clear the staged writes of `tensors` with one kernel."""
group_ids: list[int] = []
indices: list[int] = []
starts: list[int] = []
contents: list[int | float] = []
cu_lens: list[int] = []
for group_id, t in enumerate(tensors):
n = len(t._staged_write_indices)
if n == 0:
continue
group_ids.extend([group_id] * n)
indices.extend(t._staged_write_indices)
starts.extend(t._staged_write_starts)
content_base = len(contents)
contents.extend(t._staged_write_contents)
cu_lens.extend(content_base + cu_len for cu_len in t._staged_write_cu_lens)
if not group_ids:
return
group_ids_uva = self.group_ids.copy_to_uva(group_ids)
indices_uva = self.indices.copy_to_uva(indices)
starts_uva = self.starts.copy_to_uva(starts)
cu_lens_uva = self.cu_lens.copy_to_uva(cu_lens)
contents_gpu = async_tensor_h2d(contents, torch.int32, self.device)
_apply_write_kernel[(len(group_ids),)](
output_ptrs,
output_strides,
indices_uva,
starts_uva,
contents_gpu,
cu_lens_uva,
group_ids_uva,
BLOCK_SIZE=1024,
MULTI_GROUP=True,
)
for t in tensors:
t.clear_staged_writes()
@triton.jit
def _apply_write_kernel(
output_ptr,
output_stride,
output_ptr, # MULTI_GROUP: ptr-to-ptrs [num_groups]; else: data ptr
output_stride, # MULTI_GROUP: ptr-to-strides [num_groups]; else: row stride
write_indices_ptr,
write_starts_ptr,
write_contents_ptr,
write_cu_lens_ptr,
write_group_ids_ptr, # [num_writes], used only when MULTI_GROUP
BLOCK_SIZE: tl.constexpr,
MULTI_GROUP: tl.constexpr,
):
pid = tl.program_id(0)
row_idx = tl.load(write_indices_ptr + pid)
@@ -224,10 +292,26 @@ def _apply_write_kernel(
cu_end = tl.load(write_cu_lens_ptr + pid)
content_len = cu_end - cu_start
if MULTI_GROUP:
# Each write targets a different output tensor (KV cache group);
# resolve its base pointer and row stride per write.
group_id = tl.load(write_group_ids_ptr + pid)
row_ptr = _load_ptr(output_ptr + group_id, tl.int32)
row_stride = tl.load(output_stride + group_id)
else:
row_ptr = output_ptr
row_stride = output_stride
row_ptr += row_idx * row_stride + start_idx
for i in range(0, content_len, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < content_len
content = tl.load(write_contents_ptr + cu_start + block, mask=mask)
tl.store(
output_ptr + row_idx * output_stride + start_idx + block, content, mask=mask
)
tl.store(row_ptr + block, content, mask=mask)
@triton.jit
def _load_ptr(ptr_to_ptr, elem_dtype):
ptr = tl.load(ptr_to_ptr)
ptr = tl.cast(ptr, tl.pointer_type(elem_dtype))
return tl.multiple_of(ptr, 16)