[PERF] Fuse multi-group block table staged writes (#44944)
Signed-off-by: jesse <szxfml@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -0,0 +1,132 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="requires CUDA",
|
||||
)
|
||||
|
||||
|
||||
def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
|
||||
device = torch.device("cuda")
|
||||
block_tables = BlockTables(
|
||||
block_sizes=[16, 32, 8],
|
||||
max_num_reqs=4,
|
||||
max_num_batched_tokens=64,
|
||||
max_num_blocks_per_group=[8, 8, 8],
|
||||
device=device,
|
||||
kernel_block_sizes=[16, 16, 8],
|
||||
)
|
||||
|
||||
def fail_if_apply_write_called():
|
||||
pytest.fail("multi-group writes should use the fused apply kernel")
|
||||
|
||||
for block_table in block_tables.block_tables:
|
||||
monkeypatch.setattr(block_table, "apply_write", fail_if_apply_write_called)
|
||||
|
||||
block_tables.append_block_ids(
|
||||
req_index=0,
|
||||
new_block_ids=([1, 2], [10, 11], []),
|
||||
overwrite=True,
|
||||
)
|
||||
block_tables.append_block_ids(
|
||||
req_index=1,
|
||||
new_block_ids=([3], [12], [5, 6]),
|
||||
overwrite=True,
|
||||
)
|
||||
block_tables.apply_staged_writes()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[0].gpu[0, :2],
|
||||
torch.tensor([1, 2], dtype=torch.int32, device=device),
|
||||
)
|
||||
# Group 1 has blocks_per_kv_block == 2, so each KV block expands to two
|
||||
# kernel block IDs.
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[1].gpu[0, :4],
|
||||
torch.tensor([20, 21, 22, 23], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[0].gpu[1, :1],
|
||||
torch.tensor([3], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[1].gpu[1, :2],
|
||||
torch.tensor([24, 25], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[2].gpu[1, :2],
|
||||
torch.tensor([5, 6], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert block_tables.num_blocks.np[0, 0] == 2
|
||||
assert block_tables.num_blocks.np[1, 0] == 4
|
||||
assert block_tables.num_blocks.np[2, 0] == 0
|
||||
assert block_tables.num_blocks.np[0, 1] == 1
|
||||
assert block_tables.num_blocks.np[1, 1] == 2
|
||||
assert block_tables.num_blocks.np[2, 1] == 2
|
||||
assert torch.equal(
|
||||
block_tables.num_blocks.gpu[:, :2],
|
||||
torch.tensor([[2, 1], [4, 2], [0, 2]], dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
for block_table in block_tables.block_tables:
|
||||
assert not block_table._staged_write_indices
|
||||
assert not block_table._staged_write_starts
|
||||
assert not block_table._staged_write_contents
|
||||
assert not block_table._staged_write_cu_lens
|
||||
|
||||
block_tables.append_block_ids(
|
||||
req_index=0,
|
||||
new_block_ids=([7], [13], [8]),
|
||||
overwrite=False,
|
||||
)
|
||||
block_tables.apply_staged_writes()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[0].gpu[0, :3],
|
||||
torch.tensor([1, 2, 7], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[1].gpu[0, :6],
|
||||
torch.tensor([20, 21, 22, 23, 26, 27], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[2].gpu[0, :1],
|
||||
torch.tensor([8], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert block_tables.num_blocks.np[0, 0] == 3
|
||||
assert block_tables.num_blocks.np[1, 0] == 6
|
||||
assert block_tables.num_blocks.np[2, 0] == 1
|
||||
|
||||
|
||||
def test_block_tables_apply_staged_writes_single_group():
|
||||
device = torch.device("cuda")
|
||||
block_tables = BlockTables(
|
||||
block_sizes=[16],
|
||||
max_num_reqs=2,
|
||||
max_num_batched_tokens=16,
|
||||
max_num_blocks_per_group=[4],
|
||||
device=device,
|
||||
kernel_block_sizes=[16],
|
||||
)
|
||||
|
||||
block_tables.append_block_ids(
|
||||
req_index=0,
|
||||
new_block_ids=([1, 2],),
|
||||
overwrite=True,
|
||||
)
|
||||
block_tables.apply_staged_writes()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[0].gpu[0, :2],
|
||||
torch.tensor([1, 2], dtype=torch.int32, device=device),
|
||||
)
|
||||
@@ -6,7 +6,12 @@ import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
|
||||
from vllm.v1.worker.gpu.buffer_utils import (
|
||||
FusedStagedWriter,
|
||||
StagedWriteTensor,
|
||||
UvaBackedTensor,
|
||||
_load_ptr,
|
||||
)
|
||||
|
||||
|
||||
class BlockTables:
|
||||
@@ -52,6 +57,12 @@ class BlockTables:
|
||||
(self.num_kv_cache_groups, self.max_num_reqs),
|
||||
dtype=torch.int32,
|
||||
)
|
||||
self.fused_writer: FusedStagedWriter | None = None
|
||||
if self.num_kv_cache_groups > 1:
|
||||
# Only the multi-group path uses the fused writer.
|
||||
self.fused_writer = FusedStagedWriter(
|
||||
self.device, self.num_kv_cache_groups * self.max_num_reqs
|
||||
)
|
||||
|
||||
# Block tables used for model's forward pass.
|
||||
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
|
||||
@@ -109,10 +120,15 @@ class BlockTables:
|
||||
self.num_blocks.np[i, req_index] = start + len(block_ids)
|
||||
|
||||
def apply_staged_writes(self) -> None:
|
||||
# TODO(woosuk): This can be inefficient since it launches one kernel per
|
||||
# block table. Implement a kernel to handle all block tables at once.
|
||||
for block_table in self.block_tables:
|
||||
block_table.apply_write()
|
||||
if self.num_kv_cache_groups == 1:
|
||||
# Single group: write directly, skipping the per-write group lookup.
|
||||
self.block_tables[0].apply_write()
|
||||
else:
|
||||
# Multiple groups: apply all block tables with one fused kernel.
|
||||
assert self.fused_writer is not None
|
||||
self.fused_writer.apply(
|
||||
self.block_tables, self.block_table_ptrs, self.block_table_strides
|
||||
)
|
||||
self.num_blocks.copy_to_uva()
|
||||
|
||||
def gather_block_tables(
|
||||
@@ -283,10 +299,3 @@ def _compute_slot_mappings_kernel(
|
||||
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
|
||||
|
||||
tl.store(slot_mapping_ptr + offset, slot_ids, mask=offset < end_idx)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _load_ptr(ptr_to_ptr, elem_dtype):
|
||||
ptr = tl.load(ptr_to_ptr)
|
||||
ptr = tl.cast(ptr, tl.pointer_type(elem_dtype))
|
||||
return tl.multiple_of(ptr, 16)
|
||||
|
||||
@@ -194,7 +194,9 @@ class StagedWriteTensor:
|
||||
starts_uva,
|
||||
write_contents,
|
||||
cu_lens_uva,
|
||||
None,
|
||||
BLOCK_SIZE=1024,
|
||||
MULTI_GROUP=False,
|
||||
)
|
||||
# Clear the staged writes
|
||||
self.clear_staged_writes()
|
||||
@@ -206,15 +208,81 @@ class StagedWriteTensor:
|
||||
self._staged_write_cu_lens.clear()
|
||||
|
||||
|
||||
class FusedStagedWriter:
|
||||
"""Applies the staged writes of several `StagedWriteTensor`s at once."""
|
||||
|
||||
def __init__(
|
||||
self, device: torch.device, max_writes: int, max_concurrency: int | None = None
|
||||
):
|
||||
new_pool = partial(
|
||||
UvaBufferPool, dtype=torch.int32, max_concurrency=max_concurrency
|
||||
)
|
||||
self.group_ids = new_pool(max_writes)
|
||||
self.indices = new_pool(max_writes)
|
||||
self.starts = new_pool(max_writes)
|
||||
self.cu_lens = new_pool(max_writes)
|
||||
self.device = device
|
||||
|
||||
def apply(
|
||||
self,
|
||||
tensors: Sequence[StagedWriteTensor],
|
||||
output_ptrs: torch.Tensor,
|
||||
output_strides: torch.Tensor,
|
||||
) -> None:
|
||||
"""Apply and clear the staged writes of `tensors` with one kernel."""
|
||||
group_ids: list[int] = []
|
||||
indices: list[int] = []
|
||||
starts: list[int] = []
|
||||
contents: list[int | float] = []
|
||||
cu_lens: list[int] = []
|
||||
|
||||
for group_id, t in enumerate(tensors):
|
||||
n = len(t._staged_write_indices)
|
||||
if n == 0:
|
||||
continue
|
||||
|
||||
group_ids.extend([group_id] * n)
|
||||
indices.extend(t._staged_write_indices)
|
||||
starts.extend(t._staged_write_starts)
|
||||
content_base = len(contents)
|
||||
contents.extend(t._staged_write_contents)
|
||||
cu_lens.extend(content_base + cu_len for cu_len in t._staged_write_cu_lens)
|
||||
|
||||
if not group_ids:
|
||||
return
|
||||
|
||||
group_ids_uva = self.group_ids.copy_to_uva(group_ids)
|
||||
indices_uva = self.indices.copy_to_uva(indices)
|
||||
starts_uva = self.starts.copy_to_uva(starts)
|
||||
cu_lens_uva = self.cu_lens.copy_to_uva(cu_lens)
|
||||
contents_gpu = async_tensor_h2d(contents, torch.int32, self.device)
|
||||
|
||||
_apply_write_kernel[(len(group_ids),)](
|
||||
output_ptrs,
|
||||
output_strides,
|
||||
indices_uva,
|
||||
starts_uva,
|
||||
contents_gpu,
|
||||
cu_lens_uva,
|
||||
group_ids_uva,
|
||||
BLOCK_SIZE=1024,
|
||||
MULTI_GROUP=True,
|
||||
)
|
||||
for t in tensors:
|
||||
t.clear_staged_writes()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _apply_write_kernel(
|
||||
output_ptr,
|
||||
output_stride,
|
||||
output_ptr, # MULTI_GROUP: ptr-to-ptrs [num_groups]; else: data ptr
|
||||
output_stride, # MULTI_GROUP: ptr-to-strides [num_groups]; else: row stride
|
||||
write_indices_ptr,
|
||||
write_starts_ptr,
|
||||
write_contents_ptr,
|
||||
write_cu_lens_ptr,
|
||||
write_group_ids_ptr, # [num_writes], used only when MULTI_GROUP
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
MULTI_GROUP: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
row_idx = tl.load(write_indices_ptr + pid)
|
||||
@@ -224,10 +292,26 @@ def _apply_write_kernel(
|
||||
cu_end = tl.load(write_cu_lens_ptr + pid)
|
||||
content_len = cu_end - cu_start
|
||||
|
||||
if MULTI_GROUP:
|
||||
# Each write targets a different output tensor (KV cache group);
|
||||
# resolve its base pointer and row stride per write.
|
||||
group_id = tl.load(write_group_ids_ptr + pid)
|
||||
row_ptr = _load_ptr(output_ptr + group_id, tl.int32)
|
||||
row_stride = tl.load(output_stride + group_id)
|
||||
else:
|
||||
row_ptr = output_ptr
|
||||
row_stride = output_stride
|
||||
row_ptr += row_idx * row_stride + start_idx
|
||||
|
||||
for i in range(0, content_len, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < content_len
|
||||
content = tl.load(write_contents_ptr + cu_start + block, mask=mask)
|
||||
tl.store(
|
||||
output_ptr + row_idx * output_stride + start_idx + block, content, mask=mask
|
||||
)
|
||||
tl.store(row_ptr + block, content, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _load_ptr(ptr_to_ptr, elem_dtype):
|
||||
ptr = tl.load(ptr_to_ptr)
|
||||
ptr = tl.cast(ptr, tl.pointer_type(elem_dtype))
|
||||
return tl.multiple_of(ptr, 16)
|
||||
|
||||
Reference in New Issue
Block a user