[PERF] Fuse multi-group block table staged writes (#44944)

Signed-off-by: jesse <szxfml@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Song Zhixin
2026-06-16 10:53:27 -07:00
committed by GitHub
co-authored by Nick Hill
parent 7d567172fc
commit 8e27a9c215
3 changed files with 242 additions and 17 deletions
+89 -5
View File
@@ -194,7 +194,9 @@ class StagedWriteTensor:
starts_uva,
write_contents,
cu_lens_uva,
None,
BLOCK_SIZE=1024,
MULTI_GROUP=False,
)
# Clear the staged writes
self.clear_staged_writes()
@@ -206,15 +208,81 @@ class StagedWriteTensor:
self._staged_write_cu_lens.clear()
class FusedStagedWriter:
"""Applies the staged writes of several `StagedWriteTensor`s at once."""
def __init__(
self, device: torch.device, max_writes: int, max_concurrency: int | None = None
):
new_pool = partial(
UvaBufferPool, dtype=torch.int32, max_concurrency=max_concurrency
)
self.group_ids = new_pool(max_writes)
self.indices = new_pool(max_writes)
self.starts = new_pool(max_writes)
self.cu_lens = new_pool(max_writes)
self.device = device
def apply(
self,
tensors: Sequence[StagedWriteTensor],
output_ptrs: torch.Tensor,
output_strides: torch.Tensor,
) -> None:
"""Apply and clear the staged writes of `tensors` with one kernel."""
group_ids: list[int] = []
indices: list[int] = []
starts: list[int] = []
contents: list[int | float] = []
cu_lens: list[int] = []
for group_id, t in enumerate(tensors):
n = len(t._staged_write_indices)
if n == 0:
continue
group_ids.extend([group_id] * n)
indices.extend(t._staged_write_indices)
starts.extend(t._staged_write_starts)
content_base = len(contents)
contents.extend(t._staged_write_contents)
cu_lens.extend(content_base + cu_len for cu_len in t._staged_write_cu_lens)
if not group_ids:
return
group_ids_uva = self.group_ids.copy_to_uva(group_ids)
indices_uva = self.indices.copy_to_uva(indices)
starts_uva = self.starts.copy_to_uva(starts)
cu_lens_uva = self.cu_lens.copy_to_uva(cu_lens)
contents_gpu = async_tensor_h2d(contents, torch.int32, self.device)
_apply_write_kernel[(len(group_ids),)](
output_ptrs,
output_strides,
indices_uva,
starts_uva,
contents_gpu,
cu_lens_uva,
group_ids_uva,
BLOCK_SIZE=1024,
MULTI_GROUP=True,
)
for t in tensors:
t.clear_staged_writes()
@triton.jit
def _apply_write_kernel(
output_ptr,
output_stride,
output_ptr, # MULTI_GROUP: ptr-to-ptrs [num_groups]; else: data ptr
output_stride, # MULTI_GROUP: ptr-to-strides [num_groups]; else: row stride
write_indices_ptr,
write_starts_ptr,
write_contents_ptr,
write_cu_lens_ptr,
write_group_ids_ptr, # [num_writes], used only when MULTI_GROUP
BLOCK_SIZE: tl.constexpr,
MULTI_GROUP: tl.constexpr,
):
pid = tl.program_id(0)
row_idx = tl.load(write_indices_ptr + pid)
@@ -224,10 +292,26 @@ def _apply_write_kernel(
cu_end = tl.load(write_cu_lens_ptr + pid)
content_len = cu_end - cu_start
if MULTI_GROUP:
# Each write targets a different output tensor (KV cache group);
# resolve its base pointer and row stride per write.
group_id = tl.load(write_group_ids_ptr + pid)
row_ptr = _load_ptr(output_ptr + group_id, tl.int32)
row_stride = tl.load(output_stride + group_id)
else:
row_ptr = output_ptr
row_stride = output_stride
row_ptr += row_idx * row_stride + start_idx
for i in range(0, content_len, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < content_len
content = tl.load(write_contents_ptr + cu_start + block, mask=mask)
tl.store(
output_ptr + row_idx * output_stride + start_idx + block, content, mask=mask
)
tl.store(row_ptr + block, content, mask=mask)
@triton.jit
def _load_ptr(ptr_to_ptr, elem_dtype):
ptr = tl.load(ptr_to_ptr)
ptr = tl.cast(ptr, tl.pointer_type(elem_dtype))
return tl.multiple_of(ptr, 16)