[Bugfix][ROCm] Pre-allocate wvSplitKrc static workspaces before KV init (#54247)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Kimi Code <noreply@moonshot.ai>
This commit is contained in:
Nick Hill
2026-08-28 14:22:53 -07:00
committed by GitHub
co-authored by Kimi Code
parent 21fa2c5a2a
commit 70732942c6
2 changed files with 37 additions and 6 deletions
+21
View File
@@ -368,6 +368,27 @@ direct_register_custom_op(
)
@functools.cache
def warmup_rocm_skinny_gemm_workspaces(device: torch.device) -> None:
"""Eagerly allocate wvSplitKrc's process-lifetime static workspaces.
They are otherwise created lazily on the first qualifying GEMM
(csrc/rocm/skinny_gemms.cu), which can be the first real request — after
the KV cache backing buffer exists. If one landed in that segment's
rounding tail, it would pin the entire segment at engine shutdown.
"""
from vllm.platforms.rocm import on_gfx950
if not on_gfx950():
return
try:
x = torch.zeros(16, 1024, dtype=torch.bfloat16, device=device)
weight = torch.zeros(32, 1024, dtype=torch.bfloat16, device=device)
ops.wvSplitKrc(x, weight, num_compute_units())
except Exception:
logger.debug("wvSplitKrc workspace warmup failed", exc_info=True)
# Above this weight size, oneDNN's onednn_mm consistently matches or beats
# the SGL AMX kernel once M grows past decode-sized batches, and is within
# noise of it at decode-sized M -- so larger weights default to oneDNN
+16 -6
View File
@@ -13,6 +13,7 @@ import torch
from vllm.config import CacheConfig, VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.utils import warmup_rocm_skinny_gemm_workspaces
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
from vllm.model_executor.models.utils import extract_layer_index
from vllm.platforms import current_platform
@@ -393,12 +394,21 @@ def allocate_kv_cache(
sizes = {tensor.size for tensor in kv_cache_config.kv_cache_tensors}
assert len(sizes) == 1, "KV cache tensors must share one backing allocation."
raw_size = sizes.pop()
page_size = 4096
buf = torch.zeros(
((raw_size + page_size - 1) // page_size) * page_size,
dtype=torch.int8,
device=device,
)
# wvSplitKrc's process-lifetime static workspaces (csrc/rocm/skinny_gemms.cu)
# are created lazily on the first qualifying GEMM. Force that now, before
# the giant backing allocation below: if one landed in this segment's
# rounding tail it would pin the whole segment at engine shutdown.
if current_platform.is_rocm():
warmup_rocm_skinny_gemm_workspaces(device)
# Pad to the page granularity MoRIIO needs to register the shared
# backing as a single RDMA memory region. Other platforms keep the
# exact-size allocation: NIXL and SimpleCPUOffload rely on
# storage.nbytes() matching the logical KV size (see #53974).
page_size = 4096
buf_size = ((raw_size + page_size - 1) // page_size) * page_size
else:
buf_size = raw_size
buf = torch.zeros(buf_size, dtype=torch.int8, device=device)
kv_caches: dict[str, torch.Tensor] = {}
for tensor in kv_cache_config.kv_cache_tensors: