Allow native KV cache dtype in Triton cache update (#43330)

Signed-off-by: Michael Gschwind <mgschwind@nvidia.com>
Co-authored-by: Michael Gschwind <mgschwind@nvidia.com>
This commit is contained in:
Mike G
2026-05-28 16:51:40 +00:00
committed by GitHub
co-authored by Michael Gschwind
parent 099024762c
commit 53a2088675
@@ -13,6 +13,14 @@ from vllm.utils.torch_utils import is_quantized_kv_cache
FP8_MIN, FP8_MAX = get_fp8_min_max()
_NATIVE_KV_CACHE_DTYPES = {"auto", "float16", "bfloat16", "float32", "half", "float"}
def _is_supported_kv_cache_dtype(kv_cache_dtype: str) -> bool:
return kv_cache_dtype in _NATIVE_KV_CACHE_DTYPES or is_quantized_kv_cache(
kv_cache_dtype
)
@triton.jit
def reshape_and_cache_kernel_flash(
@@ -350,7 +358,7 @@ def triton_reshape_and_cache_flash(
block_stride = key_cache.stride()[0]
page_stride = key_cache.stride()[1]
assert kv_cache_dtype == "auto" or is_quantized_kv_cache(kv_cache_dtype), (
assert _is_supported_kv_cache_dtype(kv_cache_dtype), (
f"unsupported kv_cache_dtype (str), got {kv_cache_dtype}."
)
kv_cache_torch_dtype = (
@@ -529,7 +537,7 @@ def triton_reshape_and_cache_flash_diffkv(
block_stride = kv_cache.stride()[0]
page_stride = kv_cache.stride()[1]
assert kv_cache_dtype == "auto" or is_quantized_kv_cache(kv_cache_dtype), (
assert _is_supported_kv_cache_dtype(kv_cache_dtype), (
f"unsupported kv_cache_dtype (str), got {kv_cache_dtype}."
)
kv_cache_torch_dtype = (