From 53a20886752bdf8f26c4c5121458c514096da107 Mon Sep 17 00:00:00 2001 From: Mike G <180722391+mikekg@users.noreply.github.com> Date: Thu, 28 May 2026 09:51:40 -0700 Subject: [PATCH] Allow native KV cache dtype in Triton cache update (#43330) Signed-off-by: Michael Gschwind Co-authored-by: Michael Gschwind --- .../attention/ops/triton_reshape_and_cache_flash.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py b/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py index 6e696fdb51..08c6673fb5 100644 --- a/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py +++ b/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py @@ -13,6 +13,14 @@ from vllm.utils.torch_utils import is_quantized_kv_cache FP8_MIN, FP8_MAX = get_fp8_min_max() +_NATIVE_KV_CACHE_DTYPES = {"auto", "float16", "bfloat16", "float32", "half", "float"} + + +def _is_supported_kv_cache_dtype(kv_cache_dtype: str) -> bool: + return kv_cache_dtype in _NATIVE_KV_CACHE_DTYPES or is_quantized_kv_cache( + kv_cache_dtype + ) + @triton.jit def reshape_and_cache_kernel_flash( @@ -350,7 +358,7 @@ def triton_reshape_and_cache_flash( block_stride = key_cache.stride()[0] page_stride = key_cache.stride()[1] - assert kv_cache_dtype == "auto" or is_quantized_kv_cache(kv_cache_dtype), ( + assert _is_supported_kv_cache_dtype(kv_cache_dtype), ( f"unsupported kv_cache_dtype (str), got {kv_cache_dtype}." ) kv_cache_torch_dtype = ( @@ -529,7 +537,7 @@ def triton_reshape_and_cache_flash_diffkv( block_stride = kv_cache.stride()[0] page_stride = kv_cache.stride()[1] - assert kv_cache_dtype == "auto" or is_quantized_kv_cache(kv_cache_dtype), ( + assert _is_supported_kv_cache_dtype(kv_cache_dtype), ( f"unsupported kv_cache_dtype (str), got {kv_cache_dtype}." ) kv_cache_torch_dtype = (