From 48f17b77bba9430ffedc3297186a2c9d932d94c7 Mon Sep 17 00:00:00 2001 From: Akshat Anand Date: Thu, 15 Oct 2026 13:36:59 +0530 Subject: [PATCH] [Bugfix] Reject mismatched float16/bfloat16 KV cache dtype (#60885) Signed-off-by: cipheraxat Co-authored-by: cipheraxat Co-authored-by: Claude Opus 5.5 --- tests/test_config.py | 22 ++++++++++++++++++++++ vllm/config/cache.py | 1 + vllm/config/vllm.py | 19 +++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index a7e36ada62..c029b0a520 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -290,6 +290,28 @@ def test_mamba_cache_mode_all_is_rejected(): CacheConfig(mamba_cache_mode="all") +@pytest.mark.parametrize( + ("cache_dtype", "model_dtype", "should_raise"), + [ + ("float16", torch.bfloat16, True), + ("bfloat16", torch.float16, True), + ("float16", torch.float32, True), + ("float16", torch.float16, False), + ("auto", torch.bfloat16, False), + ("fp8", torch.bfloat16, False), + ], +) +def test_mismatched_float_kv_cache_is_rejected(cache_dtype, model_dtype, should_raise): + config = VllmConfig(cache_config=CacheConfig(cache_dtype=cache_dtype)) + config.model_config = SimpleNamespace(dtype=model_dtype) + + if should_raise: + with pytest.raises(ValueError, match="kv_cache_dtype"): + config._reject_mismatched_float_kv_cache() + else: + config._reject_mismatched_float_kv_cache() + + def test_per_request_spec_decode_metrics_requires_spec_decode(): # The flag only makes sense with speculative decoding configured; enabling # it without --speculative-config should fail fast rather than silently diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 9bb7166215..fb00622ed0 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -121,6 +121,7 @@ class CacheConfig: cache_dtype: CacheDType = "auto" """Data type for kv cache storage. If "auto", will use model data type. + An explicit float16 or bfloat16 must match the model data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports fp8 (=fp8_e4m3). Intel Gaudi (HPU) supports fp8 (using fp8_inc). Some models (namely DeepSeekV3.2) default to fp8, set to bfloat16 to use diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 87d2187459..5d025a589d 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1426,6 +1426,24 @@ class VllmConfig: self.engram_config.verify_parallel_config(self.parallel_config) logger.info_once("Resolved Engram configuration: %s", str(self.engram_config)) + def _reject_mismatched_float_kv_cache(self) -> None: + """Reject an explicit float16/bfloat16 KV dtype that differs from the model.""" + cache_dtype = self.cache_config.cache_dtype + if self.model_config is None or cache_dtype not in ("float16", "bfloat16"): + return + model_dtype = str(self.model_config.dtype).removeprefix("torch.") + if model_dtype == cache_dtype: + return + allowed = ( + f"`auto` or `{model_dtype}`" + if model_dtype in ("float16", "bfloat16") + else "`auto`" + ) + raise ValueError( + f'kv_cache_dtype="{cache_dtype}" is incompatible with ' + f'dtype="{model_dtype}". Please set `--kv-cache-dtype` to {allowed}.' + ) + def __post_init__(self): """Verify configs are valid & consistent with each other.""" # To give each torch profile run a unique instance name. @@ -1456,6 +1474,7 @@ class VllmConfig: logger.info_once("Performance mode set to '%s'.", self.performance_mode) self.try_verify_and_update_config() + self._reject_mismatched_float_kv_cache() self._resolve_and_verify_engram_config() self._check_supports_watermarking()