[Bugfix] Reject mismatched float16/bfloat16 KV cache dtype (#60885)

Signed-off-by: cipheraxat <cipheraxat@users.noreply.github.com>
Co-authored-by: cipheraxat <cipheraxat@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Akshat Anand
2026-10-15 08:06:59 +00:00
committed by GitHub
co-authored by cipheraxat Claude Opus 5.5
parent fc8e08a2a6
commit 48f17b77bb
3 changed files with 42 additions and 0 deletions
+22
View File
@@ -290,6 +290,28 @@ def test_mamba_cache_mode_all_is_rejected():
CacheConfig(mamba_cache_mode="all")
@pytest.mark.parametrize(
("cache_dtype", "model_dtype", "should_raise"),
[
("float16", torch.bfloat16, True),
("bfloat16", torch.float16, True),
("float16", torch.float32, True),
("float16", torch.float16, False),
("auto", torch.bfloat16, False),
("fp8", torch.bfloat16, False),
],
)
def test_mismatched_float_kv_cache_is_rejected(cache_dtype, model_dtype, should_raise):
config = VllmConfig(cache_config=CacheConfig(cache_dtype=cache_dtype))
config.model_config = SimpleNamespace(dtype=model_dtype)
if should_raise:
with pytest.raises(ValueError, match="kv_cache_dtype"):
config._reject_mismatched_float_kv_cache()
else:
config._reject_mismatched_float_kv_cache()
def test_per_request_spec_decode_metrics_requires_spec_decode():
# The flag only makes sense with speculative decoding configured; enabling
# it without --speculative-config should fail fast rather than silently
+1
View File
@@ -121,6 +121,7 @@ class CacheConfig:
cache_dtype: CacheDType = "auto"
"""Data type for kv cache storage. If "auto", will use model data type.
An explicit float16 or bfloat16 must match the model data type.
CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports
fp8 (=fp8_e4m3). Intel Gaudi (HPU) supports fp8 (using fp8_inc).
Some models (namely DeepSeekV3.2) default to fp8, set to bfloat16 to use
+19
View File
@@ -1426,6 +1426,24 @@ class VllmConfig:
self.engram_config.verify_parallel_config(self.parallel_config)
logger.info_once("Resolved Engram configuration: %s", str(self.engram_config))
def _reject_mismatched_float_kv_cache(self) -> None:
"""Reject an explicit float16/bfloat16 KV dtype that differs from the model."""
cache_dtype = self.cache_config.cache_dtype
if self.model_config is None or cache_dtype not in ("float16", "bfloat16"):
return
model_dtype = str(self.model_config.dtype).removeprefix("torch.")
if model_dtype == cache_dtype:
return
allowed = (
f"`auto` or `{model_dtype}`"
if model_dtype in ("float16", "bfloat16")
else "`auto`"
)
raise ValueError(
f'kv_cache_dtype="{cache_dtype}" is incompatible with '
f'dtype="{model_dtype}". Please set `--kv-cache-dtype` to {allowed}.'
)
def __post_init__(self):
"""Verify configs are valid & consistent with each other."""
# To give each torch profile run a unique instance name.
@@ -1456,6 +1474,7 @@ class VllmConfig:
logger.info_once("Performance mode set to '%s'.", self.performance_mode)
self.try_verify_and_update_config()
self._reject_mismatched_float_kv_cache()
self._resolve_and_verify_engram_config()
self._check_supports_watermarking()