[Bugfix] Reject mismatched float16/bfloat16 KV cache dtype (#60885)
Signed-off-by: cipheraxat <cipheraxat@users.noreply.github.com> Co-authored-by: cipheraxat <cipheraxat@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
cipheraxat
Claude Opus 5.5
parent
fc8e08a2a6
commit
48f17b77bb
@@ -290,6 +290,28 @@ def test_mamba_cache_mode_all_is_rejected():
|
||||
CacheConfig(mamba_cache_mode="all")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cache_dtype", "model_dtype", "should_raise"),
|
||||
[
|
||||
("float16", torch.bfloat16, True),
|
||||
("bfloat16", torch.float16, True),
|
||||
("float16", torch.float32, True),
|
||||
("float16", torch.float16, False),
|
||||
("auto", torch.bfloat16, False),
|
||||
("fp8", torch.bfloat16, False),
|
||||
],
|
||||
)
|
||||
def test_mismatched_float_kv_cache_is_rejected(cache_dtype, model_dtype, should_raise):
|
||||
config = VllmConfig(cache_config=CacheConfig(cache_dtype=cache_dtype))
|
||||
config.model_config = SimpleNamespace(dtype=model_dtype)
|
||||
|
||||
if should_raise:
|
||||
with pytest.raises(ValueError, match="kv_cache_dtype"):
|
||||
config._reject_mismatched_float_kv_cache()
|
||||
else:
|
||||
config._reject_mismatched_float_kv_cache()
|
||||
|
||||
|
||||
def test_per_request_spec_decode_metrics_requires_spec_decode():
|
||||
# The flag only makes sense with speculative decoding configured; enabling
|
||||
# it without --speculative-config should fail fast rather than silently
|
||||
|
||||
@@ -121,6 +121,7 @@ class CacheConfig:
|
||||
|
||||
cache_dtype: CacheDType = "auto"
|
||||
"""Data type for kv cache storage. If "auto", will use model data type.
|
||||
An explicit float16 or bfloat16 must match the model data type.
|
||||
CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports
|
||||
fp8 (=fp8_e4m3). Intel Gaudi (HPU) supports fp8 (using fp8_inc).
|
||||
Some models (namely DeepSeekV3.2) default to fp8, set to bfloat16 to use
|
||||
|
||||
@@ -1426,6 +1426,24 @@ class VllmConfig:
|
||||
self.engram_config.verify_parallel_config(self.parallel_config)
|
||||
logger.info_once("Resolved Engram configuration: %s", str(self.engram_config))
|
||||
|
||||
def _reject_mismatched_float_kv_cache(self) -> None:
|
||||
"""Reject an explicit float16/bfloat16 KV dtype that differs from the model."""
|
||||
cache_dtype = self.cache_config.cache_dtype
|
||||
if self.model_config is None or cache_dtype not in ("float16", "bfloat16"):
|
||||
return
|
||||
model_dtype = str(self.model_config.dtype).removeprefix("torch.")
|
||||
if model_dtype == cache_dtype:
|
||||
return
|
||||
allowed = (
|
||||
f"`auto` or `{model_dtype}`"
|
||||
if model_dtype in ("float16", "bfloat16")
|
||||
else "`auto`"
|
||||
)
|
||||
raise ValueError(
|
||||
f'kv_cache_dtype="{cache_dtype}" is incompatible with '
|
||||
f'dtype="{model_dtype}". Please set `--kv-cache-dtype` to {allowed}.'
|
||||
)
|
||||
|
||||
def __post_init__(self):
|
||||
"""Verify configs are valid & consistent with each other."""
|
||||
# To give each torch profile run a unique instance name.
|
||||
@@ -1456,6 +1474,7 @@ class VllmConfig:
|
||||
logger.info_once("Performance mode set to '%s'.", self.performance_mode)
|
||||
|
||||
self.try_verify_and_update_config()
|
||||
self._reject_mismatched_float_kv_cache()
|
||||
self._resolve_and_verify_engram_config()
|
||||
|
||||
self._check_supports_watermarking()
|
||||
|
||||
Reference in New Issue
Block a user