[Bugfix] Respect explicit --kv-cache-dtype over checkpoint kv_cache_scheme (#42782)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -228,9 +228,14 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
kv_cache_dtype = "auto"
|
||||
calculate_kv_scales = False
|
||||
|
||||
# llm-compressor mdls need to set cache_dtype to "fp8" manually.
|
||||
# llm-compressor models declare an FP8 KV-cache scheme in their
|
||||
# checkpoint config. Honor it only when the user did not explicitly
|
||||
# pick a kv_cache_dtype; an explicit choice (e.g. bfloat16) must win.
|
||||
# The "auto" case is normally resolved upstream in
|
||||
# resolve_kv_cache_dtype_string, but we re-apply here defensively in
|
||||
# case anything bypassed that path.
|
||||
kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None)
|
||||
if kv_cache_scheme is not None:
|
||||
if kv_cache_scheme is not None and kv_cache_dtype == "auto":
|
||||
kv_cache_dtype = "fp8"
|
||||
calculate_kv_scales = False
|
||||
if cache_config is not None:
|
||||
|
||||
Reference in New Issue
Block a user