[Spec Decode] Add kv_cache_dtype to speculative_config to control separately from target (#48787)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-07-15 23:49:23 -06:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 7dc2698632
commit 8bfd683901
6 changed files with 42 additions and 1 deletions
@@ -27,6 +27,14 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
backend=speculative_config.attention_backend,
),
cache_config=(
replace(
vllm_config.cache_config,
cache_dtype=speculative_config.kv_cache_dtype,
)
if speculative_config.kv_cache_dtype is not None
else vllm_config.cache_config
),
)
with set_model_tag("dflash_head"):
dflash_model = get_model(