[Spec Decode] Add kv_cache_dtype to speculative_config to control separately from target (#48787)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
7dc2698632
commit
8bfd683901
@@ -27,6 +27,14 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
cache_config=(
|
||||
replace(
|
||||
vllm_config.cache_config,
|
||||
cache_dtype=speculative_config.kv_cache_dtype,
|
||||
)
|
||||
if speculative_config.kv_cache_dtype is not None
|
||||
else vllm_config.cache_config
|
||||
),
|
||||
)
|
||||
with set_model_tag("dflash_head"):
|
||||
dflash_model = get_model(
|
||||
|
||||
Reference in New Issue
Block a user