[Spec Decode] Add kv_cache_dtype to speculative_config to control separately from target (#48787)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
7dc2698632
commit
8bfd683901
@@ -10,6 +10,7 @@ from pydantic import Field, SkipValidation, field_validator, model_validator
|
||||
from typing_extensions import Self
|
||||
|
||||
from vllm.config import LoadConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.model import HfOverrides, ModelConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
@@ -118,6 +119,9 @@ class SpeculativeConfig:
|
||||
"""Attention backend to use for the draft model. When `None`, the backend is
|
||||
automatically selected. Useful when the drafter requires a different attention
|
||||
backend (e.g. DFlash needs a non-causal-capable backend like FLASH_ATTN)."""
|
||||
kv_cache_dtype: CacheDType | None = None
|
||||
"""KV cache dtype for the draft model. When `None`, the draft inherits the
|
||||
target model's `--kv-cache-dtype`."""
|
||||
max_model_len: int | None = Field(default=None, ge=1)
|
||||
"""The maximum model length of the draft model. Used when testing the
|
||||
ability to skip speculation for some sequences."""
|
||||
|
||||
@@ -1754,6 +1754,10 @@ class EngineArgs:
|
||||
if self.speculative_config is None:
|
||||
return None
|
||||
|
||||
self.speculative_config = {
|
||||
k.replace("-", "_"): v for k, v in self.speculative_config.items()
|
||||
}
|
||||
|
||||
# Note(Shangming): These parameters are not obtained from the cli arg
|
||||
# '--speculative-config' and must be passed in when creating the engine
|
||||
# config.
|
||||
|
||||
@@ -1300,6 +1300,15 @@ class SpecDecodeBaseProposer:
|
||||
),
|
||||
)
|
||||
|
||||
if spec_cfg.kv_cache_dtype is not None:
|
||||
base = replace(
|
||||
base,
|
||||
cache_config=replace(
|
||||
base.cache_config,
|
||||
cache_dtype=spec_cfg.kv_cache_dtype,
|
||||
),
|
||||
)
|
||||
|
||||
return base
|
||||
|
||||
def _get_model(self) -> nn.Module:
|
||||
|
||||
@@ -27,6 +27,14 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
cache_config=(
|
||||
replace(
|
||||
vllm_config.cache_config,
|
||||
cache_dtype=speculative_config.kv_cache_dtype,
|
||||
)
|
||||
if speculative_config.kv_cache_dtype is not None
|
||||
else vllm_config.cache_config
|
||||
),
|
||||
)
|
||||
with set_model_tag("dflash_head"):
|
||||
dflash_model = get_model(
|
||||
|
||||
@@ -27,6 +27,14 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
cache_config=(
|
||||
replace(
|
||||
vllm_config.cache_config,
|
||||
cache_dtype=speculative_config.kv_cache_dtype,
|
||||
)
|
||||
if speculative_config.kv_cache_dtype is not None
|
||||
else vllm_config.cache_config
|
||||
),
|
||||
)
|
||||
|
||||
with set_model_tag("dspark_head"):
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.lora.layers.base import BaseLayerWithLoRA
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
@@ -39,6 +39,14 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
if speculative_config.kv_cache_dtype is not None:
|
||||
vllm_config = replace(
|
||||
vllm_config,
|
||||
cache_config=replace(
|
||||
vllm_config.cache_config,
|
||||
cache_dtype=speculative_config.kv_cache_dtype,
|
||||
),
|
||||
)
|
||||
with set_model_tag("eagle_head"):
|
||||
eagle_model = get_model(
|
||||
vllm_config=vllm_config, model_config=draft_model_config
|
||||
|
||||
Reference in New Issue
Block a user