[Bug] Honor spec decode draft model config (#61240)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-10-12 15:13:23 -04:00
committed by GitHub
parent d6aca837c3
commit d0948f754d
2 changed files with 46 additions and 0 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
from dataclasses import dataclass
from types import SimpleNamespace
from unittest.mock import Mock
@@ -9,6 +10,7 @@ import pytest
import torch
import vllm.v1.worker.gpu.model_runner as model_runner_module
from vllm.config import CacheConfig, ParallelConfig
from vllm.model_executor.warmup.jit_warmup import JitWarmupRegistry
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
@@ -21,6 +23,7 @@ from vllm.v1.kv_cache_interface import (
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.model_runner import ExecuteModelState, GPUModelRunner
from vllm.v1.worker.gpu.spec_decode.standalone_ar import speculator as ar_module
from vllm.v1.worker.gpu.spec_decode.utils import get_drafter_hidden_states
@@ -366,3 +369,34 @@ def test_async_copy_to_np_does_not_alias_reused_buffer():
buffer.fill_(1)
assert snapshot.tolist() == [0, 0, 0, 0]
@pytest.mark.parametrize(
("draft_dtype", "expected_dtype"),
[("auto", "auto"), ("bfloat16", "bfloat16"), (None, "fp8_e4m3")],
)
def test_standalone_draft_kv_dtype_override(monkeypatch, draft_dtype, expected_dtype):
@dataclass
class Config:
cache_config: CacheConfig
parallel_config: ParallelConfig
model_config: object
quant_config: object | None = None
target = Config(CacheConfig(cache_dtype="fp8_e4m3"), ParallelConfig(), object())
speculator = object.__new__(ar_module.StandaloneARSpeculator)
speculator.vllm_config = target
speculator.draft_model_config = object()
speculator.speculative_config = SimpleNamespace(
kv_cache_dtype=draft_dtype, draft_parallel_config=ParallelConfig()
)
get_model = Mock(return_value=object())
monkeypatch.setattr(ar_module, "get_model", get_model)
monkeypatch.setattr(ar_module, "set_model_tag", lambda _: contextlib.nullcontext())
speculator.load_draft_model(object(), set())
draft_config = get_model.call_args.kwargs["vllm_config"]
assert draft_config.cache_config.cache_dtype == expected_dtype
assert speculator.attn_vllm_config.cache_config.cache_dtype == expected_dtype
assert target.cache_config.cache_dtype == "fp8_e4m3"
@@ -70,6 +70,12 @@ class StandaloneARSpeculator(DraftModelSpeculator):
def capture(self) -> None:
pass # CUDA graph not yet supported for standalone AR speculator.
@property
def attn_vllm_config(self) -> VllmConfig:
if self.speculative_config.kv_cache_dtype is not None:
return self._draft_vllm_config
return self.vllm_config
def set_attn(
self,
model_state: ModelState,
@@ -99,15 +105,21 @@ class StandaloneARSpeculator(DraftModelSpeculator):
) -> nn.Module:
spec = self.speculative_config
assert spec is not None
cache_config = self.vllm_config.cache_config
if spec.kv_cache_dtype is not None:
cache_config = replace(cache_config, cache_dtype=spec.kv_cache_dtype)
draft_vllm_config = replace(
self.vllm_config,
model_config=self.draft_model_config,
quant_config=None,
cache_config=cache_config,
parallel_config=replace(
spec.draft_parallel_config,
rank=self.vllm_config.parallel_config.rank,
),
)
self._draft_vllm_config = draft_vllm_config
with set_model_tag("draft_model"):
return get_model(
vllm_config=draft_vllm_config,