From d0948f754dd1fa9a77721c5e9d3722036e74e6a7 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:13:23 -0400 Subject: [PATCH] [Bug] Honor spec decode draft model config (#61240) Signed-off-by: yewentao256 --- tests/v1/worker/test_gpu_model_runner_v2.py | 34 +++++++++++++++++++ .../spec_decode/standalone_ar/speculator.py | 12 +++++++ 2 files changed, 46 insertions(+) diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 85c3ddfce4..8ccc3731d9 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import contextlib +from dataclasses import dataclass from types import SimpleNamespace from unittest.mock import Mock @@ -9,6 +10,7 @@ import pytest import torch import vllm.v1.worker.gpu.model_runner as model_runner_module +from vllm.config import CacheConfig, ParallelConfig from vllm.model_executor.warmup.jit_warmup import JitWarmupRegistry from vllm.v1.kv_cache_interface import ( CircularBufferSpec, @@ -21,6 +23,7 @@ from vllm.v1.kv_cache_interface import ( from vllm.v1.worker.gpu.async_utils import async_copy_to_np from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.model_runner import ExecuteModelState, GPUModelRunner +from vllm.v1.worker.gpu.spec_decode.standalone_ar import speculator as ar_module from vllm.v1.worker.gpu.spec_decode.utils import get_drafter_hidden_states @@ -366,3 +369,34 @@ def test_async_copy_to_np_does_not_alias_reused_buffer(): buffer.fill_(1) assert snapshot.tolist() == [0, 0, 0, 0] + + +@pytest.mark.parametrize( + ("draft_dtype", "expected_dtype"), + [("auto", "auto"), ("bfloat16", "bfloat16"), (None, "fp8_e4m3")], +) +def test_standalone_draft_kv_dtype_override(monkeypatch, draft_dtype, expected_dtype): + @dataclass + class Config: + cache_config: CacheConfig + parallel_config: ParallelConfig + model_config: object + quant_config: object | None = None + + target = Config(CacheConfig(cache_dtype="fp8_e4m3"), ParallelConfig(), object()) + speculator = object.__new__(ar_module.StandaloneARSpeculator) + speculator.vllm_config = target + speculator.draft_model_config = object() + speculator.speculative_config = SimpleNamespace( + kv_cache_dtype=draft_dtype, draft_parallel_config=ParallelConfig() + ) + get_model = Mock(return_value=object()) + monkeypatch.setattr(ar_module, "get_model", get_model) + monkeypatch.setattr(ar_module, "set_model_tag", lambda _: contextlib.nullcontext()) + + speculator.load_draft_model(object(), set()) + + draft_config = get_model.call_args.kwargs["vllm_config"] + assert draft_config.cache_config.cache_dtype == expected_dtype + assert speculator.attn_vllm_config.cache_config.cache_dtype == expected_dtype + assert target.cache_config.cache_dtype == "fp8_e4m3" diff --git a/vllm/v1/worker/gpu/spec_decode/standalone_ar/speculator.py b/vllm/v1/worker/gpu/spec_decode/standalone_ar/speculator.py index 04e79debb8..bd404ef38f 100644 --- a/vllm/v1/worker/gpu/spec_decode/standalone_ar/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/standalone_ar/speculator.py @@ -70,6 +70,12 @@ class StandaloneARSpeculator(DraftModelSpeculator): def capture(self) -> None: pass # CUDA graph not yet supported for standalone AR speculator. + @property + def attn_vllm_config(self) -> VllmConfig: + if self.speculative_config.kv_cache_dtype is not None: + return self._draft_vllm_config + return self.vllm_config + def set_attn( self, model_state: ModelState, @@ -99,15 +105,21 @@ class StandaloneARSpeculator(DraftModelSpeculator): ) -> nn.Module: spec = self.speculative_config assert spec is not None + cache_config = self.vllm_config.cache_config + if spec.kv_cache_dtype is not None: + cache_config = replace(cache_config, cache_dtype=spec.kv_cache_dtype) + draft_vllm_config = replace( self.vllm_config, model_config=self.draft_model_config, quant_config=None, + cache_config=cache_config, parallel_config=replace( spec.draft_parallel_config, rank=self.vllm_config.parallel_config.rank, ), ) + self._draft_vllm_config = draft_vllm_config with set_model_tag("draft_model"): return get_model( vllm_config=draft_vllm_config,