[Bug] Honor spec decode draft model config (#61240)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import contextlib
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -9,6 +10,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.worker.gpu.model_runner as model_runner_module
|
||||
from vllm.config import CacheConfig, ParallelConfig
|
||||
from vllm.model_executor.warmup.jit_warmup import JitWarmupRegistry
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
@@ -21,6 +23,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.model_runner import ExecuteModelState, GPUModelRunner
|
||||
from vllm.v1.worker.gpu.spec_decode.standalone_ar import speculator as ar_module
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_drafter_hidden_states
|
||||
|
||||
|
||||
@@ -366,3 +369,34 @@ def test_async_copy_to_np_does_not_alias_reused_buffer():
|
||||
buffer.fill_(1)
|
||||
|
||||
assert snapshot.tolist() == [0, 0, 0, 0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("draft_dtype", "expected_dtype"),
|
||||
[("auto", "auto"), ("bfloat16", "bfloat16"), (None, "fp8_e4m3")],
|
||||
)
|
||||
def test_standalone_draft_kv_dtype_override(monkeypatch, draft_dtype, expected_dtype):
|
||||
@dataclass
|
||||
class Config:
|
||||
cache_config: CacheConfig
|
||||
parallel_config: ParallelConfig
|
||||
model_config: object
|
||||
quant_config: object | None = None
|
||||
|
||||
target = Config(CacheConfig(cache_dtype="fp8_e4m3"), ParallelConfig(), object())
|
||||
speculator = object.__new__(ar_module.StandaloneARSpeculator)
|
||||
speculator.vllm_config = target
|
||||
speculator.draft_model_config = object()
|
||||
speculator.speculative_config = SimpleNamespace(
|
||||
kv_cache_dtype=draft_dtype, draft_parallel_config=ParallelConfig()
|
||||
)
|
||||
get_model = Mock(return_value=object())
|
||||
monkeypatch.setattr(ar_module, "get_model", get_model)
|
||||
monkeypatch.setattr(ar_module, "set_model_tag", lambda _: contextlib.nullcontext())
|
||||
|
||||
speculator.load_draft_model(object(), set())
|
||||
|
||||
draft_config = get_model.call_args.kwargs["vllm_config"]
|
||||
assert draft_config.cache_config.cache_dtype == expected_dtype
|
||||
assert speculator.attn_vllm_config.cache_config.cache_dtype == expected_dtype
|
||||
assert target.cache_config.cache_dtype == "fp8_e4m3"
|
||||
|
||||
@@ -70,6 +70,12 @@ class StandaloneARSpeculator(DraftModelSpeculator):
|
||||
def capture(self) -> None:
|
||||
pass # CUDA graph not yet supported for standalone AR speculator.
|
||||
|
||||
@property
|
||||
def attn_vllm_config(self) -> VllmConfig:
|
||||
if self.speculative_config.kv_cache_dtype is not None:
|
||||
return self._draft_vllm_config
|
||||
return self.vllm_config
|
||||
|
||||
def set_attn(
|
||||
self,
|
||||
model_state: ModelState,
|
||||
@@ -99,15 +105,21 @@ class StandaloneARSpeculator(DraftModelSpeculator):
|
||||
) -> nn.Module:
|
||||
spec = self.speculative_config
|
||||
assert spec is not None
|
||||
cache_config = self.vllm_config.cache_config
|
||||
if spec.kv_cache_dtype is not None:
|
||||
cache_config = replace(cache_config, cache_dtype=spec.kv_cache_dtype)
|
||||
|
||||
draft_vllm_config = replace(
|
||||
self.vllm_config,
|
||||
model_config=self.draft_model_config,
|
||||
quant_config=None,
|
||||
cache_config=cache_config,
|
||||
parallel_config=replace(
|
||||
spec.draft_parallel_config,
|
||||
rank=self.vllm_config.parallel_config.rank,
|
||||
),
|
||||
)
|
||||
self._draft_vllm_config = draft_vllm_config
|
||||
with set_model_tag("draft_model"):
|
||||
return get_model(
|
||||
vllm_config=draft_vllm_config,
|
||||
|
||||
Reference in New Issue
Block a user