[Spec Decode] Fix max_model_len logging in speculative config for draft model (#41571)
Signed-off-by: Lanze Liu <lanzetech@gmail.com>
This commit is contained in:
@@ -6,6 +6,7 @@ import pytest
|
|||||||
|
|
||||||
from tests.utils import get_attn_backend_list_based_on_platform
|
from tests.utils import get_attn_backend_list_based_on_platform
|
||||||
from vllm import LLM, SamplingParams
|
from vllm import LLM, SamplingParams
|
||||||
|
from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.sampling_params import StructuredOutputsParams
|
from vllm.sampling_params import StructuredOutputsParams
|
||||||
|
|
||||||
@@ -77,3 +78,23 @@ def test_eagle_max_len(
|
|||||||
"is longer than the eagle max length"
|
"is longer than the eagle max length"
|
||||||
)
|
)
|
||||||
assert o.outputs[0].text == "a b c d e " * 15
|
assert o.outputs[0].text == "a b c d e " * 15
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("spec_max_model_len", [80, 150])
|
||||||
|
def test_mtp_speculative_config_max_model_len(spec_max_model_len: int):
|
||||||
|
"""Regression test for #41456: max_model_len in speculative config
|
||||||
|
should be respected for the draft model."""
|
||||||
|
model_config = ModelConfig(
|
||||||
|
model="XiaomiMiMo/MiMo-7B-Base",
|
||||||
|
runner="generate",
|
||||||
|
max_model_len=200,
|
||||||
|
trust_remote_code=True,
|
||||||
|
)
|
||||||
|
spec_config = SpeculativeConfig(
|
||||||
|
target_model_config=model_config,
|
||||||
|
target_parallel_config=ParallelConfig(),
|
||||||
|
method="mtp",
|
||||||
|
num_speculative_tokens=1,
|
||||||
|
max_model_len=spec_max_model_len,
|
||||||
|
)
|
||||||
|
assert spec_config.draft_model_config.max_model_len == spec_max_model_len
|
||||||
|
|||||||
@@ -626,6 +626,7 @@ class SpeculativeConfig:
|
|||||||
revision=self.revision,
|
revision=self.revision,
|
||||||
code_revision=self.code_revision,
|
code_revision=self.code_revision,
|
||||||
tokenizer_revision=self.target_model_config.tokenizer_revision,
|
tokenizer_revision=self.target_model_config.tokenizer_revision,
|
||||||
|
max_model_len=self.max_model_len, # type: ignore[arg-type]
|
||||||
spec_target_max_model_len=self.target_model_config.max_model_len,
|
spec_target_max_model_len=self.target_model_config.max_model_len,
|
||||||
quantization=self.quantization,
|
quantization=self.quantization,
|
||||||
enforce_eager=self.target_model_config.enforce_eager,
|
enforce_eager=self.target_model_config.enforce_eager,
|
||||||
@@ -837,10 +838,17 @@ class SpeculativeConfig:
|
|||||||
|
|
||||||
return speculative_max_model_len
|
return speculative_max_model_len
|
||||||
|
|
||||||
return min(
|
result = min(
|
||||||
draft_max_model_len,
|
draft_max_model_len,
|
||||||
target_max_model_len,
|
target_max_model_len,
|
||||||
)
|
)
|
||||||
|
if result != draft_max_model_len:
|
||||||
|
logger.info(
|
||||||
|
"Overriding draft model max model len from %d to %d",
|
||||||
|
draft_max_model_len,
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _verify_and_get_draft_tp(
|
def _verify_and_get_draft_tp(
|
||||||
|
|||||||
Reference in New Issue
Block a user