[Bugfix] Fix MTP draft model using local cache path instead of S3 URL with runai_streamer (#42079)
Signed-off-by: SoluMilken <ypiheyn.imm02g@g2.nctu.edu.tw>
This commit is contained in:
@@ -2065,6 +2065,41 @@ def test_draft_sample_method_gumbel_is_rejected():
|
||||
)
|
||||
|
||||
|
||||
@patch("vllm.config.speculative.ModelConfig")
|
||||
def test_mtp_draft_uses_model_weights_not_local_cache(mock_model_config_cls):
|
||||
"""Regression test: MTP + runai_streamer should use model_weights (original
|
||||
S3 URL) for the draft model, not model (local cache dir set by
|
||||
pull_runai_model_from_obj_storage)."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
s3_url = "s3://my-bucket/Qwen3-35B-A3B-FP8"
|
||||
local_cache = "/root/.cache/vllm/assets/model_streamer/abcd1234"
|
||||
|
||||
mock_draft = MagicMock()
|
||||
mock_draft.model = local_cache
|
||||
mock_draft.hf_config.model_type = "deepseek_mtp"
|
||||
mock_draft.hf_config.n_predict = None
|
||||
mock_draft.max_model_len = 4096
|
||||
mock_model_config_cls.return_value = mock_draft
|
||||
|
||||
target_config = MagicMock()
|
||||
target_config.model = local_cache
|
||||
target_config.model_weights = s3_url
|
||||
target_config.hf_text_config.model_type = "deepseek_v3"
|
||||
target_config.quantization = None
|
||||
target_config.max_model_len = 4096
|
||||
|
||||
SpeculativeConfig(
|
||||
method="mtp",
|
||||
num_speculative_tokens=1,
|
||||
target_model_config=target_config,
|
||||
target_parallel_config=ParallelConfig(),
|
||||
)
|
||||
|
||||
actual_model = mock_model_config_cls.call_args.kwargs["model"]
|
||||
assert actual_model == s3_url
|
||||
|
||||
|
||||
def test_ir_op_priority_default():
|
||||
"""Test that IR op priority defaults are set correctly."""
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
|
||||
@@ -769,7 +769,14 @@ class SpeculativeConfig:
|
||||
if self.target_model_config is None:
|
||||
raise ValueError("target_model_config must be present for mtp")
|
||||
# use the draft model from the same model:
|
||||
self.model = self.target_model_config.model
|
||||
# Use model_weights if set (e.g. runai_streamer replaces
|
||||
# model with a local cache dir but keeps the original path
|
||||
# in model_weights), so the draft model can load weights
|
||||
# from the same source as the target model.
|
||||
self.model = (
|
||||
self.target_model_config.model_weights
|
||||
or self.target_model_config.model
|
||||
)
|
||||
# Align the quantization of draft model for cases such as
|
||||
# --quantization fp8 with a bf16 checkpoint.
|
||||
if not self.quantization:
|
||||
|
||||
Reference in New Issue
Block a user