[Bugfix] Fix MTP draft model using local cache path instead of S3 URL with runai_streamer (#42079)

Signed-off-by: SoluMilken <ypiheyn.imm02g@g2.nctu.edu.tw>
This commit is contained in:
SoluMilken
2026-08-21 09:07:08 -07:00
committed by GitHub
parent 72aedcc426
commit 27ec8ac626
2 changed files with 43 additions and 1 deletions
+35
View File
@@ -2065,6 +2065,41 @@ def test_draft_sample_method_gumbel_is_rejected():
)
@patch("vllm.config.speculative.ModelConfig")
def test_mtp_draft_uses_model_weights_not_local_cache(mock_model_config_cls):
"""Regression test: MTP + runai_streamer should use model_weights (original
S3 URL) for the draft model, not model (local cache dir set by
pull_runai_model_from_obj_storage)."""
from unittest.mock import MagicMock
s3_url = "s3://my-bucket/Qwen3-35B-A3B-FP8"
local_cache = "/root/.cache/vllm/assets/model_streamer/abcd1234"
mock_draft = MagicMock()
mock_draft.model = local_cache
mock_draft.hf_config.model_type = "deepseek_mtp"
mock_draft.hf_config.n_predict = None
mock_draft.max_model_len = 4096
mock_model_config_cls.return_value = mock_draft
target_config = MagicMock()
target_config.model = local_cache
target_config.model_weights = s3_url
target_config.hf_text_config.model_type = "deepseek_v3"
target_config.quantization = None
target_config.max_model_len = 4096
SpeculativeConfig(
method="mtp",
num_speculative_tokens=1,
target_model_config=target_config,
target_parallel_config=ParallelConfig(),
)
actual_model = mock_model_config_cls.call_args.kwargs["model"]
assert actual_model == s3_url
def test_ir_op_priority_default():
"""Test that IR op priority defaults are set correctly."""
from vllm.config.kernel import IrOpPriorityConfig
+8 -1
View File
@@ -769,7 +769,14 @@ class SpeculativeConfig:
if self.target_model_config is None:
raise ValueError("target_model_config must be present for mtp")
# use the draft model from the same model:
self.model = self.target_model_config.model
# Use model_weights if set (e.g. runai_streamer replaces
# model with a local cache dir but keeps the original path
# in model_weights), so the draft model can load weights
# from the same source as the target model.
self.model = (
self.target_model_config.model_weights
or self.target_model_config.model
)
# Align the quantization of draft model for cases such as
# --quantization fp8 with a bf16 checkpoint.
if not self.quantization: