From 27ec8ac626345498fdac0527a4fcd1451c24bebc Mon Sep 17 00:00:00 2001 From: SoluMilken Date: Sat, 22 Aug 2026 00:07:08 +0800 Subject: [PATCH] [Bugfix] Fix MTP draft model using local cache path instead of S3 URL with runai_streamer (#42079) Signed-off-by: SoluMilken --- tests/test_config.py | 35 +++++++++++++++++++++++++++++++++++ vllm/config/speculative.py | 9 ++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 3288e250b1..eac6a95195 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2065,6 +2065,41 @@ def test_draft_sample_method_gumbel_is_rejected(): ) +@patch("vllm.config.speculative.ModelConfig") +def test_mtp_draft_uses_model_weights_not_local_cache(mock_model_config_cls): + """Regression test: MTP + runai_streamer should use model_weights (original + S3 URL) for the draft model, not model (local cache dir set by + pull_runai_model_from_obj_storage).""" + from unittest.mock import MagicMock + + s3_url = "s3://my-bucket/Qwen3-35B-A3B-FP8" + local_cache = "/root/.cache/vllm/assets/model_streamer/abcd1234" + + mock_draft = MagicMock() + mock_draft.model = local_cache + mock_draft.hf_config.model_type = "deepseek_mtp" + mock_draft.hf_config.n_predict = None + mock_draft.max_model_len = 4096 + mock_model_config_cls.return_value = mock_draft + + target_config = MagicMock() + target_config.model = local_cache + target_config.model_weights = s3_url + target_config.hf_text_config.model_type = "deepseek_v3" + target_config.quantization = None + target_config.max_model_len = 4096 + + SpeculativeConfig( + method="mtp", + num_speculative_tokens=1, + target_model_config=target_config, + target_parallel_config=ParallelConfig(), + ) + + actual_model = mock_model_config_cls.call_args.kwargs["model"] + assert actual_model == s3_url + + def test_ir_op_priority_default(): """Test that IR op priority defaults are set correctly.""" from vllm.config.kernel import IrOpPriorityConfig diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index e378906535..f0eee88429 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -769,7 +769,14 @@ class SpeculativeConfig: if self.target_model_config is None: raise ValueError("target_model_config must be present for mtp") # use the draft model from the same model: - self.model = self.target_model_config.model + # Use model_weights if set (e.g. runai_streamer replaces + # model with a local cache dir but keeps the original path + # in model_weights), so the draft model can load weights + # from the same source as the target model. + self.model = ( + self.target_model_config.model_weights + or self.target_model_config.model + ) # Align the quantization of draft model for cases such as # --quantization fp8 with a bf16 checkpoint. if not self.quantization: