From 924707f1bf94ff583d89bff7522ee12ff032c286 Mon Sep 17 00:00:00 2001 From: Thang Nguyen <69278249+Thangnguyenvn98@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:01:31 -0400 Subject: [PATCH] [CI] Split (H200 MIG 35GB) Spec Decode Speculators + MTP into 4 named jobs (#57237) Signed-off-by: Thang Nguyen Co-authored-by: Thang Nguyen Co-authored-by: Kimi Code Co-authored-by: Claude Opus 5.5 --- .buildkite/test_areas/spec_decode.yaml | 104 +++++++- tests/v1/e2e/spec_decode/mtp/_correctness.py | 120 +++++++++ .../e2e/spec_decode/mtp/deepseek/__init__.py | 2 + .../e2e/spec_decode/mtp/deepseek/test_mtp.py | 44 +++ .../v1/e2e/spec_decode/mtp/gemma4/__init__.py | 2 + .../v1/e2e/spec_decode/mtp/gemma4/test_mtp.py | 39 +++ tests/v1/e2e/spec_decode/mtp/mimo/__init__.py | 2 + tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py | 36 +++ .../e2e/spec_decode/mtp/qwen3_5/__init__.py | 2 + .../e2e/spec_decode/mtp/qwen3_5/test_mtp.py | 39 +++ .../mtp/qwen3_5/test_rejected_placeholders.py | 115 ++++++++ tests/v1/e2e/spec_decode/mtp/test_mtp.py | 252 ------------------ 12 files changed, 499 insertions(+), 258 deletions(-) create mode 100644 tests/v1/e2e/spec_decode/mtp/_correctness.py create mode 100644 tests/v1/e2e/spec_decode/mtp/deepseek/__init__.py create mode 100644 tests/v1/e2e/spec_decode/mtp/deepseek/test_mtp.py create mode 100644 tests/v1/e2e/spec_decode/mtp/gemma4/__init__.py create mode 100644 tests/v1/e2e/spec_decode/mtp/gemma4/test_mtp.py create mode 100644 tests/v1/e2e/spec_decode/mtp/mimo/__init__.py create mode 100644 tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py create mode 100644 tests/v1/e2e/spec_decode/mtp/qwen3_5/__init__.py create mode 100644 tests/v1/e2e/spec_decode/mtp/qwen3_5/test_mtp.py create mode 100644 tests/v1/e2e/spec_decode/mtp/qwen3_5/test_rejected_placeholders.py delete mode 100644 tests/v1/e2e/spec_decode/mtp/test_mtp.py diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 8a41a4c993..73d944a7a4 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -98,9 +98,11 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode/eagle/ -- label: ":nvidia: (H200 MIG 35GB) Spec Decode Speculators + MTP" - key: spec-decode-speculators-mtp - timeout_in_minutes: 50 +# MTP correctness is one parametrized test split into per-model folders; +# each job below runs whole directories. +- label: ":nvidia: (H200 MIG 35GB) Spec Decode Speculators" + key: spec-decode-speculators + timeout_in_minutes: 30 device: h200_35gb source_file_dependencies: - vllm/v1/spec_decode/ @@ -110,13 +112,103 @@ steps: - tests/v1/e2e/spec_decode/ commands: - pytest -v -s v1/e2e/spec_decode/speculators/ - - pytest -v -s v1/e2e/spec_decode/mtp/ mirror: amd: - label: ":amd: (MI300) Spec Decode Speculators + MTP" + label: ":amd: (MI300) Spec Decode Speculators" dind: false device: mi300_1 - timeout_in_minutes: 75 + timeout_in_minutes: 40 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + +- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP Gemma4" + key: spec-decode-mtp-gemma4 + timeout_in_minutes: 30 + device: h200_35gb + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/attention/backends/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode/mtp/gemma4/ + mirror: + amd: + label: ":amd: (MI300) Spec Decode MTP Gemma4" + dind: false + device: mi300_1 + timeout_in_minutes: 40 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + +- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP DeepSeek + MiMo" + key: spec-decode-mtp-deepseek-mimo + timeout_in_minutes: 30 + device: h200_35gb + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/attention/backends/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode/mtp/deepseek/ + - pytest -v -s v1/e2e/spec_decode/mtp/mimo/ + mirror: + amd: + label: ":amd: (MI300) Spec Decode MTP DeepSeek + MiMo" + dind: false + device: mi300_1 + timeout_in_minutes: 40 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + +- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP Qwen3.5" + key: spec-decode-mtp-qwen3-5 + timeout_in_minutes: 30 + device: h200_35gb + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/attention/backends/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode/mtp/qwen3_5/ + mirror: + amd: + label: ":amd: (MI300) Spec Decode MTP Qwen3.5" + dind: false + device: mi300_1 + timeout_in_minutes: 40 depends_on: - image-build-amd source_file_dependencies: diff --git a/tests/v1/e2e/spec_decode/mtp/_correctness.py b/tests/v1/e2e/spec_decode/mtp/_correctness.py new file mode 100644 index 0000000000..b88c1cce7a --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/_correctness.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +import pytest + +from vllm import SamplingParams +from vllm.config import CompilationConfig +from vllm.platforms import current_platform + +from ..utils import ( + _skip_if_insufficient_gpus_for_tp, + assert_request_outputs_match, + evaluate_llm_for_gsm8k, + get_test_prompts, +) + + +def check_mtp_correctness( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_setup: tuple[str, str, int, str | None], + mm_enabled: bool, + expected_accuracy_threshold: float, + vllm_runner, +): + """Compare the outputs of a original LLM and a speculative LLM + which should be the same when using MTP speculative decoding. Due to some variance + in the engine, it is possible for some outputs to differ, so we expect that at least + 6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed + reference threshold for each model. + """ + method, model_name, tp_size, draft_model = model_setup + _skip_if_insufficient_gpus_for_tp(tp_size) + + # Generate test prompts inside the function instead of using fixture + test_prompts = get_test_prompts(mm_enabled) + with monkeypatch.context() as m: + m.setenv("VLLM_MLA_DISABLE", "1") + + attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto" + + # Skip multimodal profiling for models that don't need it in this test. + extra_kwargs: dict[str, Any] = {} + if "Qwen3.5" in model_name: + extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0} + elif "gemma-4" in model_name: + extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0} + + if draft_model is not None and "gemma-4" in draft_model: + import transformers + from packaging.version import Version + + if Version(transformers.__version__) < Version("5.8.0"): + pytest.skip( + "Gemma4 MTP assistant requires transformers>=5.8.0, " + f"got {transformers.__version__}" + ) + + with vllm_runner( + model_name, + block_size=None, + max_model_len=2048, + tensor_parallel_size=tp_size, + trust_remote_code=True, + attention_backend=attn_backend, + enable_chunked_prefill=None, + compilation_config=CompilationConfig(), + **extra_kwargs, + ) as ref_runner: + ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config) + evaluate_llm_for_gsm8k( + ref_runner.llm, + expected_accuracy_threshold=expected_accuracy_threshold, + ) + + speculative_config: dict[str, Any] = { + "method": method, + "num_speculative_tokens": 1, + "max_model_len": 2048, + } + if draft_model is not None: + speculative_config["model"] = draft_model + speculative_config["num_speculative_tokens"] = 2 + + with vllm_runner( + model_name, + block_size=None, + trust_remote_code=True, + tensor_parallel_size=tp_size, + speculative_config=speculative_config, + max_model_len=2048, + attention_backend=attn_backend, + enable_chunked_prefill=None, + compilation_config=CompilationConfig(), + **extra_kwargs, + ) as spec_runner: + # MTP supports async scheduling by default. + has_async = ( + spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling + ) + assert has_async, ( + f"Expected async scheduling for {method}: target={model_name}, " + f"draft={draft_model}; got {has_async}" + ) + evaluate_llm_for_gsm8k( + spec_runner.llm, + expected_accuracy_threshold=expected_accuracy_threshold, + ) + spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config) + + # Heuristic: expect at least 80% of the prompts to match exactly + # Upon failure, inspect the outputs to check for inaccuracy. + assert_request_outputs_match( + ref_outputs, + spec_outputs, + required_matches=int(0.8 * len(ref_outputs)) + 1, + context=f"{method} target={model_name}, draft={draft_model}", + ) diff --git a/tests/v1/e2e/spec_decode/mtp/deepseek/__init__.py b/tests/v1/e2e/spec_decode/mtp/deepseek/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/deepseek/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/v1/e2e/spec_decode/mtp/deepseek/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/deepseek/test_mtp.py new file mode 100644 index 0000000000..a23c3ee26e --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/deepseek/test_mtp.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.utils import single_gpu_only +from vllm import SamplingParams +from vllm.platforms import current_platform + +from .._correctness import check_mtp_correctness + + +@pytest.mark.parametrize( + ["model_setup", "mm_enabled", "expected_accuracy_threshold"], + [ + pytest.param( + ("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None), + False, + 0.0, + marks=pytest.mark.skipif( + current_platform.is_device_capability_family(100), + reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell", + ), + ), # dummy model + ], + ids=["deepseek"], +) +@single_gpu_only +def test_mtp_correctness( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_setup: tuple[str, str, int, str | None], + mm_enabled: bool, + expected_accuracy_threshold: float, + vllm_runner, +): + check_mtp_correctness( + monkeypatch, + sampling_config, + model_setup, + mm_enabled, + expected_accuracy_threshold, + vllm_runner, + ) diff --git a/tests/v1/e2e/spec_decode/mtp/gemma4/__init__.py b/tests/v1/e2e/spec_decode/mtp/gemma4/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/gemma4/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/v1/e2e/spec_decode/mtp/gemma4/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/gemma4/test_mtp.py new file mode 100644 index 0000000000..66fb49fcef --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/gemma4/test_mtp.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.utils import single_gpu_only +from vllm import SamplingParams + +from .._correctness import check_mtp_correctness + + +@pytest.mark.parametrize( + ["model_setup", "mm_enabled", "expected_accuracy_threshold"], + [ + ( + ("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"), + False, + 0.50, + ), # gemma4 MTP with assistant model, ref: ~62% + ], + ids=["gemma4-e4b"], +) +@single_gpu_only +def test_mtp_correctness( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_setup: tuple[str, str, int, str | None], + mm_enabled: bool, + expected_accuracy_threshold: float, + vllm_runner, +): + check_mtp_correctness( + monkeypatch, + sampling_config, + model_setup, + mm_enabled, + expected_accuracy_threshold, + vllm_runner, + ) diff --git a/tests/v1/e2e/spec_decode/mtp/mimo/__init__.py b/tests/v1/e2e/spec_decode/mtp/mimo/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/mimo/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py new file mode 100644 index 0000000000..8a4e7e1236 --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.utils import single_gpu_only +from vllm import SamplingParams + +from .._correctness import check_mtp_correctness + + +@pytest.mark.parametrize( + ["model_setup", "mm_enabled", "expected_accuracy_threshold"], + [ + # Reference accuracy: 65%-70%. + (("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5), + ], + ids=["mimo"], +) +@single_gpu_only +def test_mtp_correctness( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_setup: tuple[str, str, int, str | None], + mm_enabled: bool, + expected_accuracy_threshold: float, + vllm_runner, +): + check_mtp_correctness( + monkeypatch, + sampling_config, + model_setup, + mm_enabled, + expected_accuracy_threshold, + vllm_runner, + ) diff --git a/tests/v1/e2e/spec_decode/mtp/qwen3_5/__init__.py b/tests/v1/e2e/spec_decode/mtp/qwen3_5/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/qwen3_5/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_mtp.py new file mode 100644 index 0000000000..c61838a655 --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_mtp.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.utils import single_gpu_only +from vllm import SamplingParams + +from .._correctness import check_mtp_correctness + + +@pytest.mark.parametrize( + ["model_setup", "mm_enabled", "expected_accuracy_threshold"], + [ + ( + ("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None), + False, + 0.20, + ), # hybrid + MTP, ref: ~34%-35% + ], + ids=["qwen3_5-hybrid"], +) +@single_gpu_only +def test_mtp_correctness( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_setup: tuple[str, str, int, str | None], + mm_enabled: bool, + expected_accuracy_threshold: float, + vllm_runner, +): + check_mtp_correctness( + monkeypatch, + sampling_config, + model_setup, + mm_enabled, + expected_accuracy_threshold, + vllm_runner, + ) diff --git a/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_rejected_placeholders.py b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_rejected_placeholders.py new file mode 100644 index 0000000000..5116f7db27 --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_rejected_placeholders.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +import pytest + +from tests.utils import single_gpu_only +from vllm import SamplingParams +from vllm.config import CompilationConfig, CUDAGraphMode +from vllm.platforms import current_platform + +from ...utils import get_spec_decode_metric_value + +PLACEHOLDER_REGRESSION_MODEL = "Qwen/Qwen3.5-0.8B-Base" +PLACEHOLDER_REGRESSION_PROMPT = "The capital of France is" +PLACEHOLDER_REGRESSION_MAX_TOKENS = 16 +PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID = 42 +PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS = 3 + + +@pytest.mark.parametrize( + "cudagraph_mode", + [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE], + ids=["piecewise", "full-and-piecewise"], +) +@single_gpu_only +def test_mtp_rejected_placeholders_match_greedy_token_ids( + cudagraph_mode: CUDAGraphMode, + vllm_runner, +): + """Rejected MTP padding must not alter the non-speculative token stream.""" + common_kwargs: dict[str, Any] = { + "block_size": None, + "max_model_len": 256, + "max_num_seqs": 1, + "trust_remote_code": True, + "enable_chunked_prefill": None, + "limit_mm_per_prompt": {"image": 0, "video": 0}, + "attention_backend": ("TRITON_ATTN" if current_platform.is_rocm() else "auto"), + } + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=PLACEHOLDER_REGRESSION_MAX_TOKENS, + ignore_eos=True, + allowed_token_ids=[PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID], + logprobs=1, + ) + + def generate_token_ids(runner) -> tuple[int, ...]: + outputs = runner.llm.generate([PLACEHOLDER_REGRESSION_PROMPT], sampling_params) + assert outputs and outputs[0].outputs, "Expected one completion" + token_ids = tuple(outputs[0].outputs[0].token_ids) + assert len(token_ids) == PLACEHOLDER_REGRESSION_MAX_TOKENS + return token_ids + + def compilation_config() -> CompilationConfig: + return CompilationConfig( + cudagraph_mode=cudagraph_mode, + cudagraph_capture_sizes=[1, PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS + 1], + ) + + with vllm_runner( + PLACEHOLDER_REGRESSION_MODEL, + compilation_config=compilation_config(), + **common_kwargs, + ) as ref_runner: + ref_mode = ( + ref_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode + ) + assert ref_mode == cudagraph_mode + ref_token_ids = generate_token_ids(ref_runner) + expected_token_ids = (PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID,) * ( + PLACEHOLDER_REGRESSION_MAX_TOKENS + ) + assert ref_token_ids == expected_token_ids + + with vllm_runner( + PLACEHOLDER_REGRESSION_MODEL, + speculative_config={ + "method": "mtp", + "num_speculative_tokens": PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS, + "max_model_len": 256, + "rejection_sample_method": "synthetic", + "synthetic_acceptance_rates": [0.0] + * PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS, + }, + disable_log_stats=False, + compilation_config=compilation_config(), + **common_kwargs, + ) as spec_runner: + spec_mode = ( + spec_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode + ) + assert spec_mode == cudagraph_mode + spec_token_ids = generate_token_ids(spec_runner) + metrics = spec_runner.llm.get_metrics() + num_drafts = get_spec_decode_metric_value( + metrics, "vllm:spec_decode_num_drafts" + ) + num_draft_tokens = get_spec_decode_metric_value( + metrics, "vllm:spec_decode_num_draft_tokens" + ) + num_accepted_tokens = get_spec_decode_metric_value( + metrics, "vllm:spec_decode_num_accepted_tokens" + ) + + assert num_drafts > 0, "MTP drafter did not run" + assert num_draft_tokens > 0, "MTP drafter did not produce draft tokens" + assert num_accepted_tokens == 0, "Synthetic sampler accepted a draft token" + assert spec_token_ids == ref_token_ids, ( + f"MTP changed greedy token ids under {cudagraph_mode.name}:\n" + f" reference={ref_token_ids}\n" + f" speculative={spec_token_ids}" + ) diff --git a/tests/v1/e2e/spec_decode/mtp/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/test_mtp.py deleted file mode 100644 index 92f4a2a773..0000000000 --- a/tests/v1/e2e/spec_decode/mtp/test_mtp.py +++ /dev/null @@ -1,252 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import Any - -import pytest - -from tests.utils import single_gpu_only -from vllm import SamplingParams -from vllm.config import CompilationConfig, CUDAGraphMode -from vllm.platforms import current_platform - -from ..utils import ( - _skip_if_insufficient_gpus_for_tp, - assert_request_outputs_match, - evaluate_llm_for_gsm8k, - get_spec_decode_metric_value, - get_test_prompts, -) - -PLACEHOLDER_REGRESSION_MODEL = "Qwen/Qwen3.5-0.8B-Base" -PLACEHOLDER_REGRESSION_PROMPT = "The capital of France is" -PLACEHOLDER_REGRESSION_MAX_TOKENS = 16 -PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID = 42 -PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS = 3 - - -@pytest.mark.parametrize( - "cudagraph_mode", - [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE], - ids=["piecewise", "full-and-piecewise"], -) -@single_gpu_only -def test_mtp_rejected_placeholders_match_greedy_token_ids( - cudagraph_mode: CUDAGraphMode, - vllm_runner, -): - """Rejected MTP padding must not alter the non-speculative token stream.""" - common_kwargs: dict[str, Any] = { - "block_size": None, - "max_model_len": 256, - "max_num_seqs": 1, - "trust_remote_code": True, - "enable_chunked_prefill": None, - "limit_mm_per_prompt": {"image": 0, "video": 0}, - "attention_backend": ("TRITON_ATTN" if current_platform.is_rocm() else "auto"), - } - sampling_params = SamplingParams( - temperature=0.0, - max_tokens=PLACEHOLDER_REGRESSION_MAX_TOKENS, - ignore_eos=True, - allowed_token_ids=[PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID], - logprobs=1, - ) - - def generate_token_ids(runner) -> tuple[int, ...]: - outputs = runner.llm.generate([PLACEHOLDER_REGRESSION_PROMPT], sampling_params) - assert outputs and outputs[0].outputs, "Expected one completion" - token_ids = tuple(outputs[0].outputs[0].token_ids) - assert len(token_ids) == PLACEHOLDER_REGRESSION_MAX_TOKENS - return token_ids - - def compilation_config() -> CompilationConfig: - return CompilationConfig( - cudagraph_mode=cudagraph_mode, - cudagraph_capture_sizes=[1, PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS + 1], - ) - - with vllm_runner( - PLACEHOLDER_REGRESSION_MODEL, - compilation_config=compilation_config(), - **common_kwargs, - ) as ref_runner: - ref_mode = ( - ref_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode - ) - assert ref_mode == cudagraph_mode - ref_token_ids = generate_token_ids(ref_runner) - expected_token_ids = (PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID,) * ( - PLACEHOLDER_REGRESSION_MAX_TOKENS - ) - assert ref_token_ids == expected_token_ids - - with vllm_runner( - PLACEHOLDER_REGRESSION_MODEL, - speculative_config={ - "method": "mtp", - "num_speculative_tokens": PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS, - "max_model_len": 256, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_rates": [0.0] - * PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS, - }, - disable_log_stats=False, - compilation_config=compilation_config(), - **common_kwargs, - ) as spec_runner: - spec_mode = ( - spec_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode - ) - assert spec_mode == cudagraph_mode - spec_token_ids = generate_token_ids(spec_runner) - metrics = spec_runner.llm.get_metrics() - num_drafts = get_spec_decode_metric_value( - metrics, "vllm:spec_decode_num_drafts" - ) - num_draft_tokens = get_spec_decode_metric_value( - metrics, "vllm:spec_decode_num_draft_tokens" - ) - num_accepted_tokens = get_spec_decode_metric_value( - metrics, "vllm:spec_decode_num_accepted_tokens" - ) - - assert num_drafts > 0, "MTP drafter did not run" - assert num_draft_tokens > 0, "MTP drafter did not produce draft tokens" - assert num_accepted_tokens == 0, "Synthetic sampler accepted a draft token" - assert spec_token_ids == ref_token_ids, ( - f"MTP changed greedy token ids under {cudagraph_mode.name}:\n" - f" reference={ref_token_ids}\n" - f" speculative={spec_token_ids}" - ) - - -@pytest.mark.parametrize( - ["model_setup", "mm_enabled", "expected_accuracy_threshold"], - [ - # Reference accuracy: 65%-70%. - (("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5), - pytest.param( - ("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None), - False, - 0.0, - marks=pytest.mark.skipif( - current_platform.is_device_capability_family(100), - reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell", - ), - ), # dummy model - ( - ("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None), - False, - 0.20, - ), # hybrid + MTP, ref: ~34%-35% - ( - ("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"), - False, - 0.50, - ), # gemma4 MTP with assistant model, ref: ~62% - ], - ids=["mimo", "deepseek", "qwen3_5-hybrid", "gemma4-e4b"], -) -@single_gpu_only -def test_mtp_correctness( - monkeypatch: pytest.MonkeyPatch, - sampling_config: SamplingParams, - model_setup: tuple[str, str, int, str | None], - mm_enabled: bool, - expected_accuracy_threshold: float, - vllm_runner, -): - """Compare the outputs of a original LLM and a speculative LLM - which should be the same when using MTP speculative decoding. Due to some variance - in the engine, it is possible for some outputs to differ, so we expect that at least - 6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed - reference threshold for each model. - """ - method, model_name, tp_size, draft_model = model_setup - _skip_if_insufficient_gpus_for_tp(tp_size) - - # Generate test prompts inside the function instead of using fixture - test_prompts = get_test_prompts(mm_enabled) - with monkeypatch.context() as m: - m.setenv("VLLM_MLA_DISABLE", "1") - - attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto" - - # Skip multimodal profiling for models that don't need it in this test. - extra_kwargs: dict[str, Any] = {} - if "Qwen3.5" in model_name: - extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0} - elif "gemma-4" in model_name: - extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0} - - if draft_model is not None and "gemma-4" in draft_model: - import transformers - from packaging.version import Version - - if Version(transformers.__version__) < Version("5.8.0"): - pytest.skip( - "Gemma4 MTP assistant requires transformers>=5.8.0, " - f"got {transformers.__version__}" - ) - - with vllm_runner( - model_name, - block_size=None, - max_model_len=2048, - tensor_parallel_size=tp_size, - trust_remote_code=True, - attention_backend=attn_backend, - enable_chunked_prefill=None, - compilation_config=CompilationConfig(), - **extra_kwargs, - ) as ref_runner: - ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config) - evaluate_llm_for_gsm8k( - ref_runner.llm, - expected_accuracy_threshold=expected_accuracy_threshold, - ) - - speculative_config: dict[str, Any] = { - "method": method, - "num_speculative_tokens": 1, - "max_model_len": 2048, - } - if draft_model is not None: - speculative_config["model"] = draft_model - speculative_config["num_speculative_tokens"] = 2 - - with vllm_runner( - model_name, - block_size=None, - trust_remote_code=True, - tensor_parallel_size=tp_size, - speculative_config=speculative_config, - max_model_len=2048, - attention_backend=attn_backend, - enable_chunked_prefill=None, - compilation_config=CompilationConfig(), - **extra_kwargs, - ) as spec_runner: - # MTP supports async scheduling by default. - has_async = ( - spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling - ) - assert has_async, ( - f"Expected async scheduling for {method}: target={model_name}, " - f"draft={draft_model}; got {has_async}" - ) - evaluate_llm_for_gsm8k( - spec_runner.llm, - expected_accuracy_threshold=expected_accuracy_threshold, - ) - spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config) - - # Heuristic: expect at least 80% of the prompts to match exactly - # Upon failure, inspect the outputs to check for inaccuracy. - assert_request_outputs_match( - ref_outputs, - spec_outputs, - required_matches=int(0.8 * len(ref_outputs)) + 1, - context=f"{method} target={model_name}, draft={draft_model}", - )