[CI] Add MTP coverage: Qwen3.5 correctness + no-sync spec decode (#40472)
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
307b17ce33
commit
92a7c121b6
@@ -97,3 +97,16 @@ steps:
|
||||
commands:
|
||||
- export VLLM_ALLOW_INSECURE_SERIALIZATION=1
|
||||
- pytest -v -s v1/spec_decode/test_speculators_dflash.py -m slow_test
|
||||
|
||||
- label: Spec Decode MTP hybrid (B200)
|
||||
timeout_in_minutes: 30
|
||||
device: b200
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- vllm/model_executor/models/qwen3_5.py
|
||||
- vllm/model_executor/models/qwen3_5_mtp.py
|
||||
- tests/v1/e2e/spec_decode/
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode -k "qwen3_5-hybrid"
|
||||
|
||||
@@ -82,6 +82,13 @@ SPEC_DECODE_CONFIGS = [
|
||||
2,
|
||||
id="eagle-mla-deepseek",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
"mtp",
|
||||
1,
|
||||
id="mtp-qwen3_5-hybrid",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -104,6 +111,14 @@ def test_no_sync_with_spec_decode(
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
# Qwen3.5 is a VLM; without this, profile_run runs the ViT warmup
|
||||
# and peaks well above the 18GB MIG slice used by one of the CI lanes.
|
||||
# This test only exercises text generation, so the vision tower is
|
||||
# never needed.
|
||||
extra_kwargs: dict = {}
|
||||
if "Qwen3.5" in model:
|
||||
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
|
||||
|
||||
llm = LLM(
|
||||
model=model,
|
||||
max_model_len=256,
|
||||
@@ -114,6 +129,7 @@ def test_no_sync_with_spec_decode(
|
||||
},
|
||||
enforce_eager=True,
|
||||
async_scheduling=True,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
# Assert async scheduling is actually active before running inference.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
import random
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
@@ -723,8 +724,13 @@ def test_eagle_correctness_heavy(
|
||||
[
|
||||
(("mtp", "XiaomiMiMo/MiMo-7B-Base", 1), False, 0.5), # ref: 65%-70%
|
||||
(("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1), False, 0.0), # dummy model
|
||||
(
|
||||
("mtp", "Qwen/Qwen3.5-0.8B-Base", 1),
|
||||
False,
|
||||
0.20,
|
||||
), # hybrid + MTP, ref: ~34%-35%
|
||||
],
|
||||
ids=["mimo", "deepseek"],
|
||||
ids=["mimo", "deepseek", "qwen3_5-hybrid"],
|
||||
)
|
||||
@single_gpu_only
|
||||
@large_gpu_mark(min_gb=20)
|
||||
@@ -750,13 +756,29 @@ def test_mtp_correctness(
|
||||
method, model_name, tp_size = model_setup
|
||||
_skip_if_insufficient_gpus_for_tp(tp_size)
|
||||
|
||||
if "Qwen3.5" in model_name and os.environ.get("VLLM_USE_V2_MODEL_RUNNER"):
|
||||
pytest.skip(
|
||||
"Model Runner V2 does not yet support hybrid models "
|
||||
"(Qwen3.5 mixes Mamba-style GDN with attention layers)."
|
||||
)
|
||||
|
||||
attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
|
||||
|
||||
# Qwen3.5 is a VLM; without this, profile_run runs the ViT warmup
|
||||
# and peaks well above the 18GB MIG slice used by one of the CI
|
||||
# lanes. This test only exercises text generation, so the vision
|
||||
# tower is never needed.
|
||||
extra_kwargs: dict[str, Any] = {}
|
||||
if "Qwen3.5" in model_name:
|
||||
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
|
||||
|
||||
ref_llm = LLM(
|
||||
model=model_name,
|
||||
max_model_len=2048,
|
||||
tensor_parallel_size=tp_size,
|
||||
trust_remote_code=True,
|
||||
attention_backend=attn_backend,
|
||||
**extra_kwargs,
|
||||
)
|
||||
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
|
||||
evaluate_llm_for_gsm8k(
|
||||
@@ -777,6 +799,7 @@ def test_mtp_correctness(
|
||||
},
|
||||
max_model_len=2048,
|
||||
attention_backend=attn_backend,
|
||||
**extra_kwargs,
|
||||
)
|
||||
# MTP supports async scheduling; assert it is active by default.
|
||||
assert spec_llm.llm_engine.vllm_config.scheduler_config.async_scheduling
|
||||
|
||||
Reference in New Issue
Block a user