[Qwen4Exp] Fix MTP buffer placement, config parsing, Triton warmup (PR #56742)
Intake of open upstream PR vllm-project/vllm#56742. Every rig here runs MTP on both Qwen3.8 models, so these three bugs sit on our hot path. The MTP hidden buffer was allocated with torch.empty and no device, so it followed the ambient default instead of device_config.device; on a CPU-default process the draft buffer lands on the wrong device. Both NVIDIA and AMD backends now call make_mtp_hidden_buffer() in vllm/models/qwen4_exp/common/mtp.py, which pins it to the configured device. Config parsing now accepts configs carrying M-RoPE keys and the Transformers spelling of layer_types; both previously failed at load. Triton warmup: Qwen4Exp was missing from the Qwen warmup dispatch, so its Triton kernels compiled during the first served requests instead of at startup. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from importlib import import_module
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
@@ -13,6 +16,7 @@ from vllm.model_executor.models.config import (
|
||||
Qwen3_5ForConditionalGenerationConfig,
|
||||
Qwen4ExpForConditionalGenerationConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.common.mtp import make_mtp_hidden_buffer
|
||||
from vllm.models.qwen4_exp.config import (
|
||||
Qwen4ExpConfig,
|
||||
Qwen4ExpTextConfig,
|
||||
@@ -281,3 +285,121 @@ def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
||||
)
|
||||
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
|
||||
assert second["ngram_context"].data_ptr() == ngram_context_ptr
|
||||
|
||||
|
||||
def test_text_config_normalizes_transformers_sparse_attention_spelling():
|
||||
"""Checkpoints re-exported through transformers' Qwen4Exp config class
|
||||
serialize "qwen_sparse_attention"; vLLM keys QSA off "full_attention"."""
|
||||
from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig
|
||||
|
||||
config = Qwen4ExpTextConfig(
|
||||
hc_count=4,
|
||||
layer_types=["linear_attention", "qwen_sparse_attention"],
|
||||
num_hidden_layers=2,
|
||||
)
|
||||
|
||||
assert config.layer_types == ["linear_attention", "full_attention"]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _captured_transformers_warnings() -> Iterator[list[str]]:
|
||||
"""Records what transformers logs, without relying on propagation.
|
||||
|
||||
transformers keeps its own logger hierarchy, and the rope validator reaches
|
||||
it through ``warning_once``, so a root-level capture can miss the record
|
||||
entirely.
|
||||
"""
|
||||
records: list[str] = []
|
||||
|
||||
class _Collect(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
records.append(record.getMessage())
|
||||
|
||||
handler = _Collect()
|
||||
logger = logging.getLogger("transformers")
|
||||
previous_level = logger.level
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(logging.WARNING)
|
||||
try:
|
||||
yield records
|
||||
finally:
|
||||
logger.removeHandler(handler)
|
||||
logger.setLevel(previous_level)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapped_config", [False, True])
|
||||
def test_rope_validation_accepts_mrope_keys_and_still_warns_for_unknown_keys(
|
||||
wrapped_config: bool,
|
||||
) -> None:
|
||||
"""M-RoPE is driven through two rope_parameters keys transformers does not
|
||||
know for rope_type="default". Declaring them must not silence the
|
||||
unrecognized-key warning for every other key."""
|
||||
|
||||
def build(rope_parameters: dict) -> None:
|
||||
text_config = _text_config(
|
||||
rope_parameters=dict(rope_parameters), rope_theta=10_000.0
|
||||
)
|
||||
if wrapped_config:
|
||||
Qwen4ExpConfig(
|
||||
architectures=["Qwen4ExpForConditionalGeneration"],
|
||||
text_config=text_config.to_dict(),
|
||||
)
|
||||
|
||||
with _captured_transformers_warnings() as records:
|
||||
build(
|
||||
{
|
||||
"rope_type": "default",
|
||||
"mrope_section": [16, 24, 24],
|
||||
"mrope_interleaved": True,
|
||||
}
|
||||
)
|
||||
assert not [line for line in records if "Unrecognized keys" in line], records
|
||||
|
||||
# warning_once dedupes on the message, so each parametrization needs a key
|
||||
# no earlier run has reported.
|
||||
unknown_key = f"not_a_rope_key_{int(wrapped_config)}"
|
||||
with _captured_transformers_warnings() as records:
|
||||
build({"rope_type": "default", unknown_key: 1})
|
||||
assert [line for line in records if unknown_key in line], records
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_last_rank", [True, False])
|
||||
def test_mtp_hidden_buffer_is_allocated_on_the_configured_device(
|
||||
is_last_rank: bool,
|
||||
) -> None:
|
||||
"""The drafter's hidden buffer belongs on the device the config names, not
|
||||
on whichever device happens to be current."""
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(method="mtp"),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
||||
# "meta" is never the ambient default, so an implicit allocation lands
|
||||
# somewhere else and this test sees it.
|
||||
device_config=SimpleNamespace(device=torch.device("meta")),
|
||||
)
|
||||
config = SimpleNamespace(hc_count=2, hidden_size=4)
|
||||
|
||||
buffer = make_mtp_hidden_buffer(vllm_config, config, is_last_rank=is_last_rank)
|
||||
|
||||
if not is_last_rank:
|
||||
assert buffer is None
|
||||
return
|
||||
assert buffer is not None
|
||||
assert buffer.device.type == "meta"
|
||||
assert buffer.shape == (8, 8)
|
||||
assert buffer.dtype is torch.bfloat16
|
||||
|
||||
|
||||
def test_mtp_hidden_buffer_is_absent_without_mtp_speculation() -> None:
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(method="eagle"),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
||||
device_config=SimpleNamespace(device=torch.device("meta")),
|
||||
)
|
||||
config = SimpleNamespace(hc_count=2, hidden_size=4)
|
||||
|
||||
assert make_mtp_hidden_buffer(vllm_config, config, is_last_rank=True) is None
|
||||
|
||||
vllm_config.speculative_config = None
|
||||
assert make_mtp_hidden_buffer(vllm_config, config, is_last_rank=True) is None
|
||||
|
||||
Reference in New Issue
Block a user