- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
433 lines
17 KiB
Python
433 lines
17 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
import logging
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from importlib import import_module
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import torch
|
|
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
|
|
|
from vllm.config import CacheConfig, ParallelConfig
|
|
from vllm.config.speculative import SpeculativeConfig
|
|
from vllm.model_executor.models.config import (
|
|
Qwen3_5ForConditionalGenerationConfig,
|
|
Qwen4ExpForConditionalGenerationConfig,
|
|
)
|
|
from vllm.models.qwen4_exp.common.mtp import make_mtp_hidden_buffer
|
|
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
|
|
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
|
|
|
from ...utils import spawn_new_process_for_each_test
|
|
from .test_ple import _make_file_gather_embedding
|
|
|
|
|
|
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
|
values = {
|
|
"vocab_size": 64,
|
|
"hidden_size": 16,
|
|
"intermediate_size": 32,
|
|
"num_hidden_layers": 2,
|
|
"num_attention_heads": 2,
|
|
"num_key_value_heads": 1,
|
|
"head_dim": 8,
|
|
"layer_types": ["linear_attention", "full_attention"],
|
|
"linear_num_key_heads": 2,
|
|
"linear_num_value_heads": 2,
|
|
"linear_key_head_dim": 8,
|
|
"linear_value_head_dim": 8,
|
|
"num_experts": 4,
|
|
"num_experts_per_tok": 2,
|
|
# `Qwen4ExpTextConfig` requires an EOS token whenever PLE is enabled.
|
|
"eos_token_id": 1,
|
|
"hc_count": 2,
|
|
"hc_lowrank": 4,
|
|
"ple_layer_ids": [1],
|
|
"mtp_num_hidden_layers": 1,
|
|
"mtp": {"hybrid": True},
|
|
}
|
|
values.update(kwargs)
|
|
return Qwen4ExpTextConfig(**values)
|
|
|
|
|
|
def test_moe_sp_rejects_pipeline_parallel() -> None:
|
|
"""Automatic MoE SP must validate HC topology even without --enable-hc-sp."""
|
|
config = SimpleNamespace(
|
|
model_config=SimpleNamespace(
|
|
hf_text_config=_text_config(num_experts=4, ple_layer_ids=[]),
|
|
),
|
|
cache_config=CacheConfig(),
|
|
parallel_config=ParallelConfig(
|
|
tensor_parallel_size=2,
|
|
pipeline_parallel_size=2,
|
|
data_parallel_size=2,
|
|
enable_expert_parallel=True,
|
|
all2all_backend="allgather_reducescatter",
|
|
),
|
|
)
|
|
with (
|
|
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=True),
|
|
pytest.raises(ValueError, match="MoE SP requires PP=1"),
|
|
):
|
|
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
|
|
|
|
|
|
@spawn_new_process_for_each_test
|
|
@pytest.mark.parametrize("backend", ["nvidia"])
|
|
@pytest.mark.parametrize("use_sp", [False, True])
|
|
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(
|
|
backend: str, use_sp: bool
|
|
) -> None:
|
|
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
|
|
mtp_module = f"vllm.models.qwen4_exp.{backend}.mtp"
|
|
model = object.__new__(import_module(mtp_module).Qwen4ExpMultiTokenPredictor)
|
|
model.use_hc_sequence_parallel = use_sp
|
|
model.use_sequence_parallel = False
|
|
torch.nn.Module.__init__(model)
|
|
model.hc_count = 2
|
|
model.hidden_size = 4
|
|
model.num_mtp_layers = 1
|
|
model.pre_fc_norm_embedding = torch.nn.Identity()
|
|
model.pre_fc_norm_hidden = torch.nn.Identity()
|
|
model.fc_embedding = torch.nn.Identity()
|
|
model.fc_hidden = torch.nn.Identity()
|
|
|
|
def decoder(**kwargs):
|
|
"""Check that the embedding residual shares the hidden state's token slice."""
|
|
hidden = kwargs["hidden_states"]
|
|
if backend == "nvidia":
|
|
prev_block_output = kwargs["prev_block_output"]
|
|
assert prev_block_output.shape == (hidden.shape[0], model.hidden_size)
|
|
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
|
|
|
|
model.layers = [decoder]
|
|
model.hyper_connection_mixer = SimpleNamespace(
|
|
combine_and_mix=lambda hidden_states, block_output, injection: (
|
|
hidden_states,
|
|
hidden_states.unflatten(-1, (2, 4)).mean(-2),
|
|
None,
|
|
),
|
|
)
|
|
multi_hidden = torch.arange(24, dtype=torch.float32).reshape(3, 8)
|
|
expected_sample = multi_hidden.unflatten(-1, (2, 4)).mean(-2)
|
|
pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True)
|
|
|
|
def gather(local):
|
|
"""Supply rank 1's last token and padding for a simulated two-rank gather."""
|
|
assert local.shape[0] == 2
|
|
remote = torch.cat([expected_sample[2:], multi_hidden[2:]], dim=-1)
|
|
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
|
|
|
|
with (
|
|
patch(f"{mtp_module}.get_pp_group", return_value=pp_group),
|
|
patch(f"{mtp_module}.sp_shard", side_effect=lambda x: x[:2]),
|
|
patch(f"{mtp_module}.sp_all_gather", side_effect=gather),
|
|
):
|
|
sample_hidden, returned_multi_hidden = model.forward(
|
|
input_ids=None,
|
|
positions=torch.arange(3),
|
|
hidden_states=multi_hidden,
|
|
inputs_embeds=torch.zeros(3, 4),
|
|
)
|
|
|
|
torch.testing.assert_close(sample_hidden, expected_sample)
|
|
torch.testing.assert_close(returned_multi_hidden, multi_hidden)
|
|
assert sample_hidden.is_contiguous()
|
|
assert returned_multi_hidden.is_contiguous()
|
|
|
|
|
|
@spawn_new_process_for_each_test
|
|
@pytest.mark.parametrize("backend", ["nvidia"])
|
|
def test_qwen4_exp_mtp_remaps_mixed_precision_layer_indices(backend: str) -> None:
|
|
mtp_module = import_module(f"vllm.models.qwen4_exp.{backend}.mtp")
|
|
|
|
quantized_layers = {
|
|
"model.language_model.layers.0.mlp.experts": {"quant_algo": "NVFP4"},
|
|
"mtp.layers.0.mlp.experts": {
|
|
"quant_algo": "FP8_BLOCK_SCALES",
|
|
"group_size": 128,
|
|
},
|
|
}
|
|
|
|
assert mtp_module._remap_quantized_layers(quantized_layers, 48) == {
|
|
"model.language_model.layers.0.mlp.experts": {"quant_algo": "NVFP4"},
|
|
"mtp.layers.48.mlp.experts": {
|
|
"quant_algo": "FP8_BLOCK_SCALES",
|
|
"group_size": 128,
|
|
},
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("wrapped_config", [False, True])
|
|
def test_qwen4_exp_mtp_override_sets_draft_config(
|
|
wrapped_config: bool,
|
|
) -> None:
|
|
text_config = _text_config(
|
|
architectures=["Qwen4ExpForCausalLM"],
|
|
index_share_for_mtp_iteration=True,
|
|
)
|
|
config = (
|
|
Qwen4ExpConfig(
|
|
architectures=["Qwen4ExpForConditionalGeneration"],
|
|
text_config=text_config,
|
|
)
|
|
if wrapped_config
|
|
else text_config
|
|
)
|
|
|
|
draft_config = SpeculativeConfig.hf_config_override(config)
|
|
|
|
assert draft_config.index_share_for_mtp_iteration is True
|
|
assert draft_config.model_type == "qwen4_exp_mtp"
|
|
assert draft_config.architectures == ["Qwen4ExpMTP"]
|
|
assert draft_config.hc_mult == 2
|
|
assert draft_config.n_predict == 1
|
|
|
|
|
|
@pytest.mark.parametrize("ple_layer_ids", [[1], []])
|
|
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
|
|
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
|
|
rest of the architecture is PP-capable, so the refusal must be conditional
|
|
-- and must land before the engine spends time loading weights."""
|
|
vllm_config = SimpleNamespace(
|
|
model_config=SimpleNamespace(
|
|
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
|
|
multimodal_config=None,
|
|
),
|
|
parallel_config=ParallelConfig(pipeline_parallel_size=2),
|
|
speculative_config=None,
|
|
)
|
|
with patch.object(
|
|
Qwen3_5ForConditionalGenerationConfig, "verify_and_update_config"
|
|
):
|
|
if ple_layer_ids:
|
|
with pytest.raises(NotImplementedError, match="pipeline_parallel_size=1"):
|
|
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(
|
|
vllm_config
|
|
)
|
|
else:
|
|
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
|
|
|
|
|
def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
|
model_state = object.__new__(Qwen4ExpModelState)
|
|
model_state.uses_ngram_embedding = True
|
|
model_state.ngram_context_len = 3
|
|
model_state.ngram_eos_token_id = 99
|
|
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
|
|
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
|
|
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
|
|
model_state.file_gather_modules = []
|
|
|
|
input_batch = SimpleNamespace(
|
|
num_reqs=2,
|
|
num_reqs_after_padding=3,
|
|
idx_mapping=torch.tensor([1, 0]),
|
|
query_start_loc=torch.tensor([0, 2, 3, 3], dtype=torch.int32),
|
|
)
|
|
req_states = SimpleNamespace(
|
|
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([3, 1])),
|
|
all_token_ids=SimpleNamespace(
|
|
gpu=torch.tensor([[1, 2, 3, 4], [20, 21, 22, 23]], dtype=torch.int32)
|
|
),
|
|
)
|
|
|
|
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
|
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
|
|
|
expected_query_start_loc = torch.full((9,), 3, dtype=torch.int32)
|
|
expected_query_start_loc[0] = 0
|
|
expected_query_start_loc[1] = 2
|
|
torch.testing.assert_close(
|
|
model_inputs["query_start_loc"], expected_query_start_loc
|
|
)
|
|
expected_context = torch.full((8, 3), 99, dtype=torch.int32)
|
|
expected_context[:2] = torch.tensor([[99, 99, 20], [1, 2, 3]])
|
|
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
|
|
|
|
# Retain the views to detect reallocations as the request layout changes.
|
|
query_start_loc = model_inputs["query_start_loc"]
|
|
ngram_context = model_inputs["ngram_context"]
|
|
input_batch.num_reqs = 1
|
|
input_batch.num_reqs_after_padding = 1
|
|
input_batch.idx_mapping = torch.tensor([0])
|
|
input_batch.query_start_loc = torch.tensor([0, 3], dtype=torch.int32)
|
|
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
|
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
|
|
|
expected_query_start_loc.fill_(3)
|
|
expected_query_start_loc[0] = 0
|
|
torch.testing.assert_close(
|
|
model_inputs["query_start_loc"], expected_query_start_loc
|
|
)
|
|
expected_context.fill_(99)
|
|
expected_context[0] = torch.tensor([1, 2, 3])
|
|
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
|
|
assert model_inputs["query_start_loc"].data_ptr() == query_start_loc.data_ptr()
|
|
assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr()
|
|
|
|
|
|
def test_qwen4_exp_model_state_stages_file_gather_rows(monkeypatch, tmp_path) -> None:
|
|
"""Rows are staged from the same PLE inputs the forward receives."""
|
|
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
|
|
model_state = object.__new__(Qwen4ExpModelState)
|
|
model_state.uses_ngram_embedding = True
|
|
model_state.ngram_eos_token_id = 0
|
|
model_state.ngram_context = torch.empty((4, 1), dtype=torch.int32)
|
|
model_state.ngram_context_offsets = torch.arange(-1, 0, dtype=torch.int64)
|
|
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
|
model_state.file_gather_modules = [module]
|
|
input_batch = SimpleNamespace(
|
|
num_reqs=2,
|
|
num_reqs_after_padding=2,
|
|
idx_mapping=torch.tensor([0, 1]),
|
|
query_start_loc=torch.tensor([0, 4, 6], dtype=torch.int32),
|
|
input_ids=torch.tensor([5, 9, 3, 7, 2, 11]),
|
|
)
|
|
req_states = SimpleNamespace(
|
|
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([2, 1])),
|
|
all_token_ids=SimpleNamespace(gpu=torch.tensor([[1, 8, 6], [4, 3, 3]])),
|
|
)
|
|
|
|
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
|
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
|
|
|
ids = module.compute_ngram_ids(
|
|
input_batch.input_ids,
|
|
model_inputs["query_start_loc"],
|
|
model_inputs["ngram_context"],
|
|
)
|
|
assert torch.equal(module.ngram_embedding._staging[:6], table[ids])
|
|
|
|
|
|
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
|
model_state = object.__new__(Qwen4ExpModelState)
|
|
model_state.uses_ngram_embedding = True
|
|
model_state.ngram_eos_token_id = 99
|
|
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
|
|
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
|
|
|
|
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
|
|
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
|
|
# Dummy runs establish the addresses used during CUDA graph capture.
|
|
query_start_loc_ptr = first["query_start_loc"].data_ptr()
|
|
ngram_context_ptr = first["ngram_context"].data_ptr()
|
|
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
|
|
|
|
expected_query_start_loc = torch.full((9,), 4, dtype=torch.int32)
|
|
expected_query_start_loc[:4] = torch.tensor([0, 1, 2, 4], dtype=torch.int32)
|
|
torch.testing.assert_close(second["query_start_loc"], expected_query_start_loc)
|
|
torch.testing.assert_close(
|
|
second["ngram_context"], torch.full((8, 3), 99, dtype=torch.int32)
|
|
)
|
|
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
|
|
assert second["ngram_context"].data_ptr() == ngram_context_ptr
|
|
|
|
|
|
@contextmanager
|
|
def _captured_transformers_warnings() -> Iterator[list[str]]:
|
|
"""Records what transformers logs, without relying on propagation.
|
|
|
|
transformers keeps its own logger hierarchy, and the rope validator reaches
|
|
it through ``warning_once``, so a root-level capture can miss the record
|
|
entirely.
|
|
"""
|
|
records: list[str] = []
|
|
|
|
class _Collect(logging.Handler):
|
|
def emit(self, record: logging.LogRecord) -> None:
|
|
records.append(record.getMessage())
|
|
|
|
handler = _Collect()
|
|
logger = logging.getLogger("transformers")
|
|
previous_level = logger.level
|
|
logger.addHandler(handler)
|
|
logger.setLevel(logging.WARNING)
|
|
try:
|
|
yield records
|
|
finally:
|
|
logger.removeHandler(handler)
|
|
logger.setLevel(previous_level)
|
|
|
|
|
|
@pytest.mark.parametrize("wrapped_config", [False, True])
|
|
def test_rope_validation_accepts_mrope_keys_and_still_warns_for_unknown_keys(
|
|
wrapped_config: bool,
|
|
) -> None:
|
|
"""M-RoPE is driven through two rope_parameters keys transformers does not
|
|
know for rope_type="default". Declaring them must not silence the
|
|
unrecognized-key warning for every other key."""
|
|
|
|
def build(rope_parameters: dict) -> None:
|
|
text_config = _text_config(
|
|
rope_parameters=dict(rope_parameters), rope_theta=10_000.0
|
|
)
|
|
if wrapped_config:
|
|
Qwen4ExpConfig(
|
|
architectures=["Qwen4ExpForConditionalGeneration"],
|
|
text_config=text_config.to_dict(),
|
|
)
|
|
|
|
with _captured_transformers_warnings() as records:
|
|
build(
|
|
{
|
|
"rope_type": "default",
|
|
"mrope_section": [16, 24, 24],
|
|
"mrope_interleaved": True,
|
|
}
|
|
)
|
|
assert not [line for line in records if "Unrecognized keys" in line], records
|
|
|
|
# warning_once dedupes on the message, so each parametrization needs a key
|
|
# no earlier run has reported.
|
|
unknown_key = f"not_a_rope_key_{int(wrapped_config)}"
|
|
with _captured_transformers_warnings() as records:
|
|
build({"rope_type": "default", unknown_key: 1})
|
|
assert [line for line in records if unknown_key in line], records
|
|
|
|
|
|
@pytest.mark.parametrize("is_last_rank", [True, False])
|
|
def test_mtp_hidden_buffer_is_allocated_on_the_configured_device(
|
|
is_last_rank: bool,
|
|
) -> None:
|
|
"""The drafter's hidden buffer belongs on the device the config names, not
|
|
on whichever device happens to be current."""
|
|
vllm_config = SimpleNamespace(
|
|
speculative_config=SimpleNamespace(method="mtp"),
|
|
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
|
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
|
# "meta" is never the ambient default, so an implicit allocation lands
|
|
# somewhere else and this test sees it.
|
|
device_config=SimpleNamespace(device=torch.device("meta")),
|
|
)
|
|
config = SimpleNamespace(hc_count=2, hidden_size=4)
|
|
|
|
buffer = make_mtp_hidden_buffer(vllm_config, config, is_last_rank=is_last_rank)
|
|
|
|
if not is_last_rank:
|
|
assert buffer is None
|
|
return
|
|
assert buffer is not None
|
|
assert buffer.device.type == "meta"
|
|
assert buffer.shape == (8, 8)
|
|
assert buffer.dtype is torch.bfloat16
|
|
|
|
|
|
def test_mtp_hidden_buffer_is_absent_without_mtp_speculation() -> None:
|
|
vllm_config = SimpleNamespace(
|
|
speculative_config=SimpleNamespace(method="eagle"),
|
|
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
|
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
|
device_config=SimpleNamespace(device=torch.device("meta")),
|
|
)
|
|
config = SimpleNamespace(hc_count=2, hidden_size=4)
|
|
|
|
assert make_mtp_hidden_buffer(vllm_config, config, is_last_rank=True) is None
|
|
|
|
vllm_config.speculative_config = None
|
|
assert make_mtp_hidden_buffer(vllm_config, config, is_last_rank=True) is None
|