Files
MOVZXandHinata 2066d3143f [Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions,
  rl, render-launcher and batch entrypoints; keep scale-out token routes
- Drop the Swagger static bundle and the /docs, /redoc routes
- Delete device-keyed tuning configs for fused_moe, quant utils, mamba
  select-state and helion; no sm86 entry ever existed for any of them
- Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant
  quantizers and their config, metrics and loader plumbing
- Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu
  requirements and the setup.py cpu target path
- Delete 29 environment variables nothing reads and 14 chat-template
  fallbacks for models this fork no longer registers
- Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark,
  dsv3 gemm) with their kernels, bindings, wrappers and CMake entries
- Delete the connector test suites that targeted removed connectors
- Cut the Mistral tokenizer and parser stack, its tests, and the
  mistral_common and ijson dependency pins
- Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier
  cut deleted while keeping the model tree

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
2026-10-19 17:10:56 +07:00

433 lines
17 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
from collections.abc import Iterator
from contextlib import contextmanager
from importlib import import_module
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
from vllm.config import CacheConfig, ParallelConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.model_executor.models.config import (
Qwen3_5ForConditionalGenerationConfig,
Qwen4ExpForConditionalGenerationConfig,
)
from vllm.models.qwen4_exp.common.mtp import make_mtp_hidden_buffer
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from ...utils import spawn_new_process_for_each_test
from .test_ple import _make_file_gather_embedding
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
values = {
"vocab_size": 64,
"hidden_size": 16,
"intermediate_size": 32,
"num_hidden_layers": 2,
"num_attention_heads": 2,
"num_key_value_heads": 1,
"head_dim": 8,
"layer_types": ["linear_attention", "full_attention"],
"linear_num_key_heads": 2,
"linear_num_value_heads": 2,
"linear_key_head_dim": 8,
"linear_value_head_dim": 8,
"num_experts": 4,
"num_experts_per_tok": 2,
# `Qwen4ExpTextConfig` requires an EOS token whenever PLE is enabled.
"eos_token_id": 1,
"hc_count": 2,
"hc_lowrank": 4,
"ple_layer_ids": [1],
"mtp_num_hidden_layers": 1,
"mtp": {"hybrid": True},
}
values.update(kwargs)
return Qwen4ExpTextConfig(**values)
def test_moe_sp_rejects_pipeline_parallel() -> None:
"""Automatic MoE SP must validate HC topology even without --enable-hc-sp."""
config = SimpleNamespace(
model_config=SimpleNamespace(
hf_text_config=_text_config(num_experts=4, ple_layer_ids=[]),
),
cache_config=CacheConfig(),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
pipeline_parallel_size=2,
data_parallel_size=2,
enable_expert_parallel=True,
all2all_backend="allgather_reducescatter",
),
)
with (
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=True),
pytest.raises(ValueError, match="MoE SP requires PP=1"),
):
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
@spawn_new_process_for_each_test
@pytest.mark.parametrize("backend", ["nvidia"])
@pytest.mark.parametrize("use_sp", [False, True])
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(
backend: str, use_sp: bool
) -> None:
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
mtp_module = f"vllm.models.qwen4_exp.{backend}.mtp"
model = object.__new__(import_module(mtp_module).Qwen4ExpMultiTokenPredictor)
model.use_hc_sequence_parallel = use_sp
model.use_sequence_parallel = False
torch.nn.Module.__init__(model)
model.hc_count = 2
model.hidden_size = 4
model.num_mtp_layers = 1
model.pre_fc_norm_embedding = torch.nn.Identity()
model.pre_fc_norm_hidden = torch.nn.Identity()
model.fc_embedding = torch.nn.Identity()
model.fc_hidden = torch.nn.Identity()
def decoder(**kwargs):
"""Check that the embedding residual shares the hidden state's token slice."""
hidden = kwargs["hidden_states"]
if backend == "nvidia":
prev_block_output = kwargs["prev_block_output"]
assert prev_block_output.shape == (hidden.shape[0], model.hidden_size)
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
model.layers = [decoder]
model.hyper_connection_mixer = SimpleNamespace(
combine_and_mix=lambda hidden_states, block_output, injection: (
hidden_states,
hidden_states.unflatten(-1, (2, 4)).mean(-2),
None,
),
)
multi_hidden = torch.arange(24, dtype=torch.float32).reshape(3, 8)
expected_sample = multi_hidden.unflatten(-1, (2, 4)).mean(-2)
pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True)
def gather(local):
"""Supply rank 1's last token and padding for a simulated two-rank gather."""
assert local.shape[0] == 2
remote = torch.cat([expected_sample[2:], multi_hidden[2:]], dim=-1)
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
with (
patch(f"{mtp_module}.get_pp_group", return_value=pp_group),
patch(f"{mtp_module}.sp_shard", side_effect=lambda x: x[:2]),
patch(f"{mtp_module}.sp_all_gather", side_effect=gather),
):
sample_hidden, returned_multi_hidden = model.forward(
input_ids=None,
positions=torch.arange(3),
hidden_states=multi_hidden,
inputs_embeds=torch.zeros(3, 4),
)
torch.testing.assert_close(sample_hidden, expected_sample)
torch.testing.assert_close(returned_multi_hidden, multi_hidden)
assert sample_hidden.is_contiguous()
assert returned_multi_hidden.is_contiguous()
@spawn_new_process_for_each_test
@pytest.mark.parametrize("backend", ["nvidia"])
def test_qwen4_exp_mtp_remaps_mixed_precision_layer_indices(backend: str) -> None:
mtp_module = import_module(f"vllm.models.qwen4_exp.{backend}.mtp")
quantized_layers = {
"model.language_model.layers.0.mlp.experts": {"quant_algo": "NVFP4"},
"mtp.layers.0.mlp.experts": {
"quant_algo": "FP8_BLOCK_SCALES",
"group_size": 128,
},
}
assert mtp_module._remap_quantized_layers(quantized_layers, 48) == {
"model.language_model.layers.0.mlp.experts": {"quant_algo": "NVFP4"},
"mtp.layers.48.mlp.experts": {
"quant_algo": "FP8_BLOCK_SCALES",
"group_size": 128,
},
}
@pytest.mark.parametrize("wrapped_config", [False, True])
def test_qwen4_exp_mtp_override_sets_draft_config(
wrapped_config: bool,
) -> None:
text_config = _text_config(
architectures=["Qwen4ExpForCausalLM"],
index_share_for_mtp_iteration=True,
)
config = (
Qwen4ExpConfig(
architectures=["Qwen4ExpForConditionalGeneration"],
text_config=text_config,
)
if wrapped_config
else text_config
)
draft_config = SpeculativeConfig.hf_config_override(config)
assert draft_config.index_share_for_mtp_iteration is True
assert draft_config.model_type == "qwen4_exp_mtp"
assert draft_config.architectures == ["Qwen4ExpMTP"]
assert draft_config.hc_mult == 2
assert draft_config.n_predict == 1
@pytest.mark.parametrize("ple_layer_ids", [[1], []])
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
rest of the architecture is PP-capable, so the refusal must be conditional
-- and must land before the engine spends time loading weights."""
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
multimodal_config=None,
),
parallel_config=ParallelConfig(pipeline_parallel_size=2),
speculative_config=None,
)
with patch.object(
Qwen3_5ForConditionalGenerationConfig, "verify_and_update_config"
):
if ple_layer_ids:
with pytest.raises(NotImplementedError, match="pipeline_parallel_size=1"):
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(
vllm_config
)
else:
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_context_len = 3
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
model_state.file_gather_modules = []
input_batch = SimpleNamespace(
num_reqs=2,
num_reqs_after_padding=3,
idx_mapping=torch.tensor([1, 0]),
query_start_loc=torch.tensor([0, 2, 3, 3], dtype=torch.int32),
)
req_states = SimpleNamespace(
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([3, 1])),
all_token_ids=SimpleNamespace(
gpu=torch.tensor([[1, 2, 3, 4], [20, 21, 22, 23]], dtype=torch.int32)
),
)
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
expected_query_start_loc = torch.full((9,), 3, dtype=torch.int32)
expected_query_start_loc[0] = 0
expected_query_start_loc[1] = 2
torch.testing.assert_close(
model_inputs["query_start_loc"], expected_query_start_loc
)
expected_context = torch.full((8, 3), 99, dtype=torch.int32)
expected_context[:2] = torch.tensor([[99, 99, 20], [1, 2, 3]])
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
# Retain the views to detect reallocations as the request layout changes.
query_start_loc = model_inputs["query_start_loc"]
ngram_context = model_inputs["ngram_context"]
input_batch.num_reqs = 1
input_batch.num_reqs_after_padding = 1
input_batch.idx_mapping = torch.tensor([0])
input_batch.query_start_loc = torch.tensor([0, 3], dtype=torch.int32)
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
expected_query_start_loc.fill_(3)
expected_query_start_loc[0] = 0
torch.testing.assert_close(
model_inputs["query_start_loc"], expected_query_start_loc
)
expected_context.fill_(99)
expected_context[0] = torch.tensor([1, 2, 3])
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
assert model_inputs["query_start_loc"].data_ptr() == query_start_loc.data_ptr()
assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr()
def test_qwen4_exp_model_state_stages_file_gather_rows(monkeypatch, tmp_path) -> None:
"""Rows are staged from the same PLE inputs the forward receives."""
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_eos_token_id = 0
model_state.ngram_context = torch.empty((4, 1), dtype=torch.int32)
model_state.ngram_context_offsets = torch.arange(-1, 0, dtype=torch.int64)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
model_state.file_gather_modules = [module]
input_batch = SimpleNamespace(
num_reqs=2,
num_reqs_after_padding=2,
idx_mapping=torch.tensor([0, 1]),
query_start_loc=torch.tensor([0, 4, 6], dtype=torch.int32),
input_ids=torch.tensor([5, 9, 3, 7, 2, 11]),
)
req_states = SimpleNamespace(
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([2, 1])),
all_token_ids=SimpleNamespace(gpu=torch.tensor([[1, 8, 6], [4, 3, 3]])),
)
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
ids = module.compute_ngram_ids(
input_batch.input_ids,
model_inputs["query_start_loc"],
model_inputs["ngram_context"],
)
assert torch.equal(module.ngram_embedding._staging[:6], table[ids])
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
# Dummy runs establish the addresses used during CUDA graph capture.
query_start_loc_ptr = first["query_start_loc"].data_ptr()
ngram_context_ptr = first["ngram_context"].data_ptr()
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
expected_query_start_loc = torch.full((9,), 4, dtype=torch.int32)
expected_query_start_loc[:4] = torch.tensor([0, 1, 2, 4], dtype=torch.int32)
torch.testing.assert_close(second["query_start_loc"], expected_query_start_loc)
torch.testing.assert_close(
second["ngram_context"], torch.full((8, 3), 99, dtype=torch.int32)
)
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
assert second["ngram_context"].data_ptr() == ngram_context_ptr
@contextmanager
def _captured_transformers_warnings() -> Iterator[list[str]]:
"""Records what transformers logs, without relying on propagation.
transformers keeps its own logger hierarchy, and the rope validator reaches
it through ``warning_once``, so a root-level capture can miss the record
entirely.
"""
records: list[str] = []
class _Collect(logging.Handler):
def emit(self, record: logging.LogRecord) -> None:
records.append(record.getMessage())
handler = _Collect()
logger = logging.getLogger("transformers")
previous_level = logger.level
logger.addHandler(handler)
logger.setLevel(logging.WARNING)
try:
yield records
finally:
logger.removeHandler(handler)
logger.setLevel(previous_level)
@pytest.mark.parametrize("wrapped_config", [False, True])
def test_rope_validation_accepts_mrope_keys_and_still_warns_for_unknown_keys(
wrapped_config: bool,
) -> None:
"""M-RoPE is driven through two rope_parameters keys transformers does not
know for rope_type="default". Declaring them must not silence the
unrecognized-key warning for every other key."""
def build(rope_parameters: dict) -> None:
text_config = _text_config(
rope_parameters=dict(rope_parameters), rope_theta=10_000.0
)
if wrapped_config:
Qwen4ExpConfig(
architectures=["Qwen4ExpForConditionalGeneration"],
text_config=text_config.to_dict(),
)
with _captured_transformers_warnings() as records:
build(
{
"rope_type": "default",
"mrope_section": [16, 24, 24],
"mrope_interleaved": True,
}
)
assert not [line for line in records if "Unrecognized keys" in line], records
# warning_once dedupes on the message, so each parametrization needs a key
# no earlier run has reported.
unknown_key = f"not_a_rope_key_{int(wrapped_config)}"
with _captured_transformers_warnings() as records:
build({"rope_type": "default", unknown_key: 1})
assert [line for line in records if unknown_key in line], records
@pytest.mark.parametrize("is_last_rank", [True, False])
def test_mtp_hidden_buffer_is_allocated_on_the_configured_device(
is_last_rank: bool,
) -> None:
"""The drafter's hidden buffer belongs on the device the config names, not
on whichever device happens to be current."""
vllm_config = SimpleNamespace(
speculative_config=SimpleNamespace(method="mtp"),
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
model_config=SimpleNamespace(dtype=torch.bfloat16),
# "meta" is never the ambient default, so an implicit allocation lands
# somewhere else and this test sees it.
device_config=SimpleNamespace(device=torch.device("meta")),
)
config = SimpleNamespace(hc_count=2, hidden_size=4)
buffer = make_mtp_hidden_buffer(vllm_config, config, is_last_rank=is_last_rank)
if not is_last_rank:
assert buffer is None
return
assert buffer is not None
assert buffer.device.type == "meta"
assert buffer.shape == (8, 8)
assert buffer.dtype is torch.bfloat16
def test_mtp_hidden_buffer_is_absent_without_mtp_speculation() -> None:
vllm_config = SimpleNamespace(
speculative_config=SimpleNamespace(method="eagle"),
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
model_config=SimpleNamespace(dtype=torch.bfloat16),
device_config=SimpleNamespace(device=torch.device("meta")),
)
config = SimpleNamespace(hc_count=2, hidden_size=4)
assert make_mtp_hidden_buffer(vllm_config, config, is_last_rank=True) is None
vllm_config.speculative_config = None
assert make_mtp_hidden_buffer(vllm_config, config, is_last_rank=True) is None