[Qwen4Exp] support Sequence Parallelism for Qwen3.8-Flash-Next (#56322)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Huang
2026-10-12 20:56:44 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 258b97ff6b
commit c75cb63cd2
23 changed files with 870 additions and 178 deletions
@@ -9,6 +9,8 @@ each rank, and summing the per-rank outputs (what the all-reduce does)
reconstructs a full-table lookup.
"""
from unittest.mock import Mock
import pytest
import torch
@@ -124,3 +126,43 @@ def test_rank_sum_is_full_lookup(tp_size):
total += _fused(ids, _shard(full_table, si, rows, dtype), si)
torch.testing.assert_close(total, full_table[ids.long()], atol=0.0, rtol=0.0)
@requires_cuda
@torch.inference_mode()
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize("use_fused", [False, True])
def test_embedding_optionally_reduces_output(dtype, use_fused, default_vllm_config):
"""SP's deferred reduction preserves masked contributions and their dtype."""
with torch.device("cuda"):
values = torch.arange(128 * 16).reshape(128, 16)
table = ((values % 29 - 14) / 16).to(dtype)
ids = torch.tensor([0, 63, 64, 127, 12])
expected = table[ids].float()
for rank in range(2):
local = expected.clone()
local[ids // 64 != rank] = 0
remote = (expected - local).to(dtype)
group = Mock(world_size=2, rank_in_group=rank)
# Model the other vocabulary owner's contribution, including FP8 bytes.
group.all_reduce.side_effect = lambda output, remote=remote: (
output + remote.view(output.dtype)
)
for reduce_results in (True, False):
embedding = VocabParallelEmbedding(
128,
16,
params_dtype=dtype,
parallel_group=group,
reduce_results=reduce_results,
)
embedding.use_fused_embedding = use_fused
embedding.weight_loader(embedding.get_parameter("weight"), table)
actual = embedding(ids)
assert actual.dtype == dtype
torch.testing.assert_close(
actual.float(),
expected if reduce_results else local,
atol=0,
rtol=0,
)
+61 -23
View File
@@ -9,6 +9,7 @@ import pytest
import torch
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
from vllm.config import CacheConfig, ParallelConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.model_executor.models.config import (
Qwen3_5ForConditionalGenerationConfig,
@@ -48,23 +49,54 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig:
return Qwen4ExpTextConfig(**values)
def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
def test_moe_sp_rejects_pipeline_parallel() -> None:
"""Automatic MoE SP must validate HC topology even without --enable-hc-sp."""
config = SimpleNamespace(
model_config=SimpleNamespace(
hf_text_config=_text_config(num_experts=4, ple_layer_ids=[]),
),
cache_config=CacheConfig(),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
pipeline_parallel_size=2,
data_parallel_size=2,
enable_expert_parallel=True,
all2all_backend="allgather_reducescatter",
),
)
with (
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
pytest.raises(ValueError, match="MoE SP requires PP=1"),
):
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
@pytest.mark.parametrize("use_sp", [False, True])
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
from vllm.models.qwen4_exp.nvidia.mtp import (
Qwen4ExpMultiTokenPredictor,
)
model = object.__new__(Qwen4ExpMultiTokenPredictor)
model.use_hc_sequence_parallel = use_sp
model.use_sequence_parallel = False
torch.nn.Module.__init__(model)
model.hc_count = 2
model.hidden_size = 4
model.num_mtp_layers = 1
model.layers = [
lambda **kwargs: (
kwargs["hidden_states"],
kwargs["hidden_states"],
torch.zeros(kwargs["hidden_states"].shape[0], 2),
),
]
model.pre_fc_norm_embedding = torch.nn.Identity()
model.pre_fc_norm_hidden = torch.nn.Identity()
model.fc_embedding = torch.nn.Identity()
model.fc_hidden = torch.nn.Identity()
def decoder(**kwargs):
"""Check that the embedding residual shares the hidden state's token slice."""
hidden = kwargs["hidden_states"]
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
model.layers = [decoder]
model.hyper_connection_mixer = SimpleNamespace(
combine_and_mix=lambda hidden_states, block_output, injection: (
hidden_states,
@@ -72,24 +104,32 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
None,
),
)
multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8)
pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True)
multi_hidden = torch.arange(24, dtype=torch.float32).reshape(3, 8)
expected_sample = multi_hidden.unflatten(-1, (2, 4)).mean(-2)
pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True)
with patch(
"vllm.models.qwen4_exp.nvidia.mtp.get_pp_group",
return_value=pp_group,
def gather(local):
"""Supply rank 1's last token and padding for a simulated two-rank gather."""
assert local.shape[0] == 2
remote = torch.cat([expected_sample[2:], multi_hidden[2:]], dim=-1)
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
with (
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
):
sample_hidden, returned_multi_hidden = model.forward(
input_ids=None,
positions=torch.arange(2),
intermediate_tensors={"hidden_states": multi_hidden},
positions=torch.arange(3),
hidden_states=multi_hidden,
inputs_embeds=torch.zeros(3, 4),
)
torch.testing.assert_close(
sample_hidden,
multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2),
)
assert returned_multi_hidden is multi_hidden
torch.testing.assert_close(sample_hidden, expected_sample)
torch.testing.assert_close(returned_multi_hidden, multi_hidden)
assert sample_hidden.is_contiguous()
assert returned_multi_hidden.is_contiguous()
@spawn_new_process_for_each_test
@@ -150,9 +190,7 @@ def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> Non
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
multimodal_config=None,
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1
),
parallel_config=ParallelConfig(pipeline_parallel_size=2),
speculative_config=None,
)
with patch.object(
+63 -40
View File
@@ -611,46 +611,65 @@ def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
)
def test_pinned_embedding_forward_finalizes_prefetched_output(
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize("reduce_results", [False, True])
def test_pinned_embedding_finalize_prefetch_optionally_reduces_output(
monkeypatch: pytest.MonkeyPatch,
dtype: torch.dtype,
reduce_results: bool,
) -> None:
"""Deferring reduction must preserve full token rows and wait for the lookup."""
total_tokens = 3
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding._prefetch_buffer = torch.empty(4, 2, 3, dtype=torch.float8_e4m3fn)
values = torch.arange(1, 13).reshape(4, 3)
embedding._prefetch_buffer = torch.stack(
(values, torch.zeros_like(values)), dim=1
).to(dtype)
embedding._output_dim = 6
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
expected = torch.arange(12).reshape(2, 6).to(torch.float8_e4m3fn)
def finalize_prefetched(
prefetch_output: torch.Tensor,
output: torch.Tensor,
) -> None:
assert prefetch_output is embedding._prefetch_buffer
output.copy_(expected)
embedding.tp_size = 2
embedding.reduce_results = reduce_results
embedding.etp_data_parallel_size = 1
embedding._prefetch_stream = object()
waited_streams: list[object] = []
reduced_dtypes = []
monkeypatch.setattr(
embedding,
"_finalize_prefetch",
finalize_prefetched,
torch.cuda,
"current_stream",
lambda: SimpleNamespace(wait_stream=waited_streams.append),
)
output = embedding(hidden_states)
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
assert waited_streams == [embedding._prefetch_stream]
reduced_dtypes.append(tensor.dtype)
# The remote rank owns the other head's vocabulary rows.
return tensor + tensor.flip(1)
assert output.dtype == embedding._prefetch_buffer.dtype
assert torch.equal(output, expected)
embedding.parallel_group = SimpleNamespace(all_reduce=all_reduce)
output = embedding.finalize_prefetch(total_tokens=total_tokens)
expected = (
values.unsqueeze(1).expand(-1, 2, -1).to(dtype)
if reduce_results
else embedding._prefetch_buffer
)
expected = expected[:total_tokens].flatten(-2)
comm_dtype = torch.int8 if dtype == torch.float8_e4m3fn else dtype
assert reduced_dtypes == ([comm_dtype] if reduce_results else [])
assert waited_streams == [embedding._prefetch_stream]
assert output.dtype == dtype
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
def test_pinned_embedding_forward_requires_prior_start_prefetch() -> None:
"""Forward before any start_prefetch must fail loudly, not read garbage."""
def test_pinned_embedding_finalize_prefetch_requires_prior_start_prefetch() -> None:
"""Finalizing before start_prefetch must fail instead of reading garbage."""
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding._prefetch_buffer = None
embedding._prefetch_stream = None
embedding._output_dim = 6
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
with pytest.raises(RuntimeError, match="prior start_prefetch"):
embedding(hidden_states)
embedding.finalize_prefetch(total_tokens=2)
def test_pinned_embedding_finalize_requires_prior_start_prefetch() -> None:
@@ -781,15 +800,10 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
def test_pinned_embedding_start_prefetch_allocates_lazily(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""start_prefetch lazily allocates the stream/buffer, then forward consumes it.
Exercises the full pinned-prefetch contract end to end: first call
allocates the side stream and buffer with the constructor geometry,
and forward finalizes the lookup into the returned output.
"""
"""Allocate prefetch storage on first use and finalize the lookup results."""
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
@@ -825,8 +839,7 @@ def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
)
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16, device="cuda:0")
embedding.start_prefetch(hidden_states, ngram_ids)
embedding.start_prefetch(ngram_ids)
buffer = embedding._prefetch_buffer
assert buffer is not None
@@ -835,7 +848,7 @@ def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
assert buffer.device.type == "cuda"
assert embedding._prefetch_stream is not None
output = embedding(hidden_states)
output = embedding.finalize_prefetch(total_tokens=ngram_ids.shape[0])
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
@@ -1149,7 +1162,7 @@ def test_ngram_prefetch_ids_outlive_start_prefetch() -> None:
prefetched: list[torch.Tensor] = []
module.ngram_embedding = SimpleNamespace(
supports_prefetch=True,
start_prefetch=lambda _, ids: prefetched.append(weak_ref_tensor(ids)),
start_prefetch=lambda ids: prefetched.append(weak_ref_tensor(ids)),
)
expected = _reference_ngram_ids(input_ids, query_start_loc, ngram_context, **params)
@@ -1157,7 +1170,7 @@ def test_ngram_prefetch_ids_outlive_start_prefetch() -> None:
# like a later CUDA graph segment would.
pool = torch.cuda.MemPool()
with torch.cuda.use_mem_pool(pool):
module.start_prefetch(None, input_ids, query_start_loc, ngram_context)
module.start_prefetch(input_ids, query_start_loc, ngram_context)
torch.full_like(expected, -1)
assert torch.equal(prefetched[0], expected)
@@ -1859,12 +1872,18 @@ def _make_conv_metadata(
],
)
@pytest.mark.parametrize(
"layer_cls", [Qwen4ExpPLELayer, Qwen4ExpPLELayerAMD], ids=["nvidia", "amd"]
("layer_cls", "fuse_residual"),
[
pytest.param(Qwen4ExpPLELayer, True, id="nvidia-fused"),
pytest.param(Qwen4ExpPLELayer, False, id="nvidia-unfused"),
pytest.param(Qwen4ExpPLELayerAMD, True, id="amd-fused"),
],
)
def test_fused_conv_correctness(
case: _ConvBatchCase,
state_layout: str,
layer_cls: type[nn.Module],
fuse_residual: bool,
) -> None:
device = torch.device("cuda")
metadata, num_real_tokens = _make_conv_metadata(case, device)
@@ -1906,12 +1925,15 @@ def test_fused_conv_correctness(
)
null_state = conv_state[NULL_BLOCK_ID].clone()
residual_kernel = residual.clone()
residual_reference = residual.clone()
# The unfused kernel must overwrite the buffer without reading its contents.
residual_reference = (
residual.clone() if fuse_residual else torch.zeros_like(residual)
)
module._short_conv_dilated_dispatch(
inputs=inputs,
residual=residual_kernel,
outer_residual=outer_residual,
outer_residual=outer_residual if fuse_residual else None,
metadata=metadata,
conv_state=conv_state,
conv_weights=weights,
@@ -1925,7 +1947,8 @@ def test_fused_conv_correctness(
conv_state_len=module.conv_state_len,
dilation=module.short_conv_dilation,
)
residual_reference = outer_residual + residual_reference
if fuse_residual:
residual_reference = outer_residual + residual_reference
assert torch.equal(
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
@@ -1935,7 +1958,7 @@ def test_fused_conv_correctness(
if case.graph_padding:
assert torch.equal(
residual_kernel[num_real_tokens:],
(outer_residual + residual)[num_real_tokens:],
residual_reference[num_real_tokens:],
)
+267
View File
@@ -0,0 +1,267 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Multi-GPU equivalence tests for Qwen4Exp HC and MoE sequence parallelism."""
from dataclasses import MISSING, fields
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.multiprocessing as mp
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.platforms import current_platform
if not current_platform.is_cuda():
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
from tests.utils import multi_gpu_test
from vllm.config import (
EngramConfig,
ParallelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
tensor_model_parallel_all_reduce,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.linear import RowParallelLinear
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
from vllm.utils.network_utils import get_open_port
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
from .test_ple import _ConvBatchCase, _make_conv_metadata
class _TPAttentionProjection(nn.Module):
"""Keep real TP communication without setting up an attention KV cache."""
def __init__(self, config, *, reduce_results: bool, **kwargs) -> None:
super().__init__()
self.proj = RowParallelLinear(
config.hidden_size,
config.hidden_size,
bias=False,
input_is_parallel=False,
reduce_results=reduce_results,
)
def forward(self, hidden_states: torch.Tensor):
return self.proj(hidden_states)[0]
def _make_config(rank: int, *, hc_sp: bool = False, moe_sp: bool = False) -> VllmConfig:
"""Configure HC SP explicitly or MoE SP through TP=2, DP=2 and EP."""
config = VllmConfig(
parallel_config=ParallelConfig(
tensor_parallel_size=2,
data_parallel_size=2 if moe_sp else 1,
data_parallel_rank=rank // 2,
enable_expert_parallel=moe_sp,
enable_hc_sp=hc_sp,
all2all_backend="allgather_reducescatter",
),
)
config.model_config = SimpleNamespace(
architecture="Qwen4ExpForCausalLM",
is_moe=True,
sleep_mode_offload_cudagraph=False,
hf_text_config=Qwen4ExpTextConfig(
vocab_size=128,
eos_token_id=0,
hidden_size=128,
hc_count=2,
hc_lowrank=16,
num_hidden_layers=1,
layer_types=["linear_attention"],
num_experts=4,
num_experts_per_tok=2,
moe_intermediate_size=256,
shared_expert_intermediate_size=256,
ple_layer_ids=[] if moe_sp else [1],
ple_embed_dim=128,
heads_per_ngram=2,
ngram_vocab_size_base=32,
make_ngram_vocab_size_divisible_by=64,
),
dtype=torch.bfloat16,
)
config.engram_config = EngramConfig(cpu_offload=False)
return config
def _make_decoder(config: VllmConfig, rank: int) -> Qwen4ExpDecoderLayer:
"""Use the production constructor and identical initial weights for both modes."""
with (
set_current_vllm_config(config),
patch.object(qwen4_model, "QwenGatedDeltaNetAttention", _TPAttentionProjection),
):
layer = Qwen4ExpDecoderLayer(
config, config.model_config.hf_text_config.layer_types[0], "model.layers.0"
)
torch.manual_seed(12)
for param in layer.parameters():
param.normal_(std=0.08)
# Distinct rank contributions expose missing or duplicate reductions.
layer.linear_attn.proj.weight.add_(rank * 0.01)
experts = layer.mlp.experts.routed_experts
if config.parallel_config.enable_expert_parallel:
experts.w13_weight.add_(rank * 0.01)
experts.quant_method.process_weights_after_loading(experts)
return layer
def _check_decoder_ple(rank: int) -> None:
"""Compare HC SP with full tokens across a PLE prefill and cached decode."""
configs = [_make_config(rank, hc_sp=enabled) for enabled in (False, True)]
layers = [_make_decoder(config, rank) for config in configs]
states = []
for layer in layers:
ple = layer.ple
assert ple is not None
state = torch.zeros(2, ple.hc_hidden_size, ple.conv_state_len)
if not is_conv_state_dim_first():
state = state.transpose(-1, -2).contiguous()
ple.kv_cache = (state,)
states.append(state)
# Three prefill tokens straddle the TP=2 boundary and require SP padding.
for case in (
_ConvBatchCase(prefill_query_lens=(3,)),
_ConvBatchCase(num_decodes=1),
):
layout, num_tokens = _make_conv_metadata(case, torch.device("cuda", rank))
metadata_args = {
f.name: None
for f in fields(PleShortConvAttentionMetadata)
if f.default is MISSING and f.default_factory is MISSING
}
metadata = PleShortConvAttentionMetadata(**(metadata_args | vars(layout)))
ids = torch.randint(0, 128, (num_tokens,))
positions = torch.arange(num_tokens).expand(3, -1)
qsl = torch.tensor([0, num_tokens], dtype=torch.int32)
context = torch.zeros(1, layers[0].config.ngram_size - 1, dtype=torch.int64)
hidden = torch.randn(num_tokens, 256)
block = torch.randn(num_tokens, 128)
injection = torch.randn(num_tokens, 2)
outputs = []
for config, layer in zip(configs, layers):
assert layer.ple is not None
inputs = (hidden, block, injection)
if config.parallel_config.enable_hc_sp:
inputs = tuple(sp_shard(x) for x in inputs)
with (
set_current_vllm_config(config),
set_forward_context({layer.ple.prefix: metadata}, config),
):
layer.ple.start_prefetch(ids, qsl, context)
output = layer(
*inputs,
positions,
input_ids=ids,
query_start_loc=qsl,
ngram_context=context,
)
if config.parallel_config.enable_hc_sp:
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
# A later MoE call may reuse workspace-backed output storage.
outputs.append(tuple(x.clone() for x in output))
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
torch.testing.assert_close(states[1], states[0], atol=0.002, rtol=0.02)
def _check_moe_sp(rank: int, config: VllmConfig) -> None:
"""Compare native SP with full-token GR and the MoE wrapper's public interface."""
layer = _make_decoder(config, rank)
dp_rank = config.parallel_config.data_parallel_rank
counts = torch.tensor([3, 5], device="cpu")
num_tokens = int(counts[dp_rank])
torch.manual_seed(30 + dp_rank)
hidden = torch.randn(num_tokens, 256)
positions = torch.arange(num_tokens)
# Include runner padding in addition to the padding introduced by SP.
padding = sp_padding_mask(positions == num_tokens - 1, hidden)
with (
set_current_vllm_config(config),
set_forward_context(
None,
config,
num_tokens=num_tokens,
num_tokens_across_dp=counts,
is_padding=padding,
),
):
# The reference keeps full GR rows. MoE handles its own chunk and gather.
full_hidden, attn_input, injection = layer.attn_hyper_connection.mix(hidden)
attn_output = tensor_model_parallel_all_reduce(layer.linear_attn(attn_input))
full_hidden, moe_input, injection = layer.mlp_hyper_connection.combine_and_mix(
full_hidden, attn_output, injection
)
expected = tuple(
x.clone() for x in (full_hidden, layer.mlp(moe_input), injection)
)
output = layer(
sp_shard(hidden),
None,
None,
positions,
input_ids=None,
query_start_loc=None,
ngram_context=None,
)
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
"""Initialize rank-local runtime state and run one equivalence check."""
torch.accelerator.set_device_index(rank)
torch.set_num_threads(1)
config = _make_config(rank, moe_sp=moe_sp)
init_distributed_environment(
world_size=4 if moe_sp else 2,
rank=rank,
local_rank=rank,
distributed_init_method=f"tcp://127.0.0.1:{port}",
)
try:
# Spawned ranks bypass the GPU worker's workspace initialization.
init_workspace_manager(torch.device(f"cuda:{rank}"))
with set_current_vllm_config(config):
initialize_model_parallel(tensor_model_parallel_size=2)
with torch.device(f"cuda:{rank}"), torch.inference_mode():
torch.set_default_dtype(torch.bfloat16)
if moe_sp:
_check_moe_sp(rank, config)
else:
_check_decoder_ple(rank)
finally:
reset_workspace_manager()
cleanup_dist_env_and_memory()
@multi_gpu_test(num_gpus=2)
def test_hc_sp_decoder_and_ple() -> None:
"""HC SP preserves decoder output and PLE cache across a token-shard boundary."""
mp.spawn(_run_sp, args=(get_open_port(), False), nprocs=2)
@multi_gpu_test(num_gpus=4)
def test_moe_sp_equivalence(monkeypatch) -> None:
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
+16
View File
@@ -127,6 +127,22 @@ def test_registry_is_pp(model_arch, is_pp):
assert model_info.supports_pp is is_pp
@pytest.mark.parametrize(
"model_arch,supported",
[
("Qwen4ExpForCausalLM", not current_platform.is_rocm()),
("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()),
("Qwen4ExpMTP", not current_platform.is_rocm()),
("LlamaForCausalLM", False),
],
)
def test_registry_supports_hc_sp(model_arch, supported):
"""HC SP is opt-in for each model implementation, including MTP."""
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
assert model_info is not None
assert model_info.supports_hc_sp is supported
@create_new_process_for_each_test()
@pytest.mark.parametrize(
"model_arch",
+41
View File
@@ -4109,6 +4109,47 @@ def test_revision_resolved_for_model(mock_resolve):
mock_resolve.assert_any_call(model, None, config.hf_token)
@pytest.mark.parametrize(
"enabled,supported,is_cuda,tp_size,pp_size,error",
[
(True, True, True, 2, 1, None),
(True, False, True, 2, 1, "not supported for this model"),
(True, True, False, 2, 1, "requires CUDA"),
(True, True, True, 1, 1, None),
(True, False, False, 1, 2, None),
(True, True, True, 2, 2, "requires PP=1"),
(False, False, False, 2, 1, None),
],
)
def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, error):
"""Validate active HC SP configurations and ignore the flag with TP=1."""
from vllm.model_executor.models.registry import ModelRegistry
model = SimpleNamespace(
architectures=["HCModel"],
registry=ModelRegistry,
model_arch_config=SimpleNamespace(total_num_attention_heads=8),
multimodal_config=None,
)
parallel = ParallelConfig(
enable_hc_sp=enabled,
tensor_parallel_size=tp_size,
pipeline_parallel_size=pp_size,
# Avoid backend auto-detection in this config-only test.
distributed_executor_backend="mp",
)
with (
patch("vllm.config.model.current_platform.is_cuda", return_value=is_cuda),
patch.object(ModelRegistry, "is_hc_sp_supported_model", return_value=supported),
patch.object(ModelRegistry, "is_pp_supported_model", return_value=True),
):
if error:
with pytest.raises(ValueError, match=error):
ModelConfig.verify_with_parallel_config(model, parallel)
else:
ModelConfig.verify_with_parallel_config(model, parallel)
@pytest.mark.parametrize(
("layer_types", "expected_attention"),
[