[Qwen4Exp] support Sequence Parallelism for Qwen3.8-Flash-Next (#56322)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
258b97ff6b
commit
c75cb63cd2
@@ -9,6 +9,8 @@ each rank, and summing the per-rank outputs (what the all-reduce does)
|
||||
reconstructs a full-table lookup.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -124,3 +126,43 @@ def test_rank_sum_is_full_lookup(tp_size):
|
||||
total += _fused(ids, _shard(full_table, si, rows, dtype), si)
|
||||
|
||||
torch.testing.assert_close(total, full_table[ids.long()], atol=0.0, rtol=0.0)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_embedding_optionally_reduces_output(dtype, use_fused, default_vllm_config):
|
||||
"""SP's deferred reduction preserves masked contributions and their dtype."""
|
||||
with torch.device("cuda"):
|
||||
values = torch.arange(128 * 16).reshape(128, 16)
|
||||
table = ((values % 29 - 14) / 16).to(dtype)
|
||||
ids = torch.tensor([0, 63, 64, 127, 12])
|
||||
expected = table[ids].float()
|
||||
for rank in range(2):
|
||||
local = expected.clone()
|
||||
local[ids // 64 != rank] = 0
|
||||
remote = (expected - local).to(dtype)
|
||||
group = Mock(world_size=2, rank_in_group=rank)
|
||||
# Model the other vocabulary owner's contribution, including FP8 bytes.
|
||||
group.all_reduce.side_effect = lambda output, remote=remote: (
|
||||
output + remote.view(output.dtype)
|
||||
)
|
||||
for reduce_results in (True, False):
|
||||
embedding = VocabParallelEmbedding(
|
||||
128,
|
||||
16,
|
||||
params_dtype=dtype,
|
||||
parallel_group=group,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
embedding.use_fused_embedding = use_fused
|
||||
embedding.weight_loader(embedding.get_parameter("weight"), table)
|
||||
actual = embedding(ids)
|
||||
assert actual.dtype == dtype
|
||||
torch.testing.assert_close(
|
||||
actual.float(),
|
||||
expected if reduce_results else local,
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
@@ -9,6 +9,7 @@ import pytest
|
||||
import torch
|
||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import CacheConfig, ParallelConfig
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
from vllm.model_executor.models.config import (
|
||||
Qwen3_5ForConditionalGenerationConfig,
|
||||
@@ -48,23 +49,54 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
||||
return Qwen4ExpTextConfig(**values)
|
||||
|
||||
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
|
||||
def test_moe_sp_rejects_pipeline_parallel() -> None:
|
||||
"""Automatic MoE SP must validate HC topology even without --enable-hc-sp."""
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=_text_config(num_experts=4, ple_layer_ids=[]),
|
||||
),
|
||||
cache_config=CacheConfig(),
|
||||
parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=2,
|
||||
data_parallel_size=2,
|
||||
enable_expert_parallel=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
|
||||
pytest.raises(ValueError, match="MoE SP requires PP=1"),
|
||||
):
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_sp", [False, True])
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
|
||||
from vllm.models.qwen4_exp.nvidia.mtp import (
|
||||
Qwen4ExpMultiTokenPredictor,
|
||||
)
|
||||
|
||||
model = object.__new__(Qwen4ExpMultiTokenPredictor)
|
||||
model.use_hc_sequence_parallel = use_sp
|
||||
model.use_sequence_parallel = False
|
||||
torch.nn.Module.__init__(model)
|
||||
model.hc_count = 2
|
||||
model.hidden_size = 4
|
||||
model.num_mtp_layers = 1
|
||||
model.layers = [
|
||||
lambda **kwargs: (
|
||||
kwargs["hidden_states"],
|
||||
kwargs["hidden_states"],
|
||||
torch.zeros(kwargs["hidden_states"].shape[0], 2),
|
||||
),
|
||||
]
|
||||
model.pre_fc_norm_embedding = torch.nn.Identity()
|
||||
model.pre_fc_norm_hidden = torch.nn.Identity()
|
||||
model.fc_embedding = torch.nn.Identity()
|
||||
model.fc_hidden = torch.nn.Identity()
|
||||
|
||||
def decoder(**kwargs):
|
||||
"""Check that the embedding residual shares the hidden state's token slice."""
|
||||
hidden = kwargs["hidden_states"]
|
||||
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
|
||||
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
|
||||
|
||||
model.layers = [decoder]
|
||||
model.hyper_connection_mixer = SimpleNamespace(
|
||||
combine_and_mix=lambda hidden_states, block_output, injection: (
|
||||
hidden_states,
|
||||
@@ -72,24 +104,32 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
|
||||
None,
|
||||
),
|
||||
)
|
||||
multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8)
|
||||
pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True)
|
||||
multi_hidden = torch.arange(24, dtype=torch.float32).reshape(3, 8)
|
||||
expected_sample = multi_hidden.unflatten(-1, (2, 4)).mean(-2)
|
||||
pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True)
|
||||
|
||||
with patch(
|
||||
"vllm.models.qwen4_exp.nvidia.mtp.get_pp_group",
|
||||
return_value=pp_group,
|
||||
def gather(local):
|
||||
"""Supply rank 1's last token and padding for a simulated two-rank gather."""
|
||||
assert local.shape[0] == 2
|
||||
remote = torch.cat([expected_sample[2:], multi_hidden[2:]], dim=-1)
|
||||
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
|
||||
|
||||
with (
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
|
||||
):
|
||||
sample_hidden, returned_multi_hidden = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.arange(2),
|
||||
intermediate_tensors={"hidden_states": multi_hidden},
|
||||
positions=torch.arange(3),
|
||||
hidden_states=multi_hidden,
|
||||
inputs_embeds=torch.zeros(3, 4),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
sample_hidden,
|
||||
multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2),
|
||||
)
|
||||
assert returned_multi_hidden is multi_hidden
|
||||
torch.testing.assert_close(sample_hidden, expected_sample)
|
||||
torch.testing.assert_close(returned_multi_hidden, multi_hidden)
|
||||
assert sample_hidden.is_contiguous()
|
||||
assert returned_multi_hidden.is_contiguous()
|
||||
|
||||
|
||||
@spawn_new_process_for_each_test
|
||||
@@ -150,9 +190,7 @@ def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> Non
|
||||
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
|
||||
multimodal_config=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1
|
||||
),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=2),
|
||||
speculative_config=None,
|
||||
)
|
||||
with patch.object(
|
||||
|
||||
@@ -611,46 +611,65 @@ def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_pinned_embedding_forward_finalizes_prefetched_output(
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize("reduce_results", [False, True])
|
||||
def test_pinned_embedding_finalize_prefetch_optionally_reduces_output(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dtype: torch.dtype,
|
||||
reduce_results: bool,
|
||||
) -> None:
|
||||
"""Deferring reduction must preserve full token rows and wait for the lookup."""
|
||||
total_tokens = 3
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = torch.empty(4, 2, 3, dtype=torch.float8_e4m3fn)
|
||||
values = torch.arange(1, 13).reshape(4, 3)
|
||||
embedding._prefetch_buffer = torch.stack(
|
||||
(values, torch.zeros_like(values)), dim=1
|
||||
).to(dtype)
|
||||
embedding._output_dim = 6
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
|
||||
expected = torch.arange(12).reshape(2, 6).to(torch.float8_e4m3fn)
|
||||
|
||||
def finalize_prefetched(
|
||||
prefetch_output: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
assert prefetch_output is embedding._prefetch_buffer
|
||||
output.copy_(expected)
|
||||
|
||||
embedding.tp_size = 2
|
||||
embedding.reduce_results = reduce_results
|
||||
embedding.etp_data_parallel_size = 1
|
||||
embedding._prefetch_stream = object()
|
||||
waited_streams: list[object] = []
|
||||
reduced_dtypes = []
|
||||
monkeypatch.setattr(
|
||||
embedding,
|
||||
"_finalize_prefetch",
|
||||
finalize_prefetched,
|
||||
torch.cuda,
|
||||
"current_stream",
|
||||
lambda: SimpleNamespace(wait_stream=waited_streams.append),
|
||||
)
|
||||
|
||||
output = embedding(hidden_states)
|
||||
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
|
||||
assert waited_streams == [embedding._prefetch_stream]
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
# The remote rank owns the other head's vocabulary rows.
|
||||
return tensor + tensor.flip(1)
|
||||
|
||||
assert output.dtype == embedding._prefetch_buffer.dtype
|
||||
assert torch.equal(output, expected)
|
||||
embedding.parallel_group = SimpleNamespace(all_reduce=all_reduce)
|
||||
output = embedding.finalize_prefetch(total_tokens=total_tokens)
|
||||
|
||||
expected = (
|
||||
values.unsqueeze(1).expand(-1, 2, -1).to(dtype)
|
||||
if reduce_results
|
||||
else embedding._prefetch_buffer
|
||||
)
|
||||
expected = expected[:total_tokens].flatten(-2)
|
||||
comm_dtype = torch.int8 if dtype == torch.float8_e4m3fn else dtype
|
||||
assert reduced_dtypes == ([comm_dtype] if reduce_results else [])
|
||||
assert waited_streams == [embedding._prefetch_stream]
|
||||
assert output.dtype == dtype
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_pinned_embedding_forward_requires_prior_start_prefetch() -> None:
|
||||
"""Forward before any start_prefetch must fail loudly, not read garbage."""
|
||||
def test_pinned_embedding_finalize_prefetch_requires_prior_start_prefetch() -> None:
|
||||
"""Finalizing before start_prefetch must fail instead of reading garbage."""
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = None
|
||||
embedding._prefetch_stream = None
|
||||
embedding._output_dim = 6
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
|
||||
|
||||
with pytest.raises(RuntimeError, match="prior start_prefetch"):
|
||||
embedding(hidden_states)
|
||||
embedding.finalize_prefetch(total_tokens=2)
|
||||
|
||||
|
||||
def test_pinned_embedding_finalize_requires_prior_start_prefetch() -> None:
|
||||
@@ -781,15 +800,10 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
|
||||
def test_pinned_embedding_start_prefetch_allocates_lazily(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""start_prefetch lazily allocates the stream/buffer, then forward consumes it.
|
||||
|
||||
Exercises the full pinned-prefetch contract end to end: first call
|
||||
allocates the side stream and buffer with the constructor geometry,
|
||||
and forward finalizes the lookup into the returned output.
|
||||
"""
|
||||
"""Allocate prefetch storage on first use and finalize the lookup results."""
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
@@ -825,8 +839,7 @@ def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
|
||||
)
|
||||
|
||||
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16, device="cuda:0")
|
||||
embedding.start_prefetch(hidden_states, ngram_ids)
|
||||
embedding.start_prefetch(ngram_ids)
|
||||
|
||||
buffer = embedding._prefetch_buffer
|
||||
assert buffer is not None
|
||||
@@ -835,7 +848,7 @@ def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
|
||||
assert buffer.device.type == "cuda"
|
||||
assert embedding._prefetch_stream is not None
|
||||
|
||||
output = embedding(hidden_states)
|
||||
output = embedding.finalize_prefetch(total_tokens=ngram_ids.shape[0])
|
||||
|
||||
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
@@ -1149,7 +1162,7 @@ def test_ngram_prefetch_ids_outlive_start_prefetch() -> None:
|
||||
prefetched: list[torch.Tensor] = []
|
||||
module.ngram_embedding = SimpleNamespace(
|
||||
supports_prefetch=True,
|
||||
start_prefetch=lambda _, ids: prefetched.append(weak_ref_tensor(ids)),
|
||||
start_prefetch=lambda ids: prefetched.append(weak_ref_tensor(ids)),
|
||||
)
|
||||
expected = _reference_ngram_ids(input_ids, query_start_loc, ngram_context, **params)
|
||||
|
||||
@@ -1157,7 +1170,7 @@ def test_ngram_prefetch_ids_outlive_start_prefetch() -> None:
|
||||
# like a later CUDA graph segment would.
|
||||
pool = torch.cuda.MemPool()
|
||||
with torch.cuda.use_mem_pool(pool):
|
||||
module.start_prefetch(None, input_ids, query_start_loc, ngram_context)
|
||||
module.start_prefetch(input_ids, query_start_loc, ngram_context)
|
||||
torch.full_like(expected, -1)
|
||||
|
||||
assert torch.equal(prefetched[0], expected)
|
||||
@@ -1859,12 +1872,18 @@ def _make_conv_metadata(
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"layer_cls", [Qwen4ExpPLELayer, Qwen4ExpPLELayerAMD], ids=["nvidia", "amd"]
|
||||
("layer_cls", "fuse_residual"),
|
||||
[
|
||||
pytest.param(Qwen4ExpPLELayer, True, id="nvidia-fused"),
|
||||
pytest.param(Qwen4ExpPLELayer, False, id="nvidia-unfused"),
|
||||
pytest.param(Qwen4ExpPLELayerAMD, True, id="amd-fused"),
|
||||
],
|
||||
)
|
||||
def test_fused_conv_correctness(
|
||||
case: _ConvBatchCase,
|
||||
state_layout: str,
|
||||
layer_cls: type[nn.Module],
|
||||
fuse_residual: bool,
|
||||
) -> None:
|
||||
device = torch.device("cuda")
|
||||
metadata, num_real_tokens = _make_conv_metadata(case, device)
|
||||
@@ -1906,12 +1925,15 @@ def test_fused_conv_correctness(
|
||||
)
|
||||
null_state = conv_state[NULL_BLOCK_ID].clone()
|
||||
residual_kernel = residual.clone()
|
||||
residual_reference = residual.clone()
|
||||
# The unfused kernel must overwrite the buffer without reading its contents.
|
||||
residual_reference = (
|
||||
residual.clone() if fuse_residual else torch.zeros_like(residual)
|
||||
)
|
||||
|
||||
module._short_conv_dilated_dispatch(
|
||||
inputs=inputs,
|
||||
residual=residual_kernel,
|
||||
outer_residual=outer_residual,
|
||||
outer_residual=outer_residual if fuse_residual else None,
|
||||
metadata=metadata,
|
||||
conv_state=conv_state,
|
||||
conv_weights=weights,
|
||||
@@ -1925,7 +1947,8 @@ def test_fused_conv_correctness(
|
||||
conv_state_len=module.conv_state_len,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
residual_reference = outer_residual + residual_reference
|
||||
if fuse_residual:
|
||||
residual_reference = outer_residual + residual_reference
|
||||
|
||||
assert torch.equal(
|
||||
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
|
||||
@@ -1935,7 +1958,7 @@ def test_fused_conv_correctness(
|
||||
if case.graph_padding:
|
||||
assert torch.equal(
|
||||
residual_kernel[num_real_tokens:],
|
||||
(outer_residual + residual)[num_real_tokens:],
|
||||
residual_reference[num_real_tokens:],
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Multi-GPU equivalence tests for Qwen4Exp HC and MoE sequence parallelism."""
|
||||
|
||||
from dataclasses import MISSING, fields
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import (
|
||||
EngramConfig,
|
||||
ParallelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.linear import RowParallelLinear
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
|
||||
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
|
||||
|
||||
from .test_ple import _ConvBatchCase, _make_conv_metadata
|
||||
|
||||
|
||||
class _TPAttentionProjection(nn.Module):
|
||||
"""Keep real TP communication without setting up an attention KV cache."""
|
||||
|
||||
def __init__(self, config, *, reduce_results: bool, **kwargs) -> None:
|
||||
super().__init__()
|
||||
self.proj = RowParallelLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=False,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor):
|
||||
return self.proj(hidden_states)[0]
|
||||
|
||||
|
||||
def _make_config(rank: int, *, hc_sp: bool = False, moe_sp: bool = False) -> VllmConfig:
|
||||
"""Configure HC SP explicitly or MoE SP through TP=2, DP=2 and EP."""
|
||||
config = VllmConfig(
|
||||
parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=2 if moe_sp else 1,
|
||||
data_parallel_rank=rank // 2,
|
||||
enable_expert_parallel=moe_sp,
|
||||
enable_hc_sp=hc_sp,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
),
|
||||
)
|
||||
config.model_config = SimpleNamespace(
|
||||
architecture="Qwen4ExpForCausalLM",
|
||||
is_moe=True,
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
hf_text_config=Qwen4ExpTextConfig(
|
||||
vocab_size=128,
|
||||
eos_token_id=0,
|
||||
hidden_size=128,
|
||||
hc_count=2,
|
||||
hc_lowrank=16,
|
||||
num_hidden_layers=1,
|
||||
layer_types=["linear_attention"],
|
||||
num_experts=4,
|
||||
num_experts_per_tok=2,
|
||||
moe_intermediate_size=256,
|
||||
shared_expert_intermediate_size=256,
|
||||
ple_layer_ids=[] if moe_sp else [1],
|
||||
ple_embed_dim=128,
|
||||
heads_per_ngram=2,
|
||||
ngram_vocab_size_base=32,
|
||||
make_ngram_vocab_size_divisible_by=64,
|
||||
),
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
config.engram_config = EngramConfig(cpu_offload=False)
|
||||
return config
|
||||
|
||||
|
||||
def _make_decoder(config: VllmConfig, rank: int) -> Qwen4ExpDecoderLayer:
|
||||
"""Use the production constructor and identical initial weights for both modes."""
|
||||
with (
|
||||
set_current_vllm_config(config),
|
||||
patch.object(qwen4_model, "QwenGatedDeltaNetAttention", _TPAttentionProjection),
|
||||
):
|
||||
layer = Qwen4ExpDecoderLayer(
|
||||
config, config.model_config.hf_text_config.layer_types[0], "model.layers.0"
|
||||
)
|
||||
torch.manual_seed(12)
|
||||
for param in layer.parameters():
|
||||
param.normal_(std=0.08)
|
||||
# Distinct rank contributions expose missing or duplicate reductions.
|
||||
layer.linear_attn.proj.weight.add_(rank * 0.01)
|
||||
experts = layer.mlp.experts.routed_experts
|
||||
if config.parallel_config.enable_expert_parallel:
|
||||
experts.w13_weight.add_(rank * 0.01)
|
||||
experts.quant_method.process_weights_after_loading(experts)
|
||||
return layer
|
||||
|
||||
|
||||
def _check_decoder_ple(rank: int) -> None:
|
||||
"""Compare HC SP with full tokens across a PLE prefill and cached decode."""
|
||||
configs = [_make_config(rank, hc_sp=enabled) for enabled in (False, True)]
|
||||
layers = [_make_decoder(config, rank) for config in configs]
|
||||
states = []
|
||||
for layer in layers:
|
||||
ple = layer.ple
|
||||
assert ple is not None
|
||||
state = torch.zeros(2, ple.hc_hidden_size, ple.conv_state_len)
|
||||
if not is_conv_state_dim_first():
|
||||
state = state.transpose(-1, -2).contiguous()
|
||||
ple.kv_cache = (state,)
|
||||
states.append(state)
|
||||
|
||||
# Three prefill tokens straddle the TP=2 boundary and require SP padding.
|
||||
for case in (
|
||||
_ConvBatchCase(prefill_query_lens=(3,)),
|
||||
_ConvBatchCase(num_decodes=1),
|
||||
):
|
||||
layout, num_tokens = _make_conv_metadata(case, torch.device("cuda", rank))
|
||||
metadata_args = {
|
||||
f.name: None
|
||||
for f in fields(PleShortConvAttentionMetadata)
|
||||
if f.default is MISSING and f.default_factory is MISSING
|
||||
}
|
||||
metadata = PleShortConvAttentionMetadata(**(metadata_args | vars(layout)))
|
||||
ids = torch.randint(0, 128, (num_tokens,))
|
||||
positions = torch.arange(num_tokens).expand(3, -1)
|
||||
qsl = torch.tensor([0, num_tokens], dtype=torch.int32)
|
||||
context = torch.zeros(1, layers[0].config.ngram_size - 1, dtype=torch.int64)
|
||||
hidden = torch.randn(num_tokens, 256)
|
||||
block = torch.randn(num_tokens, 128)
|
||||
injection = torch.randn(num_tokens, 2)
|
||||
outputs = []
|
||||
for config, layer in zip(configs, layers):
|
||||
assert layer.ple is not None
|
||||
inputs = (hidden, block, injection)
|
||||
if config.parallel_config.enable_hc_sp:
|
||||
inputs = tuple(sp_shard(x) for x in inputs)
|
||||
with (
|
||||
set_current_vllm_config(config),
|
||||
set_forward_context({layer.ple.prefix: metadata}, config),
|
||||
):
|
||||
layer.ple.start_prefetch(ids, qsl, context)
|
||||
output = layer(
|
||||
*inputs,
|
||||
positions,
|
||||
input_ids=ids,
|
||||
query_start_loc=qsl,
|
||||
ngram_context=context,
|
||||
)
|
||||
if config.parallel_config.enable_hc_sp:
|
||||
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
# A later MoE call may reuse workspace-backed output storage.
|
||||
outputs.append(tuple(x.clone() for x in output))
|
||||
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
|
||||
torch.testing.assert_close(states[1], states[0], atol=0.002, rtol=0.02)
|
||||
|
||||
|
||||
def _check_moe_sp(rank: int, config: VllmConfig) -> None:
|
||||
"""Compare native SP with full-token GR and the MoE wrapper's public interface."""
|
||||
layer = _make_decoder(config, rank)
|
||||
dp_rank = config.parallel_config.data_parallel_rank
|
||||
counts = torch.tensor([3, 5], device="cpu")
|
||||
num_tokens = int(counts[dp_rank])
|
||||
torch.manual_seed(30 + dp_rank)
|
||||
hidden = torch.randn(num_tokens, 256)
|
||||
positions = torch.arange(num_tokens)
|
||||
# Include runner padding in addition to the padding introduced by SP.
|
||||
padding = sp_padding_mask(positions == num_tokens - 1, hidden)
|
||||
with (
|
||||
set_current_vllm_config(config),
|
||||
set_forward_context(
|
||||
None,
|
||||
config,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_across_dp=counts,
|
||||
is_padding=padding,
|
||||
),
|
||||
):
|
||||
# The reference keeps full GR rows. MoE handles its own chunk and gather.
|
||||
full_hidden, attn_input, injection = layer.attn_hyper_connection.mix(hidden)
|
||||
attn_output = tensor_model_parallel_all_reduce(layer.linear_attn(attn_input))
|
||||
full_hidden, moe_input, injection = layer.mlp_hyper_connection.combine_and_mix(
|
||||
full_hidden, attn_output, injection
|
||||
)
|
||||
expected = tuple(
|
||||
x.clone() for x in (full_hidden, layer.mlp(moe_input), injection)
|
||||
)
|
||||
output = layer(
|
||||
sp_shard(hidden),
|
||||
None,
|
||||
None,
|
||||
positions,
|
||||
input_ids=None,
|
||||
query_start_loc=None,
|
||||
ngram_context=None,
|
||||
)
|
||||
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
|
||||
|
||||
|
||||
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
|
||||
"""Initialize rank-local runtime state and run one equivalence check."""
|
||||
torch.accelerator.set_device_index(rank)
|
||||
torch.set_num_threads(1)
|
||||
config = _make_config(rank, moe_sp=moe_sp)
|
||||
init_distributed_environment(
|
||||
world_size=4 if moe_sp else 2,
|
||||
rank=rank,
|
||||
local_rank=rank,
|
||||
distributed_init_method=f"tcp://127.0.0.1:{port}",
|
||||
)
|
||||
try:
|
||||
# Spawned ranks bypass the GPU worker's workspace initialization.
|
||||
init_workspace_manager(torch.device(f"cuda:{rank}"))
|
||||
with set_current_vllm_config(config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=2)
|
||||
with torch.device(f"cuda:{rank}"), torch.inference_mode():
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
if moe_sp:
|
||||
_check_moe_sp(rank, config)
|
||||
else:
|
||||
_check_decoder_ple(rank)
|
||||
finally:
|
||||
reset_workspace_manager()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_hc_sp_decoder_and_ple() -> None:
|
||||
"""HC SP preserves decoder output and PLE cache across a token-shard boundary."""
|
||||
mp.spawn(_run_sp, args=(get_open_port(), False), nprocs=2)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_moe_sp_equivalence(monkeypatch) -> None:
|
||||
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
|
||||
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
|
||||
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
|
||||
@@ -127,6 +127,22 @@ def test_registry_is_pp(model_arch, is_pp):
|
||||
assert model_info.supports_pp is is_pp
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,supported",
|
||||
[
|
||||
("Qwen4ExpForCausalLM", not current_platform.is_rocm()),
|
||||
("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()),
|
||||
("Qwen4ExpMTP", not current_platform.is_rocm()),
|
||||
("LlamaForCausalLM", False),
|
||||
],
|
||||
)
|
||||
def test_registry_supports_hc_sp(model_arch, supported):
|
||||
"""HC SP is opt-in for each model implementation, including MTP."""
|
||||
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
|
||||
assert model_info is not None
|
||||
assert model_info.supports_hc_sp is supported
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch",
|
||||
|
||||
@@ -4109,6 +4109,47 @@ def test_revision_resolved_for_model(mock_resolve):
|
||||
mock_resolve.assert_any_call(model, None, config.hf_token)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,supported,is_cuda,tp_size,pp_size,error",
|
||||
[
|
||||
(True, True, True, 2, 1, None),
|
||||
(True, False, True, 2, 1, "not supported for this model"),
|
||||
(True, True, False, 2, 1, "requires CUDA"),
|
||||
(True, True, True, 1, 1, None),
|
||||
(True, False, False, 1, 2, None),
|
||||
(True, True, True, 2, 2, "requires PP=1"),
|
||||
(False, False, False, 2, 1, None),
|
||||
],
|
||||
)
|
||||
def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, error):
|
||||
"""Validate active HC SP configurations and ignore the flag with TP=1."""
|
||||
from vllm.model_executor.models.registry import ModelRegistry
|
||||
|
||||
model = SimpleNamespace(
|
||||
architectures=["HCModel"],
|
||||
registry=ModelRegistry,
|
||||
model_arch_config=SimpleNamespace(total_num_attention_heads=8),
|
||||
multimodal_config=None,
|
||||
)
|
||||
parallel = ParallelConfig(
|
||||
enable_hc_sp=enabled,
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
# Avoid backend auto-detection in this config-only test.
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
with (
|
||||
patch("vllm.config.model.current_platform.is_cuda", return_value=is_cuda),
|
||||
patch.object(ModelRegistry, "is_hc_sp_supported_model", return_value=supported),
|
||||
patch.object(ModelRegistry, "is_pp_supported_model", return_value=True),
|
||||
):
|
||||
if error:
|
||||
with pytest.raises(ValueError, match=error):
|
||||
ModelConfig.verify_with_parallel_config(model, parallel)
|
||||
else:
|
||||
ModelConfig.verify_with_parallel_config(model, parallel)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("layer_types", "expected_attention"),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user