[Qwen4Exp] support Sequence Parallelism for Qwen3.8-Flash-Next (#56322)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Huang
2026-10-12 20:56:44 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 258b97ff6b
commit c75cb63cd2
23 changed files with 870 additions and 178 deletions
@@ -9,6 +9,8 @@ each rank, and summing the per-rank outputs (what the all-reduce does)
reconstructs a full-table lookup.
"""
from unittest.mock import Mock
import pytest
import torch
@@ -124,3 +126,43 @@ def test_rank_sum_is_full_lookup(tp_size):
total += _fused(ids, _shard(full_table, si, rows, dtype), si)
torch.testing.assert_close(total, full_table[ids.long()], atol=0.0, rtol=0.0)
@requires_cuda
@torch.inference_mode()
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize("use_fused", [False, True])
def test_embedding_optionally_reduces_output(dtype, use_fused, default_vllm_config):
"""SP's deferred reduction preserves masked contributions and their dtype."""
with torch.device("cuda"):
values = torch.arange(128 * 16).reshape(128, 16)
table = ((values % 29 - 14) / 16).to(dtype)
ids = torch.tensor([0, 63, 64, 127, 12])
expected = table[ids].float()
for rank in range(2):
local = expected.clone()
local[ids // 64 != rank] = 0
remote = (expected - local).to(dtype)
group = Mock(world_size=2, rank_in_group=rank)
# Model the other vocabulary owner's contribution, including FP8 bytes.
group.all_reduce.side_effect = lambda output, remote=remote: (
output + remote.view(output.dtype)
)
for reduce_results in (True, False):
embedding = VocabParallelEmbedding(
128,
16,
params_dtype=dtype,
parallel_group=group,
reduce_results=reduce_results,
)
embedding.use_fused_embedding = use_fused
embedding.weight_loader(embedding.get_parameter("weight"), table)
actual = embedding(ids)
assert actual.dtype == dtype
torch.testing.assert_close(
actual.float(),
expected if reduce_results else local,
atol=0,
rtol=0,
)