[ROCm][Qwen4Exp] Support HC and MoE sequence parallelism on ROCm (#61471)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Andreas Karatzas
2026-10-15 20:11:17 -05:00
committed by GitHub
co-authored by Claude
parent a956a8126e
commit 29345f42a6
17 changed files with 361 additions and 83 deletions
+20
View File
@@ -3439,6 +3439,26 @@ steps:
commands:
- pytest -v -s tests/kernels/test_rocm_gemm_rs_ar.py
- label: ":amd: (MI355) Qwen4 Exp SP"
timeout_in_minutes: 30
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/config/parallel.py
- vllm/models/common/ops/sequence_parallel.py
- vllm/models/qwen4_exp/
- vllm/model_executor/layers/vocab_parallel_embedding.py
- vllm/model_executor/models/qwen3_next.py
- vllm/platforms/rocm.py
- tests/models/qwen4_exp/test_ple.py
- tests/models/qwen4_exp/test_tpsp.py
commands:
- pytest -v -s models/qwen4_exp/test_tpsp.py
- label: ":amd: (MI355) Distributed DP Basic" # TBD
timeout_in_minutes: 45
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -18,6 +18,22 @@ steps:
- tests/models/qwen4_exp/test_tpsp.py
commands:
- pytest -v -s models/qwen4_exp/test_tpsp.py
mirror:
amd:
label: ":amd: (MI355) Qwen4 Exp SP"
dind: false
device: mi355_4
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/config/parallel.py
- vllm/models/common/ops/sequence_parallel.py
- vllm/models/qwen4_exp/
- vllm/model_executor/layers/vocab_parallel_embedding.py
- vllm/model_executor/models/qwen3_next.py
- vllm/platforms/rocm.py
- tests/models/qwen4_exp/test_ple.py
- tests/models/qwen4_exp/test_tpsp.py
- label: ":nvidia: (L4) Distributed Models Shard %N"
key: distributed-model-tests-2-gpus
+14 -11
View File
@@ -65,20 +65,21 @@ def test_moe_sp_rejects_pipeline_parallel() -> None:
),
)
with (
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=True),
pytest.raises(ValueError, match="MoE SP requires PP=1"),
):
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
@spawn_new_process_for_each_test
@pytest.mark.parametrize("backend", ["amd", "nvidia"])
@pytest.mark.parametrize("use_sp", [False, True])
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(
backend: str, use_sp: bool
) -> None:
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
from vllm.models.qwen4_exp.nvidia.mtp import (
Qwen4ExpMultiTokenPredictor,
)
model = object.__new__(Qwen4ExpMultiTokenPredictor)
mtp_module = f"vllm.models.qwen4_exp.{backend}.mtp"
model = object.__new__(import_module(mtp_module).Qwen4ExpMultiTokenPredictor)
model.use_hc_sequence_parallel = use_sp
model.use_sequence_parallel = False
torch.nn.Module.__init__(model)
@@ -93,7 +94,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
def decoder(**kwargs):
"""Check that the embedding residual shares the hidden state's token slice."""
hidden = kwargs["hidden_states"]
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
if backend == "nvidia":
prev_block_output = kwargs["prev_block_output"]
assert prev_block_output.shape == (hidden.shape[0], model.hidden_size)
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
model.layers = [decoder]
@@ -115,9 +118,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
with (
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
patch(f"{mtp_module}.get_pp_group", return_value=pp_group),
patch(f"{mtp_module}.sp_shard", side_effect=lambda x: x[:2]),
patch(f"{mtp_module}.sp_all_gather", side_effect=gather),
):
sample_hidden, returned_multi_hidden = model.forward(
input_ids=None,
+53 -8
View File
@@ -14,8 +14,8 @@ from transformers import Qwen4ExpTextConfig
from vllm.platforms import current_platform
if not current_platform.is_cuda():
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
if not current_platform.is_cuda_alike():
pytest.skip("Qwen4Exp SP requires CUDA or ROCm", allow_module_level=True)
from tests.utils import multi_gpu_test
from vllm.config import (
@@ -38,14 +38,19 @@ from vllm.models.common.ops.sequence_parallel import (
sp_padding_mask,
sp_shard,
)
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
from vllm.utils.network_utils import get_open_port
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
from .test_ple import _ConvBatchCase, _make_conv_metadata
if current_platform.is_rocm():
from vllm.models.qwen4_exp.amd import model as qwen4_model
from vllm.models.qwen4_exp.amd.model import Qwen4ExpDecoderLayer
else:
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
class _TPAttentionProjection(nn.Module):
"""Keep real TP communication without setting up an attention KV cache."""
@@ -168,7 +173,8 @@ def _check_decoder_ple(rank: int) -> None:
set_current_vllm_config(config),
set_forward_context({layer.ple.prefix: metadata}, config),
):
layer.ple.start_prefetch(ids, qsl, context)
if hasattr(layer.ple, "start_prefetch"):
layer.ple.start_prefetch(ids, qsl, context)
output = layer(
*inputs,
positions,
@@ -177,7 +183,10 @@ def _check_decoder_ple(rank: int) -> None:
ngram_context=context,
)
if config.parallel_config.enable_hc_sp:
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
# ROCm all-gather requires contiguous inputs.
output = tuple(
sp_all_gather(x.contiguous())[:num_tokens] for x in output
)
# A later MoE call may reuse workspace-backed output storage.
outputs.append(tuple(x.clone() for x in output))
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
@@ -223,11 +232,38 @@ def _check_moe_sp(rank: int, config: VllmConfig) -> None:
query_start_loc=None,
ngram_context=None,
)
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
actual = tuple(sp_all_gather(x.contiguous())[:num_tokens] for x in output)
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
def _check_compiled_sp_ops() -> None:
"""Compiled SP collectives stay correct for token counts unseen at trace time."""
from vllm.models.qwen4_exp.amd.ops.sequence_parallel import (
sp_reduce_scatter,
sp_shard,
)
def collectives(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
gathered = sp_all_gather(sp_shard(x) * 2)[: x.shape[0]]
reduced = sp_all_gather(sp_reduce_scatter(x))[: x.shape[0]]
return gathered, reduced
# vLLM drops Dynamo guards, so one trace must serve every token count.
compiled = torch.compile(
collectives,
fullgraph=True,
options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe},
)
traced = torch.randn(8, 16)
torch._dynamo.mark_dynamic(traced, 0)
compiled(traced)
for num_tokens in (7, 3, 1):
torch.manual_seed(num_tokens)
x = torch.randn(num_tokens, 16)
torch.testing.assert_close(compiled(x), collectives(x), atol=0, rtol=0)
def _run_sp(rank: int, port: int, moe_sp: bool, compiled_ops: bool = False) -> None:
"""Initialize rank-local runtime state and run one equivalence check."""
torch.accelerator.set_device_index(rank)
torch.set_num_threads(1)
@@ -247,6 +283,8 @@ def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
torch.set_default_dtype(torch.bfloat16)
if moe_sp:
_check_moe_sp(rank, config)
elif compiled_ops:
_check_compiled_sp_ops()
else:
_check_decoder_ple(rank)
finally:
@@ -265,3 +303,10 @@ def test_moe_sp_equivalence(monkeypatch) -> None:
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only SP ops")
@multi_gpu_test(num_gpus=2)
def test_compiled_sp_ops_handle_untraced_token_counts() -> None:
"""Padding in compiled SP collectives must not specialize on the traced size."""
mp.spawn(_run_sp, args=(get_open_port(), False, True), nprocs=2)
+3 -3
View File
@@ -127,9 +127,9 @@ def test_registry_is_pp(model_arch, is_pp):
@pytest.mark.parametrize(
"model_arch,supported",
[
("Qwen4ExpForCausalLM", not current_platform.is_rocm()),
("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()),
("Qwen4ExpMTP", not current_platform.is_rocm()),
("Qwen4ExpForCausalLM", True),
("Qwen4ExpForConditionalGeneration", True),
("Qwen4ExpMTP", True),
("LlamaForCausalLM", False),
],
)
+6 -3
View File
@@ -4144,7 +4144,7 @@ def test_revision_resolved_for_model(mock_resolve):
@pytest.mark.parametrize(
"enabled,supported,is_cuda,tp_size,pp_size,error",
"enabled,supported,cuda_alike,tp_size,pp_size,error",
[
(True, True, True, 2, 1, None),
(True, False, True, 2, 1, "not supported for this model"),
@@ -4155,7 +4155,7 @@ def test_revision_resolved_for_model(mock_resolve):
(False, False, False, 2, 1, None),
],
)
def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, error):
def test_hc_sp_model_support(enabled, supported, cuda_alike, tp_size, pp_size, error):
"""Validate active HC SP configurations and ignore the flag with TP=1."""
from vllm.model_executor.models.registry import ModelRegistry
@@ -4173,7 +4173,10 @@ def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, erro
distributed_executor_backend="mp",
)
with (
patch("vllm.config.model.current_platform.is_cuda", return_value=is_cuda),
patch(
"vllm.config.model.current_platform.is_cuda_alike",
return_value=cuda_alike,
),
patch.object(ModelRegistry, "is_hc_sp_supported_model", return_value=supported),
patch.object(ModelRegistry, "is_pp_supported_model", return_value=True),
):
+2 -2
View File
@@ -1534,8 +1534,8 @@ class ModelConfig:
parallel_config: ParallelConfig,
) -> None:
if parallel_config.enable_hc_sp and parallel_config.tensor_parallel_size > 1:
if not current_platform.is_cuda():
raise ValueError("--enable-hc-sp currently requires CUDA")
if not current_platform.is_cuda_alike():
raise ValueError("--enable-hc-sp currently requires CUDA or ROCm")
if not self.registry.is_hc_sp_supported_model(self.architectures, self):
raise ValueError("--enable-hc-sp is not supported for this model")
if parallel_config.pipeline_parallel_size != 1:
+1 -1
View File
@@ -184,7 +184,7 @@ class ParallelConfig:
"""Whether the deployed model is MoE (if known)."""
enable_hc_sp: bool = False
"""Enable sequence parallelism for HyperConnection (HC) computation.
Currently supported by Qwen4Exp on CUDA with TP>1 and PP=1 (ignored when TP=1).
Currently supported by Qwen4Exp on CUDA/ROCm with TP>1 and PP=1 (ignored when TP=1).
Does not change MoE parallelism. MoE SP may enable HC SP independently of this flag.
"""
enable_expert_parallel: bool = False
+1 -1
View File
@@ -992,7 +992,7 @@ class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConf
parallel_config = vllm_config.parallel_config
# MoE SP also enables HC SP without the explicit HC flag.
if (
current_platform.is_cuda()
current_platform.is_cuda_alike()
and parallel_config.use_sequence_parallel_moe
and parallel_config.pipeline_parallel_size != 1
):
+78 -13
View File
@@ -37,6 +37,7 @@ from vllm.model_executor.models.interfaces import (
IsHybrid,
MixtureOfExperts,
MultiModalEmbeddings,
SupportsHCSP,
SupportsLoRA,
SupportsMRoPE,
SupportsPP,
@@ -67,6 +68,7 @@ from vllm.model_executor.models.utils import (
maybe_fuse_shared_experts,
maybe_prefix,
)
from vllm.models.common.ops.sequence_parallel import sp_all_gather
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
@@ -74,8 +76,10 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .ops.sequence_parallel import sp_reduce_scatter, sp_shard
from .ple_layer import Qwen4ExpPLELayer
from .qsa import Qwen4ExpQSAAttention
@@ -161,13 +165,19 @@ _EXTRA_WEIGHTS_MAPPER = WeightsMapper(
class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock):
"""Qwen3Next MoE with Qwen4Exp HC validation."""
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
parallel_config = vllm_config.parallel_config
if parallel_config.use_sequence_parallel_moe:
raise NotImplementedError(
"Qwen4Exp HC does not support sequence-parallel MoE"
)
super().__init__(vllm_config=vllm_config, prefix=prefix)
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
reduce_results: bool = True,
) -> None:
super().__init__(
vllm_config=vllm_config, prefix=prefix, reduce_results=reduce_results
)
if self.replicate_shared_expert and not self.is_sequence_parallel:
# Reduce routed outputs before adding replicated shared outputs.
# SP then slices the result to avoid summing shared outputs again via RS.
self.experts.moe_config.skip_final_all_reduce = False
config = vllm_config.model_config.hf_text_config
self.n_shared_experts = int(config.shared_expert_intermediate_size > 0)
@@ -185,11 +195,12 @@ class Qwen4ExpDecoderLayer(nn.Module):
self.config = config
self.layer_type = layer_type
# MoE SP also enables HC SP, while HC SP can be enabled independently.
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
self.layer_idx = extract_layer_index(prefix)
if vllm_config.parallel_config.use_sequence_parallel_moe:
raise NotImplementedError(
"Qwen4Exp HC does not support sequence-parallel MoE"
)
self.use_sequence_parallel = (
vllm_config.parallel_config.use_sequence_parallel_moe
)
self.ple: Qwen4ExpPLELayer | None = None
ple_layer_ids = config.ple_layer_ids
if (self.layer_idx + 1) in ple_layer_ids:
@@ -204,6 +215,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
layer_idx=self.layer_idx,
ple_dense_layer_id=ple_dense_layer_id,
prefix=f"{prefix}.ple",
use_sequence_parallel=self.use_hc_sequence_parallel,
)
if layer_type == "linear_attention":
@@ -212,6 +224,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
vllm_config=vllm_config,
prefix=f"{prefix}.linear_attn",
gqa_interleaved_layout=False,
reduce_results=not self.use_hc_sequence_parallel,
)
elif layer_type in _QSA_LAYER_TYPES:
use_qsa = getattr(config, "indexer_n_heads", None) is not None
@@ -222,6 +235,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
cache_config=vllm_config.cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
reduce_results=not self.use_hc_sequence_parallel,
)
else:
self.self_attn = Qwen4ExpQSAAttention(
@@ -230,12 +244,15 @@ class Qwen4ExpDecoderLayer(nn.Module):
layer_id=self.layer_idx,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
reduce_results=not self.use_hc_sequence_parallel,
)
else:
raise ValueError(f"Invalid layer_type {layer_type}")
self.mlp = Qwen4ExpSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
vllm_config=vllm_config,
prefix=f"{prefix}.mlp",
reduce_results=not self.use_hc_sequence_parallel,
)
hc_config = HyperConnectionConfig(
@@ -293,6 +310,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
else:
hidden_states, block_input, injection = attn_hc.mix(hidden_states)
if self.use_hc_sequence_parallel:
# Positions retain the full token axis for both text and MRoPE inputs.
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
if self.layer_type == "linear_attention":
attn_out = self.linear_attn(hidden_states=block_input)
elif self.layer_type in _QSA_LAYER_TYPES:
@@ -304,10 +324,25 @@ class Qwen4ExpDecoderLayer(nn.Module):
raise ValueError("Invalid layer_type")
mlp_hc = self.mlp_hyper_connection
if self.use_hc_sequence_parallel:
attn_out = sp_reduce_scatter(attn_out)
hidden_states, block_input, injection = mlp_hc.combine_and_mix(
hidden_states, attn_out, injection
)
mlp_out = self.mlp(block_input)
if self.use_sequence_parallel:
# MoE consumes local tokens and returns complete local outputs.
mlp_out = self.mlp(block_input, already_sequence_parallel=True)
else:
if self.use_hc_sequence_parallel:
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
mlp_out = self.mlp(block_input)
if self.use_hc_sequence_parallel:
assert isinstance(self.mlp, Qwen4ExpSparseMoeBlock)
reduce_output = self.mlp.experts.moe_config.skip_final_all_reduce
# Only sum partial outputs. Reduced outputs already include all ranks.
mlp_out = (
sp_reduce_scatter(mlp_out) if reduce_output else sp_shard(mlp_out)
)
return hidden_states, mlp_out, injection
@@ -382,6 +417,8 @@ class Qwen4ExpModel(nn.Module):
self.num_redundant_experts = (
vllm_config.parallel_config.eplb_config.num_redundant_experts
)
# MoE SP also enables HC SP, while HC SP can be enabled independently.
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
self.vocab_size = config.vocab_size
self._qsa_layer_ids = frozenset(
layer_idx
@@ -470,12 +507,18 @@ class Qwen4ExpModel(nn.Module):
if input_ids is None:
raise ValueError("input_ids or inputs_embeds is required")
hidden_states = self.embed_input_ids(input_ids)
if self.use_hc_sequence_parallel:
# The MoE padding mask stays unsharded: under torch.compile its
# update would land only after the graph has run.
hidden_states = sp_shard(hidden_states)
# Expand only the local token rows when SP is enabled.
hidden_states = hidden_states.repeat(1, self.config.hc_count)
else:
if intermediate_tensors is None:
raise ValueError("pipeline stage requires intermediate tensors")
hidden_states = intermediate_tensors["hidden_states"]
full_num_tokens = positions.shape[-1]
block_output = None
injection = None
last_layer = None
@@ -498,6 +541,8 @@ class Qwen4ExpModel(nn.Module):
deepstack_embed = deepstack_input_embeds[
f"deepstack_input_embeds_{layer_idx}"
]
if self.use_hc_sequence_parallel:
deepstack_embed = sp_shard(deepstack_embed)
deepstack_embed = (
deepstack_embed.unsqueeze(-2)
.expand(
@@ -532,6 +577,24 @@ class Qwen4ExpModel(nn.Module):
multi_hidden, sample_hidden_states, _ = final_mixer.combine_and_mix(
hidden_states, block_output, injection
)
if self.use_hc_sequence_parallel:
if self._mtp_hidden_buffer is not None:
# Gather LM-head and MTP states together when both are needed.
hidden_size = self.config.hidden_size
packed_hidden_states = torch.cat(
[sample_hidden_states, multi_hidden], dim=-1
)
packed_hidden_states = sp_all_gather(packed_hidden_states)[
:full_num_tokens
]
sample_hidden_states, multi_hidden = packed_hidden_states.split(
[hidden_size, self.config.hc_count * hidden_size], dim=-1
)
sample_hidden_states = sample_hidden_states.contiguous()
else:
sample_hidden_states = sp_all_gather(sample_hidden_states)[
:full_num_tokens
]
if self._mtp_hidden_buffer is not None:
# Capture the pre-final-mixer multi-stream hidden state
# [T, hc_count*H] for the MTP drafter (zero extra compute:
@@ -587,6 +650,7 @@ class Qwen4ExpModel(nn.Module):
class Qwen4ExpForCausalLM(
nn.Module,
HasInnerState,
SupportsHCSP,
SupportsLoRA,
SupportsMRoPE,
SupportsPP,
@@ -819,6 +883,7 @@ class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo):
class Qwen4ExpForConditionalGeneration(
Qwen3_5ForConditionalGeneration,
HasInnerState,
SupportsHCSP,
Qwen4ExpMixtureOfExperts,
):
"""Qwen3-VL vision tower backed by the Qwen4Exp language model."""
+20 -2
View File
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.utils import configure_quant_config
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.interfaces import SupportsHCSP, SupportsPP
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
@@ -45,8 +45,10 @@ from vllm.model_executor.models.utils import (
maybe_fuse_shared_experts,
maybe_prefix,
)
from vllm.models.common.ops.sequence_parallel import sp_all_gather, sp_shard
from vllm.sequence import IntermediateTensors
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .model import (
@@ -181,6 +183,8 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
config: Qwen4ExpTextConfig = model_config.hf_text_config
self.config = config
# MoE SP also enables HC SP, while HC SP can be enabled independently.
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
self.vocab_size = config.vocab_size
self.mtp_start_layer_idx = config.num_hidden_layers
@@ -321,6 +325,9 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
hidden_states = intermediate_tensors["hidden_states"]
current_step_idx = spec_step_idx % self.num_mtp_layers
full_num_tokens = positions.shape[-1]
if self.use_hc_sequence_parallel:
hidden_states = sp_shard(hidden_states)
layer = self.layers[current_step_idx]
hidden_states, block_output, injection = layer(
hidden_states=hidden_states,
@@ -348,6 +355,17 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
hidden_states, block_output, injection
)
)
if self.use_hc_sequence_parallel:
# Gather both outputs together, then restore their contiguous layouts.
packed_hidden_states = torch.cat(
[sample_hidden_states, multi_hidden], dim=-1
)
packed_hidden_states = sp_all_gather(packed_hidden_states)[:full_num_tokens]
sample_hidden_states, multi_hidden = packed_hidden_states.split(
[hidden_size, hc_count * hidden_size], dim=-1
)
sample_hidden_states = sample_hidden_states.contiguous()
multi_hidden = multi_hidden.contiguous()
return sample_hidden_states, multi_hidden
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -377,7 +395,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
"hidden_states": 0,
}
)
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
class Qwen4ExpMTP(nn.Module, SupportsHCSP, SupportsPP, Qwen4ExpMixtureOfExperts):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.models.common.ops import sequence_parallel as common_sp
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import direct_register_custom_op
def _sp_shard_op(x: torch.Tensor) -> torch.Tensor:
output = common_sp.sp_shard(x)
# A functional custom op must not return a view of its input.
unpadded = x.shape[0] % get_tensor_model_parallel_world_size() == 0
return output.clone() if unpadded else output
def _sp_local_fake(x: torch.Tensor) -> torch.Tensor:
tp_size = get_tensor_model_parallel_world_size()
return x.new_empty((cdiv(x.shape[0], tp_size), *x.shape[1:]))
# Dynamo would specialize the padding branch on the traced token count, and
# vLLM drops the resulting guards. Keep the branch opaque when compiling.
direct_register_custom_op(
op_name="sp_reduce_scatter",
op_func=common_sp.sp_reduce_scatter,
fake_impl=_sp_local_fake,
)
direct_register_custom_op(
op_name="sp_shard",
op_func=_sp_shard_op,
fake_impl=_sp_local_fake,
)
def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor:
if torch.compiler.is_compiling():
return torch.ops.vllm.sp_reduce_scatter(x)
return common_sp.sp_reduce_scatter(x)
def sp_shard(x: torch.Tensor) -> torch.Tensor:
if torch.compiler.is_compiling():
return torch.ops.vllm.sp_shard(x)
return common_sp.sp_shard(x)
+83 -29
View File
@@ -9,6 +9,7 @@ from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.distributed import get_etp_group, get_tp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
@@ -21,7 +22,13 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_reduce_scatter,
sp_shard,
)
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import (
@@ -182,8 +189,10 @@ class Qwen4ExpNGramEmbedding(nn.Module):
data_parallel_rank: int = 0,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
use_sequence_parallel: bool = False,
) -> None:
super().__init__()
self.use_sequence_parallel = use_sequence_parallel
self.embedding_dim = embedding_dim
self.layer_name = layer_name
self.ngram_size = int(config.ngram_size)
@@ -253,6 +262,11 @@ class Qwen4ExpNGramEmbedding(nn.Module):
if engram_config is not None and engram_config.cpu_offload
else Qwen4ExpPLEDeviceEmbedding
)
# A TP reduce-scatter covers all vocabulary owners only when ETP=TP.
self.use_reduce_scatter = (
self.use_sequence_parallel
and get_etp_group().world_size == get_tp_group().world_size
)
self.ngram_embedding = embedding_cls(
padded_vocab_size,
self.head_dim,
@@ -263,6 +277,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
num_ngram_heads=self.ngram_heads,
max_total_tokens=max_total_tokens,
data_parallel_rank=data_parallel_rank,
reduce_results=not self.use_reduce_scatter,
)
logger.info(
"Initialized AMD PLE embedding %s: quantization_method=%s, "
@@ -274,12 +289,25 @@ class Qwen4ExpNGramEmbedding(nn.Module):
self.ngram_embedding.weight.is_pinned(),
)
def _sp_shard_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor:
"""Shard embedding token rows, optionally reducing vocabulary contributions."""
# Each embedding element has one vocabulary owner, including FP8 bytes.
output_dtype = embeddings.dtype
if is_fp8(embeddings):
embeddings = embeddings.view(torch.int8)
if self.use_reduce_scatter:
embeddings = sp_reduce_scatter(embeddings)
else:
embeddings = sp_shard(embeddings)
return embeddings.view(output_dtype)
def forward(
self,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
"""Return full embeddings or a token shard after combining vocabulary owners."""
input_ids = input_ids.reshape(-1)
ngram_ids = input_ids.new_empty(
(input_ids.shape[0], self.ngram_heads), dtype=torch.int64
@@ -302,17 +330,19 @@ class Qwen4ExpNGramEmbedding(nn.Module):
output,
self.layer_name,
)
else:
output = ngram_ids.new_empty(
(ngram_ids.shape[0], self.embedding_dim),
dtype=embedding.params_dtype,
)
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding(
ngram_ids,
output,
self.layer_name,
)
if not self.use_sequence_parallel:
return output
output = ngram_ids.new_empty(
(ngram_ids.shape[0], self.embedding_dim),
dtype=embedding.params_dtype,
)
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding(
ngram_ids,
output,
self.layer_name,
)
return output
return self._sp_shard_embeddings(output)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load hash buffers and checkpoint-split embedding rows."""
@@ -388,8 +418,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
layer_idx: int = 0,
ple_dense_layer_id: int | None = None,
prefix: str = "",
use_sequence_parallel: bool = False,
) -> None:
super().__init__()
self.use_sequence_parallel = use_sequence_parallel
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
@@ -420,8 +452,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
quant_config=quant_config,
params_dtype=model_config.dtype,
use_sequence_parallel=self.use_sequence_parallel,
)
# The PLE cache is TP-replicated, so this merged projection is too.
# Projection weights stay replicated even when their token rows are sharded.
self.kv_proj = MergedColumnParallelLinear(
int(config.ple_embed_dim),
[self.hc_hidden_size, self.hidden_size],
@@ -486,7 +519,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
self,
inputs: torch.Tensor,
residual: torch.Tensor,
outer_residual: torch.Tensor,
outer_residual: torch.Tensor | None,
metadata: PleShortConvAttentionMetadata,
conv_state: torch.Tensor,
conv_weights: torch.Tensor,
@@ -501,7 +534,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
has_non_spec = has_prefill or has_decode
inputs = inputs[: metadata.num_actual_tokens]
residual = residual[: metadata.num_actual_tokens]
outer_residual = outer_residual[: metadata.num_actual_tokens]
if outer_residual is not None:
outer_residual = outer_residual[: metadata.num_actual_tokens]
spec_token_indices = None
non_spec_token_indices = None
@@ -553,11 +587,14 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
residual_d, residual_p = torch.split(
residual, [num_decode_tokens, num_prefill_tokens], dim=0
)
outer_residual_d, outer_residual_p = torch.split(
outer_residual,
[num_decode_tokens, num_prefill_tokens],
dim=0,
)
outer_residual_d: torch.Tensor | None = None
outer_residual_p: torch.Tensor | None = None
if outer_residual is not None:
outer_residual_d, outer_residual_p = torch.split(
outer_residual,
[num_decode_tokens, num_prefill_tokens],
dim=0,
)
token_indices_d = None
token_indices_p = None
else:
@@ -628,14 +665,16 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
self,
inputs: torch.Tensor,
residual: torch.Tensor,
outer_residual: torch.Tensor,
outer_residual: torch.Tensor | None,
) -> None:
"""Update convolution state, optionally fusing both residual additions."""
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
# Profiling omits all metadata or this Mamba entry. Short convolution
# is a no-op there, but preserve the outer residual addition.
if attn_metadata is None:
residual.add_(outer_residual)
if outer_residual is not None:
residual.add_(outer_residual)
return
if not isinstance(attn_metadata, dict):
@@ -646,7 +685,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
layer_attn_metadata = attn_metadata.get(self.prefix)
if layer_attn_metadata is None:
residual.add_(outer_residual)
if outer_residual is not None:
residual.add_(outer_residual)
return
if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata):
raise TypeError(
@@ -687,7 +727,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
ngram_context: torch.Tensor,
) -> torch.Tensor:
input_ids = input_ids.reshape(-1)
if input_ids.shape[0] != hidden_states.shape[0]:
if (
not self.use_sequence_parallel
and input_ids.shape[0] != hidden_states.shape[0]
):
raise ValueError(
"PLE expects input_ids and hidden_states to have the same "
f"token length, got {input_ids.shape[0]} and "
@@ -711,12 +754,23 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
# The short-conv op is a piecewise-graph splitting op: state routing
# reads runtime request metadata. It accumulates the convolution and
# the outer residual into gated_output.
torch.ops.vllm.qwen4_exp_ple_short_conv(
conv_input,
gated_output,
hidden_states,
self.prefix,
)
if self.use_sequence_parallel:
# Only convolution needs full tokens. Add residuals on local shards.
conv_input = sp_all_gather(conv_input)[: input_ids.shape[0]]
# Metadata may omit padding tokens or all tokens during profiling.
conv_output = torch.zeros_like(conv_input)
torch.ops.vllm.qwen4_exp_ple_short_conv(
conv_input, conv_output, None, self.prefix
)
gated_output.add_(sp_shard(conv_output))
gated_output.add_(hidden_states)
else:
torch.ops.vllm.qwen4_exp_ple_short_conv(
conv_input,
gated_output,
hidden_states,
self.prefix,
)
return gated_output
@@ -787,7 +841,7 @@ def qwen4_exp_amd_ple_ngram_embedding_pinned(
def qwen4_exp_ple_short_conv(
inputs: torch.Tensor,
residual: torch.Tensor,
outer_residual: torch.Tensor,
outer_residual: torch.Tensor | None,
layer_name: str,
) -> None:
layer = get_forward_context().no_compile_layers[layer_name]
@@ -35,6 +35,16 @@ import torch
import torch.nn.functional as F
from torch import nn
from vllm.config import VllmConfig
def is_hc_sequence_parallel_enabled(vllm_config: VllmConfig) -> bool:
"""Enable GR/PLE SP explicitly or automatically with sequence-parallel MoE."""
parallel_config = vllm_config.parallel_config
return parallel_config.tensor_parallel_size > 1 and (
parallel_config.enable_hc_sp or parallel_config.use_sequence_parallel_moe
)
# ---------------------------------------------------------------------------
# Config
@@ -711,7 +711,11 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0])
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
embeddings = self._lookup(gathered_ids)
embeddings = self._reduce_etp_embeddings(embeddings)
if self.reduce_results:
embeddings = self._reduce_etp_embeddings(embeddings)
else:
# The outer TP reduce-scatter requires ETP=TP.
assert self.etp_data_parallel_size == 1
return self._select_embeddings(
embeddings,
ngram_ids.shape[0],
+1 -8
View File
@@ -82,6 +82,7 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
@@ -93,14 +94,6 @@ from .qsa import Qwen4ExpQSAAttention
_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention")
def is_hc_sequence_parallel_enabled(vllm_config: VllmConfig) -> bool:
"""Enable GR/PLE SP explicitly or automatically with sequence-parallel MoE."""
parallel_config = vllm_config.parallel_config
return parallel_config.tensor_parallel_size > 1 and (
parallel_config.enable_hc_sp or parallel_config.use_sequence_parallel_moe
)
def without_modelopt_fp4(
quant_config: QuantizationConfig | None,
) -> QuantizationConfig | None:
+1 -1
View File
@@ -53,6 +53,7 @@ from vllm.models.common.ops.sequence_parallel import (
)
from vllm.sequence import IntermediateTensors
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .model import (
@@ -61,7 +62,6 @@ from .model import (
Qwen4ExpDecoderLayer,
Qwen4ExpMixtureOfExperts,
Qwen4ExpSparseMoeBlock,
is_hc_sequence_parallel_enabled,
)