From 29345f42a6f90297ae8db427013206b1bd0154e6 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Thu, 15 Oct 2026 20:11:17 -0500 Subject: [PATCH] [ROCm][Qwen4Exp] Support HC and MoE sequence parallelism on ROCm (#61471) Signed-off-by: Andreas Karatzas Co-authored-by: Claude --- .buildkite/test-amd.yaml | 20 ++++ .buildkite/test_areas/models_distributed.yaml | 16 +++ tests/models/qwen4_exp/test_config.py | 25 ++-- tests/models/qwen4_exp/test_tpsp.py | 61 ++++++++-- tests/models/test_registry.py | 6 +- tests/test_config.py | 9 +- vllm/config/model.py | 4 +- vllm/config/parallel.py | 2 +- vllm/model_executor/models/config.py | 2 +- vllm/models/qwen4_exp/amd/model.py | 91 ++++++++++++-- vllm/models/qwen4_exp/amd/mtp.py | 22 +++- .../qwen4_exp/amd/ops/sequence_parallel.py | 47 ++++++++ vllm/models/qwen4_exp/amd/ple_layer.py | 112 +++++++++++++----- .../qwen4_exp/common/hyperconnection.py | 10 ++ .../qwen4_exp/common/ngram_embedding.py | 6 +- vllm/models/qwen4_exp/nvidia/model.py | 9 +- vllm/models/qwen4_exp/nvidia/mtp.py | 2 +- 17 files changed, 361 insertions(+), 83 deletions(-) create mode 100644 vllm/models/qwen4_exp/amd/ops/sequence_parallel.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 39ac57b8af..56cba27ceb 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3439,6 +3439,26 @@ steps: commands: - pytest -v -s tests/kernels/test_rocm_gemm_rs_ar.py +- label: ":amd: (MI355) Qwen4 Exp SP" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/config/parallel.py + - vllm/models/common/ops/sequence_parallel.py + - vllm/models/qwen4_exp/ + - vllm/model_executor/layers/vocab_parallel_embedding.py + - vllm/model_executor/models/qwen3_next.py + - vllm/platforms/rocm.py + - tests/models/qwen4_exp/test_ple.py + - tests/models/qwen4_exp/test_tpsp.py + commands: + - pytest -v -s models/qwen4_exp/test_tpsp.py + - label: ":amd: (MI355) Distributed DP Basic" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] diff --git a/.buildkite/test_areas/models_distributed.yaml b/.buildkite/test_areas/models_distributed.yaml index 8d2bc5dbaf..38a2e7c09f 100644 --- a/.buildkite/test_areas/models_distributed.yaml +++ b/.buildkite/test_areas/models_distributed.yaml @@ -18,6 +18,22 @@ steps: - tests/models/qwen4_exp/test_tpsp.py commands: - pytest -v -s models/qwen4_exp/test_tpsp.py + mirror: + amd: + label: ":amd: (MI355) Qwen4 Exp SP" + dind: false + device: mi355_4 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/config/parallel.py + - vllm/models/common/ops/sequence_parallel.py + - vllm/models/qwen4_exp/ + - vllm/model_executor/layers/vocab_parallel_embedding.py + - vllm/model_executor/models/qwen3_next.py + - vllm/platforms/rocm.py + - tests/models/qwen4_exp/test_ple.py + - tests/models/qwen4_exp/test_tpsp.py - label: ":nvidia: (L4) Distributed Models Shard %N" key: distributed-model-tests-2-gpus diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.py index 818f515438..177ebaf3f6 100644 --- a/tests/models/qwen4_exp/test_config.py +++ b/tests/models/qwen4_exp/test_config.py @@ -65,20 +65,21 @@ def test_moe_sp_rejects_pipeline_parallel() -> None: ), ) with ( - patch("vllm.platforms.current_platform.is_cuda", return_value=True), + patch("vllm.platforms.current_platform.is_cuda_alike", return_value=True), pytest.raises(ValueError, match="MoE SP requires PP=1"), ): Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config) +@spawn_new_process_for_each_test +@pytest.mark.parametrize("backend", ["amd", "nvidia"]) @pytest.mark.parametrize("use_sp", [False, True]) -def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None: +def test_qwen4_exp_mtp_returns_sample_and_multi_streams( + backend: str, use_sp: bool +) -> None: """MTP returns complete, contiguous outputs after gathering padded SP shards.""" - from vllm.models.qwen4_exp.nvidia.mtp import ( - Qwen4ExpMultiTokenPredictor, - ) - - model = object.__new__(Qwen4ExpMultiTokenPredictor) + mtp_module = f"vllm.models.qwen4_exp.{backend}.mtp" + model = object.__new__(import_module(mtp_module).Qwen4ExpMultiTokenPredictor) model.use_hc_sequence_parallel = use_sp model.use_sequence_parallel = False torch.nn.Module.__init__(model) @@ -93,7 +94,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None: def decoder(**kwargs): """Check that the embedding residual shares the hidden state's token slice.""" hidden = kwargs["hidden_states"] - assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size) + if backend == "nvidia": + prev_block_output = kwargs["prev_block_output"] + assert prev_block_output.shape == (hidden.shape[0], model.hidden_size) return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count) model.layers = [decoder] @@ -115,9 +118,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None: return torch.cat([local, remote, torch.full_like(remote, float("nan"))]) with ( - patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group), - patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]), - patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather), + patch(f"{mtp_module}.get_pp_group", return_value=pp_group), + patch(f"{mtp_module}.sp_shard", side_effect=lambda x: x[:2]), + patch(f"{mtp_module}.sp_all_gather", side_effect=gather), ): sample_hidden, returned_multi_hidden = model.forward( input_ids=None, diff --git a/tests/models/qwen4_exp/test_tpsp.py b/tests/models/qwen4_exp/test_tpsp.py index 5182ab09c3..33e01ab24f 100644 --- a/tests/models/qwen4_exp/test_tpsp.py +++ b/tests/models/qwen4_exp/test_tpsp.py @@ -14,8 +14,8 @@ from transformers import Qwen4ExpTextConfig from vllm.platforms import current_platform -if not current_platform.is_cuda(): - pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True) +if not current_platform.is_cuda_alike(): + pytest.skip("Qwen4Exp SP requires CUDA or ROCm", allow_module_level=True) from tests.utils import multi_gpu_test from vllm.config import ( @@ -38,14 +38,19 @@ from vllm.models.common.ops.sequence_parallel import ( sp_padding_mask, sp_shard, ) -from vllm.models.qwen4_exp.nvidia import model as qwen4_model -from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer from vllm.utils.network_utils import get_open_port from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager from .test_ple import _ConvBatchCase, _make_conv_metadata +if current_platform.is_rocm(): + from vllm.models.qwen4_exp.amd import model as qwen4_model + from vllm.models.qwen4_exp.amd.model import Qwen4ExpDecoderLayer +else: + from vllm.models.qwen4_exp.nvidia import model as qwen4_model + from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer + class _TPAttentionProjection(nn.Module): """Keep real TP communication without setting up an attention KV cache.""" @@ -168,7 +173,8 @@ def _check_decoder_ple(rank: int) -> None: set_current_vllm_config(config), set_forward_context({layer.ple.prefix: metadata}, config), ): - layer.ple.start_prefetch(ids, qsl, context) + if hasattr(layer.ple, "start_prefetch"): + layer.ple.start_prefetch(ids, qsl, context) output = layer( *inputs, positions, @@ -177,7 +183,10 @@ def _check_decoder_ple(rank: int) -> None: ngram_context=context, ) if config.parallel_config.enable_hc_sp: - output = tuple(sp_all_gather(x)[:num_tokens] for x in output) + # ROCm all-gather requires contiguous inputs. + output = tuple( + sp_all_gather(x.contiguous())[:num_tokens] for x in output + ) # A later MoE call may reuse workspace-backed output storage. outputs.append(tuple(x.clone() for x in output)) torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02) @@ -223,11 +232,38 @@ def _check_moe_sp(rank: int, config: VllmConfig) -> None: query_start_loc=None, ngram_context=None, ) - actual = tuple(sp_all_gather(x)[:num_tokens] for x in output) + actual = tuple(sp_all_gather(x.contiguous())[:num_tokens] for x in output) torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02) -def _run_sp(rank: int, port: int, moe_sp: bool) -> None: +def _check_compiled_sp_ops() -> None: + """Compiled SP collectives stay correct for token counts unseen at trace time.""" + from vllm.models.qwen4_exp.amd.ops.sequence_parallel import ( + sp_reduce_scatter, + sp_shard, + ) + + def collectives(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + gathered = sp_all_gather(sp_shard(x) * 2)[: x.shape[0]] + reduced = sp_all_gather(sp_reduce_scatter(x))[: x.shape[0]] + return gathered, reduced + + # vLLM drops Dynamo guards, so one trace must serve every token count. + compiled = torch.compile( + collectives, + fullgraph=True, + options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe}, + ) + traced = torch.randn(8, 16) + torch._dynamo.mark_dynamic(traced, 0) + compiled(traced) + for num_tokens in (7, 3, 1): + torch.manual_seed(num_tokens) + x = torch.randn(num_tokens, 16) + torch.testing.assert_close(compiled(x), collectives(x), atol=0, rtol=0) + + +def _run_sp(rank: int, port: int, moe_sp: bool, compiled_ops: bool = False) -> None: """Initialize rank-local runtime state and run one equivalence check.""" torch.accelerator.set_device_index(rank) torch.set_num_threads(1) @@ -247,6 +283,8 @@ def _run_sp(rank: int, port: int, moe_sp: bool) -> None: torch.set_default_dtype(torch.bfloat16) if moe_sp: _check_moe_sp(rank, config) + elif compiled_ops: + _check_compiled_sp_ops() else: _check_decoder_ple(rank) finally: @@ -265,3 +303,10 @@ def test_moe_sp_equivalence(monkeypatch) -> None: """Native MoE SP preserves outputs with unequal DP batches and padding.""" monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1") mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only SP ops") +@multi_gpu_test(num_gpus=2) +def test_compiled_sp_ops_handle_untraced_token_counts() -> None: + """Padding in compiled SP collectives must not specialize on the traced size.""" + mp.spawn(_run_sp, args=(get_open_port(), False, True), nprocs=2) diff --git a/tests/models/test_registry.py b/tests/models/test_registry.py index 37a95d9f7f..354e7fd689 100644 --- a/tests/models/test_registry.py +++ b/tests/models/test_registry.py @@ -127,9 +127,9 @@ def test_registry_is_pp(model_arch, is_pp): @pytest.mark.parametrize( "model_arch,supported", [ - ("Qwen4ExpForCausalLM", not current_platform.is_rocm()), - ("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()), - ("Qwen4ExpMTP", not current_platform.is_rocm()), + ("Qwen4ExpForCausalLM", True), + ("Qwen4ExpForConditionalGeneration", True), + ("Qwen4ExpMTP", True), ("LlamaForCausalLM", False), ], ) diff --git a/tests/test_config.py b/tests/test_config.py index fc482bcf4e..c68ecca605 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4144,7 +4144,7 @@ def test_revision_resolved_for_model(mock_resolve): @pytest.mark.parametrize( - "enabled,supported,is_cuda,tp_size,pp_size,error", + "enabled,supported,cuda_alike,tp_size,pp_size,error", [ (True, True, True, 2, 1, None), (True, False, True, 2, 1, "not supported for this model"), @@ -4155,7 +4155,7 @@ def test_revision_resolved_for_model(mock_resolve): (False, False, False, 2, 1, None), ], ) -def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, error): +def test_hc_sp_model_support(enabled, supported, cuda_alike, tp_size, pp_size, error): """Validate active HC SP configurations and ignore the flag with TP=1.""" from vllm.model_executor.models.registry import ModelRegistry @@ -4173,7 +4173,10 @@ def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, erro distributed_executor_backend="mp", ) with ( - patch("vllm.config.model.current_platform.is_cuda", return_value=is_cuda), + patch( + "vllm.config.model.current_platform.is_cuda_alike", + return_value=cuda_alike, + ), patch.object(ModelRegistry, "is_hc_sp_supported_model", return_value=supported), patch.object(ModelRegistry, "is_pp_supported_model", return_value=True), ): diff --git a/vllm/config/model.py b/vllm/config/model.py index c4016e82bc..1dcf092229 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -1534,8 +1534,8 @@ class ModelConfig: parallel_config: ParallelConfig, ) -> None: if parallel_config.enable_hc_sp and parallel_config.tensor_parallel_size > 1: - if not current_platform.is_cuda(): - raise ValueError("--enable-hc-sp currently requires CUDA") + if not current_platform.is_cuda_alike(): + raise ValueError("--enable-hc-sp currently requires CUDA or ROCm") if not self.registry.is_hc_sp_supported_model(self.architectures, self): raise ValueError("--enable-hc-sp is not supported for this model") if parallel_config.pipeline_parallel_size != 1: diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 975c0cd1f3..6ea95142c1 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -184,7 +184,7 @@ class ParallelConfig: """Whether the deployed model is MoE (if known).""" enable_hc_sp: bool = False """Enable sequence parallelism for HyperConnection (HC) computation. - Currently supported by Qwen4Exp on CUDA with TP>1 and PP=1 (ignored when TP=1). + Currently supported by Qwen4Exp on CUDA/ROCm with TP>1 and PP=1 (ignored when TP=1). Does not change MoE parallelism. MoE SP may enable HC SP independently of this flag. """ enable_expert_parallel: bool = False diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index f796bcf615..12ba253671 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -992,7 +992,7 @@ class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConf parallel_config = vllm_config.parallel_config # MoE SP also enables HC SP without the explicit HC flag. if ( - current_platform.is_cuda() + current_platform.is_cuda_alike() and parallel_config.use_sequence_parallel_moe and parallel_config.pipeline_parallel_size != 1 ): diff --git a/vllm/models/qwen4_exp/amd/model.py b/vllm/models/qwen4_exp/amd/model.py index f36f70bee2..b8e8e39a64 100644 --- a/vllm/models/qwen4_exp/amd/model.py +++ b/vllm/models/qwen4_exp/amd/model.py @@ -37,6 +37,7 @@ from vllm.model_executor.models.interfaces import ( IsHybrid, MixtureOfExperts, MultiModalEmbeddings, + SupportsHCSP, SupportsLoRA, SupportsMRoPE, SupportsPP, @@ -67,6 +68,7 @@ from vllm.model_executor.models.utils import ( maybe_fuse_shared_experts, maybe_prefix, ) +from vllm.models.common.ops.sequence_parallel import sp_all_gather from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors @@ -74,8 +76,10 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.kv_cache_interface import MambaSpec +from ..common.hyperconnection import is_hc_sequence_parallel_enabled from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm +from .ops.sequence_parallel import sp_reduce_scatter, sp_shard from .ple_layer import Qwen4ExpPLELayer from .qsa import Qwen4ExpQSAAttention @@ -161,13 +165,19 @@ _EXTRA_WEIGHTS_MAPPER = WeightsMapper( class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock): """Qwen3Next MoE with Qwen4Exp HC validation.""" - def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None: - parallel_config = vllm_config.parallel_config - if parallel_config.use_sequence_parallel_moe: - raise NotImplementedError( - "Qwen4Exp HC does not support sequence-parallel MoE" - ) - super().__init__(vllm_config=vllm_config, prefix=prefix) + def __init__( + self, + vllm_config: VllmConfig, + prefix: str = "", + reduce_results: bool = True, + ) -> None: + super().__init__( + vllm_config=vllm_config, prefix=prefix, reduce_results=reduce_results + ) + if self.replicate_shared_expert and not self.is_sequence_parallel: + # Reduce routed outputs before adding replicated shared outputs. + # SP then slices the result to avoid summing shared outputs again via RS. + self.experts.moe_config.skip_final_all_reduce = False config = vllm_config.model_config.hf_text_config self.n_shared_experts = int(config.shared_expert_intermediate_size > 0) @@ -185,11 +195,12 @@ class Qwen4ExpDecoderLayer(nn.Module): self.config = config self.layer_type = layer_type + # MoE SP also enables HC SP, while HC SP can be enabled independently. + self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config) self.layer_idx = extract_layer_index(prefix) - if vllm_config.parallel_config.use_sequence_parallel_moe: - raise NotImplementedError( - "Qwen4Exp HC does not support sequence-parallel MoE" - ) + self.use_sequence_parallel = ( + vllm_config.parallel_config.use_sequence_parallel_moe + ) self.ple: Qwen4ExpPLELayer | None = None ple_layer_ids = config.ple_layer_ids if (self.layer_idx + 1) in ple_layer_ids: @@ -204,6 +215,7 @@ class Qwen4ExpDecoderLayer(nn.Module): layer_idx=self.layer_idx, ple_dense_layer_id=ple_dense_layer_id, prefix=f"{prefix}.ple", + use_sequence_parallel=self.use_hc_sequence_parallel, ) if layer_type == "linear_attention": @@ -212,6 +224,7 @@ class Qwen4ExpDecoderLayer(nn.Module): vllm_config=vllm_config, prefix=f"{prefix}.linear_attn", gqa_interleaved_layout=False, + reduce_results=not self.use_hc_sequence_parallel, ) elif layer_type in _QSA_LAYER_TYPES: use_qsa = getattr(config, "indexer_n_heads", None) is not None @@ -222,6 +235,7 @@ class Qwen4ExpDecoderLayer(nn.Module): cache_config=vllm_config.cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", + reduce_results=not self.use_hc_sequence_parallel, ) else: self.self_attn = Qwen4ExpQSAAttention( @@ -230,12 +244,15 @@ class Qwen4ExpDecoderLayer(nn.Module): layer_id=self.layer_idx, quant_config=quant_config, prefix=f"{prefix}.self_attn", + reduce_results=not self.use_hc_sequence_parallel, ) else: raise ValueError(f"Invalid layer_type {layer_type}") self.mlp = Qwen4ExpSparseMoeBlock( - vllm_config=vllm_config, prefix=f"{prefix}.mlp" + vllm_config=vllm_config, + prefix=f"{prefix}.mlp", + reduce_results=not self.use_hc_sequence_parallel, ) hc_config = HyperConnectionConfig( @@ -293,6 +310,9 @@ class Qwen4ExpDecoderLayer(nn.Module): else: hidden_states, block_input, injection = attn_hc.mix(hidden_states) + if self.use_hc_sequence_parallel: + # Positions retain the full token axis for both text and MRoPE inputs. + block_input = sp_all_gather(block_input)[: positions.shape[-1]] if self.layer_type == "linear_attention": attn_out = self.linear_attn(hidden_states=block_input) elif self.layer_type in _QSA_LAYER_TYPES: @@ -304,10 +324,25 @@ class Qwen4ExpDecoderLayer(nn.Module): raise ValueError("Invalid layer_type") mlp_hc = self.mlp_hyper_connection + if self.use_hc_sequence_parallel: + attn_out = sp_reduce_scatter(attn_out) hidden_states, block_input, injection = mlp_hc.combine_and_mix( hidden_states, attn_out, injection ) - mlp_out = self.mlp(block_input) + if self.use_sequence_parallel: + # MoE consumes local tokens and returns complete local outputs. + mlp_out = self.mlp(block_input, already_sequence_parallel=True) + else: + if self.use_hc_sequence_parallel: + block_input = sp_all_gather(block_input)[: positions.shape[-1]] + mlp_out = self.mlp(block_input) + if self.use_hc_sequence_parallel: + assert isinstance(self.mlp, Qwen4ExpSparseMoeBlock) + reduce_output = self.mlp.experts.moe_config.skip_final_all_reduce + # Only sum partial outputs. Reduced outputs already include all ranks. + mlp_out = ( + sp_reduce_scatter(mlp_out) if reduce_output else sp_shard(mlp_out) + ) return hidden_states, mlp_out, injection @@ -382,6 +417,8 @@ class Qwen4ExpModel(nn.Module): self.num_redundant_experts = ( vllm_config.parallel_config.eplb_config.num_redundant_experts ) + # MoE SP also enables HC SP, while HC SP can be enabled independently. + self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config) self.vocab_size = config.vocab_size self._qsa_layer_ids = frozenset( layer_idx @@ -470,12 +507,18 @@ class Qwen4ExpModel(nn.Module): if input_ids is None: raise ValueError("input_ids or inputs_embeds is required") hidden_states = self.embed_input_ids(input_ids) + if self.use_hc_sequence_parallel: + # The MoE padding mask stays unsharded: under torch.compile its + # update would land only after the graph has run. + hidden_states = sp_shard(hidden_states) + # Expand only the local token rows when SP is enabled. hidden_states = hidden_states.repeat(1, self.config.hc_count) else: if intermediate_tensors is None: raise ValueError("pipeline stage requires intermediate tensors") hidden_states = intermediate_tensors["hidden_states"] + full_num_tokens = positions.shape[-1] block_output = None injection = None last_layer = None @@ -498,6 +541,8 @@ class Qwen4ExpModel(nn.Module): deepstack_embed = deepstack_input_embeds[ f"deepstack_input_embeds_{layer_idx}" ] + if self.use_hc_sequence_parallel: + deepstack_embed = sp_shard(deepstack_embed) deepstack_embed = ( deepstack_embed.unsqueeze(-2) .expand( @@ -532,6 +577,24 @@ class Qwen4ExpModel(nn.Module): multi_hidden, sample_hidden_states, _ = final_mixer.combine_and_mix( hidden_states, block_output, injection ) + if self.use_hc_sequence_parallel: + if self._mtp_hidden_buffer is not None: + # Gather LM-head and MTP states together when both are needed. + hidden_size = self.config.hidden_size + packed_hidden_states = torch.cat( + [sample_hidden_states, multi_hidden], dim=-1 + ) + packed_hidden_states = sp_all_gather(packed_hidden_states)[ + :full_num_tokens + ] + sample_hidden_states, multi_hidden = packed_hidden_states.split( + [hidden_size, self.config.hc_count * hidden_size], dim=-1 + ) + sample_hidden_states = sample_hidden_states.contiguous() + else: + sample_hidden_states = sp_all_gather(sample_hidden_states)[ + :full_num_tokens + ] if self._mtp_hidden_buffer is not None: # Capture the pre-final-mixer multi-stream hidden state # [T, hc_count*H] for the MTP drafter (zero extra compute: @@ -587,6 +650,7 @@ class Qwen4ExpModel(nn.Module): class Qwen4ExpForCausalLM( nn.Module, HasInnerState, + SupportsHCSP, SupportsLoRA, SupportsMRoPE, SupportsPP, @@ -819,6 +883,7 @@ class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo): class Qwen4ExpForConditionalGeneration( Qwen3_5ForConditionalGeneration, HasInnerState, + SupportsHCSP, Qwen4ExpMixtureOfExperts, ): """Qwen3-VL vision tower backed by the Qwen4Exp language model.""" diff --git a/vllm/models/qwen4_exp/amd/mtp.py b/vllm/models/qwen4_exp/amd/mtp.py index b6513e802d..e9d68d9cdc 100644 --- a/vllm/models/qwen4_exp/amd/mtp.py +++ b/vllm/models/qwen4_exp/amd/mtp.py @@ -34,7 +34,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from vllm.model_executor.model_loader.utils import configure_quant_config -from vllm.model_executor.models.interfaces import SupportsPP +from vllm.model_executor.models.interfaces import SupportsHCSP, SupportsPP from vllm.model_executor.models.qwen3_5 import Qwen3_5Model from vllm.model_executor.models.utils import ( AutoWeightsLoader, @@ -45,8 +45,10 @@ from vllm.model_executor.models.utils import ( maybe_fuse_shared_experts, maybe_prefix, ) +from vllm.models.common.ops.sequence_parallel import sp_all_gather, sp_shard from vllm.sequence import IntermediateTensors +from ..common.hyperconnection import is_hc_sequence_parallel_enabled from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm from .model import ( @@ -181,6 +183,8 @@ class Qwen4ExpMultiTokenPredictor(nn.Module): config: Qwen4ExpTextConfig = model_config.hf_text_config self.config = config + # MoE SP also enables HC SP, while HC SP can be enabled independently. + self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config) self.vocab_size = config.vocab_size self.mtp_start_layer_idx = config.num_hidden_layers @@ -321,6 +325,9 @@ class Qwen4ExpMultiTokenPredictor(nn.Module): hidden_states = intermediate_tensors["hidden_states"] current_step_idx = spec_step_idx % self.num_mtp_layers + full_num_tokens = positions.shape[-1] + if self.use_hc_sequence_parallel: + hidden_states = sp_shard(hidden_states) layer = self.layers[current_step_idx] hidden_states, block_output, injection = layer( hidden_states=hidden_states, @@ -348,6 +355,17 @@ class Qwen4ExpMultiTokenPredictor(nn.Module): hidden_states, block_output, injection ) ) + if self.use_hc_sequence_parallel: + # Gather both outputs together, then restore their contiguous layouts. + packed_hidden_states = torch.cat( + [sample_hidden_states, multi_hidden], dim=-1 + ) + packed_hidden_states = sp_all_gather(packed_hidden_states)[:full_num_tokens] + sample_hidden_states, multi_hidden = packed_hidden_states.split( + [hidden_size, hc_count * hidden_size], dim=-1 + ) + sample_hidden_states = sample_hidden_states.contiguous() + multi_hidden = multi_hidden.contiguous() return sample_hidden_states, multi_hidden def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: @@ -377,7 +395,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module): "hidden_states": 0, } ) -class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts): +class Qwen4ExpMTP(nn.Module, SupportsHCSP, SupportsPP, Qwen4ExpMixtureOfExperts): packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], "gate_up_proj": ["gate_proj", "up_proj"], diff --git a/vllm/models/qwen4_exp/amd/ops/sequence_parallel.py b/vllm/models/qwen4_exp/amd/ops/sequence_parallel.py new file mode 100644 index 0000000000..2efcd56d54 --- /dev/null +++ b/vllm/models/qwen4_exp/amd/ops/sequence_parallel.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.models.common.ops import sequence_parallel as common_sp +from vllm.utils.math_utils import cdiv +from vllm.utils.torch_utils import direct_register_custom_op + + +def _sp_shard_op(x: torch.Tensor) -> torch.Tensor: + output = common_sp.sp_shard(x) + # A functional custom op must not return a view of its input. + unpadded = x.shape[0] % get_tensor_model_parallel_world_size() == 0 + return output.clone() if unpadded else output + + +def _sp_local_fake(x: torch.Tensor) -> torch.Tensor: + tp_size = get_tensor_model_parallel_world_size() + return x.new_empty((cdiv(x.shape[0], tp_size), *x.shape[1:])) + + +# Dynamo would specialize the padding branch on the traced token count, and +# vLLM drops the resulting guards. Keep the branch opaque when compiling. +direct_register_custom_op( + op_name="sp_reduce_scatter", + op_func=common_sp.sp_reduce_scatter, + fake_impl=_sp_local_fake, +) +direct_register_custom_op( + op_name="sp_shard", + op_func=_sp_shard_op, + fake_impl=_sp_local_fake, +) + + +def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor: + if torch.compiler.is_compiling(): + return torch.ops.vllm.sp_reduce_scatter(x) + return common_sp.sp_reduce_scatter(x) + + +def sp_shard(x: torch.Tensor) -> torch.Tensor: + if torch.compiler.is_compiling(): + return torch.ops.vllm.sp_shard(x) + return common_sp.sp_shard(x) diff --git a/vllm/models/qwen4_exp/amd/ple_layer.py b/vllm/models/qwen4_exp/amd/ple_layer.py index 30595df32d..c1cfc0f246 100644 --- a/vllm/models/qwen4_exp/amd/ple_layer.py +++ b/vllm/models/qwen4_exp/amd/ple_layer.py @@ -9,6 +9,7 @@ from torch import nn from transformers import Qwen4ExpTextConfig from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config +from vllm.distributed import get_etp_group, get_tp_group from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.linear import MergedColumnParallelLinear @@ -21,7 +22,13 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, ) +from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8 from vllm.model_executor.models.utils import AutoWeightsLoader +from vllm.models.common.ops.sequence_parallel import ( + sp_all_gather, + sp_reduce_scatter, + sp_shard, +) from vllm.utils.torch_utils import direct_register_custom_op from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.short_conv_attn import ( @@ -182,8 +189,10 @@ class Qwen4ExpNGramEmbedding(nn.Module): data_parallel_rank: int = 0, quant_config: QuantizationConfig | None = None, params_dtype: torch.dtype | None = None, + use_sequence_parallel: bool = False, ) -> None: super().__init__() + self.use_sequence_parallel = use_sequence_parallel self.embedding_dim = embedding_dim self.layer_name = layer_name self.ngram_size = int(config.ngram_size) @@ -253,6 +262,11 @@ class Qwen4ExpNGramEmbedding(nn.Module): if engram_config is not None and engram_config.cpu_offload else Qwen4ExpPLEDeviceEmbedding ) + # A TP reduce-scatter covers all vocabulary owners only when ETP=TP. + self.use_reduce_scatter = ( + self.use_sequence_parallel + and get_etp_group().world_size == get_tp_group().world_size + ) self.ngram_embedding = embedding_cls( padded_vocab_size, self.head_dim, @@ -263,6 +277,7 @@ class Qwen4ExpNGramEmbedding(nn.Module): num_ngram_heads=self.ngram_heads, max_total_tokens=max_total_tokens, data_parallel_rank=data_parallel_rank, + reduce_results=not self.use_reduce_scatter, ) logger.info( "Initialized AMD PLE embedding %s: quantization_method=%s, " @@ -274,12 +289,25 @@ class Qwen4ExpNGramEmbedding(nn.Module): self.ngram_embedding.weight.is_pinned(), ) + def _sp_shard_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor: + """Shard embedding token rows, optionally reducing vocabulary contributions.""" + # Each embedding element has one vocabulary owner, including FP8 bytes. + output_dtype = embeddings.dtype + if is_fp8(embeddings): + embeddings = embeddings.view(torch.int8) + if self.use_reduce_scatter: + embeddings = sp_reduce_scatter(embeddings) + else: + embeddings = sp_shard(embeddings) + return embeddings.view(output_dtype) + def forward( self, input_ids: torch.Tensor, query_start_loc: torch.Tensor, ngram_context: torch.Tensor, ) -> torch.Tensor: + """Return full embeddings or a token shard after combining vocabulary owners.""" input_ids = input_ids.reshape(-1) ngram_ids = input_ids.new_empty( (input_ids.shape[0], self.ngram_heads), dtype=torch.int64 @@ -302,17 +330,19 @@ class Qwen4ExpNGramEmbedding(nn.Module): output, self.layer_name, ) + else: + output = ngram_ids.new_empty( + (ngram_ids.shape[0], self.embedding_dim), + dtype=embedding.params_dtype, + ) + torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding( + ngram_ids, + output, + self.layer_name, + ) + if not self.use_sequence_parallel: return output - output = ngram_ids.new_empty( - (ngram_ids.shape[0], self.embedding_dim), - dtype=embedding.params_dtype, - ) - torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding( - ngram_ids, - output, - self.layer_name, - ) - return output + return self._sp_shard_embeddings(output) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: """Load hash buffers and checkpoint-split embedding rows.""" @@ -388,8 +418,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): layer_idx: int = 0, ple_dense_layer_id: int | None = None, prefix: str = "", + use_sequence_parallel: bool = False, ) -> None: super().__init__() + self.use_sequence_parallel = use_sequence_parallel model_config = vllm_config.model_config cache_config = vllm_config.cache_config quant_config = vllm_config.quant_config @@ -420,8 +452,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): data_parallel_rank=vllm_config.parallel_config.data_parallel_rank, quant_config=quant_config, params_dtype=model_config.dtype, + use_sequence_parallel=self.use_sequence_parallel, ) - # The PLE cache is TP-replicated, so this merged projection is too. + # Projection weights stay replicated even when their token rows are sharded. self.kv_proj = MergedColumnParallelLinear( int(config.ple_embed_dim), [self.hc_hidden_size, self.hidden_size], @@ -486,7 +519,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): self, inputs: torch.Tensor, residual: torch.Tensor, - outer_residual: torch.Tensor, + outer_residual: torch.Tensor | None, metadata: PleShortConvAttentionMetadata, conv_state: torch.Tensor, conv_weights: torch.Tensor, @@ -501,7 +534,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): has_non_spec = has_prefill or has_decode inputs = inputs[: metadata.num_actual_tokens] residual = residual[: metadata.num_actual_tokens] - outer_residual = outer_residual[: metadata.num_actual_tokens] + if outer_residual is not None: + outer_residual = outer_residual[: metadata.num_actual_tokens] spec_token_indices = None non_spec_token_indices = None @@ -553,11 +587,14 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): residual_d, residual_p = torch.split( residual, [num_decode_tokens, num_prefill_tokens], dim=0 ) - outer_residual_d, outer_residual_p = torch.split( - outer_residual, - [num_decode_tokens, num_prefill_tokens], - dim=0, - ) + outer_residual_d: torch.Tensor | None = None + outer_residual_p: torch.Tensor | None = None + if outer_residual is not None: + outer_residual_d, outer_residual_p = torch.split( + outer_residual, + [num_decode_tokens, num_prefill_tokens], + dim=0, + ) token_indices_d = None token_indices_p = None else: @@ -628,14 +665,16 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): self, inputs: torch.Tensor, residual: torch.Tensor, - outer_residual: torch.Tensor, + outer_residual: torch.Tensor | None, ) -> None: + """Update convolution state, optionally fusing both residual additions.""" forward_context = get_forward_context() attn_metadata = forward_context.attn_metadata # Profiling omits all metadata or this Mamba entry. Short convolution # is a no-op there, but preserve the outer residual addition. if attn_metadata is None: - residual.add_(outer_residual) + if outer_residual is not None: + residual.add_(outer_residual) return if not isinstance(attn_metadata, dict): @@ -646,7 +685,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): layer_attn_metadata = attn_metadata.get(self.prefix) if layer_attn_metadata is None: - residual.add_(outer_residual) + if outer_residual is not None: + residual.add_(outer_residual) return if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata): raise TypeError( @@ -687,7 +727,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): ngram_context: torch.Tensor, ) -> torch.Tensor: input_ids = input_ids.reshape(-1) - if input_ids.shape[0] != hidden_states.shape[0]: + if ( + not self.use_sequence_parallel + and input_ids.shape[0] != hidden_states.shape[0] + ): raise ValueError( "PLE expects input_ids and hidden_states to have the same " f"token length, got {input_ids.shape[0]} and " @@ -711,12 +754,23 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): # The short-conv op is a piecewise-graph splitting op: state routing # reads runtime request metadata. It accumulates the convolution and # the outer residual into gated_output. - torch.ops.vllm.qwen4_exp_ple_short_conv( - conv_input, - gated_output, - hidden_states, - self.prefix, - ) + if self.use_sequence_parallel: + # Only convolution needs full tokens. Add residuals on local shards. + conv_input = sp_all_gather(conv_input)[: input_ids.shape[0]] + # Metadata may omit padding tokens or all tokens during profiling. + conv_output = torch.zeros_like(conv_input) + torch.ops.vllm.qwen4_exp_ple_short_conv( + conv_input, conv_output, None, self.prefix + ) + gated_output.add_(sp_shard(conv_output)) + gated_output.add_(hidden_states) + else: + torch.ops.vllm.qwen4_exp_ple_short_conv( + conv_input, + gated_output, + hidden_states, + self.prefix, + ) return gated_output @@ -787,7 +841,7 @@ def qwen4_exp_amd_ple_ngram_embedding_pinned( def qwen4_exp_ple_short_conv( inputs: torch.Tensor, residual: torch.Tensor, - outer_residual: torch.Tensor, + outer_residual: torch.Tensor | None, layer_name: str, ) -> None: layer = get_forward_context().no_compile_layers[layer_name] diff --git a/vllm/models/qwen4_exp/common/hyperconnection.py b/vllm/models/qwen4_exp/common/hyperconnection.py index 5f8bde0479..42c3fba4ff 100644 --- a/vllm/models/qwen4_exp/common/hyperconnection.py +++ b/vllm/models/qwen4_exp/common/hyperconnection.py @@ -35,6 +35,16 @@ import torch import torch.nn.functional as F from torch import nn +from vllm.config import VllmConfig + + +def is_hc_sequence_parallel_enabled(vllm_config: VllmConfig) -> bool: + """Enable GR/PLE SP explicitly or automatically with sequence-parallel MoE.""" + parallel_config = vllm_config.parallel_config + return parallel_config.tensor_parallel_size > 1 and ( + parallel_config.enable_hc_sp or parallel_config.use_sequence_parallel_moe + ) + # --------------------------------------------------------------------------- # Config diff --git a/vllm/models/qwen4_exp/common/ngram_embedding.py b/vllm/models/qwen4_exp/common/ngram_embedding.py index 5bf75cb551..0348b24224 100644 --- a/vllm/models/qwen4_exp/common/ngram_embedding.py +++ b/vllm/models/qwen4_exp/common/ngram_embedding.py @@ -711,7 +711,11 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0]) gathered_ids = self._gather_dp_ids(ngram_ids, slot_size) embeddings = self._lookup(gathered_ids) - embeddings = self._reduce_etp_embeddings(embeddings) + if self.reduce_results: + embeddings = self._reduce_etp_embeddings(embeddings) + else: + # The outer TP reduce-scatter requires ETP=TP. + assert self.etp_data_parallel_size == 1 return self._select_embeddings( embeddings, ngram_ids.shape[0], diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index 2fb056bde4..826db4efd3 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -82,6 +82,7 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.kv_cache_interface import MambaSpec +from ..common.hyperconnection import is_hc_sequence_parallel_enabled from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup @@ -93,14 +94,6 @@ from .qsa import Qwen4ExpQSAAttention _QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention") -def is_hc_sequence_parallel_enabled(vllm_config: VllmConfig) -> bool: - """Enable GR/PLE SP explicitly or automatically with sequence-parallel MoE.""" - parallel_config = vllm_config.parallel_config - return parallel_config.tensor_parallel_size > 1 and ( - parallel_config.enable_hc_sp or parallel_config.use_sequence_parallel_moe - ) - - def without_modelopt_fp4( quant_config: QuantizationConfig | None, ) -> QuantizationConfig | None: diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py index 32e95faa0d..e9aa87659a 100644 --- a/vllm/models/qwen4_exp/nvidia/mtp.py +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -53,6 +53,7 @@ from vllm.models.common.ops.sequence_parallel import ( ) from vllm.sequence import IntermediateTensors +from ..common.hyperconnection import is_hc_sequence_parallel_enabled from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm from .model import ( @@ -61,7 +62,6 @@ from .model import ( Qwen4ExpDecoderLayer, Qwen4ExpMixtureOfExperts, Qwen4ExpSparseMoeBlock, - is_hc_sequence_parallel_enabled, )