[ROCm][Qwen4Exp] Support HC and MoE sequence parallelism on ROCm (#61471)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
a956a8126e
commit
29345f42a6
@@ -3439,6 +3439,26 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s tests/kernels/test_rocm_gemm_rs_ar.py
|
||||
|
||||
- label: ":amd: (MI355) Qwen4 Exp SP"
|
||||
timeout_in_minutes: 30
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_4
|
||||
num_gpus: 4
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/config/parallel.py
|
||||
- vllm/models/common/ops/sequence_parallel.py
|
||||
- vllm/models/qwen4_exp/
|
||||
- vllm/model_executor/layers/vocab_parallel_embedding.py
|
||||
- vllm/model_executor/models/qwen3_next.py
|
||||
- vllm/platforms/rocm.py
|
||||
- tests/models/qwen4_exp/test_ple.py
|
||||
- tests/models/qwen4_exp/test_tpsp.py
|
||||
commands:
|
||||
- pytest -v -s models/qwen4_exp/test_tpsp.py
|
||||
|
||||
- label: ":amd: (MI355) Distributed DP Basic" # TBD
|
||||
timeout_in_minutes: 45
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
|
||||
@@ -18,6 +18,22 @@ steps:
|
||||
- tests/models/qwen4_exp/test_tpsp.py
|
||||
commands:
|
||||
- pytest -v -s models/qwen4_exp/test_tpsp.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Qwen4 Exp SP"
|
||||
dind: false
|
||||
device: mi355_4
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/config/parallel.py
|
||||
- vllm/models/common/ops/sequence_parallel.py
|
||||
- vllm/models/qwen4_exp/
|
||||
- vllm/model_executor/layers/vocab_parallel_embedding.py
|
||||
- vllm/model_executor/models/qwen3_next.py
|
||||
- vllm/platforms/rocm.py
|
||||
- tests/models/qwen4_exp/test_ple.py
|
||||
- tests/models/qwen4_exp/test_tpsp.py
|
||||
|
||||
- label: ":nvidia: (L4) Distributed Models Shard %N"
|
||||
key: distributed-model-tests-2-gpus
|
||||
|
||||
@@ -65,20 +65,21 @@ def test_moe_sp_rejects_pipeline_parallel() -> None:
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
|
||||
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=True),
|
||||
pytest.raises(ValueError, match="MoE SP requires PP=1"),
|
||||
):
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
|
||||
|
||||
|
||||
@spawn_new_process_for_each_test
|
||||
@pytest.mark.parametrize("backend", ["amd", "nvidia"])
|
||||
@pytest.mark.parametrize("use_sp", [False, True])
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(
|
||||
backend: str, use_sp: bool
|
||||
) -> None:
|
||||
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
|
||||
from vllm.models.qwen4_exp.nvidia.mtp import (
|
||||
Qwen4ExpMultiTokenPredictor,
|
||||
)
|
||||
|
||||
model = object.__new__(Qwen4ExpMultiTokenPredictor)
|
||||
mtp_module = f"vllm.models.qwen4_exp.{backend}.mtp"
|
||||
model = object.__new__(import_module(mtp_module).Qwen4ExpMultiTokenPredictor)
|
||||
model.use_hc_sequence_parallel = use_sp
|
||||
model.use_sequence_parallel = False
|
||||
torch.nn.Module.__init__(model)
|
||||
@@ -93,7 +94,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
def decoder(**kwargs):
|
||||
"""Check that the embedding residual shares the hidden state's token slice."""
|
||||
hidden = kwargs["hidden_states"]
|
||||
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
|
||||
if backend == "nvidia":
|
||||
prev_block_output = kwargs["prev_block_output"]
|
||||
assert prev_block_output.shape == (hidden.shape[0], model.hidden_size)
|
||||
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
|
||||
|
||||
model.layers = [decoder]
|
||||
@@ -115,9 +118,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
|
||||
|
||||
with (
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
|
||||
patch(f"{mtp_module}.get_pp_group", return_value=pp_group),
|
||||
patch(f"{mtp_module}.sp_shard", side_effect=lambda x: x[:2]),
|
||||
patch(f"{mtp_module}.sp_all_gather", side_effect=gather),
|
||||
):
|
||||
sample_hidden, returned_multi_hidden = model.forward(
|
||||
input_ids=None,
|
||||
|
||||
@@ -14,8 +14,8 @@ from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Qwen4Exp SP requires CUDA or ROCm", allow_module_level=True)
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import (
|
||||
@@ -38,14 +38,19 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
|
||||
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
|
||||
|
||||
from .test_ple import _ConvBatchCase, _make_conv_metadata
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.qwen4_exp.amd import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.amd.model import Qwen4ExpDecoderLayer
|
||||
else:
|
||||
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
|
||||
|
||||
|
||||
class _TPAttentionProjection(nn.Module):
|
||||
"""Keep real TP communication without setting up an attention KV cache."""
|
||||
@@ -168,7 +173,8 @@ def _check_decoder_ple(rank: int) -> None:
|
||||
set_current_vllm_config(config),
|
||||
set_forward_context({layer.ple.prefix: metadata}, config),
|
||||
):
|
||||
layer.ple.start_prefetch(ids, qsl, context)
|
||||
if hasattr(layer.ple, "start_prefetch"):
|
||||
layer.ple.start_prefetch(ids, qsl, context)
|
||||
output = layer(
|
||||
*inputs,
|
||||
positions,
|
||||
@@ -177,7 +183,10 @@ def _check_decoder_ple(rank: int) -> None:
|
||||
ngram_context=context,
|
||||
)
|
||||
if config.parallel_config.enable_hc_sp:
|
||||
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
# ROCm all-gather requires contiguous inputs.
|
||||
output = tuple(
|
||||
sp_all_gather(x.contiguous())[:num_tokens] for x in output
|
||||
)
|
||||
# A later MoE call may reuse workspace-backed output storage.
|
||||
outputs.append(tuple(x.clone() for x in output))
|
||||
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
|
||||
@@ -223,11 +232,38 @@ def _check_moe_sp(rank: int, config: VllmConfig) -> None:
|
||||
query_start_loc=None,
|
||||
ngram_context=None,
|
||||
)
|
||||
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
actual = tuple(sp_all_gather(x.contiguous())[:num_tokens] for x in output)
|
||||
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
|
||||
|
||||
|
||||
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
|
||||
def _check_compiled_sp_ops() -> None:
|
||||
"""Compiled SP collectives stay correct for token counts unseen at trace time."""
|
||||
from vllm.models.qwen4_exp.amd.ops.sequence_parallel import (
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
|
||||
def collectives(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
gathered = sp_all_gather(sp_shard(x) * 2)[: x.shape[0]]
|
||||
reduced = sp_all_gather(sp_reduce_scatter(x))[: x.shape[0]]
|
||||
return gathered, reduced
|
||||
|
||||
# vLLM drops Dynamo guards, so one trace must serve every token count.
|
||||
compiled = torch.compile(
|
||||
collectives,
|
||||
fullgraph=True,
|
||||
options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe},
|
||||
)
|
||||
traced = torch.randn(8, 16)
|
||||
torch._dynamo.mark_dynamic(traced, 0)
|
||||
compiled(traced)
|
||||
for num_tokens in (7, 3, 1):
|
||||
torch.manual_seed(num_tokens)
|
||||
x = torch.randn(num_tokens, 16)
|
||||
torch.testing.assert_close(compiled(x), collectives(x), atol=0, rtol=0)
|
||||
|
||||
|
||||
def _run_sp(rank: int, port: int, moe_sp: bool, compiled_ops: bool = False) -> None:
|
||||
"""Initialize rank-local runtime state and run one equivalence check."""
|
||||
torch.accelerator.set_device_index(rank)
|
||||
torch.set_num_threads(1)
|
||||
@@ -247,6 +283,8 @@ def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
if moe_sp:
|
||||
_check_moe_sp(rank, config)
|
||||
elif compiled_ops:
|
||||
_check_compiled_sp_ops()
|
||||
else:
|
||||
_check_decoder_ple(rank)
|
||||
finally:
|
||||
@@ -265,3 +303,10 @@ def test_moe_sp_equivalence(monkeypatch) -> None:
|
||||
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
|
||||
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
|
||||
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only SP ops")
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_compiled_sp_ops_handle_untraced_token_counts() -> None:
|
||||
"""Padding in compiled SP collectives must not specialize on the traced size."""
|
||||
mp.spawn(_run_sp, args=(get_open_port(), False, True), nprocs=2)
|
||||
|
||||
@@ -127,9 +127,9 @@ def test_registry_is_pp(model_arch, is_pp):
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,supported",
|
||||
[
|
||||
("Qwen4ExpForCausalLM", not current_platform.is_rocm()),
|
||||
("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()),
|
||||
("Qwen4ExpMTP", not current_platform.is_rocm()),
|
||||
("Qwen4ExpForCausalLM", True),
|
||||
("Qwen4ExpForConditionalGeneration", True),
|
||||
("Qwen4ExpMTP", True),
|
||||
("LlamaForCausalLM", False),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -4144,7 +4144,7 @@ def test_revision_resolved_for_model(mock_resolve):
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,supported,is_cuda,tp_size,pp_size,error",
|
||||
"enabled,supported,cuda_alike,tp_size,pp_size,error",
|
||||
[
|
||||
(True, True, True, 2, 1, None),
|
||||
(True, False, True, 2, 1, "not supported for this model"),
|
||||
@@ -4155,7 +4155,7 @@ def test_revision_resolved_for_model(mock_resolve):
|
||||
(False, False, False, 2, 1, None),
|
||||
],
|
||||
)
|
||||
def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, error):
|
||||
def test_hc_sp_model_support(enabled, supported, cuda_alike, tp_size, pp_size, error):
|
||||
"""Validate active HC SP configurations and ignore the flag with TP=1."""
|
||||
from vllm.model_executor.models.registry import ModelRegistry
|
||||
|
||||
@@ -4173,7 +4173,10 @@ def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, erro
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
with (
|
||||
patch("vllm.config.model.current_platform.is_cuda", return_value=is_cuda),
|
||||
patch(
|
||||
"vllm.config.model.current_platform.is_cuda_alike",
|
||||
return_value=cuda_alike,
|
||||
),
|
||||
patch.object(ModelRegistry, "is_hc_sp_supported_model", return_value=supported),
|
||||
patch.object(ModelRegistry, "is_pp_supported_model", return_value=True),
|
||||
):
|
||||
|
||||
@@ -1534,8 +1534,8 @@ class ModelConfig:
|
||||
parallel_config: ParallelConfig,
|
||||
) -> None:
|
||||
if parallel_config.enable_hc_sp and parallel_config.tensor_parallel_size > 1:
|
||||
if not current_platform.is_cuda():
|
||||
raise ValueError("--enable-hc-sp currently requires CUDA")
|
||||
if not current_platform.is_cuda_alike():
|
||||
raise ValueError("--enable-hc-sp currently requires CUDA or ROCm")
|
||||
if not self.registry.is_hc_sp_supported_model(self.architectures, self):
|
||||
raise ValueError("--enable-hc-sp is not supported for this model")
|
||||
if parallel_config.pipeline_parallel_size != 1:
|
||||
|
||||
@@ -184,7 +184,7 @@ class ParallelConfig:
|
||||
"""Whether the deployed model is MoE (if known)."""
|
||||
enable_hc_sp: bool = False
|
||||
"""Enable sequence parallelism for HyperConnection (HC) computation.
|
||||
Currently supported by Qwen4Exp on CUDA with TP>1 and PP=1 (ignored when TP=1).
|
||||
Currently supported by Qwen4Exp on CUDA/ROCm with TP>1 and PP=1 (ignored when TP=1).
|
||||
Does not change MoE parallelism. MoE SP may enable HC SP independently of this flag.
|
||||
"""
|
||||
enable_expert_parallel: bool = False
|
||||
|
||||
@@ -992,7 +992,7 @@ class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConf
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# MoE SP also enables HC SP without the explicit HC flag.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
current_platform.is_cuda_alike()
|
||||
and parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size != 1
|
||||
):
|
||||
|
||||
@@ -37,6 +37,7 @@ from vllm.model_executor.models.interfaces import (
|
||||
IsHybrid,
|
||||
MixtureOfExperts,
|
||||
MultiModalEmbeddings,
|
||||
SupportsHCSP,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsPP,
|
||||
@@ -67,6 +68,7 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.common.ops.sequence_parallel import sp_all_gather
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -74,8 +76,10 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .ops.sequence_parallel import sp_reduce_scatter, sp_shard
|
||||
from .ple_layer import Qwen4ExpPLELayer
|
||||
from .qsa import Qwen4ExpQSAAttention
|
||||
|
||||
@@ -161,13 +165,19 @@ _EXTRA_WEIGHTS_MAPPER = WeightsMapper(
|
||||
class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock):
|
||||
"""Qwen3Next MoE with Qwen4Exp HC validation."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if parallel_config.use_sequence_parallel_moe:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp HC does not support sequence-parallel MoE"
|
||||
)
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, prefix=prefix, reduce_results=reduce_results
|
||||
)
|
||||
if self.replicate_shared_expert and not self.is_sequence_parallel:
|
||||
# Reduce routed outputs before adding replicated shared outputs.
|
||||
# SP then slices the result to avoid summing shared outputs again via RS.
|
||||
self.experts.moe_config.skip_final_all_reduce = False
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.n_shared_experts = int(config.shared_expert_intermediate_size > 0)
|
||||
|
||||
@@ -185,11 +195,12 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
|
||||
self.config = config
|
||||
self.layer_type = layer_type
|
||||
# MoE SP also enables HC SP, while HC SP can be enabled independently.
|
||||
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
|
||||
self.layer_idx = extract_layer_index(prefix)
|
||||
if vllm_config.parallel_config.use_sequence_parallel_moe:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp HC does not support sequence-parallel MoE"
|
||||
)
|
||||
self.use_sequence_parallel = (
|
||||
vllm_config.parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
self.ple: Qwen4ExpPLELayer | None = None
|
||||
ple_layer_ids = config.ple_layer_ids
|
||||
if (self.layer_idx + 1) in ple_layer_ids:
|
||||
@@ -204,6 +215,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
layer_idx=self.layer_idx,
|
||||
ple_dense_layer_id=ple_dense_layer_id,
|
||||
prefix=f"{prefix}.ple",
|
||||
use_sequence_parallel=self.use_hc_sequence_parallel,
|
||||
)
|
||||
|
||||
if layer_type == "linear_attention":
|
||||
@@ -212,6 +224,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
elif layer_type in _QSA_LAYER_TYPES:
|
||||
use_qsa = getattr(config, "indexer_n_heads", None) is not None
|
||||
@@ -222,6 +235,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
else:
|
||||
self.self_attn = Qwen4ExpQSAAttention(
|
||||
@@ -230,12 +244,15 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
layer_id=self.layer_idx,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid layer_type {layer_type}")
|
||||
|
||||
self.mlp = Qwen4ExpSparseMoeBlock(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
|
||||
hc_config = HyperConnectionConfig(
|
||||
@@ -293,6 +310,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
else:
|
||||
hidden_states, block_input, injection = attn_hc.mix(hidden_states)
|
||||
|
||||
if self.use_hc_sequence_parallel:
|
||||
# Positions retain the full token axis for both text and MRoPE inputs.
|
||||
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
|
||||
if self.layer_type == "linear_attention":
|
||||
attn_out = self.linear_attn(hidden_states=block_input)
|
||||
elif self.layer_type in _QSA_LAYER_TYPES:
|
||||
@@ -304,10 +324,25 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
raise ValueError("Invalid layer_type")
|
||||
|
||||
mlp_hc = self.mlp_hyper_connection
|
||||
if self.use_hc_sequence_parallel:
|
||||
attn_out = sp_reduce_scatter(attn_out)
|
||||
hidden_states, block_input, injection = mlp_hc.combine_and_mix(
|
||||
hidden_states, attn_out, injection
|
||||
)
|
||||
mlp_out = self.mlp(block_input)
|
||||
if self.use_sequence_parallel:
|
||||
# MoE consumes local tokens and returns complete local outputs.
|
||||
mlp_out = self.mlp(block_input, already_sequence_parallel=True)
|
||||
else:
|
||||
if self.use_hc_sequence_parallel:
|
||||
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
|
||||
mlp_out = self.mlp(block_input)
|
||||
if self.use_hc_sequence_parallel:
|
||||
assert isinstance(self.mlp, Qwen4ExpSparseMoeBlock)
|
||||
reduce_output = self.mlp.experts.moe_config.skip_final_all_reduce
|
||||
# Only sum partial outputs. Reduced outputs already include all ranks.
|
||||
mlp_out = (
|
||||
sp_reduce_scatter(mlp_out) if reduce_output else sp_shard(mlp_out)
|
||||
)
|
||||
return hidden_states, mlp_out, injection
|
||||
|
||||
|
||||
@@ -382,6 +417,8 @@ class Qwen4ExpModel(nn.Module):
|
||||
self.num_redundant_experts = (
|
||||
vllm_config.parallel_config.eplb_config.num_redundant_experts
|
||||
)
|
||||
# MoE SP also enables HC SP, while HC SP can be enabled independently.
|
||||
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
|
||||
self.vocab_size = config.vocab_size
|
||||
self._qsa_layer_ids = frozenset(
|
||||
layer_idx
|
||||
@@ -470,12 +507,18 @@ class Qwen4ExpModel(nn.Module):
|
||||
if input_ids is None:
|
||||
raise ValueError("input_ids or inputs_embeds is required")
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
if self.use_hc_sequence_parallel:
|
||||
# The MoE padding mask stays unsharded: under torch.compile its
|
||||
# update would land only after the graph has run.
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
# Expand only the local token rows when SP is enabled.
|
||||
hidden_states = hidden_states.repeat(1, self.config.hc_count)
|
||||
else:
|
||||
if intermediate_tensors is None:
|
||||
raise ValueError("pipeline stage requires intermediate tensors")
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
full_num_tokens = positions.shape[-1]
|
||||
block_output = None
|
||||
injection = None
|
||||
last_layer = None
|
||||
@@ -498,6 +541,8 @@ class Qwen4ExpModel(nn.Module):
|
||||
deepstack_embed = deepstack_input_embeds[
|
||||
f"deepstack_input_embeds_{layer_idx}"
|
||||
]
|
||||
if self.use_hc_sequence_parallel:
|
||||
deepstack_embed = sp_shard(deepstack_embed)
|
||||
deepstack_embed = (
|
||||
deepstack_embed.unsqueeze(-2)
|
||||
.expand(
|
||||
@@ -532,6 +577,24 @@ class Qwen4ExpModel(nn.Module):
|
||||
multi_hidden, sample_hidden_states, _ = final_mixer.combine_and_mix(
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
if self.use_hc_sequence_parallel:
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
# Gather LM-head and MTP states together when both are needed.
|
||||
hidden_size = self.config.hidden_size
|
||||
packed_hidden_states = torch.cat(
|
||||
[sample_hidden_states, multi_hidden], dim=-1
|
||||
)
|
||||
packed_hidden_states = sp_all_gather(packed_hidden_states)[
|
||||
:full_num_tokens
|
||||
]
|
||||
sample_hidden_states, multi_hidden = packed_hidden_states.split(
|
||||
[hidden_size, self.config.hc_count * hidden_size], dim=-1
|
||||
)
|
||||
sample_hidden_states = sample_hidden_states.contiguous()
|
||||
else:
|
||||
sample_hidden_states = sp_all_gather(sample_hidden_states)[
|
||||
:full_num_tokens
|
||||
]
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
# Capture the pre-final-mixer multi-stream hidden state
|
||||
# [T, hc_count*H] for the MTP drafter (zero extra compute:
|
||||
@@ -587,6 +650,7 @@ class Qwen4ExpModel(nn.Module):
|
||||
class Qwen4ExpForCausalLM(
|
||||
nn.Module,
|
||||
HasInnerState,
|
||||
SupportsHCSP,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsPP,
|
||||
@@ -819,6 +883,7 @@ class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo):
|
||||
class Qwen4ExpForConditionalGeneration(
|
||||
Qwen3_5ForConditionalGeneration,
|
||||
HasInnerState,
|
||||
SupportsHCSP,
|
||||
Qwen4ExpMixtureOfExperts,
|
||||
):
|
||||
"""Qwen3-VL vision tower backed by the Qwen4Exp language model."""
|
||||
|
||||
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import configure_quant_config
|
||||
from vllm.model_executor.models.interfaces import SupportsPP
|
||||
from vllm.model_executor.models.interfaces import SupportsHCSP, SupportsPP
|
||||
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
@@ -45,8 +45,10 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.common.ops.sequence_parallel import sp_all_gather, sp_shard
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .model import (
|
||||
@@ -181,6 +183,8 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
config: Qwen4ExpTextConfig = model_config.hf_text_config
|
||||
|
||||
self.config = config
|
||||
# MoE SP also enables HC SP, while HC SP can be enabled independently.
|
||||
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
@@ -321,6 +325,9 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
full_num_tokens = positions.shape[-1]
|
||||
if self.use_hc_sequence_parallel:
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
layer = self.layers[current_step_idx]
|
||||
hidden_states, block_output, injection = layer(
|
||||
hidden_states=hidden_states,
|
||||
@@ -348,6 +355,17 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
)
|
||||
if self.use_hc_sequence_parallel:
|
||||
# Gather both outputs together, then restore their contiguous layouts.
|
||||
packed_hidden_states = torch.cat(
|
||||
[sample_hidden_states, multi_hidden], dim=-1
|
||||
)
|
||||
packed_hidden_states = sp_all_gather(packed_hidden_states)[:full_num_tokens]
|
||||
sample_hidden_states, multi_hidden = packed_hidden_states.split(
|
||||
[hidden_size, hc_count * hidden_size], dim=-1
|
||||
)
|
||||
sample_hidden_states = sample_hidden_states.contiguous()
|
||||
multi_hidden = multi_hidden.contiguous()
|
||||
return sample_hidden_states, multi_hidden
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -377,7 +395,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
||||
class Qwen4ExpMTP(nn.Module, SupportsHCSP, SupportsPP, Qwen4ExpMixtureOfExperts):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.models.common.ops import sequence_parallel as common_sp
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def _sp_shard_op(x: torch.Tensor) -> torch.Tensor:
|
||||
output = common_sp.sp_shard(x)
|
||||
# A functional custom op must not return a view of its input.
|
||||
unpadded = x.shape[0] % get_tensor_model_parallel_world_size() == 0
|
||||
return output.clone() if unpadded else output
|
||||
|
||||
|
||||
def _sp_local_fake(x: torch.Tensor) -> torch.Tensor:
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
return x.new_empty((cdiv(x.shape[0], tp_size), *x.shape[1:]))
|
||||
|
||||
|
||||
# Dynamo would specialize the padding branch on the traced token count, and
|
||||
# vLLM drops the resulting guards. Keep the branch opaque when compiling.
|
||||
direct_register_custom_op(
|
||||
op_name="sp_reduce_scatter",
|
||||
op_func=common_sp.sp_reduce_scatter,
|
||||
fake_impl=_sp_local_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="sp_shard",
|
||||
op_func=_sp_shard_op,
|
||||
fake_impl=_sp_local_fake,
|
||||
)
|
||||
|
||||
|
||||
def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor:
|
||||
if torch.compiler.is_compiling():
|
||||
return torch.ops.vllm.sp_reduce_scatter(x)
|
||||
return common_sp.sp_reduce_scatter(x)
|
||||
|
||||
|
||||
def sp_shard(x: torch.Tensor) -> torch.Tensor:
|
||||
if torch.compiler.is_compiling():
|
||||
return torch.ops.vllm.sp_shard(x)
|
||||
return common_sp.sp_shard(x)
|
||||
@@ -9,6 +9,7 @@ from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_etp_group, get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
@@ -21,7 +22,13 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
@@ -182,8 +189,10 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
data_parallel_rank: int = 0,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
use_sequence_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
self.embedding_dim = embedding_dim
|
||||
self.layer_name = layer_name
|
||||
self.ngram_size = int(config.ngram_size)
|
||||
@@ -253,6 +262,11 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
if engram_config is not None and engram_config.cpu_offload
|
||||
else Qwen4ExpPLEDeviceEmbedding
|
||||
)
|
||||
# A TP reduce-scatter covers all vocabulary owners only when ETP=TP.
|
||||
self.use_reduce_scatter = (
|
||||
self.use_sequence_parallel
|
||||
and get_etp_group().world_size == get_tp_group().world_size
|
||||
)
|
||||
self.ngram_embedding = embedding_cls(
|
||||
padded_vocab_size,
|
||||
self.head_dim,
|
||||
@@ -263,6 +277,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
num_ngram_heads=self.ngram_heads,
|
||||
max_total_tokens=max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
reduce_results=not self.use_reduce_scatter,
|
||||
)
|
||||
logger.info(
|
||||
"Initialized AMD PLE embedding %s: quantization_method=%s, "
|
||||
@@ -274,12 +289,25 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
self.ngram_embedding.weight.is_pinned(),
|
||||
)
|
||||
|
||||
def _sp_shard_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor:
|
||||
"""Shard embedding token rows, optionally reducing vocabulary contributions."""
|
||||
# Each embedding element has one vocabulary owner, including FP8 bytes.
|
||||
output_dtype = embeddings.dtype
|
||||
if is_fp8(embeddings):
|
||||
embeddings = embeddings.view(torch.int8)
|
||||
if self.use_reduce_scatter:
|
||||
embeddings = sp_reduce_scatter(embeddings)
|
||||
else:
|
||||
embeddings = sp_shard(embeddings)
|
||||
return embeddings.view(output_dtype)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Return full embeddings or a token shard after combining vocabulary owners."""
|
||||
input_ids = input_ids.reshape(-1)
|
||||
ngram_ids = input_ids.new_empty(
|
||||
(input_ids.shape[0], self.ngram_heads), dtype=torch.int64
|
||||
@@ -302,17 +330,19 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
output,
|
||||
self.layer_name,
|
||||
)
|
||||
else:
|
||||
output = ngram_ids.new_empty(
|
||||
(ngram_ids.shape[0], self.embedding_dim),
|
||||
dtype=embedding.params_dtype,
|
||||
)
|
||||
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding(
|
||||
ngram_ids,
|
||||
output,
|
||||
self.layer_name,
|
||||
)
|
||||
if not self.use_sequence_parallel:
|
||||
return output
|
||||
output = ngram_ids.new_empty(
|
||||
(ngram_ids.shape[0], self.embedding_dim),
|
||||
dtype=embedding.params_dtype,
|
||||
)
|
||||
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding(
|
||||
ngram_ids,
|
||||
output,
|
||||
self.layer_name,
|
||||
)
|
||||
return output
|
||||
return self._sp_shard_embeddings(output)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load hash buffers and checkpoint-split embedding rows."""
|
||||
@@ -388,8 +418,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
layer_idx: int = 0,
|
||||
ple_dense_layer_id: int | None = None,
|
||||
prefix: str = "",
|
||||
use_sequence_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
@@ -420,8 +452,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
)
|
||||
# The PLE cache is TP-replicated, so this merged projection is too.
|
||||
# Projection weights stay replicated even when their token rows are sharded.
|
||||
self.kv_proj = MergedColumnParallelLinear(
|
||||
int(config.ple_embed_dim),
|
||||
[self.hc_hidden_size, self.hidden_size],
|
||||
@@ -486,7 +519,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
self,
|
||||
inputs: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor | None,
|
||||
metadata: PleShortConvAttentionMetadata,
|
||||
conv_state: torch.Tensor,
|
||||
conv_weights: torch.Tensor,
|
||||
@@ -501,7 +534,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
has_non_spec = has_prefill or has_decode
|
||||
inputs = inputs[: metadata.num_actual_tokens]
|
||||
residual = residual[: metadata.num_actual_tokens]
|
||||
outer_residual = outer_residual[: metadata.num_actual_tokens]
|
||||
if outer_residual is not None:
|
||||
outer_residual = outer_residual[: metadata.num_actual_tokens]
|
||||
|
||||
spec_token_indices = None
|
||||
non_spec_token_indices = None
|
||||
@@ -553,11 +587,14 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
residual_d, residual_p = torch.split(
|
||||
residual, [num_decode_tokens, num_prefill_tokens], dim=0
|
||||
)
|
||||
outer_residual_d, outer_residual_p = torch.split(
|
||||
outer_residual,
|
||||
[num_decode_tokens, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
outer_residual_d: torch.Tensor | None = None
|
||||
outer_residual_p: torch.Tensor | None = None
|
||||
if outer_residual is not None:
|
||||
outer_residual_d, outer_residual_p = torch.split(
|
||||
outer_residual,
|
||||
[num_decode_tokens, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
token_indices_d = None
|
||||
token_indices_p = None
|
||||
else:
|
||||
@@ -628,14 +665,16 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
self,
|
||||
inputs: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor | None,
|
||||
) -> None:
|
||||
"""Update convolution state, optionally fusing both residual additions."""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
# Profiling omits all metadata or this Mamba entry. Short convolution
|
||||
# is a no-op there, but preserve the outer residual addition.
|
||||
if attn_metadata is None:
|
||||
residual.add_(outer_residual)
|
||||
if outer_residual is not None:
|
||||
residual.add_(outer_residual)
|
||||
return
|
||||
|
||||
if not isinstance(attn_metadata, dict):
|
||||
@@ -646,7 +685,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
|
||||
layer_attn_metadata = attn_metadata.get(self.prefix)
|
||||
if layer_attn_metadata is None:
|
||||
residual.add_(outer_residual)
|
||||
if outer_residual is not None:
|
||||
residual.add_(outer_residual)
|
||||
return
|
||||
if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata):
|
||||
raise TypeError(
|
||||
@@ -687,7 +727,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
input_ids = input_ids.reshape(-1)
|
||||
if input_ids.shape[0] != hidden_states.shape[0]:
|
||||
if (
|
||||
not self.use_sequence_parallel
|
||||
and input_ids.shape[0] != hidden_states.shape[0]
|
||||
):
|
||||
raise ValueError(
|
||||
"PLE expects input_ids and hidden_states to have the same "
|
||||
f"token length, got {input_ids.shape[0]} and "
|
||||
@@ -711,12 +754,23 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
# The short-conv op is a piecewise-graph splitting op: state routing
|
||||
# reads runtime request metadata. It accumulates the convolution and
|
||||
# the outer residual into gated_output.
|
||||
torch.ops.vllm.qwen4_exp_ple_short_conv(
|
||||
conv_input,
|
||||
gated_output,
|
||||
hidden_states,
|
||||
self.prefix,
|
||||
)
|
||||
if self.use_sequence_parallel:
|
||||
# Only convolution needs full tokens. Add residuals on local shards.
|
||||
conv_input = sp_all_gather(conv_input)[: input_ids.shape[0]]
|
||||
# Metadata may omit padding tokens or all tokens during profiling.
|
||||
conv_output = torch.zeros_like(conv_input)
|
||||
torch.ops.vllm.qwen4_exp_ple_short_conv(
|
||||
conv_input, conv_output, None, self.prefix
|
||||
)
|
||||
gated_output.add_(sp_shard(conv_output))
|
||||
gated_output.add_(hidden_states)
|
||||
else:
|
||||
torch.ops.vllm.qwen4_exp_ple_short_conv(
|
||||
conv_input,
|
||||
gated_output,
|
||||
hidden_states,
|
||||
self.prefix,
|
||||
)
|
||||
return gated_output
|
||||
|
||||
|
||||
@@ -787,7 +841,7 @@ def qwen4_exp_amd_ple_ngram_embedding_pinned(
|
||||
def qwen4_exp_ple_short_conv(
|
||||
inputs: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor | None,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
|
||||
@@ -35,6 +35,16 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
def is_hc_sequence_parallel_enabled(vllm_config: VllmConfig) -> bool:
|
||||
"""Enable GR/PLE SP explicitly or automatically with sequence-parallel MoE."""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
return parallel_config.tensor_parallel_size > 1 and (
|
||||
parallel_config.enable_hc_sp or parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
|
||||
@@ -711,7 +711,11 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0])
|
||||
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
|
||||
embeddings = self._lookup(gathered_ids)
|
||||
embeddings = self._reduce_etp_embeddings(embeddings)
|
||||
if self.reduce_results:
|
||||
embeddings = self._reduce_etp_embeddings(embeddings)
|
||||
else:
|
||||
# The outer TP reduce-scatter requires ETP=TP.
|
||||
assert self.etp_data_parallel_size == 1
|
||||
return self._select_embeddings(
|
||||
embeddings,
|
||||
ngram_ids.shape[0],
|
||||
|
||||
@@ -82,6 +82,7 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
|
||||
@@ -93,14 +94,6 @@ from .qsa import Qwen4ExpQSAAttention
|
||||
_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention")
|
||||
|
||||
|
||||
def is_hc_sequence_parallel_enabled(vllm_config: VllmConfig) -> bool:
|
||||
"""Enable GR/PLE SP explicitly or automatically with sequence-parallel MoE."""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
return parallel_config.tensor_parallel_size > 1 and (
|
||||
parallel_config.enable_hc_sp or parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
|
||||
|
||||
def without_modelopt_fp4(
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> QuantizationConfig | None:
|
||||
|
||||
@@ -53,6 +53,7 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .model import (
|
||||
@@ -61,7 +62,6 @@ from .model import (
|
||||
Qwen4ExpDecoderLayer,
|
||||
Qwen4ExpMixtureOfExperts,
|
||||
Qwen4ExpSparseMoeBlock,
|
||||
is_hc_sequence_parallel_enabled,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user