[Qwen4Exp] support Sequence Parallelism for Qwen3.8-Flash-Next (#56322)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Huang
2026-10-12 20:56:44 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 258b97ff6b
commit c75cb63cd2
23 changed files with 870 additions and 178 deletions
+34 -22
View File
@@ -70,6 +70,7 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
num_ngram_heads: int = 1,
max_total_tokens: int = 0,
data_parallel_rank: int = 0,
reduce_results: bool = True,
) -> None:
del num_ngram_heads, max_total_tokens
super().__init__(
@@ -80,6 +81,7 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
prefix=prefix,
quant_method=embedding_method,
parallel_group=get_etp_group(),
reduce_results=reduce_results,
)
self.embedding_method = embedding_method
self.data_parallel_rank = data_parallel_rank
@@ -153,14 +155,15 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
return embeddings.narrow(0, slot_offset, local_num_tokens)
@abstractmethod
def start_prefetch(
self,
hidden_states: torch.Tensor,
ngram_ids: torch.Tensor,
) -> None:
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
"""Start an asynchronous lookup when supported."""
raise NotImplementedError
@abstractmethod
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
"""Return prefetched embeddings for the full input token set."""
raise NotImplementedError
class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
"""Quantization interface shared by resident and pinned PLE tables."""
@@ -351,16 +354,16 @@ class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding):
"""Allocate the complete PLE weight on the active device."""
return torch.empty(num_embeddings, embedding_dim, dtype=dtype)
def start_prefetch(
self,
hidden_states: torch.Tensor,
ngram_ids: torch.Tensor,
) -> None:
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
"""Resident embedding prefetch is a no-op."""
return None
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
"""Reject prefetch finalization for resident embeddings."""
raise RuntimeError("Device PLE embedding does not support prefetch")
def forward(self, ngram_ids: torch.Tensor) -> torch.Tensor:
"""Gather ETP inputs, look up embeddings, and select local rows."""
"""Gather ETP IDs, optionally reduce lookup results, and select local rows."""
slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0])
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
embeddings = super().forward(gathered_ids)
@@ -422,6 +425,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
num_ngram_heads: int = 1,
max_total_tokens: int = 0,
data_parallel_rank: int = 0,
reduce_results: bool = True,
) -> None:
if not is_uva_available():
raise RuntimeError("Engram CPU offload requires UVA support")
@@ -435,6 +439,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
num_ngram_heads=num_ngram_heads,
max_total_tokens=max_total_tokens,
data_parallel_rank=data_parallel_rank,
reduce_results=reduce_results,
)
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
self._row_bytes = self.embedding_dim * self.weight.element_size()
@@ -521,11 +526,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
return self.parallel_group.all_reduce(embeddings)
@eager_break_during_capture
def start_prefetch(
self,
hidden_states: torch.Tensor,
ngram_ids: torch.Tensor,
) -> None:
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
"""Gather ETP IDs and launch their UVA lookup on the side stream."""
buffer = self._prefetch_buffer
if buffer is None:
@@ -574,14 +575,18 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
prefetch_output: torch.Tensor,
output: torch.Tensor,
) -> None:
"""Join the side stream, reduce ETP shards, and select local rows."""
"""Join the side stream, optionally reduce ETP shards, and select rows."""
prefetch_stream = self._prefetch_stream
if prefetch_stream is None:
raise RuntimeError("pinned PLE finalize requires a prior start_prefetch")
torch.cuda.current_stream().wait_stream(prefetch_stream)
slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0])
active_output = prefetch_output[: slot_size * self.etp_data_parallel_size]
embeddings = self._reduce_etp_embeddings(active_output)
embeddings = prefetch_output[: slot_size * self.etp_data_parallel_size]
if self.reduce_results:
embeddings = self._reduce_etp_embeddings(embeddings)
else:
# The outer TP reduce-scatter requires ETP=TP.
assert self.etp_data_parallel_size == 1
embeddings = self._select_embeddings(
embeddings,
output.shape[0],
@@ -589,11 +594,18 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
)
output.copy_(embeddings.flatten(-2))
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Finish the pinned lookup into graph-owned output storage."""
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
"""Finalize the full-token pinned lookup with optional ETP reduction."""
buffer = self._prefetch_buffer
if buffer is None:
raise RuntimeError("pinned PLE lookup requires a prior start_prefetch")
output = buffer.new_empty((hidden_states.shape[0], self._output_dim))
# total_tokens counts this DP rank's tokens before SP sharding.
output = buffer.new_empty((total_tokens, self._output_dim))
self._finalize_prefetch(buffer, output)
return output
def forward(self, input_: torch.Tensor) -> torch.Tensor:
raise RuntimeError(
"Pinned-host PLE embedding requires "
"start_prefetch() and finalize_prefetch()."
)
+23 -17
View File
@@ -344,6 +344,7 @@ def _ple_conv_kernel(
MODE: tl.constexpr,
HAS_INIT: tl.constexpr,
NULL_STATE_ID: tl.constexpr,
FUSE_RESIDUAL: tl.constexpr,
launch_pdl: tl.constexpr,
):
t = tl.program_id(0)
@@ -438,21 +439,23 @@ def _ple_conv_kernel(
conv = acc.to(residual_ptr.dtype.element_ty).to(tl.float32)
y = conv * tl.sigmoid(conv)
conv_output = tl.where(out_ok, y, 0.0).to(residual_ptr.dtype.element_ty)
residual = tl.load(
residual_ptr + output_t * C + c_offs,
mask=c_mask,
other=0.0,
)
# Preserve the original eager operation boundaries: short convolution is
# first accumulated into the BF16/FP16 PLE output, then the outer residual
# is added to that rounded value.
ple_output = (residual + conv_output).to(residual_ptr.dtype.element_ty)
outer_residual = tl.load(
outer_residual_ptr + output_t * C + c_offs,
mask=c_mask,
other=0.0,
)
ple_output = outer_residual.to(tl.float32) + ple_output.to(tl.float32)
ple_output = conv_output
if FUSE_RESIDUAL:
residual = tl.load(
residual_ptr + output_t * C + c_offs,
mask=c_mask,
other=0.0,
)
# Preserve the original eager operation boundaries: short convolution is
# first accumulated into the BF16/FP16 PLE output, then the outer residual
# is added to that rounded value.
ple_output = (residual + conv_output).to(residual_ptr.dtype.element_ty)
outer_residual = tl.load(
outer_residual_ptr + output_t * C + c_offs,
mask=c_mask,
other=0.0,
)
ple_output = outer_residual.to(tl.float32) + ple_output.to(tl.float32)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(
@@ -575,7 +578,7 @@ def ple_conv(
conv_state: torch.Tensor,
conv_weights: torch.Tensor,
state_indices: torch.Tensor,
outer_residual: torch.Tensor,
outer_residual: torch.Tensor | None,
*,
mode: Literal["decode", "spec", "prefill"],
dilation: int,
@@ -585,7 +588,9 @@ def ple_conv(
spec_query_len: int = 1,
token_indices: torch.Tensor | None = None,
) -> None:
"""Add short convolution and the outer residual; update state."""
"""Update state and write activated convolution into residual.
A non-None outer_residual enables both fused residual additions.
"""
BLOCK_C = 512
kernel_spec_query_len = spec_query_len if mode == "spec" else 1
T, C = inputs.shape
@@ -657,6 +662,7 @@ def ple_conv(
SPEC_QUERY_LEN=kernel_spec_query_len,
MODE=mode,
HAS_INIT=has_initial_states_arg,
FUSE_RESIDUAL=outer_residual is not None,
NULL_STATE_ID=NULL_BLOCK_ID,
launch_pdl=launch_pdl,
num_warps=num_warps,
+102 -22
View File
@@ -9,8 +9,10 @@ import torch
from torch import nn
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
from vllm import envs
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
@@ -36,6 +38,7 @@ from vllm.model_executor.models.interfaces import (
IsHybrid,
MixtureOfExperts,
MultiModalEmbeddings,
SupportsHCSP,
SupportsLoRA,
SupportsMRoPE,
SupportsPP,
@@ -66,6 +69,12 @@ from vllm.model_executor.models.utils import (
maybe_fuse_shared_experts,
maybe_prefix,
)
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_reduce_scatter,
sp_shard,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
@@ -84,6 +93,14 @@ from .qsa import Qwen4ExpQSAAttention
_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention")
def is_hc_sequence_parallel_enabled(vllm_config: VllmConfig) -> bool:
"""Enable GR/PLE SP explicitly or automatically with sequence-parallel MoE."""
parallel_config = vllm_config.parallel_config
return parallel_config.tensor_parallel_size > 1 and (
parallel_config.enable_hc_sp or parallel_config.use_sequence_parallel_moe
)
def without_modelopt_fp4(
quant_config: QuantizationConfig | None,
) -> QuantizationConfig | None:
@@ -161,13 +178,19 @@ _EXTRA_WEIGHTS_MAPPER = WeightsMapper(
class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock):
"""Qwen3Next MoE with Qwen4Exp HC validation."""
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
parallel_config = vllm_config.parallel_config
if parallel_config.use_sequence_parallel_moe:
raise NotImplementedError(
"Qwen4Exp HC does not support sequence-parallel MoE"
)
super().__init__(vllm_config=vllm_config, prefix=prefix)
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
reduce_results: bool = True,
) -> None:
super().__init__(
vllm_config=vllm_config, prefix=prefix, reduce_results=reduce_results
)
if self.replicate_shared_expert and not self.is_sequence_parallel:
# Reduce routed outputs before adding replicated shared outputs.
# SP then slices the result to avoid summing shared outputs again via RS.
self.experts.moe_config.skip_final_all_reduce = False
config = vllm_config.model_config.hf_text_config
self.n_shared_experts = int(config.shared_expert_intermediate_size > 0)
@@ -185,11 +208,12 @@ class Qwen4ExpDecoderLayer(nn.Module):
self.config = config
self.layer_type = layer_type
# MoE SP also enables HC SP, while HC SP can be enabled independently.
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
self.layer_idx = extract_layer_index(prefix)
if vllm_config.parallel_config.use_sequence_parallel_moe:
raise NotImplementedError(
"Qwen4Exp HC does not support sequence-parallel MoE"
)
self.use_sequence_parallel = (
vllm_config.parallel_config.use_sequence_parallel_moe
)
self.ple: Qwen4ExpPLELayer | None = None
ple_layer_ids = config.ple_layer_ids
if (self.layer_idx + 1) in ple_layer_ids:
@@ -204,6 +228,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
layer_idx=self.layer_idx,
ple_dense_layer_id=ple_dense_layer_id,
prefix=f"{prefix}.ple",
use_sequence_parallel=self.use_hc_sequence_parallel,
)
if layer_type == "linear_attention":
@@ -212,6 +237,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
vllm_config=vllm_config,
prefix=f"{prefix}.linear_attn",
gqa_interleaved_layout=False,
reduce_results=not self.use_hc_sequence_parallel,
)
elif layer_type in _QSA_LAYER_TYPES:
use_qsa = getattr(config, "indexer_n_heads", None) is not None
@@ -222,6 +248,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
cache_config=vllm_config.cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
reduce_results=not self.use_hc_sequence_parallel,
)
else:
self.self_attn = Qwen4ExpQSAAttention(
@@ -230,12 +257,15 @@ class Qwen4ExpDecoderLayer(nn.Module):
layer_id=self.layer_idx,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
reduce_results=not self.use_hc_sequence_parallel,
)
else:
raise ValueError(f"Invalid layer_type {layer_type}")
self.mlp = Qwen4ExpSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
vllm_config=vllm_config,
prefix=f"{prefix}.mlp",
reduce_results=not self.use_hc_sequence_parallel,
)
hc_config = HyperConnectionConfig(
@@ -295,6 +325,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
else:
hidden_states, block_input, injection = attn_hc.mix(hidden_states)
if self.use_hc_sequence_parallel:
# Positions retain the full token axis for both text and MRoPE inputs.
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
if self.layer_type == "linear_attention":
attn_out = self.linear_attn(hidden_states=block_input)
elif self.layer_type in _QSA_LAYER_TYPES:
@@ -306,10 +339,25 @@ class Qwen4ExpDecoderLayer(nn.Module):
raise ValueError("Invalid layer_type")
mlp_hc = self.mlp_hyper_connection
if self.use_hc_sequence_parallel:
attn_out = sp_reduce_scatter(attn_out)
hidden_states, block_input, injection = mlp_hc.combine_and_mix(
hidden_states, attn_out, injection
)
mlp_out = self.mlp(block_input)
if self.use_sequence_parallel:
# MoE consumes local tokens and returns complete local outputs.
mlp_out = self.mlp(block_input, already_sequence_parallel=True)
else:
if self.use_hc_sequence_parallel:
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
mlp_out = self.mlp(block_input)
if self.use_hc_sequence_parallel:
assert isinstance(self.mlp, Qwen4ExpSparseMoeBlock)
reduce_output = self.mlp.experts.moe_config.skip_final_all_reduce
# Only sum partial outputs. Reduced outputs already include all ranks.
mlp_out = (
sp_reduce_scatter(mlp_out) if reduce_output else sp_shard(mlp_out)
)
return hidden_states, mlp_out, injection
@@ -373,6 +421,11 @@ class Qwen4ExpModel(nn.Module):
self.num_redundant_experts = (
vllm_config.parallel_config.eplb_config.num_redundant_experts
)
# MoE SP also enables HC SP, while HC SP can be enabled independently.
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
self.use_sequence_parallel = (
vllm_config.parallel_config.use_sequence_parallel_moe
)
self.vocab_size = config.vocab_size
self._qsa_layer_ids = frozenset(
layer_idx
@@ -447,7 +500,6 @@ class Qwen4ExpModel(nn.Module):
@staticmethod
def _start_layer_ple_prefetch(
layer: nn.Module,
hidden_states: torch.Tensor,
input_ids: torch.Tensor | None,
query_start_loc: torch.Tensor | None,
ngram_context: torch.Tensor | None,
@@ -458,12 +510,7 @@ class Qwen4ExpModel(nn.Module):
return
if input_ids is None or query_start_loc is None or ngram_context is None:
raise RuntimeError("PLE inputs were not prepared")
ple.start_prefetch(
hidden_states,
input_ids,
query_start_loc,
ngram_context,
)
ple.start_prefetch(input_ids, query_start_loc, ngram_context)
def forward(
self,
@@ -482,19 +529,31 @@ class Qwen4ExpModel(nn.Module):
if input_ids is None:
raise ValueError("input_ids or inputs_embeds is required")
hidden_states = self.embed_input_ids(input_ids)
if self.use_hc_sequence_parallel:
if (
self.use_sequence_parallel
and envs.VLLM_MOE_SKIP_PADDING
and is_forward_context_available()
):
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, hidden_states
)
hidden_states = sp_shard(hidden_states)
# Expand only the local token rows when SP is enabled.
hidden_states = hidden_states.repeat(1, self.config.hc_count)
else:
if intermediate_tensors is None:
raise ValueError("pipeline stage requires intermediate tensors")
hidden_states = intermediate_tensors["hidden_states"]
full_num_tokens = positions.shape[-1]
block_output = None
injection = None
last_layer = None
if self.start_layer < self.end_layer:
self._start_layer_ple_prefetch(
self.layers[self.start_layer],
hidden_states,
input_ids,
query_start_loc,
ngram_context,
@@ -506,7 +565,6 @@ class Qwen4ExpModel(nn.Module):
if layer_idx + 1 < self.end_layer:
self._start_layer_ple_prefetch(
self.layers[layer_idx + 1],
hidden_states,
input_ids,
query_start_loc,
ngram_context,
@@ -526,6 +584,8 @@ class Qwen4ExpModel(nn.Module):
deepstack_embed = deepstack_input_embeds[
f"deepstack_input_embeds_{layer_idx}"
]
if self.use_hc_sequence_parallel:
deepstack_embed = sp_shard(deepstack_embed)
deepstack_embed = (
deepstack_embed.unsqueeze(-2)
.expand(
@@ -560,6 +620,24 @@ class Qwen4ExpModel(nn.Module):
multi_hidden, sample_hidden_states, _ = final_mixer.combine_and_mix(
hidden_states, block_output, injection
)
if self.use_hc_sequence_parallel:
if self._mtp_hidden_buffer is not None:
# Gather LM-head and MTP states together when both are needed.
hidden_size = self.config.hidden_size
packed_hidden_states = torch.cat(
[sample_hidden_states, multi_hidden], dim=-1
)
packed_hidden_states = sp_all_gather(packed_hidden_states)[
:full_num_tokens
]
sample_hidden_states, multi_hidden = packed_hidden_states.split(
[hidden_size, self.config.hc_count * hidden_size], dim=-1
)
sample_hidden_states = sample_hidden_states.contiguous()
else:
sample_hidden_states = sp_all_gather(sample_hidden_states)[
:full_num_tokens
]
if self._mtp_hidden_buffer is not None:
# Capture the pre-final-mixer multi-stream hidden state
# [T, hc_count*H] for the MTP drafter (zero extra compute:
@@ -615,6 +693,7 @@ class Qwen4ExpModel(nn.Module):
class Qwen4ExpForCausalLM(
nn.Module,
HasInnerState,
SupportsHCSP,
SupportsLoRA,
SupportsMRoPE,
SupportsPP,
@@ -865,6 +944,7 @@ class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo):
class Qwen4ExpForConditionalGeneration(
Qwen3_5ForConditionalGeneration,
HasInnerState,
SupportsHCSP,
Qwen4ExpMixtureOfExperts,
):
"""Qwen3-VL vision tower backed by the Qwen4Exp language model."""
+42 -2
View File
@@ -20,8 +20,10 @@ import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm import envs
from vllm.config import VllmConfig, replace, set_current_vllm_config
from vllm.distributed import get_pp_group
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
@@ -33,7 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.utils import configure_quant_config
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.interfaces import SupportsHCSP, SupportsPP
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
@@ -44,6 +46,11 @@ from vllm.model_executor.models.utils import (
maybe_fuse_shared_experts,
maybe_prefix,
)
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from vllm.sequence import IntermediateTensors
from .hyperconnection import GatedResidual, HyperConnectionConfig
@@ -54,6 +61,7 @@ from .model import (
Qwen4ExpDecoderLayer,
Qwen4ExpMixtureOfExperts,
Qwen4ExpSparseMoeBlock,
is_hc_sequence_parallel_enabled,
)
@@ -171,6 +179,11 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
config: Qwen4ExpTextConfig = model_config.hf_text_config
self.config = config
# MoE SP also enables HC SP, while HC SP can be enabled independently.
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
self.use_sequence_parallel = (
vllm_config.parallel_config.use_sequence_parallel_moe
)
self.vocab_size = config.vocab_size
self.mtp_start_layer_idx = config.num_hidden_layers
@@ -310,6 +323,22 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
hidden_states = intermediate_tensors["hidden_states"]
current_step_idx = spec_step_idx % self.num_mtp_layers
full_num_tokens = positions.shape[-1]
if self.use_hc_sequence_parallel:
# SP requires PP=1, so the first-rank branch sets prev_block_output.
assert prev_block_output is not None
if (
self.use_sequence_parallel
and envs.VLLM_MOE_SKIP_PADDING
and is_forward_context_available()
):
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, hidden_states
)
hidden_states = sp_shard(hidden_states)
# The FC output is already complete and needs only a slice.
prev_block_output = sp_shard(prev_block_output)
layer = self.layers[current_step_idx]
hidden_states, block_output, injection = layer(
hidden_states=hidden_states,
@@ -337,6 +366,17 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
hidden_states, block_output, injection
)
)
if self.use_hc_sequence_parallel:
# Gather both outputs together, then restore their contiguous layouts.
packed_hidden_states = torch.cat(
[sample_hidden_states, multi_hidden], dim=-1
)
packed_hidden_states = sp_all_gather(packed_hidden_states)[:full_num_tokens]
sample_hidden_states, multi_hidden = packed_hidden_states.split(
[hidden_size, hc_count * hidden_size], dim=-1
)
sample_hidden_states = sample_hidden_states.contiguous()
multi_hidden = multi_hidden.contiguous()
return sample_hidden_states, multi_hidden
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -357,7 +397,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
return loader.load_weights(weights, mapper=mapper)
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
class Qwen4ExpMTP(nn.Module, SupportsHCSP, SupportsPP, Qwen4ExpMixtureOfExperts):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
+35 -10
View File
@@ -9,11 +9,14 @@ from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import get_current_vllm_config
from vllm.distributed import get_etp_group, get_tp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter, sp_shard
from ..common.ngram_embedding import (
Qwen4ExpPLEDeviceEmbedding,
@@ -146,8 +149,10 @@ class Qwen4ExpNGramEmbedding(nn.Module):
prefix: str,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
use_sequence_parallel: bool = False,
) -> None:
super().__init__()
self.use_sequence_parallel = use_sequence_parallel
self.embedding_dim = embedding_dim
self.ngram_size = int(config.ngram_size)
self.heads_per_ngram = int(config.heads_per_ngram)
@@ -211,6 +216,11 @@ class Qwen4ExpNGramEmbedding(nn.Module):
if engram_config is not None and engram_config.cpu_offload
else Qwen4ExpPLEDeviceEmbedding
)
# A TP reduce-scatter covers all vocabulary owners only when ETP=TP.
self.use_reduce_scatter = (
self.use_sequence_parallel
and get_etp_group().world_size == get_tp_group().world_size
)
self.ngram_embedding = embedding_cls(
padded_vocab_size,
self.head_dim,
@@ -221,6 +231,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
num_ngram_heads=self.ngram_heads,
max_total_tokens=max_total_tokens,
data_parallel_rank=data_parallel_rank,
reduce_results=not self.use_reduce_scatter,
)
if self.ngram_embedding.supports_prefetch:
# The side-stream lookup outlives eager-break args, whose
@@ -348,6 +359,18 @@ class Qwen4ExpNGramEmbedding(nn.Module):
id_blocks.append(ids[request_indices, adjusted_columns])
return torch.cat(id_blocks, dim=-1)
def _sp_shard_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor:
"""Shard embedding token rows, optionally reducing vocabulary contributions."""
# Each embedding element has one vocabulary owner, including FP8 bytes.
output_dtype = embeddings.dtype
if is_fp8(embeddings):
embeddings = embeddings.view(torch.int8)
if self.use_reduce_scatter:
embeddings = sp_reduce_scatter(embeddings)
else:
embeddings = sp_shard(embeddings)
return embeddings.view(output_dtype)
def forward(
self,
hidden_states: torch.Tensor,
@@ -355,15 +378,22 @@ class Qwen4ExpNGramEmbedding(nn.Module):
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
"""Return full embeddings or a token shard after combining vocabulary owners."""
embedding = self.ngram_embedding
if embedding.supports_prefetch:
return embedding(hidden_states)
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
return self.ngram_embedding(ngram_ids).flatten(-2)
# Prefetch contains full tokens even when hidden states are sharded.
embeddings = embedding.finalize_prefetch(total_tokens=input_ids.shape[0])
else:
ngram_ids = self.compute_ngram_ids(
input_ids, query_start_loc, ngram_context
)
embeddings = embedding(ngram_ids).flatten(-2)
if not self.use_sequence_parallel:
return embeddings
return self._sp_shard_embeddings(embeddings)
def start_prefetch(
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
@@ -378,7 +408,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
ngram_context,
output=self._prefetch_ids[: input_ids.numel()],
)
embedding.start_prefetch(hidden_states, ngram_ids)
embedding.start_prefetch(ngram_ids)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load hash buffers and checkpoint-split embedding rows."""
@@ -444,8 +474,3 @@ class Qwen4ExpNGramEmbedding(nn.Module):
if regular_weights:
loaded.update(AutoWeightsLoader(self).load_weights(regular_weights))
return loaded
__all__ = [
"Qwen4ExpNGramEmbedding",
]
+38 -22
View File
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.models.common.ops.sequence_parallel import sp_all_gather, sp_shard
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionBackend,
@@ -69,8 +70,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
layer_idx: int = 0,
ple_dense_layer_id: int | None = None,
prefix: str = "",
use_sequence_parallel: bool = False,
) -> None:
super().__init__()
self.use_sequence_parallel = use_sequence_parallel
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
@@ -100,8 +103,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
prefix=f"{prefix}.ple_embedding",
quant_config=quant_config,
params_dtype=model_config.dtype,
use_sequence_parallel=self.use_sequence_parallel,
)
# The PLE cache is TP-replicated, so this merged projection is too.
# Projection weights stay replicated even when their token rows are sharded.
self.kv_proj = MergedColumnParallelLinear(
int(config.ple_embed_dim),
[self.hc_hidden_size, self.hidden_size],
@@ -151,18 +155,12 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
def start_prefetch(
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> None:
"""Start the pinned PLE lookup while the preceding decoder layer runs."""
self.ple_embedding.start_prefetch(
hidden_states,
input_ids,
query_start_loc,
ngram_context,
)
self.ple_embedding.start_prefetch(input_ids, query_start_loc, ngram_context)
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
@@ -192,7 +190,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
self,
inputs: torch.Tensor,
residual: torch.Tensor,
outer_residual: torch.Tensor,
outer_residual: torch.Tensor | None,
metadata: PleShortConvAttentionMetadata,
conv_state: torch.Tensor,
conv_weights: torch.Tensor,
@@ -207,7 +205,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
has_non_spec = has_prefill or has_decode
inputs = inputs[: metadata.num_actual_tokens]
residual = residual[: metadata.num_actual_tokens]
outer_residual = outer_residual[: metadata.num_actual_tokens]
if outer_residual is not None:
outer_residual = outer_residual[: metadata.num_actual_tokens]
spec_token_indices = None
non_spec_token_indices = None
@@ -259,11 +258,14 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
residual_d, residual_p = torch.split(
residual, [num_decode_tokens, num_prefill_tokens], dim=0
)
outer_residual_d, outer_residual_p = torch.split(
outer_residual,
[num_decode_tokens, num_prefill_tokens],
dim=0,
)
outer_residual_d: torch.Tensor | None = None
outer_residual_p: torch.Tensor | None = None
if outer_residual is not None:
outer_residual_d, outer_residual_p = torch.split(
outer_residual,
[num_decode_tokens, num_prefill_tokens],
dim=0,
)
token_indices_d = None
token_indices_p = None
else:
@@ -336,14 +338,16 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
self,
inputs: torch.Tensor,
residual: torch.Tensor,
outer_residual: torch.Tensor,
outer_residual: torch.Tensor | None,
) -> None:
"""Update convolution state, optionally fusing both residual additions."""
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
# Profiling omits all metadata or this Mamba entry. Short convolution
# is a no-op there, but preserve the outer residual addition.
if attn_metadata is None:
residual.add_(outer_residual)
if outer_residual is not None:
residual.add_(outer_residual)
return
if not isinstance(attn_metadata, dict):
@@ -354,7 +358,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
layer_attn_metadata = attn_metadata.get(self.prefix)
if layer_attn_metadata is None:
residual.add_(outer_residual)
if outer_residual is not None:
residual.add_(outer_residual)
return
if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata):
raise TypeError(
@@ -396,7 +401,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
ngram_context: torch.Tensor,
) -> torch.Tensor:
input_ids = input_ids.reshape(-1)
if input_ids.shape[0] != hidden_states.shape[0]:
if (
not self.use_sequence_parallel
and input_ids.shape[0] != hidden_states.shape[0]
):
raise ValueError(
"PLE expects input_ids and hidden_states to have the same "
f"token length, got {input_ids.shape[0]} and "
@@ -408,6 +416,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
query_start_loc,
ngram_context,
)
assert embeddings.shape[0] == hidden_states.shape[0]
embeddings = self._dequantize_embeddings(embeddings, hidden_states.dtype)
kv, _ = self.kv_proj(embeddings)
key, value = kv.split(self.kv_proj.output_sizes, dim=-1)
@@ -420,9 +429,16 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
self.norm_conv.weight,
self.norm_key.eps,
)
# State routing depends on runtime request metadata and remains outside
# the graph; short convolution accumulates into gated_output.
self._short_conv(conv_input, gated_output, hidden_states)
if self.use_sequence_parallel:
# Only convolution needs full tokens. Add residuals on local shards.
conv_input = sp_all_gather(conv_input)[: input_ids.shape[0]]
# Metadata may omit padding tokens or all tokens during profiling.
conv_output = torch.zeros_like(conv_input)
self._short_conv(conv_input, conv_output, outer_residual=None)
gated_output.add_(sp_shard(conv_output))
gated_output.add_(hidden_states)
else:
self._short_conv(conv_input, gated_output, hidden_states)
return gated_output