[Qwen4Exp] support Sequence Parallelism for Qwen3.8-Flash-Next (#56322)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
258b97ff6b
commit
c75cb63cd2
@@ -70,6 +70,7 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
|
||||
num_ngram_heads: int = 1,
|
||||
max_total_tokens: int = 0,
|
||||
data_parallel_rank: int = 0,
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
del num_ngram_heads, max_total_tokens
|
||||
super().__init__(
|
||||
@@ -80,6 +81,7 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
|
||||
prefix=prefix,
|
||||
quant_method=embedding_method,
|
||||
parallel_group=get_etp_group(),
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
self.embedding_method = embedding_method
|
||||
self.data_parallel_rank = data_parallel_rank
|
||||
@@ -153,14 +155,15 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
|
||||
return embeddings.narrow(0, slot_offset, local_num_tokens)
|
||||
|
||||
@abstractmethod
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
ngram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
|
||||
"""Start an asynchronous lookup when supported."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
|
||||
"""Return prefetched embeddings for the full input token set."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
||||
"""Quantization interface shared by resident and pinned PLE tables."""
|
||||
@@ -351,16 +354,16 @@ class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding):
|
||||
"""Allocate the complete PLE weight on the active device."""
|
||||
return torch.empty(num_embeddings, embedding_dim, dtype=dtype)
|
||||
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
ngram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
|
||||
"""Resident embedding prefetch is a no-op."""
|
||||
return None
|
||||
|
||||
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
|
||||
"""Reject prefetch finalization for resident embeddings."""
|
||||
raise RuntimeError("Device PLE embedding does not support prefetch")
|
||||
|
||||
def forward(self, ngram_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Gather ETP inputs, look up embeddings, and select local rows."""
|
||||
"""Gather ETP IDs, optionally reduce lookup results, and select local rows."""
|
||||
slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0])
|
||||
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
|
||||
embeddings = super().forward(gathered_ids)
|
||||
@@ -422,6 +425,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
num_ngram_heads: int = 1,
|
||||
max_total_tokens: int = 0,
|
||||
data_parallel_rank: int = 0,
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
if not is_uva_available():
|
||||
raise RuntimeError("Engram CPU offload requires UVA support")
|
||||
@@ -435,6 +439,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
num_ngram_heads=num_ngram_heads,
|
||||
max_total_tokens=max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
|
||||
self._row_bytes = self.embedding_dim * self.weight.element_size()
|
||||
@@ -521,11 +526,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
return self.parallel_group.all_reduce(embeddings)
|
||||
|
||||
@eager_break_during_capture
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
ngram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
|
||||
"""Gather ETP IDs and launch their UVA lookup on the side stream."""
|
||||
buffer = self._prefetch_buffer
|
||||
if buffer is None:
|
||||
@@ -574,14 +575,18 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
prefetch_output: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
"""Join the side stream, reduce ETP shards, and select local rows."""
|
||||
"""Join the side stream, optionally reduce ETP shards, and select rows."""
|
||||
prefetch_stream = self._prefetch_stream
|
||||
if prefetch_stream is None:
|
||||
raise RuntimeError("pinned PLE finalize requires a prior start_prefetch")
|
||||
torch.cuda.current_stream().wait_stream(prefetch_stream)
|
||||
slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0])
|
||||
active_output = prefetch_output[: slot_size * self.etp_data_parallel_size]
|
||||
embeddings = self._reduce_etp_embeddings(active_output)
|
||||
embeddings = prefetch_output[: slot_size * self.etp_data_parallel_size]
|
||||
if self.reduce_results:
|
||||
embeddings = self._reduce_etp_embeddings(embeddings)
|
||||
else:
|
||||
# The outer TP reduce-scatter requires ETP=TP.
|
||||
assert self.etp_data_parallel_size == 1
|
||||
embeddings = self._select_embeddings(
|
||||
embeddings,
|
||||
output.shape[0],
|
||||
@@ -589,11 +594,18 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
)
|
||||
output.copy_(embeddings.flatten(-2))
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Finish the pinned lookup into graph-owned output storage."""
|
||||
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
|
||||
"""Finalize the full-token pinned lookup with optional ETP reduction."""
|
||||
buffer = self._prefetch_buffer
|
||||
if buffer is None:
|
||||
raise RuntimeError("pinned PLE lookup requires a prior start_prefetch")
|
||||
output = buffer.new_empty((hidden_states.shape[0], self._output_dim))
|
||||
# total_tokens counts this DP rank's tokens before SP sharding.
|
||||
output = buffer.new_empty((total_tokens, self._output_dim))
|
||||
self._finalize_prefetch(buffer, output)
|
||||
return output
|
||||
|
||||
def forward(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
raise RuntimeError(
|
||||
"Pinned-host PLE embedding requires "
|
||||
"start_prefetch() and finalize_prefetch()."
|
||||
)
|
||||
|
||||
@@ -344,6 +344,7 @@ def _ple_conv_kernel(
|
||||
MODE: tl.constexpr,
|
||||
HAS_INIT: tl.constexpr,
|
||||
NULL_STATE_ID: tl.constexpr,
|
||||
FUSE_RESIDUAL: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
t = tl.program_id(0)
|
||||
@@ -438,21 +439,23 @@ def _ple_conv_kernel(
|
||||
conv = acc.to(residual_ptr.dtype.element_ty).to(tl.float32)
|
||||
y = conv * tl.sigmoid(conv)
|
||||
conv_output = tl.where(out_ok, y, 0.0).to(residual_ptr.dtype.element_ty)
|
||||
residual = tl.load(
|
||||
residual_ptr + output_t * C + c_offs,
|
||||
mask=c_mask,
|
||||
other=0.0,
|
||||
)
|
||||
# Preserve the original eager operation boundaries: short convolution is
|
||||
# first accumulated into the BF16/FP16 PLE output, then the outer residual
|
||||
# is added to that rounded value.
|
||||
ple_output = (residual + conv_output).to(residual_ptr.dtype.element_ty)
|
||||
outer_residual = tl.load(
|
||||
outer_residual_ptr + output_t * C + c_offs,
|
||||
mask=c_mask,
|
||||
other=0.0,
|
||||
)
|
||||
ple_output = outer_residual.to(tl.float32) + ple_output.to(tl.float32)
|
||||
ple_output = conv_output
|
||||
if FUSE_RESIDUAL:
|
||||
residual = tl.load(
|
||||
residual_ptr + output_t * C + c_offs,
|
||||
mask=c_mask,
|
||||
other=0.0,
|
||||
)
|
||||
# Preserve the original eager operation boundaries: short convolution is
|
||||
# first accumulated into the BF16/FP16 PLE output, then the outer residual
|
||||
# is added to that rounded value.
|
||||
ple_output = (residual + conv_output).to(residual_ptr.dtype.element_ty)
|
||||
outer_residual = tl.load(
|
||||
outer_residual_ptr + output_t * C + c_offs,
|
||||
mask=c_mask,
|
||||
other=0.0,
|
||||
)
|
||||
ple_output = outer_residual.to(tl.float32) + ple_output.to(tl.float32)
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(
|
||||
@@ -575,7 +578,7 @@ def ple_conv(
|
||||
conv_state: torch.Tensor,
|
||||
conv_weights: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
outer_residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor | None,
|
||||
*,
|
||||
mode: Literal["decode", "spec", "prefill"],
|
||||
dilation: int,
|
||||
@@ -585,7 +588,9 @@ def ple_conv(
|
||||
spec_query_len: int = 1,
|
||||
token_indices: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Add short convolution and the outer residual; update state."""
|
||||
"""Update state and write activated convolution into residual.
|
||||
A non-None outer_residual enables both fused residual additions.
|
||||
"""
|
||||
BLOCK_C = 512
|
||||
kernel_spec_query_len = spec_query_len if mode == "spec" else 1
|
||||
T, C = inputs.shape
|
||||
@@ -657,6 +662,7 @@ def ple_conv(
|
||||
SPEC_QUERY_LEN=kernel_spec_query_len,
|
||||
MODE=mode,
|
||||
HAS_INIT=has_initial_states_arg,
|
||||
FUSE_RESIDUAL=outer_residual is not None,
|
||||
NULL_STATE_ID=NULL_BLOCK_ID,
|
||||
launch_pdl=launch_pdl,
|
||||
num_warps=num_warps,
|
||||
|
||||
@@ -9,8 +9,10 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
@@ -36,6 +38,7 @@ from vllm.model_executor.models.interfaces import (
|
||||
IsHybrid,
|
||||
MixtureOfExperts,
|
||||
MultiModalEmbeddings,
|
||||
SupportsHCSP,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsPP,
|
||||
@@ -66,6 +69,12 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -84,6 +93,14 @@ from .qsa import Qwen4ExpQSAAttention
|
||||
_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention")
|
||||
|
||||
|
||||
def is_hc_sequence_parallel_enabled(vllm_config: VllmConfig) -> bool:
|
||||
"""Enable GR/PLE SP explicitly or automatically with sequence-parallel MoE."""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
return parallel_config.tensor_parallel_size > 1 and (
|
||||
parallel_config.enable_hc_sp or parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
|
||||
|
||||
def without_modelopt_fp4(
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> QuantizationConfig | None:
|
||||
@@ -161,13 +178,19 @@ _EXTRA_WEIGHTS_MAPPER = WeightsMapper(
|
||||
class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock):
|
||||
"""Qwen3Next MoE with Qwen4Exp HC validation."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if parallel_config.use_sequence_parallel_moe:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp HC does not support sequence-parallel MoE"
|
||||
)
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, prefix=prefix, reduce_results=reduce_results
|
||||
)
|
||||
if self.replicate_shared_expert and not self.is_sequence_parallel:
|
||||
# Reduce routed outputs before adding replicated shared outputs.
|
||||
# SP then slices the result to avoid summing shared outputs again via RS.
|
||||
self.experts.moe_config.skip_final_all_reduce = False
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.n_shared_experts = int(config.shared_expert_intermediate_size > 0)
|
||||
|
||||
@@ -185,11 +208,12 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
|
||||
self.config = config
|
||||
self.layer_type = layer_type
|
||||
# MoE SP also enables HC SP, while HC SP can be enabled independently.
|
||||
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
|
||||
self.layer_idx = extract_layer_index(prefix)
|
||||
if vllm_config.parallel_config.use_sequence_parallel_moe:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp HC does not support sequence-parallel MoE"
|
||||
)
|
||||
self.use_sequence_parallel = (
|
||||
vllm_config.parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
self.ple: Qwen4ExpPLELayer | None = None
|
||||
ple_layer_ids = config.ple_layer_ids
|
||||
if (self.layer_idx + 1) in ple_layer_ids:
|
||||
@@ -204,6 +228,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
layer_idx=self.layer_idx,
|
||||
ple_dense_layer_id=ple_dense_layer_id,
|
||||
prefix=f"{prefix}.ple",
|
||||
use_sequence_parallel=self.use_hc_sequence_parallel,
|
||||
)
|
||||
|
||||
if layer_type == "linear_attention":
|
||||
@@ -212,6 +237,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
elif layer_type in _QSA_LAYER_TYPES:
|
||||
use_qsa = getattr(config, "indexer_n_heads", None) is not None
|
||||
@@ -222,6 +248,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
else:
|
||||
self.self_attn = Qwen4ExpQSAAttention(
|
||||
@@ -230,12 +257,15 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
layer_id=self.layer_idx,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid layer_type {layer_type}")
|
||||
|
||||
self.mlp = Qwen4ExpSparseMoeBlock(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
|
||||
hc_config = HyperConnectionConfig(
|
||||
@@ -295,6 +325,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
else:
|
||||
hidden_states, block_input, injection = attn_hc.mix(hidden_states)
|
||||
|
||||
if self.use_hc_sequence_parallel:
|
||||
# Positions retain the full token axis for both text and MRoPE inputs.
|
||||
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
|
||||
if self.layer_type == "linear_attention":
|
||||
attn_out = self.linear_attn(hidden_states=block_input)
|
||||
elif self.layer_type in _QSA_LAYER_TYPES:
|
||||
@@ -306,10 +339,25 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
raise ValueError("Invalid layer_type")
|
||||
|
||||
mlp_hc = self.mlp_hyper_connection
|
||||
if self.use_hc_sequence_parallel:
|
||||
attn_out = sp_reduce_scatter(attn_out)
|
||||
hidden_states, block_input, injection = mlp_hc.combine_and_mix(
|
||||
hidden_states, attn_out, injection
|
||||
)
|
||||
mlp_out = self.mlp(block_input)
|
||||
if self.use_sequence_parallel:
|
||||
# MoE consumes local tokens and returns complete local outputs.
|
||||
mlp_out = self.mlp(block_input, already_sequence_parallel=True)
|
||||
else:
|
||||
if self.use_hc_sequence_parallel:
|
||||
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
|
||||
mlp_out = self.mlp(block_input)
|
||||
if self.use_hc_sequence_parallel:
|
||||
assert isinstance(self.mlp, Qwen4ExpSparseMoeBlock)
|
||||
reduce_output = self.mlp.experts.moe_config.skip_final_all_reduce
|
||||
# Only sum partial outputs. Reduced outputs already include all ranks.
|
||||
mlp_out = (
|
||||
sp_reduce_scatter(mlp_out) if reduce_output else sp_shard(mlp_out)
|
||||
)
|
||||
return hidden_states, mlp_out, injection
|
||||
|
||||
|
||||
@@ -373,6 +421,11 @@ class Qwen4ExpModel(nn.Module):
|
||||
self.num_redundant_experts = (
|
||||
vllm_config.parallel_config.eplb_config.num_redundant_experts
|
||||
)
|
||||
# MoE SP also enables HC SP, while HC SP can be enabled independently.
|
||||
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
|
||||
self.use_sequence_parallel = (
|
||||
vllm_config.parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
self.vocab_size = config.vocab_size
|
||||
self._qsa_layer_ids = frozenset(
|
||||
layer_idx
|
||||
@@ -447,7 +500,6 @@ class Qwen4ExpModel(nn.Module):
|
||||
@staticmethod
|
||||
def _start_layer_ple_prefetch(
|
||||
layer: nn.Module,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
query_start_loc: torch.Tensor | None,
|
||||
ngram_context: torch.Tensor | None,
|
||||
@@ -458,12 +510,7 @@ class Qwen4ExpModel(nn.Module):
|
||||
return
|
||||
if input_ids is None or query_start_loc is None or ngram_context is None:
|
||||
raise RuntimeError("PLE inputs were not prepared")
|
||||
ple.start_prefetch(
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
)
|
||||
ple.start_prefetch(input_ids, query_start_loc, ngram_context)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -482,19 +529,31 @@ class Qwen4ExpModel(nn.Module):
|
||||
if input_ids is None:
|
||||
raise ValueError("input_ids or inputs_embeds is required")
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
if self.use_hc_sequence_parallel:
|
||||
if (
|
||||
self.use_sequence_parallel
|
||||
and envs.VLLM_MOE_SKIP_PADDING
|
||||
and is_forward_context_available()
|
||||
):
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, hidden_states
|
||||
)
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
# Expand only the local token rows when SP is enabled.
|
||||
hidden_states = hidden_states.repeat(1, self.config.hc_count)
|
||||
else:
|
||||
if intermediate_tensors is None:
|
||||
raise ValueError("pipeline stage requires intermediate tensors")
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
full_num_tokens = positions.shape[-1]
|
||||
block_output = None
|
||||
injection = None
|
||||
last_layer = None
|
||||
if self.start_layer < self.end_layer:
|
||||
self._start_layer_ple_prefetch(
|
||||
self.layers[self.start_layer],
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
@@ -506,7 +565,6 @@ class Qwen4ExpModel(nn.Module):
|
||||
if layer_idx + 1 < self.end_layer:
|
||||
self._start_layer_ple_prefetch(
|
||||
self.layers[layer_idx + 1],
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
@@ -526,6 +584,8 @@ class Qwen4ExpModel(nn.Module):
|
||||
deepstack_embed = deepstack_input_embeds[
|
||||
f"deepstack_input_embeds_{layer_idx}"
|
||||
]
|
||||
if self.use_hc_sequence_parallel:
|
||||
deepstack_embed = sp_shard(deepstack_embed)
|
||||
deepstack_embed = (
|
||||
deepstack_embed.unsqueeze(-2)
|
||||
.expand(
|
||||
@@ -560,6 +620,24 @@ class Qwen4ExpModel(nn.Module):
|
||||
multi_hidden, sample_hidden_states, _ = final_mixer.combine_and_mix(
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
if self.use_hc_sequence_parallel:
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
# Gather LM-head and MTP states together when both are needed.
|
||||
hidden_size = self.config.hidden_size
|
||||
packed_hidden_states = torch.cat(
|
||||
[sample_hidden_states, multi_hidden], dim=-1
|
||||
)
|
||||
packed_hidden_states = sp_all_gather(packed_hidden_states)[
|
||||
:full_num_tokens
|
||||
]
|
||||
sample_hidden_states, multi_hidden = packed_hidden_states.split(
|
||||
[hidden_size, self.config.hc_count * hidden_size], dim=-1
|
||||
)
|
||||
sample_hidden_states = sample_hidden_states.contiguous()
|
||||
else:
|
||||
sample_hidden_states = sp_all_gather(sample_hidden_states)[
|
||||
:full_num_tokens
|
||||
]
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
# Capture the pre-final-mixer multi-stream hidden state
|
||||
# [T, hc_count*H] for the MTP drafter (zero extra compute:
|
||||
@@ -615,6 +693,7 @@ class Qwen4ExpModel(nn.Module):
|
||||
class Qwen4ExpForCausalLM(
|
||||
nn.Module,
|
||||
HasInnerState,
|
||||
SupportsHCSP,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsPP,
|
||||
@@ -865,6 +944,7 @@ class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo):
|
||||
class Qwen4ExpForConditionalGeneration(
|
||||
Qwen3_5ForConditionalGeneration,
|
||||
HasInnerState,
|
||||
SupportsHCSP,
|
||||
Qwen4ExpMixtureOfExperts,
|
||||
):
|
||||
"""Qwen3-VL vision tower backed by the Qwen4Exp language model."""
|
||||
|
||||
@@ -20,8 +20,10 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
@@ -33,7 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import configure_quant_config
|
||||
from vllm.model_executor.models.interfaces import SupportsPP
|
||||
from vllm.model_executor.models.interfaces import SupportsHCSP, SupportsPP
|
||||
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
@@ -44,6 +46,11 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
@@ -54,6 +61,7 @@ from .model import (
|
||||
Qwen4ExpDecoderLayer,
|
||||
Qwen4ExpMixtureOfExperts,
|
||||
Qwen4ExpSparseMoeBlock,
|
||||
is_hc_sequence_parallel_enabled,
|
||||
)
|
||||
|
||||
|
||||
@@ -171,6 +179,11 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
config: Qwen4ExpTextConfig = model_config.hf_text_config
|
||||
|
||||
self.config = config
|
||||
# MoE SP also enables HC SP, while HC SP can be enabled independently.
|
||||
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
|
||||
self.use_sequence_parallel = (
|
||||
vllm_config.parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
@@ -310,6 +323,22 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
full_num_tokens = positions.shape[-1]
|
||||
if self.use_hc_sequence_parallel:
|
||||
# SP requires PP=1, so the first-rank branch sets prev_block_output.
|
||||
assert prev_block_output is not None
|
||||
if (
|
||||
self.use_sequence_parallel
|
||||
and envs.VLLM_MOE_SKIP_PADDING
|
||||
and is_forward_context_available()
|
||||
):
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, hidden_states
|
||||
)
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
# The FC output is already complete and needs only a slice.
|
||||
prev_block_output = sp_shard(prev_block_output)
|
||||
layer = self.layers[current_step_idx]
|
||||
hidden_states, block_output, injection = layer(
|
||||
hidden_states=hidden_states,
|
||||
@@ -337,6 +366,17 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
)
|
||||
if self.use_hc_sequence_parallel:
|
||||
# Gather both outputs together, then restore their contiguous layouts.
|
||||
packed_hidden_states = torch.cat(
|
||||
[sample_hidden_states, multi_hidden], dim=-1
|
||||
)
|
||||
packed_hidden_states = sp_all_gather(packed_hidden_states)[:full_num_tokens]
|
||||
sample_hidden_states, multi_hidden = packed_hidden_states.split(
|
||||
[hidden_size, hc_count * hidden_size], dim=-1
|
||||
)
|
||||
sample_hidden_states = sample_hidden_states.contiguous()
|
||||
multi_hidden = multi_hidden.contiguous()
|
||||
return sample_hidden_states, multi_hidden
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -357,7 +397,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
|
||||
|
||||
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
||||
class Qwen4ExpMTP(nn.Module, SupportsHCSP, SupportsPP, Qwen4ExpMixtureOfExperts):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
|
||||
@@ -9,11 +9,14 @@ from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_etp_group, get_tp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter, sp_shard
|
||||
|
||||
from ..common.ngram_embedding import (
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
@@ -146,8 +149,10 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
use_sequence_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
self.embedding_dim = embedding_dim
|
||||
self.ngram_size = int(config.ngram_size)
|
||||
self.heads_per_ngram = int(config.heads_per_ngram)
|
||||
@@ -211,6 +216,11 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
if engram_config is not None and engram_config.cpu_offload
|
||||
else Qwen4ExpPLEDeviceEmbedding
|
||||
)
|
||||
# A TP reduce-scatter covers all vocabulary owners only when ETP=TP.
|
||||
self.use_reduce_scatter = (
|
||||
self.use_sequence_parallel
|
||||
and get_etp_group().world_size == get_tp_group().world_size
|
||||
)
|
||||
self.ngram_embedding = embedding_cls(
|
||||
padded_vocab_size,
|
||||
self.head_dim,
|
||||
@@ -221,6 +231,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
num_ngram_heads=self.ngram_heads,
|
||||
max_total_tokens=max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
reduce_results=not self.use_reduce_scatter,
|
||||
)
|
||||
if self.ngram_embedding.supports_prefetch:
|
||||
# The side-stream lookup outlives eager-break args, whose
|
||||
@@ -348,6 +359,18 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
id_blocks.append(ids[request_indices, adjusted_columns])
|
||||
return torch.cat(id_blocks, dim=-1)
|
||||
|
||||
def _sp_shard_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor:
|
||||
"""Shard embedding token rows, optionally reducing vocabulary contributions."""
|
||||
# Each embedding element has one vocabulary owner, including FP8 bytes.
|
||||
output_dtype = embeddings.dtype
|
||||
if is_fp8(embeddings):
|
||||
embeddings = embeddings.view(torch.int8)
|
||||
if self.use_reduce_scatter:
|
||||
embeddings = sp_reduce_scatter(embeddings)
|
||||
else:
|
||||
embeddings = sp_shard(embeddings)
|
||||
return embeddings.view(output_dtype)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -355,15 +378,22 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Return full embeddings or a token shard after combining vocabulary owners."""
|
||||
embedding = self.ngram_embedding
|
||||
if embedding.supports_prefetch:
|
||||
return embedding(hidden_states)
|
||||
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
|
||||
return self.ngram_embedding(ngram_ids).flatten(-2)
|
||||
# Prefetch contains full tokens even when hidden states are sharded.
|
||||
embeddings = embedding.finalize_prefetch(total_tokens=input_ids.shape[0])
|
||||
else:
|
||||
ngram_ids = self.compute_ngram_ids(
|
||||
input_ids, query_start_loc, ngram_context
|
||||
)
|
||||
embeddings = embedding(ngram_ids).flatten(-2)
|
||||
if not self.use_sequence_parallel:
|
||||
return embeddings
|
||||
return self._sp_shard_embeddings(embeddings)
|
||||
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
@@ -378,7 +408,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
ngram_context,
|
||||
output=self._prefetch_ids[: input_ids.numel()],
|
||||
)
|
||||
embedding.start_prefetch(hidden_states, ngram_ids)
|
||||
embedding.start_prefetch(ngram_ids)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load hash buffers and checkpoint-split embedding rows."""
|
||||
@@ -444,8 +474,3 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
if regular_weights:
|
||||
loaded.update(AutoWeightsLoader(self).load_weights(regular_weights))
|
||||
return loaded
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Qwen4ExpNGramEmbedding",
|
||||
]
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.models.common.ops.sequence_parallel import sp_all_gather, sp_shard
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionBackend,
|
||||
@@ -69,8 +70,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
layer_idx: int = 0,
|
||||
ple_dense_layer_id: int | None = None,
|
||||
prefix: str = "",
|
||||
use_sequence_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
@@ -100,8 +103,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
prefix=f"{prefix}.ple_embedding",
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
)
|
||||
# The PLE cache is TP-replicated, so this merged projection is too.
|
||||
# Projection weights stay replicated even when their token rows are sharded.
|
||||
self.kv_proj = MergedColumnParallelLinear(
|
||||
int(config.ple_embed_dim),
|
||||
[self.hc_hidden_size, self.hidden_size],
|
||||
@@ -151,18 +155,12 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> None:
|
||||
"""Start the pinned PLE lookup while the preceding decoder layer runs."""
|
||||
self.ple_embedding.start_prefetch(
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
)
|
||||
self.ple_embedding.start_prefetch(input_ids, query_start_loc, ngram_context)
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
@@ -192,7 +190,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
self,
|
||||
inputs: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor | None,
|
||||
metadata: PleShortConvAttentionMetadata,
|
||||
conv_state: torch.Tensor,
|
||||
conv_weights: torch.Tensor,
|
||||
@@ -207,7 +205,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
has_non_spec = has_prefill or has_decode
|
||||
inputs = inputs[: metadata.num_actual_tokens]
|
||||
residual = residual[: metadata.num_actual_tokens]
|
||||
outer_residual = outer_residual[: metadata.num_actual_tokens]
|
||||
if outer_residual is not None:
|
||||
outer_residual = outer_residual[: metadata.num_actual_tokens]
|
||||
|
||||
spec_token_indices = None
|
||||
non_spec_token_indices = None
|
||||
@@ -259,11 +258,14 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
residual_d, residual_p = torch.split(
|
||||
residual, [num_decode_tokens, num_prefill_tokens], dim=0
|
||||
)
|
||||
outer_residual_d, outer_residual_p = torch.split(
|
||||
outer_residual,
|
||||
[num_decode_tokens, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
outer_residual_d: torch.Tensor | None = None
|
||||
outer_residual_p: torch.Tensor | None = None
|
||||
if outer_residual is not None:
|
||||
outer_residual_d, outer_residual_p = torch.split(
|
||||
outer_residual,
|
||||
[num_decode_tokens, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
token_indices_d = None
|
||||
token_indices_p = None
|
||||
else:
|
||||
@@ -336,14 +338,16 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
self,
|
||||
inputs: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor | None,
|
||||
) -> None:
|
||||
"""Update convolution state, optionally fusing both residual additions."""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
# Profiling omits all metadata or this Mamba entry. Short convolution
|
||||
# is a no-op there, but preserve the outer residual addition.
|
||||
if attn_metadata is None:
|
||||
residual.add_(outer_residual)
|
||||
if outer_residual is not None:
|
||||
residual.add_(outer_residual)
|
||||
return
|
||||
|
||||
if not isinstance(attn_metadata, dict):
|
||||
@@ -354,7 +358,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
|
||||
layer_attn_metadata = attn_metadata.get(self.prefix)
|
||||
if layer_attn_metadata is None:
|
||||
residual.add_(outer_residual)
|
||||
if outer_residual is not None:
|
||||
residual.add_(outer_residual)
|
||||
return
|
||||
if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata):
|
||||
raise TypeError(
|
||||
@@ -396,7 +401,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
input_ids = input_ids.reshape(-1)
|
||||
if input_ids.shape[0] != hidden_states.shape[0]:
|
||||
if (
|
||||
not self.use_sequence_parallel
|
||||
and input_ids.shape[0] != hidden_states.shape[0]
|
||||
):
|
||||
raise ValueError(
|
||||
"PLE expects input_ids and hidden_states to have the same "
|
||||
f"token length, got {input_ids.shape[0]} and "
|
||||
@@ -408,6 +416,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
)
|
||||
assert embeddings.shape[0] == hidden_states.shape[0]
|
||||
embeddings = self._dequantize_embeddings(embeddings, hidden_states.dtype)
|
||||
kv, _ = self.kv_proj(embeddings)
|
||||
key, value = kv.split(self.kv_proj.output_sizes, dim=-1)
|
||||
@@ -420,9 +429,16 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
self.norm_conv.weight,
|
||||
self.norm_key.eps,
|
||||
)
|
||||
# State routing depends on runtime request metadata and remains outside
|
||||
# the graph; short convolution accumulates into gated_output.
|
||||
self._short_conv(conv_input, gated_output, hidden_states)
|
||||
if self.use_sequence_parallel:
|
||||
# Only convolution needs full tokens. Add residuals on local shards.
|
||||
conv_input = sp_all_gather(conv_input)[: input_ids.shape[0]]
|
||||
# Metadata may omit padding tokens or all tokens during profiling.
|
||||
conv_output = torch.zeros_like(conv_input)
|
||||
self._short_conv(conv_input, conv_output, outer_residual=None)
|
||||
gated_output.add_(sp_shard(conv_output))
|
||||
gated_output.add_(hidden_states)
|
||||
else:
|
||||
self._short_conv(conv_input, gated_output, hidden_states)
|
||||
return gated_output
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user