[Qwen4Exp] support Sequence Parallelism for Qwen3.8-Flash-Next (#56322)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
258b97ff6b
commit
c75cb63cd2
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
method_has_implemented_embedding,
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
|
||||
from vllm.model_executor.layers.utils import dispatch_unquantized_gemm
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
@@ -240,6 +241,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
quant_config: quant config for the layer
|
||||
prefix: full name of the layer in the state dict
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
reduce_results: Whether to all-reduce contributions from vocabulary shards.
|
||||
quant_method: Preselected quantization method for model-specific layers.
|
||||
parallel_group: Process group used to shard and reduce the embedding.
|
||||
|
||||
@@ -258,6 +260,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
prefix: str = "",
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
reduce_results: bool = True,
|
||||
quant_method: QuantizeMethodBase | None = None,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
):
|
||||
@@ -266,6 +269,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
# Keep the input dimensions.
|
||||
self.disable_tp = disable_tp
|
||||
self.parallel_group = parallel_group
|
||||
self.reduce_results = reduce_results
|
||||
if disable_tp:
|
||||
tp_rank, self.tp_size = 0, 1
|
||||
elif parallel_group is not None:
|
||||
@@ -516,6 +520,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
param[loaded_weight.shape[0] :].data.fill_(0)
|
||||
|
||||
def forward(self, input_):
|
||||
"""Look up IDs, optionally reducing contributions from vocabulary shards."""
|
||||
if self.tp_size == 1:
|
||||
return self.quant_method.embedding(self, input_.long())
|
||||
|
||||
@@ -543,24 +548,20 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
self.shard_indices.added_vocab_end_index,
|
||||
)
|
||||
output_parallel = self.quant_method.embedding(self, masked_input.long())
|
||||
if output_parallel.dtype in (
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e5m2,
|
||||
):
|
||||
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
|
||||
comm_output = output_parallel.view(torch.int8)
|
||||
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
output = (
|
||||
self.parallel_group.all_reduce(comm_output)
|
||||
if self.parallel_group is not None
|
||||
else tensor_model_parallel_all_reduce(comm_output)
|
||||
)
|
||||
return output.view(output_parallel.dtype)
|
||||
|
||||
output_dtype = output_parallel.dtype
|
||||
if is_fp8(output_dtype):
|
||||
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
|
||||
output_parallel = output_parallel.view(torch.int8)
|
||||
if not self.use_fused_embedding:
|
||||
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
# Reduce across all the model parallel GPUs.
|
||||
if self.parallel_group is not None:
|
||||
return self.parallel_group.all_reduce(output_parallel)
|
||||
return tensor_model_parallel_all_reduce(output_parallel)
|
||||
if self.reduce_results:
|
||||
output_parallel = (
|
||||
self.parallel_group.all_reduce(output_parallel)
|
||||
if self.parallel_group is not None
|
||||
else tensor_model_parallel_all_reduce(output_parallel)
|
||||
)
|
||||
return output_parallel.view(output_dtype)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
s = f"num_embeddings={self.num_embeddings}"
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from .interfaces import (
|
||||
HasInnerState,
|
||||
SupportsHCSP,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
@@ -10,6 +11,7 @@ from .interfaces import (
|
||||
SupportsPP,
|
||||
SupportsTranscription,
|
||||
has_inner_state,
|
||||
supports_hc_sp,
|
||||
supports_lora,
|
||||
supports_mrope,
|
||||
supports_multimodal,
|
||||
@@ -33,6 +35,8 @@ __all__ = [
|
||||
"is_text_generation_model",
|
||||
"HasInnerState",
|
||||
"has_inner_state",
|
||||
"SupportsHCSP",
|
||||
"supports_hc_sp",
|
||||
"SupportsLoRA",
|
||||
"supports_lora",
|
||||
"SupportsMultiModal",
|
||||
|
||||
@@ -912,11 +912,21 @@ class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConf
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
text_config = vllm_config.model_config.hf_text_config
|
||||
if text_config.hc_count <= 1:
|
||||
raise ValueError("Qwen4Exp requires hc_count > 1")
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# MoE SP also enables HC SP without the explicit HC flag.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size != 1
|
||||
):
|
||||
raise ValueError("Qwen4Exp MoE SP requires PP=1")
|
||||
|
||||
uses_ple_or_qsa = bool(text_config.ple_layer_ids) or (
|
||||
getattr(text_config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
|
||||
@@ -858,6 +858,27 @@ def _supports_pp_inspect(model: type[object] | object) -> bool:
|
||||
return supports_kw(model_forward, "intermediate_tensors")
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsHCSP(Protocol):
|
||||
"""Support sequence-parallel HC without changing MoE parallelism."""
|
||||
|
||||
supports_hc_sp: ClassVar[Literal[True]] = True
|
||||
|
||||
|
||||
@overload
|
||||
def supports_hc_sp(model: type[object]) -> TypeIs[type[SupportsHCSP]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_hc_sp(model: object) -> TypeIs[SupportsHCSP]: ...
|
||||
|
||||
|
||||
def supports_hc_sp(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsHCSP]] | TypeIs[SupportsHCSP]:
|
||||
return getattr(model, "supports_hc_sp", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class HasInnerState(Protocol):
|
||||
"""The interface required for all models that has inner state."""
|
||||
|
||||
@@ -129,7 +129,12 @@ def _should_replicate_misaligned_shared_expert(
|
||||
|
||||
|
||||
class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
@@ -231,6 +236,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
n_shared_experts=1 if self.shared_expert is None else None,
|
||||
reduce_results=reduce_results,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_expert",
|
||||
shared_expert_gate=self.shared_expert_gate
|
||||
|
||||
@@ -49,6 +49,7 @@ from .interfaces import (
|
||||
is_attention_free,
|
||||
is_hybrid,
|
||||
requires_raw_input_tokens,
|
||||
supports_hc_sp,
|
||||
supports_mamba_prefix_caching,
|
||||
supports_multimodal,
|
||||
supports_multimodal_encoder_tp_data,
|
||||
@@ -873,6 +874,7 @@ class _ModelInfo:
|
||||
requires_raw_input_tokens: bool
|
||||
supports_multimodal_encoder_tp_data: bool
|
||||
supports_pp: bool
|
||||
supports_hc_sp: bool
|
||||
has_inner_state: bool
|
||||
is_attention_free: bool
|
||||
is_hybrid: bool
|
||||
@@ -903,6 +905,7 @@ class _ModelInfo:
|
||||
model
|
||||
),
|
||||
supports_pp=supports_pp(model),
|
||||
supports_hc_sp=supports_hc_sp(model),
|
||||
has_inner_state=has_inner_state(model),
|
||||
is_attention_free=is_attention_free(model),
|
||||
is_hybrid=is_hybrid(model),
|
||||
@@ -1463,6 +1466,15 @@ class _ModelRegistry:
|
||||
model_info, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_info.supports_pp
|
||||
|
||||
def is_hc_sp_supported_model(
|
||||
self,
|
||||
architectures: str | list[str],
|
||||
model_config: ModelConfig,
|
||||
) -> bool:
|
||||
"""Check support for HC SP without changing MoE parallelism."""
|
||||
model_info, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_info.supports_hc_sp
|
||||
|
||||
def model_has_inner_state(
|
||||
self,
|
||||
architectures: str | list[str],
|
||||
|
||||
Reference in New Issue
Block a user