[Qwen4Exp] support Sequence Parallelism for Qwen3.8-Flash-Next (#56322)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Huang
2026-10-12 20:56:44 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 258b97ff6b
commit c75cb63cd2
23 changed files with 870 additions and 178 deletions
+8
View File
@@ -1537,6 +1537,14 @@ class ModelConfig:
self,
parallel_config: ParallelConfig,
) -> None:
if parallel_config.enable_hc_sp and parallel_config.tensor_parallel_size > 1:
if not current_platform.is_cuda():
raise ValueError("--enable-hc-sp currently requires CUDA")
if not self.registry.is_hc_sp_supported_model(self.architectures, self):
raise ValueError("--enable-hc-sp is not supported for this model")
if parallel_config.pipeline_parallel_size != 1:
raise ValueError("--enable-hc-sp requires PP=1")
total_num_attention_heads = self.model_arch_config.total_num_attention_heads
tensor_parallel_size = parallel_config.tensor_parallel_size
if total_num_attention_heads % tensor_parallel_size != 0:
+5
View File
@@ -181,6 +181,11 @@ class ParallelConfig:
--data-parallel-start-rank."""
is_moe_model: bool | None = None
"""Whether the deployed model is MoE (if known)."""
enable_hc_sp: bool = False
"""Enable sequence parallelism for HyperConnection (HC) computation.
Currently supported by Qwen4Exp on CUDA with TP>1 and PP=1 (ignored when TP=1).
Does not change MoE parallelism. MoE SP may enable HC SP independently of this flag.
"""
enable_expert_parallel: bool = False
"""Use expert parallelism instead of tensor parallelism for MoE layers."""
enable_batch_sharded_sampling: bool | None = None