[Qwen4Exp] support Sequence Parallelism for Qwen3.8-Flash-Next (#56322)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Huang
2026-10-12 20:56:44 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 258b97ff6b
commit c75cb63cd2
23 changed files with 870 additions and 178 deletions
+7 -1
View File
@@ -129,7 +129,12 @@ def _should_replicate_misaligned_shared_expert(
class Qwen3NextSparseMoeBlock(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
reduce_results: bool = True,
):
super().__init__()
config = vllm_config.model_config.hf_text_config
@@ -231,6 +236,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=1 if self.shared_expert is None else None,
reduce_results=reduce_results,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
shared_expert_prefix=f"{prefix}.shared_expert",
shared_expert_gate=self.shared_expert_gate