Disable Cascade Attention for Batch Invariance (#32561)

Signed-off-by: frankwang28 <frank.wbb@hotmail.com>
Signed-off-by: Frank Wang <41319051+frankwang28@users.noreply.github.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Frank Wang
2026-01-30 10:00:46 -05:00
committed by GitHub
co-authored by Wentao Ye
parent ae5b7aff2b
commit 8f5d51203b
6 changed files with 60 additions and 9 deletions
+14
View File
@@ -16,6 +16,20 @@ from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
MOE_LAYER_ROUTER_GATE_SUFFIXES = {
"gate",
"router",
"router_gate",
"shared_expert_gate",
"expert_gate",
}
def is_layer_moe_router_gate(prefix: str) -> bool:
if not prefix:
return False
return prefix.rsplit(".", 1)[-1] in MOE_LAYER_ROUTER_GATE_SUFFIXES
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
# Shuffle weight along the last dimension so that