Disable Cascade Attention for Batch Invariance (#32561)
Signed-off-by: frankwang28 <frank.wbb@hotmail.com> Signed-off-by: Frank Wang <41319051+frankwang28@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
@@ -16,6 +16,20 @@ from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
MOE_LAYER_ROUTER_GATE_SUFFIXES = {
|
||||
"gate",
|
||||
"router",
|
||||
"router_gate",
|
||||
"shared_expert_gate",
|
||||
"expert_gate",
|
||||
}
|
||||
|
||||
|
||||
def is_layer_moe_router_gate(prefix: str) -> bool:
|
||||
if not prefix:
|
||||
return False
|
||||
return prefix.rsplit(".", 1)[-1] in MOE_LAYER_ROUTER_GATE_SUFFIXES
|
||||
|
||||
|
||||
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
|
||||
# Shuffle weight along the last dimension so that
|
||||
|
||||
Reference in New Issue
Block a user