diff --git a/vllm/model_executor/layers/fused_moe/router/gate_linear.py b/vllm/model_executor/layers/fused_moe/router/gate_linear.py index 13eff3889c..8ecd244b61 100644 --- a/vllm/model_executor/layers/fused_moe/router/gate_linear.py +++ b/vllm/model_executor/layers/fused_moe/router/gate_linear.py @@ -4,6 +4,7 @@ import torch from torch.nn.parameter import Parameter import vllm._custom_ops as ops +import vllm.envs as envs from vllm.logger import init_logger from vllm.model_executor.custom_op import PluggableLayer from vllm.model_executor.layers.linear import ( @@ -39,6 +40,9 @@ class GateLinear(ReplicatedLinear): A ``quant_config`` that actually quantizes the gate disables every specialized tier, leaving plain ``ReplicatedLinear`` behavior. + + ``VLLM_BATCH_INVARIANT`` also disables tiers 1-5: they pick kernels and + split-K from the token count and skip the batch-invariant linear path. """ # (hidden_size, num_experts) pairs with an instantiated fp32 kernel: @@ -79,8 +83,12 @@ class GateLinear(ReplicatedLinear): input_size, output_size, ) in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES + batch_invariant = envs.VLLM_BATCH_INVARIANT can_use_specialized_kernels = ( - current_platform.is_cuda() and (is_hopper or is_blackwell) and not bias + current_platform.is_cuda() + and (is_hopper or is_blackwell) + and not bias + and not batch_invariant ) # If fp32 compute is required and no specialized kernel is available, @@ -110,6 +118,7 @@ class GateLinear(ReplicatedLinear): # fp32 specialized kernel eligibility (exact dims, fp32 weight) self.allow_fp32_router_gemm = ( self.is_unquantized + and not batch_invariant and not bias and self.weight.dtype == torch.float32 and ( @@ -123,6 +132,7 @@ class GateLinear(ReplicatedLinear): ) self.allow_bf16x3_router_gemm = ( self.is_unquantized + and not batch_invariant and not bias and self.weight.dtype == torch.float32 and current_platform.is_cuda() @@ -139,8 +149,10 @@ class GateLinear(ReplicatedLinear): # (GB10 / DGX Spark), which this tier still covers. See #49921. self._router_gemm_no_bias = self.is_unquantized and not bias self._router_gemm_cublas_capable = ( - current_platform.is_cuda() or current_platform.is_rocm() - ) and self._router_gemm_no_bias + (current_platform.is_cuda() or current_platform.is_rocm()) + and self._router_gemm_no_bias + and not batch_invariant + ) self.allow_cublas_router_gemm = ( self._router_gemm_cublas_capable and self.weight.dtype == torch.bfloat16 @@ -156,6 +168,7 @@ class GateLinear(ReplicatedLinear): self._rocm_bf16x3_weight_eligible = ( self._router_gemm_no_bias + and not batch_invariant and self.weight.dtype == torch.float32 and bf16x3_router_gemm_rocm.platform_supported() )