[Bugfix] Keep GateLinear batch invariant under VLLM_BATCH_INVARIANT (#59985)

Signed-off-by: yifanFengg <evanfengff@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
yifanFengg
2026-10-13 15:43:54 -04:00
committed by GitHub
co-authored by Claude Fable 5.1 Wentao Ye
parent b33cb81237
commit 68fc395554
@@ -4,6 +4,7 @@ import torch
from torch.nn.parameter import Parameter
import vllm._custom_ops as ops
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.linear import (
@@ -39,6 +40,9 @@ class GateLinear(ReplicatedLinear):
A ``quant_config`` that actually quantizes the gate disables every
specialized tier, leaving plain ``ReplicatedLinear`` behavior.
``VLLM_BATCH_INVARIANT`` also disables tiers 1-5: they pick kernels and
split-K from the token count and skip the batch-invariant linear path.
"""
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
@@ -79,8 +83,12 @@ class GateLinear(ReplicatedLinear):
input_size,
output_size,
) in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
batch_invariant = envs.VLLM_BATCH_INVARIANT
can_use_specialized_kernels = (
current_platform.is_cuda() and (is_hopper or is_blackwell) and not bias
current_platform.is_cuda()
and (is_hopper or is_blackwell)
and not bias
and not batch_invariant
)
# If fp32 compute is required and no specialized kernel is available,
@@ -110,6 +118,7 @@ class GateLinear(ReplicatedLinear):
# fp32 specialized kernel eligibility (exact dims, fp32 weight)
self.allow_fp32_router_gemm = (
self.is_unquantized
and not batch_invariant
and not bias
and self.weight.dtype == torch.float32
and (
@@ -123,6 +132,7 @@ class GateLinear(ReplicatedLinear):
)
self.allow_bf16x3_router_gemm = (
self.is_unquantized
and not batch_invariant
and not bias
and self.weight.dtype == torch.float32
and current_platform.is_cuda()
@@ -139,8 +149,10 @@ class GateLinear(ReplicatedLinear):
# (GB10 / DGX Spark), which this tier still covers. See #49921.
self._router_gemm_no_bias = self.is_unquantized and not bias
self._router_gemm_cublas_capable = (
current_platform.is_cuda() or current_platform.is_rocm()
) and self._router_gemm_no_bias
(current_platform.is_cuda() or current_platform.is_rocm())
and self._router_gemm_no_bias
and not batch_invariant
)
self.allow_cublas_router_gemm = (
self._router_gemm_cublas_capable
and self.weight.dtype == torch.bfloat16
@@ -156,6 +168,7 @@ class GateLinear(ReplicatedLinear):
self._rocm_bf16x3_weight_eligible = (
self._router_gemm_no_bias
and not batch_invariant
and self.weight.dtype == torch.float32
and bf16x3_router_gemm_rocm.platform_supported()
)