[Perf] Batch invariance with Cutlass fp8 support, 28.9% E2E latency improvement (#40408)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Wentao Ye
2026-05-11 12:20:58 -04:00
committed by GitHub
parent 3f9c0c25b3
commit 0d453e2336
11 changed files with 305 additions and 3 deletions
@@ -16,7 +16,10 @@ from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
init_fp8_linear_kernel,
)
from vllm.model_executor.kernels.linear.scaled_mm import MarlinFP8ScaledMMLinearKernel
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
@@ -437,8 +440,8 @@ class Fp8LinearMethod(LinearMethodBase):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
# if batch invariant mode is enabled, prefer DeepGEMM FP8 path
# we will use BF16 dequant when DeepGEMM is not supported.
# if batch invariant mode is enabled, prefer direct FP8 path
# we will use BF16 dequant when direct FP8 is not supported.
if envs.VLLM_BATCH_INVARIANT:
if self.block_quant:
assert self.weight_block_size is not None
@@ -448,6 +451,9 @@ class Fp8LinearMethod(LinearMethodBase):
bias,
)
else:
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
return self.fp8_linear.apply_weights(layer, x, bias)
# per-tensor/channel: dequant to BF16 and run GEMM
weight_fp8 = layer.weight.to(torch.bfloat16)
weight_scale = layer.weight_scale.to(torch.bfloat16)
@@ -18,6 +18,9 @@ import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.config import get_current_vllm_config
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
select_fp8_moe_backend,
)
@@ -161,6 +164,9 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
) -> torch.Tensor:
# if batch invariant mode is enabled, use BF16 dequant
if envs.VLLM_BATCH_INVARIANT:
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
return self.fp8_linear.apply_weights(layer, x, bias)
weight_fp8 = layer.weight.to(torch.bfloat16)
weight_scale = layer.weight_scale.to(torch.bfloat16)
if weight_scale.numel() == 1: