[Feature] Support batch invariance on ROCm (#52231)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Matt
2026-10-12 10:53:26 -05:00
committed by GitHub
co-authored by Claude Opus 5.5 Andreas Karatzas
parent 50b21a2667
commit e0317b28ce
31 changed files with 1106 additions and 122 deletions
@@ -9,10 +9,10 @@ import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.kernels.linear import (
fp8_scaled_mm_is_batch_invariant,
init_fp8_linear_kernel,
)
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.attention import Attention
@@ -432,7 +432,7 @@ class Fp8LinearMethod(LinearMethodBase):
bias,
)
else:
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
if fp8_scaled_mm_is_batch_invariant(self.fp8_linear):
return self.fp8_linear.apply_weights(layer, x, bias)
# per-tensor/channel: dequant to BF16 and run GEMM