[Feature] Support batch invariance on ROCm (#52231)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Andreas Karatzas
parent
50b21a2667
commit
e0317b28ce
@@ -39,6 +39,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
|
||||
ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
@@ -65,6 +66,7 @@ __all__ = [
|
||||
"PerTensorTorchFP8ScaledMMLinearKernel",
|
||||
"RowWiseTorchFP8ScaledMMLinearKernel",
|
||||
"ROCmFP8ScaledMMLinearKernel",
|
||||
"TritonFP8ScaledMMLinearKernel",
|
||||
"TritonInt8ScaledMMLinearKernel",
|
||||
"ZentorchInt8ScaledMMLinearKernel",
|
||||
"Fp8BlockScaledMMLinearKernel",
|
||||
|
||||
@@ -20,6 +20,8 @@ from .BlockScaledMMLinearKernel import (
|
||||
)
|
||||
from .cutlass import CutlassInt8ScaledMMLinearKernel
|
||||
from .ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
Int8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
|
||||
@@ -156,6 +158,46 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
|
||||
return out
|
||||
|
||||
|
||||
class TritonFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cuda_alike():
|
||||
return False, "requires ROCm or CUDA."
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
act_group = c.activation_quant_key.scale.group_shape
|
||||
weight_group = c.weight_quant_key.scale.group_shape
|
||||
if not (act_group.is_per_tensor() or act_group.is_per_token()):
|
||||
return False, "requires per-tensor or per-token activation scales."
|
||||
if not (
|
||||
weight_group.is_per_tensor()
|
||||
or weight_group.is_per_channel()
|
||||
or weight_group.is_per_token()
|
||||
):
|
||||
return False, "requires per-tensor, per-channel or per-token weight scales."
|
||||
return True, None
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
out = triton_scaled_mm(
|
||||
A, B, scale_a=As, scale_b=Bs, out_dtype=out_dtype, bias=bias
|
||||
)
|
||||
return out.view(*output_shape)
|
||||
|
||||
|
||||
class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(cls, compute_capability=None):
|
||||
|
||||
Reference in New Issue
Block a user