[Perf] Deepgemm fused layout kernel for activations, 4.3% throughput improvement, 10.7% TTFT improvement. (#29546)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -269,7 +269,11 @@ class W8A8BlockFp8LinearOp:
|
||||
weight_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
assert self.deepgemm_input_quant_op is not None
|
||||
q_input, input_scale = self.deepgemm_input_quant_op(input_2d)
|
||||
q_input, input_scale = per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
input_2d,
|
||||
group_size=self.act_quant_group_shape.col,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
output = torch.empty(
|
||||
(q_input.shape[0], weight.shape[0]),
|
||||
dtype=torch.bfloat16,
|
||||
@@ -791,6 +795,80 @@ def per_token_group_quant_fp8(
|
||||
return x_q, x_s
|
||||
|
||||
|
||||
def per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
x: torch.Tensor,
|
||||
group_size: int,
|
||||
eps: float = 1e-10,
|
||||
use_ue8m0: bool | None = None,
|
||||
out_q: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""FP8 per-token-group quantization for DeepGEMM.
|
||||
|
||||
Returns:
|
||||
(x_q, x_s_packed)
|
||||
x_q: FP8 activations, same shape as `x`.
|
||||
x_s_packed: Int32 tensor with logical shape
|
||||
[mn, ceil(num_groups_per_row / 4)], laid out with
|
||||
TMA-aligned stride along the packed-K dimension
|
||||
"""
|
||||
if use_ue8m0 is None:
|
||||
use_ue8m0 = is_deep_gemm_e8m0_used()
|
||||
# for DeepGEMM UE8M0-packed layout we *require* UE8M0 scales.
|
||||
assert use_ue8m0, (
|
||||
"per_token_group_quant_fp8_packed_for_deepgemm requires UE8M0 scales."
|
||||
)
|
||||
|
||||
dtype = current_platform.fp8_dtype()
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
f"the last dimension of `x` {x.shape[-1]} must be divisible "
|
||||
f"by `group_size` {group_size}"
|
||||
)
|
||||
assert x.stride(-1) == 1, "`x` groups must be contiguous"
|
||||
|
||||
finfo = torch.finfo(dtype)
|
||||
fp8_min, fp8_max = finfo.min, finfo.max
|
||||
|
||||
# compute DeepGEMM-style packed scale tensor shape.
|
||||
hidden_dim = x.shape[-1]
|
||||
mn = x.numel() // hidden_dim
|
||||
num_groups_per_row = hidden_dim // group_size
|
||||
k_num_packed_sf_k = (num_groups_per_row + 3) // 4
|
||||
tma_aligned_mn = ((mn + 3) // 4) * 4
|
||||
|
||||
x_s_packed = torch.empty_strided(
|
||||
(mn, k_num_packed_sf_k),
|
||||
(1, tma_aligned_mn),
|
||||
device=x.device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
# CUDA kernel path only (DeepGEMM + E8M0 is CUDA-specific).
|
||||
assert current_platform.is_cuda(), (
|
||||
"per_token_group_quant_fp8_packed_for_deepgemm is only valid on CUDA "
|
||||
"platforms using DeepGEMM."
|
||||
)
|
||||
|
||||
x_contiguous = x.contiguous()
|
||||
if out_q is not None:
|
||||
x_q_local = out_q
|
||||
else:
|
||||
x_q_local = torch.empty_like(x_contiguous, device=x.device, dtype=dtype)
|
||||
|
||||
torch.ops._C.per_token_group_fp8_quant_packed(
|
||||
x_contiguous,
|
||||
x_q_local,
|
||||
x_s_packed,
|
||||
group_size,
|
||||
eps,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
)
|
||||
|
||||
# return a tensor with the original logical shape.
|
||||
x_q = x_q_local.view_as(x)
|
||||
return x_q, x_s_packed
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _w8a8_triton_block_scaled_mm(
|
||||
# Pointers to inputs and output
|
||||
|
||||
Reference in New Issue
Block a user