[GLM5.2 Perf] fused_indexer_q_rope_quant triton kernel, 1.9% ~ 3.3% E2E Throughput improvement. (#46862)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-06-26 22:16:20 -07:00
committed by GitHub
parent 455f25aa13
commit b588f66dc2
2 changed files with 167 additions and 0 deletions
@@ -11,7 +11,11 @@ from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import (
fp8_fp4_mqa_logits,
fp8_fp4_paged_mqa_logits,
@@ -37,6 +41,133 @@ RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
MXFP4_BLOCK_SIZE = 32
@triton.jit
def _fused_indexer_q_rope_quant_kernel(
positions,
q,
q_s0,
q_s1,
cos_sin_cache,
cos_sin_s0,
q_fp8,
q_fp8_s0,
q_fp8_s1,
weights,
weights_s0,
weights_s1,
weights_out,
weights_out_s0,
weights_out_s1,
softmax_scale,
head_scale,
fp8_min: tl.constexpr,
fp8_max: tl.constexpr,
is_neox: tl.constexpr,
):
token = tl.program_id(0)
head = tl.program_id(1)
offs32 = tl.arange(0, 32)
offs64 = tl.arange(0, 64)
pos = tl.load(positions + token)
cos = tl.load(cos_sin_cache + pos * cos_sin_s0 + offs32).to(tl.float32)
sin = tl.load(cos_sin_cache + pos * cos_sin_s0 + 32 + offs32).to(tl.float32)
q_base = q + token * q_s0 + head * q_s1
out_base = q_fp8 + token * q_fp8_s0 + head * q_fp8_s1
if is_neox:
# NeoX layout, x0 = q[0:32], x1 = q[32:64]
x0 = tl.load(q_base + offs32).to(tl.float32)
x1 = tl.load(q_base + 32 + offs32).to(tl.float32)
else:
# interleaved layout
# x0 = q[0, 2, 4, ...], x1 = q[1, 3, 5, ...]
x0 = tl.load(q_base + offs32 * 2).to(tl.float32)
x1 = tl.load(q_base + offs32 * 2 + 1).to(tl.float32)
r0 = (x0 * cos - x1 * sin).to(tl.bfloat16).to(tl.float32)
r1 = (x1 * cos + x0 * sin).to(tl.bfloat16).to(tl.float32)
amax = tl.maximum(tl.max(tl.abs(r0)), tl.max(tl.abs(r1)))
q_nope = tl.load(q_base + 64 + offs64).to(tl.float32)
amax = tl.maximum(amax, tl.max(tl.abs(q_nope)))
scale_raw = tl.maximum(amax, 1e-10) * (1.0 / fp8_max)
# e8m0 format
q_scale = tl.math.exp2(tl.ceil(tl.log2(scale_raw)))
if is_neox:
tl.store(
out_base + offs32,
tl.clamp(r0 / q_scale, fp8_min, fp8_max).to(q_fp8.dtype.element_ty),
)
tl.store(
out_base + 32 + offs32,
tl.clamp(r1 / q_scale, fp8_min, fp8_max).to(q_fp8.dtype.element_ty),
)
else:
tl.store(
out_base + offs32 * 2,
tl.clamp(r0 / q_scale, fp8_min, fp8_max).to(q_fp8.dtype.element_ty),
)
tl.store(
out_base + offs32 * 2 + 1,
tl.clamp(r1 / q_scale, fp8_min, fp8_max).to(q_fp8.dtype.element_ty),
)
tl.store(
out_base + 64 + offs64,
tl.clamp(q_nope / q_scale, fp8_min, fp8_max).to(q_fp8.dtype.element_ty),
)
weight = tl.load(weights + token * weights_s0 + head * weights_s1).to(tl.float32)
tl.store(
weights_out + token * weights_out_s0 + head * weights_out_s1,
weight * q_scale * softmax_scale * head_scale,
)
def fused_indexer_q_rope_quant(
positions: torch.Tensor,
q: torch.Tensor,
cos_sin_cache: torch.Tensor,
weights: torch.Tensor,
softmax_scale: float,
head_scale: float,
is_neox: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
assert current_platform.is_cuda()
assert q.dtype == torch.bfloat16
assert q.shape[-1] == 128
assert cos_sin_cache.shape[-1] == 64
assert weights.shape == q.shape[:2]
q_fp8 = torch.empty_like(q, dtype=current_platform.fp8_dtype())
weights_out = torch.empty_like(weights, dtype=torch.float32)
fp8_min, fp8_max = get_fp8_min_max()
_fused_indexer_q_rope_quant_kernel[(q.shape[0], q.shape[1])](
positions,
q,
q.stride(0),
q.stride(1),
cos_sin_cache,
cos_sin_cache.stride(0),
q_fp8,
q_fp8.stride(0),
q_fp8.stride(1),
weights,
weights.stride(0),
weights.stride(1),
weights_out,
weights_out.stride(0),
weights_out.stride(1),
softmax_scale,
head_scale,
fp8_min=fp8_min,
fp8_max=fp8_max,
is_neox=is_neox,
num_warps=1,
)
return q_fp8, weights_out
def _gather_workspace_shapes(
total_seq_lens: int,
head_dim: int,