[GLM5.2 Perf] fused_indexer_q_rope_quant triton kernel, 1.9% ~ 3.3% E2E Throughput improvement. (#46862)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -11,7 +11,11 @@ from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_mqa_logits,
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
@@ -37,6 +41,133 @@ RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_indexer_q_rope_quant_kernel(
|
||||
positions,
|
||||
q,
|
||||
q_s0,
|
||||
q_s1,
|
||||
cos_sin_cache,
|
||||
cos_sin_s0,
|
||||
q_fp8,
|
||||
q_fp8_s0,
|
||||
q_fp8_s1,
|
||||
weights,
|
||||
weights_s0,
|
||||
weights_s1,
|
||||
weights_out,
|
||||
weights_out_s0,
|
||||
weights_out_s1,
|
||||
softmax_scale,
|
||||
head_scale,
|
||||
fp8_min: tl.constexpr,
|
||||
fp8_max: tl.constexpr,
|
||||
is_neox: tl.constexpr,
|
||||
):
|
||||
token = tl.program_id(0)
|
||||
head = tl.program_id(1)
|
||||
offs32 = tl.arange(0, 32)
|
||||
offs64 = tl.arange(0, 64)
|
||||
|
||||
pos = tl.load(positions + token)
|
||||
cos = tl.load(cos_sin_cache + pos * cos_sin_s0 + offs32).to(tl.float32)
|
||||
sin = tl.load(cos_sin_cache + pos * cos_sin_s0 + 32 + offs32).to(tl.float32)
|
||||
q_base = q + token * q_s0 + head * q_s1
|
||||
out_base = q_fp8 + token * q_fp8_s0 + head * q_fp8_s1
|
||||
|
||||
if is_neox:
|
||||
# NeoX layout, x0 = q[0:32], x1 = q[32:64]
|
||||
x0 = tl.load(q_base + offs32).to(tl.float32)
|
||||
x1 = tl.load(q_base + 32 + offs32).to(tl.float32)
|
||||
else:
|
||||
# interleaved layout
|
||||
# x0 = q[0, 2, 4, ...], x1 = q[1, 3, 5, ...]
|
||||
x0 = tl.load(q_base + offs32 * 2).to(tl.float32)
|
||||
x1 = tl.load(q_base + offs32 * 2 + 1).to(tl.float32)
|
||||
r0 = (x0 * cos - x1 * sin).to(tl.bfloat16).to(tl.float32)
|
||||
r1 = (x1 * cos + x0 * sin).to(tl.bfloat16).to(tl.float32)
|
||||
amax = tl.maximum(tl.max(tl.abs(r0)), tl.max(tl.abs(r1)))
|
||||
|
||||
q_nope = tl.load(q_base + 64 + offs64).to(tl.float32)
|
||||
amax = tl.maximum(amax, tl.max(tl.abs(q_nope)))
|
||||
scale_raw = tl.maximum(amax, 1e-10) * (1.0 / fp8_max)
|
||||
# e8m0 format
|
||||
q_scale = tl.math.exp2(tl.ceil(tl.log2(scale_raw)))
|
||||
|
||||
if is_neox:
|
||||
tl.store(
|
||||
out_base + offs32,
|
||||
tl.clamp(r0 / q_scale, fp8_min, fp8_max).to(q_fp8.dtype.element_ty),
|
||||
)
|
||||
tl.store(
|
||||
out_base + 32 + offs32,
|
||||
tl.clamp(r1 / q_scale, fp8_min, fp8_max).to(q_fp8.dtype.element_ty),
|
||||
)
|
||||
else:
|
||||
tl.store(
|
||||
out_base + offs32 * 2,
|
||||
tl.clamp(r0 / q_scale, fp8_min, fp8_max).to(q_fp8.dtype.element_ty),
|
||||
)
|
||||
tl.store(
|
||||
out_base + offs32 * 2 + 1,
|
||||
tl.clamp(r1 / q_scale, fp8_min, fp8_max).to(q_fp8.dtype.element_ty),
|
||||
)
|
||||
tl.store(
|
||||
out_base + 64 + offs64,
|
||||
tl.clamp(q_nope / q_scale, fp8_min, fp8_max).to(q_fp8.dtype.element_ty),
|
||||
)
|
||||
|
||||
weight = tl.load(weights + token * weights_s0 + head * weights_s1).to(tl.float32)
|
||||
tl.store(
|
||||
weights_out + token * weights_out_s0 + head * weights_out_s1,
|
||||
weight * q_scale * softmax_scale * head_scale,
|
||||
)
|
||||
|
||||
|
||||
def fused_indexer_q_rope_quant(
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
head_scale: float,
|
||||
is_neox: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert current_platform.is_cuda()
|
||||
assert q.dtype == torch.bfloat16
|
||||
assert q.shape[-1] == 128
|
||||
assert cos_sin_cache.shape[-1] == 64
|
||||
assert weights.shape == q.shape[:2]
|
||||
|
||||
q_fp8 = torch.empty_like(q, dtype=current_platform.fp8_dtype())
|
||||
weights_out = torch.empty_like(weights, dtype=torch.float32)
|
||||
fp8_min, fp8_max = get_fp8_min_max()
|
||||
_fused_indexer_q_rope_quant_kernel[(q.shape[0], q.shape[1])](
|
||||
positions,
|
||||
q,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
cos_sin_cache,
|
||||
cos_sin_cache.stride(0),
|
||||
q_fp8,
|
||||
q_fp8.stride(0),
|
||||
q_fp8.stride(1),
|
||||
weights,
|
||||
weights.stride(0),
|
||||
weights.stride(1),
|
||||
weights_out,
|
||||
weights_out.stride(0),
|
||||
weights_out.stride(1),
|
||||
softmax_scale,
|
||||
head_scale,
|
||||
fp8_min=fp8_min,
|
||||
fp8_max=fp8_max,
|
||||
is_neox=is_neox,
|
||||
num_warps=1,
|
||||
)
|
||||
return q_fp8, weights_out
|
||||
|
||||
|
||||
def _gather_workspace_shapes(
|
||||
total_seq_lens: int,
|
||||
head_dim: int,
|
||||
|
||||
Reference in New Issue
Block a user