[GLM5.2 Perf] fused_indexer_q_rope_quant triton kernel, 1.9% ~ 3.3% E2E Throughput improvement. (#46862)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-06-26 22:16:20 -07:00
committed by GitHub
parent 455f25aa13
commit b588f66dc2
2 changed files with 167 additions and 0 deletions
+36
View File
@@ -73,6 +73,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.sparse_attn_indexer import (
SparseAttnIndexer,
fused_indexer_q_rope_quant,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -674,6 +675,13 @@ class Indexer(nn.Module):
)
self.is_inplace_rope = is_inplace_rope
self.use_fused_indexer_q = (
current_platform.is_cuda()
and self.quant_block_size == self.head_dim
and self.head_dim == 128
and self.rope_dim == 64
and self.scale_fmt is not None
)
def forward(
self, hidden_states: torch.Tensor, qr: torch.Tensor, positions, rotary_emb
@@ -698,6 +706,34 @@ class Indexer(nn.Module):
rotary_emb(
positions, q[..., : self.rope_dim], k[..., : self.rope_dim].unsqueeze(1)
)
elif self.use_fused_indexer_q and q.dtype == torch.bfloat16:
# fused wk + weights_proj: one GEMM, then split
kw, _ = self.wk_weights_proj(hidden_states)
k = kw[:, : self.head_dim]
weights = kw[:, self.head_dim :]
k = self.k_norm(k)
k_pe, k_nope = torch.split(
k, [self.rope_dim, self.head_dim - self.rope_dim], dim=-1
)
q_fp8, weights = fused_indexer_q_rope_quant(
positions,
q,
rotary_emb.cos_sin_cache,
weights,
self.softmax_scale,
self.n_head**-0.5,
rotary_emb.is_neox_style,
)
# rotate only the MQA K
q_dummy = torch.empty_like(k_pe.unsqueeze(1))
_, k_pe = rotary_emb(positions, q_dummy, k_pe.unsqueeze(1))
k_pe = k_pe.reshape(-1, 1, self.rope_dim)
k = torch.cat([k_pe.squeeze(-2), k_nope], dim=-1)
return self.indexer_op(hidden_states, q_fp8, k, weights)
else:
q_pe, q_nope = torch.split(
q, [self.rope_dim, self.head_dim - self.rope_dim], dim=-1