Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
225 lines
7.4 KiB
Python
225 lines
7.4 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||
from dataclasses import dataclass
|
||
from typing import Any
|
||
|
||
import torch
|
||
|
||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||
LaunchSpec,
|
||
TritonWarmupTensor,
|
||
VllmTritonJitKernel,
|
||
)
|
||
from vllm.platforms import current_platform
|
||
from vllm.triton_utils import tl, triton
|
||
from vllm.utils.math_utils import next_power_of_2
|
||
|
||
|
||
class FusedQKVRMSNormKernel(VllmTritonJitKernel["FusedQKVRMSNormKernel.CompileKey"]):
|
||
@dataclass(frozen=True)
|
||
class CompileKey:
|
||
dtype: torch.dtype
|
||
q_size: int
|
||
kv_size: int
|
||
block_size: int
|
||
q_in_stride: int
|
||
q_out_stride: int
|
||
kv_in_stride: int
|
||
kv_out_stride: int
|
||
eps: float
|
||
launch_pdl: bool
|
||
|
||
@staticmethod
|
||
@triton.jit
|
||
def kernel(
|
||
q_ptr,
|
||
q_out_ptr,
|
||
q_weight_ptr,
|
||
q_in_stride,
|
||
q_out_stride,
|
||
kv_ptr,
|
||
kv_out_ptr,
|
||
kv_weight_ptr,
|
||
kv_in_stride,
|
||
kv_out_stride,
|
||
eps,
|
||
Q_SIZE: tl.constexpr,
|
||
KV_SIZE: tl.constexpr,
|
||
BLOCK_SIZE: tl.constexpr,
|
||
launch_pdl: tl.constexpr,
|
||
):
|
||
# num_tokens goes on grid-x (max 2**31 - 1); task goes on grid-y.
|
||
# CUDA's grid-y/z are capped at 65535, so putting num_tokens there crashes
|
||
# the launch at max-num-batched-tokens >= 65536 with "invalid argument".
|
||
# int64: q_in_stride can be ~24K (128 heads × 192) and overflows int32
|
||
# past num_tokens ~87K under large chunked prefill.
|
||
token_idx = tl.program_id(0).to(tl.int64)
|
||
pid_task = tl.program_id(1)
|
||
|
||
if pid_task == 0:
|
||
SIZE = Q_SIZE
|
||
row_in = q_ptr + token_idx * q_in_stride
|
||
weight_ptr = q_weight_ptr
|
||
row_out = q_out_ptr + token_idx * q_out_stride
|
||
else:
|
||
SIZE = KV_SIZE
|
||
row_in = kv_ptr + token_idx * kv_in_stride
|
||
weight_ptr = kv_weight_ptr
|
||
row_out = kv_out_ptr + token_idx * kv_out_stride
|
||
|
||
# RMSNorm in fp32 throughout — matches
|
||
# csrc/layernorm_kernels.cu's
|
||
# `(scalar_t)(x * s_variance * w)` and DeepseekV4's compressor kernel, which
|
||
# keep x, rrms, and w all in fp32 and perform a single cast at store.
|
||
block = tl.arange(0, BLOCK_SIZE)
|
||
mask = block < SIZE
|
||
# The weight load does not depend on the producer's output, so issue it
|
||
# before the PDL wait: gamma streams in while the producer finishes,
|
||
# leaving a single dependent global round trip (x) after the wait.
|
||
w = tl.load(weight_ptr + block, mask=mask, other=0.0).to(tl.float32)
|
||
|
||
if launch_pdl:
|
||
tl.extra.cuda.gdc_wait()
|
||
tl.extra.cuda.gdc_launch_dependents()
|
||
|
||
x = tl.load(row_in + block, mask=mask, other=0.0).to(tl.float32)
|
||
variance = tl.sum(x * x, axis=0) / SIZE
|
||
rrms = tl.rsqrt(variance + eps)
|
||
y = x * rrms * w
|
||
tl.store(row_out + block, y.to(row_out.dtype.element_ty), mask=mask)
|
||
|
||
def dispatch( # type: ignore[override]
|
||
self,
|
||
*,
|
||
dtype: torch.dtype,
|
||
q_size: int,
|
||
kv_size: int,
|
||
eps: float,
|
||
launch_pdl: bool,
|
||
**compile_key_fields: int,
|
||
) -> CompileKey:
|
||
max_size = q_size if q_size >= kv_size else kv_size
|
||
return self.CompileKey(
|
||
**compile_key_fields,
|
||
dtype=dtype,
|
||
q_size=q_size,
|
||
kv_size=kv_size,
|
||
block_size=next_power_of_2(max_size),
|
||
eps=eps,
|
||
launch_pdl=launch_pdl,
|
||
)
|
||
|
||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||
hf_config = vllm_config.model_config.hf_config
|
||
q_size = int(getattr(hf_config, "q_lora_rank", 0) or 0)
|
||
kv_size = int(getattr(hf_config, "head_dim", 0) or 0)
|
||
if q_size <= 0 or kv_size <= 0:
|
||
return []
|
||
|
||
input_stride = q_size + kv_size
|
||
return self._trace_dispatch(self.dispatch)(
|
||
dtype=vllm_config.model_config.dtype,
|
||
q_size=q_size,
|
||
kv_size=kv_size,
|
||
q_in_stride=input_stride,
|
||
q_out_stride=(input_stride, q_size),
|
||
kv_in_stride=input_stride,
|
||
kv_out_stride=(input_stride, kv_size),
|
||
eps=float(hf_config.rms_norm_eps),
|
||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||
)
|
||
|
||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||
return dict(
|
||
q=TritonWarmupTensor(
|
||
compile_key.dtype,
|
||
shape=(1, compile_key.q_size),
|
||
strides=(compile_key.q_in_stride, 1),
|
||
),
|
||
kv=TritonWarmupTensor(
|
||
compile_key.dtype,
|
||
shape=(1, compile_key.kv_size),
|
||
strides=(compile_key.kv_in_stride, 1),
|
||
),
|
||
q_out=TritonWarmupTensor(
|
||
compile_key.dtype,
|
||
shape=(1, compile_key.q_size),
|
||
strides=(compile_key.q_out_stride, 1),
|
||
),
|
||
kv_out=TritonWarmupTensor(
|
||
compile_key.dtype,
|
||
shape=(1, compile_key.kv_size),
|
||
strides=(compile_key.kv_out_stride, 1),
|
||
),
|
||
q_weight=TritonWarmupTensor(
|
||
compile_key.dtype,
|
||
shape=(compile_key.q_size,),
|
||
),
|
||
kv_weight=TritonWarmupTensor(
|
||
compile_key.dtype,
|
||
shape=(compile_key.kv_size,),
|
||
),
|
||
eps=compile_key.eps,
|
||
)
|
||
|
||
@kernel_launcher
|
||
def __call__(
|
||
self,
|
||
q: torch.Tensor,
|
||
kv: torch.Tensor,
|
||
q_out: torch.Tensor,
|
||
kv_out: torch.Tensor,
|
||
q_weight: torch.Tensor,
|
||
kv_weight: torch.Tensor,
|
||
eps: float,
|
||
) -> LaunchSpec:
|
||
q_size = q.shape[1]
|
||
kv_size = kv.shape[1]
|
||
block_size = next_power_of_2(max(q_size, kv_size))
|
||
return (q.shape[0], 2), dict(
|
||
q_in_stride=q.stride(0),
|
||
q_out_stride=q_out.stride(0),
|
||
kv_in_stride=kv.stride(0),
|
||
kv_out_stride=kv_out.stride(0),
|
||
Q_SIZE=q_size,
|
||
KV_SIZE=kv_size,
|
||
BLOCK_SIZE=block_size,
|
||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||
# One vectorized 128-bit access per thread for K3/DSv4-sized rows,
|
||
# instead of a serial per-thread chain at the default 4 warps.
|
||
num_warps=8 if block_size >= 2048 else 4,
|
||
)
|
||
|
||
|
||
def fused_q_kv_rmsnorm(
|
||
qr: torch.Tensor,
|
||
kv: torch.Tensor,
|
||
q_weight: torch.Tensor,
|
||
kv_weight: torch.Tensor,
|
||
eps: float,
|
||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||
assert qr.ndim == 2 and kv.ndim == 2
|
||
assert qr.shape[0] == kv.shape[0], (
|
||
f"token dim mismatch: qr={qr.shape}, kv={kv.shape}"
|
||
)
|
||
assert qr.stride(-1) == 1 and kv.stride(-1) == 1
|
||
assert q_weight.is_contiguous() and kv_weight.is_contiguous()
|
||
|
||
qr_out = torch.empty(qr.shape, dtype=qr.dtype, device=qr.device)
|
||
kv_out = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
|
||
if qr.shape[0] > 0:
|
||
_FUSED_Q_KV_RMSNORM_KERNEL(
|
||
qr,
|
||
kv,
|
||
qr_out,
|
||
kv_out,
|
||
q_weight,
|
||
kv_weight,
|
||
eps,
|
||
)
|
||
return qr_out, kv_out
|
||
|
||
|
||
_FUSED_Q_KV_RMSNORM_KERNEL = FusedQKVRMSNormKernel()
|