[ROCm][Perf][Kimi-K3] Use AITER Gluon chunk_kda for KDA prefill (#59552)
Signed-off-by: Omar Muhammad <omar.muhammad@amd.com> Signed-off-by: Nicholas Susanto <nicholas.susanto@amd.com> Co-authored-by: Nicholas Susanto <nicholas.susanto@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Nicholas Susanto
Douglas Lehr
Kevin H. Luu
parent
b088eba944
commit
28e92eeff2
@@ -19,9 +19,11 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
|
||||
def _get_kda_layer(
|
||||
worker: Worker, module: str = "vllm.models.kimi_k3.nvidia.kda"
|
||||
) -> KimiK3DeltaAttention | None:
|
||||
# Kimi model construction already imports kda. Avoid importing it here.
|
||||
kda = sys.modules.get("vllm.models.kimi_k3.nvidia.kda")
|
||||
kda = sys.modules.get(module)
|
||||
if kda is None:
|
||||
return None
|
||||
|
||||
@@ -172,9 +174,59 @@ def _warm_recurrent_kda(
|
||||
)
|
||||
|
||||
|
||||
def _warm_gluon_chunk_kda(layer, input_dtype: torch.dtype) -> None:
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import (
|
||||
chunk_kda_prefill,
|
||||
get_gluon_chunk_kda_warmup_profiles,
|
||||
)
|
||||
|
||||
if layer.kda_prefill_backend != "gluon":
|
||||
return
|
||||
kv_cache = layer.kv_cache
|
||||
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
|
||||
return
|
||||
state = kv_cache[1]
|
||||
if not isinstance(state, torch.Tensor) or not state.numel():
|
||||
return
|
||||
|
||||
logger.info("Warming up Kimi-K3 Gluon chunk KDA prefill kernels.")
|
||||
h, d = int(layer.local_num_heads), int(layer.head_dim)
|
||||
for num_seqs, seq_len in get_gluon_chunk_kda_warmup_profiles():
|
||||
num_tokens = num_seqs * seq_len
|
||||
shape = (1, num_tokens, h, d)
|
||||
chunk_kda_prefill(
|
||||
q=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
k=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
v=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
raw_g=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
raw_beta=torch.zeros(
|
||||
(1, num_tokens, h), dtype=input_dtype, device=state.device
|
||||
),
|
||||
A_log=layer.A_log,
|
||||
g_bias=layer.dt_bias,
|
||||
lower_bound=layer.gate_lower_bound,
|
||||
state_cache=state,
|
||||
state_indices=torch.zeros(num_seqs, dtype=torch.int32, device=state.device),
|
||||
has_initial_state=torch.zeros(
|
||||
num_seqs, dtype=torch.bool, device=state.device
|
||||
),
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=torch.arange(
|
||||
0, num_tokens + 1, seq_len, dtype=torch.int32, device=state.device
|
||||
),
|
||||
backend="gluon",
|
||||
out=torch.empty(shape, dtype=input_dtype, device=state.device),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def kimi_k3_triton_warmup(worker: Worker) -> None:
|
||||
"""Warm Kimi-K3 Triton kernels reachable by this server."""
|
||||
if current_platform.is_rocm():
|
||||
layer = _get_kda_layer(worker, "vllm.models.kimi_k3.amd.kda")
|
||||
if layer is not None:
|
||||
_warm_gluon_chunk_kda(layer, worker.model_config.dtype)
|
||||
return
|
||||
if not current_platform.is_cuda():
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user