[Feat][Model] Enable KDA prefill checkpoints for GLM-5.3-Flash (#56960)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.3-Flash KDA layer with separate convolutions and a bounded safe gate."""
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
@@ -14,7 +16,14 @@ from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.kda_checkpoint import (
|
||||
FlashKDAPrefillCheckpointExporter,
|
||||
kda_prefill_checkpoint_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
@@ -37,6 +46,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if current_platform.is_rocm():
|
||||
@@ -155,6 +165,16 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
num_heads: int
|
||||
conv_size: int
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> MambaSpec:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
assert isinstance(spec, MambaSpec)
|
||||
alignment = kda_prefill_checkpoint_alignment(self.kda_prefill_backend)
|
||||
return replace(
|
||||
spec,
|
||||
num_prefill_checkpoint_blocks=int(alignment is not None),
|
||||
prefill_checkpoint_alignment=alignment,
|
||||
)
|
||||
|
||||
def get_state_dtype(
|
||||
self,
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
@@ -330,6 +350,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
self._flashkda_buffer_specs: (
|
||||
tuple[tuple[tuple[int, ...], torch.dtype], ...] | None
|
||||
) = None
|
||||
self._checkpoint_exporter: FlashKDAPrefillCheckpointExporter | None = None
|
||||
if self.kda_prefill_backend == "flashkda":
|
||||
import vllm._flashkda_C # noqa: F401
|
||||
|
||||
@@ -339,6 +360,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
max_tokens, self.local_num_heads, max_seqs
|
||||
)
|
||||
self._flashkda_buffer_specs = (
|
||||
(
|
||||
(max_seqs, self.local_num_heads, self.head_dim, self.head_dim),
|
||||
self.get_state_dtype()[1],
|
||||
),
|
||||
(
|
||||
(max_seqs, self.local_num_heads, self.head_dim, self.head_dim),
|
||||
self.get_state_dtype()[1],
|
||||
@@ -351,6 +376,9 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
vllm_config.model_config.dtype,
|
||||
),
|
||||
)
|
||||
self._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(
|
||||
state_len=self.conv_size - 1
|
||||
)
|
||||
|
||||
def _flashkda_prefill(
|
||||
self,
|
||||
@@ -362,6 +390,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
out: torch.Tensor | None,
|
||||
checkpoint: MambaPrefillCheckpointMetadata | None = None,
|
||||
raw_qkv: torch.Tensor | None = None,
|
||||
conv_state: torch.Tensor | None = None,
|
||||
recurrent_state: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused KDA chunked prefill (FlashKDA). Takes the raw gate logits ``g``
|
||||
and raw ``beta`` logits, l2-normalizes q/k in-kernel and applies the
|
||||
@@ -370,10 +402,15 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
attention output into ``out`` (a workspace buffer when ``None``) and
|
||||
returns ``(out, final_state)``."""
|
||||
assert self._flashkda_buffer_specs is not None
|
||||
final_state, workspace, workspace_out = (
|
||||
final_state, checkpoint_state, workspace, workspace_out = (
|
||||
current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs)
|
||||
)
|
||||
final_state = final_state[: initial_state.shape[0]]
|
||||
checkpoint_state = (
|
||||
checkpoint_state[: initial_state.shape[0]]
|
||||
if checkpoint is not None
|
||||
else None
|
||||
)
|
||||
if out is None:
|
||||
out = workspace_out[:, : q.shape[1]]
|
||||
# FlashKDA hardcodes dense q/k/v/g strides; beta may be row-strided.
|
||||
@@ -392,9 +429,23 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
initial_state.contiguous(),
|
||||
final_state,
|
||||
cu_seqlens.contiguous(),
|
||||
None,
|
||||
None,
|
||||
checkpoint_state,
|
||||
checkpoint.checkpoint_offsets.contiguous()
|
||||
if checkpoint is not None
|
||||
else None,
|
||||
)
|
||||
if checkpoint is not None:
|
||||
assert self._checkpoint_exporter is not None
|
||||
assert raw_qkv is not None and conv_state is not None
|
||||
assert recurrent_state is not None
|
||||
self._checkpoint_exporter.export(
|
||||
checkpoint,
|
||||
raw_qkv=raw_qkv,
|
||||
conv_state=conv_state,
|
||||
recurrent_checkpoint=checkpoint_state,
|
||||
recurrent_state=recurrent_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
return out, final_state
|
||||
|
||||
def forward(
|
||||
@@ -565,6 +616,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
q_spec, k_spec, v_spec = qkv_spec.split(self.local_projection_size, dim=-1)
|
||||
|
||||
# --- causal conv1d: non-spec path (prefill or plain decode) ---
|
||||
raw_qkv_ns = qkv_ns
|
||||
q_ns = k_ns = v_ns = None
|
||||
if attn_metadata_narrowed.num_prefills > 0:
|
||||
assert qkv_ns is not None
|
||||
@@ -662,6 +714,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
initial_state=initial_state,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
out=ns_out,
|
||||
checkpoint=attn_metadata_narrowed.checkpoint,
|
||||
raw_qkv=raw_qkv_ns,
|
||||
conv_state=conv_state,
|
||||
recurrent_state=recurrent_state,
|
||||
)
|
||||
else:
|
||||
(
|
||||
|
||||
@@ -18,10 +18,6 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointBuilder,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
@@ -271,7 +267,6 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
|
||||
recoverssm_context: "KDARecoverSSMCommitContext | None" = field(
|
||||
default=None, repr=False, compare=False
|
||||
)
|
||||
checkpoint: MambaPrefillCheckpointMetadata | None = None
|
||||
|
||||
def commit_recoverssm_state(
|
||||
self, num_accepted_tokens: torch.Tensor
|
||||
@@ -319,9 +314,6 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.checkpoint_builder = MambaPrefillCheckpointBuilder(
|
||||
vllm_config, kv_cache_spec
|
||||
)
|
||||
additional_config = vllm_config.additional_config
|
||||
self.use_flashinfer_prefill = (
|
||||
isinstance(additional_config, dict)
|
||||
|
||||
Reference in New Issue
Block a user