[Bugfix][Kimi K3] Skip absent metadata during CUDA graph profiling (#53581)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
fused_recurrent_kda_packed_decode as fused_recurrent_kda_packed_decode_amd,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia import kda as nvidia_kda
|
||||
from vllm.models.kimi_k3.nvidia.kda import (
|
||||
_flashkda_prefill,
|
||||
_store_cache_checkpoints_kernel,
|
||||
@@ -59,6 +60,19 @@ PACKED_DECODE_IMPLS = {
|
||||
}
|
||||
|
||||
|
||||
def test_kda_warmup_skips_missing_metadata(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
nvidia_kda,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={}),
|
||||
)
|
||||
layer = object.__new__(nvidia_kda.KimiK3DeltaAttention)
|
||||
object.__setattr__(layer, "prefix", "language_model.model.layers.0.self_attn")
|
||||
empty = torch.empty(0, device=DEVICE)
|
||||
|
||||
assert layer._forward(empty, empty, empty, empty, empty) is None
|
||||
|
||||
|
||||
def test_kda_recoverssm_config_state_layout():
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
|
||||
@@ -313,7 +313,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
|
||||
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata_narrowed is None:
|
||||
return
|
||||
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
||||
m = attn_metadata_narrowed
|
||||
has_initial_state = m.has_initial_state
|
||||
|
||||
@@ -658,7 +658,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
)
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
|
||||
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata_narrowed is None:
|
||||
return
|
||||
assert isinstance(attn_metadata_narrowed, KimiK3KDAMetadata)
|
||||
m = attn_metadata_narrowed
|
||||
has_initial_state = m.has_initial_state
|
||||
|
||||
Reference in New Issue
Block a user