[Feat][Model] Enable KDA prefill checkpoints for GLM-5.3-Flash (#56960)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -193,3 +193,170 @@ def test_fused_recurrent_kda_rejects_unaddressable_layouts():
|
||||
broken["cu_seqlens"] = None if q.shape[0] > 1 else inputs["cu_seqlens"]
|
||||
with pytest.raises(AssertionError, match=r"torch.Size"):
|
||||
run_kernel(broken, state)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("offset", [16, 144, 256])
|
||||
@pytest.mark.parametrize("dim_first", [False, True])
|
||||
@pytest.mark.parametrize("num_spec", [0, 3])
|
||||
@torch.inference_mode()
|
||||
def test_prefill_checkpoint_resumes_suffix(monkeypatch, dim_first, num_spec, offset):
|
||||
"""Restoring both cached states must reproduce an uninterrupted prefill."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.kda_checkpoint import (
|
||||
FlashKDAPrefillCheckpointExporter,
|
||||
)
|
||||
from vllm.models.glm5next.common import kda
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import compute_causal_conv1d_metadata
|
||||
from vllm.v1.worker.workspace import WorkspaceManager
|
||||
|
||||
if kda._resolve_kda_prefill_backend("auto", D, torch.bfloat16, -5) != "flashkda":
|
||||
pytest.skip("requires FlashKDA")
|
||||
import vllm._flashkda_C # noqa: F401
|
||||
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
length, heads = 273, 2
|
||||
width, state_len = 3 * heads * D, 3 + num_spec
|
||||
layer = object.__new__(kda.Glm5NextLinearAttention)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.prefix = "layer.0"
|
||||
layer.local_num_heads = heads
|
||||
layer.head_dim = D
|
||||
layer.conv_size = 4
|
||||
layer.local_projection_size = heads * D
|
||||
layer.kda_safe_gate = True
|
||||
layer.kda_lower_bound = -5.0
|
||||
layer.kda_prefill_backend = "flashkda"
|
||||
layer._conv_state_dim_first = dim_first
|
||||
layer._merged_conv_weight = torch.randn(width, 4, device=device) * 0.1
|
||||
layer.q_conv1d = SimpleNamespace(bias=None)
|
||||
layer.A_log = torch.randn(1, 1, heads, 1, device=device) * 0.1
|
||||
layer.dt_bias = torch.randn(heads * D, device=device) * 0.1
|
||||
state_shape = (2, heads, D, D)
|
||||
layer._flashkda_buffer_specs = (
|
||||
(state_shape, torch.float32),
|
||||
(state_shape, torch.float32),
|
||||
(
|
||||
(torch.ops._flashkda_C.get_workspace_size(2 * length, heads, 2),),
|
||||
torch.uint8,
|
||||
),
|
||||
((1, 2 * length, heads, D), torch.bfloat16),
|
||||
)
|
||||
layer._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(state_len=3)
|
||||
workspace = WorkspaceManager(device)
|
||||
monkeypatch.setattr(kda, "current_workspace_manager", lambda: workspace)
|
||||
conv_shape = (width, state_len) if dim_first else (state_len, width)
|
||||
num_slots = 5 + (num_spec + 1 if num_spec else 0)
|
||||
conv_storage = torch.zeros(
|
||||
num_slots, width * state_len + 8, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
conv = conv_storage[:, : width * state_len].view(num_slots, *conv_shape)
|
||||
recurrent_storage = torch.zeros(
|
||||
num_slots, heads * D * D + 8, device=device, dtype=torch.float32
|
||||
)
|
||||
recurrent = recurrent_storage[:, : heads * D * D].view(num_slots, heads, D, D)
|
||||
layer.kv_cache = (conv, recurrent)
|
||||
projected = torch.randn(
|
||||
2 * length, width + heads, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
raw_qkv, beta = projected[:, :width], projected[:, width:].unsqueeze(0)
|
||||
raw_before = raw_qkv.clone()
|
||||
gate = torch.randn(1, 2 * length, heads, D, device=device, dtype=torch.bfloat16)
|
||||
|
||||
def run(qkv, g, b, slots, has_initial, checkpoint=None):
|
||||
n = qkv.shape[0] // 2
|
||||
cu_cpu = torch.tensor([0, n, 2 * n], dtype=torch.int32)
|
||||
nums, batch_ptr, chunk_offsets = compute_causal_conv1d_metadata(
|
||||
cu_cpu, device=device
|
||||
)
|
||||
metadata = GDNAttentionMetadata(
|
||||
num_prefills=2,
|
||||
num_prefill_tokens=2 * n,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_spec_decodes=0,
|
||||
num_spec_decode_tokens=0,
|
||||
num_actual_tokens=2 * n,
|
||||
has_initial_state=torch.tensor(has_initial, device=device),
|
||||
non_spec_query_start_loc=cu_cpu.to(device),
|
||||
non_spec_state_indices_tensor=torch.tensor(
|
||||
slots, dtype=torch.int32, device=device
|
||||
),
|
||||
checkpoint=checkpoint,
|
||||
nums_dict=nums,
|
||||
batch_ptr=batch_ptr,
|
||||
token_chunk_offset_ptr=chunk_offsets,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kda,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={layer.prefix: metadata}),
|
||||
)
|
||||
spec_tokens = num_spec + 1 if num_spec else 0
|
||||
if spec_tokens:
|
||||
qkv = torch.cat([qkv[:spec_tokens], qkv])
|
||||
g = torch.cat([g[:, :spec_tokens], g], dim=1)
|
||||
b = torch.cat([b[:, :spec_tokens], b], dim=1)
|
||||
metadata.num_spec_decodes = 1
|
||||
metadata.num_spec_decode_tokens = spec_tokens
|
||||
metadata.num_actual_tokens += spec_tokens
|
||||
metadata.spec_sequence_masks = torch.tensor(
|
||||
[True, False, False], device=device
|
||||
)
|
||||
metadata.spec_query_start_loc = torch.tensor(
|
||||
[0, spec_tokens], dtype=torch.int32, device=device
|
||||
)
|
||||
metadata.spec_state_indices_tensor = torch.arange(
|
||||
5, num_slots, dtype=torch.int32, device=device
|
||||
).unsqueeze(0)
|
||||
metadata.spec_token_indx = torch.arange(spec_tokens, device=device)
|
||||
metadata.non_spec_token_indx = torch.arange(
|
||||
spec_tokens, qkv.shape[0], device=device
|
||||
)
|
||||
metadata.num_accepted_tokens = torch.ones(
|
||||
1, dtype=torch.int32, device=device
|
||||
)
|
||||
out = torch.empty(
|
||||
1, qkv.shape[0], heads, D, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
layer._forward(qkv, g, b, out)
|
||||
return out[:, spec_tokens:]
|
||||
|
||||
checkpoint = MambaPrefillCheckpointMetadata(
|
||||
torch.tensor([offset, 0], dtype=torch.int32, device=device),
|
||||
torch.tensor([3, 0], dtype=torch.int32, device=device),
|
||||
)
|
||||
expected = run(raw_qkv, gate, beta, [1, 2], [False, False], checkpoint)
|
||||
expected_state = recurrent[1].clone()
|
||||
saved_conv, saved_recurrent = conv[3].clone(), recurrent[3].clone()
|
||||
torch.testing.assert_close(raw_qkv, raw_before)
|
||||
conv_view = conv if dim_first else conv.transpose(-1, -2)
|
||||
torch.testing.assert_close(conv_view[3, :, :3], raw_qkv[offset - 3 : offset].T)
|
||||
assert torch.count_nonzero(conv_view[3, :, 3:]) == 0
|
||||
assert torch.count_nonzero(conv[0]) == torch.count_nonzero(recurrent[0]) == 0
|
||||
|
||||
run(
|
||||
torch.cat([raw_qkv[:offset], raw_qkv[length : length + offset]]),
|
||||
torch.cat([gate[:, :offset], gate[:, length : length + offset]], dim=1),
|
||||
torch.cat([beta[:, :offset], beta[:, length : length + offset]], dim=1),
|
||||
[1, 2],
|
||||
[False, False],
|
||||
)
|
||||
torch.testing.assert_close(saved_recurrent, recurrent[1], atol=0.003, rtol=0.03)
|
||||
|
||||
suffix = slice(offset, length)
|
||||
qkv = torch.cat([raw_qkv[suffix], raw_qkv[length + offset :]])
|
||||
g = torch.cat([gate[:, suffix], gate[:, length + offset :]], dim=1)
|
||||
b = torch.cat([beta[:, suffix], beta[:, length + offset :]], dim=1)
|
||||
conv[4].copy_(saved_conv)
|
||||
recurrent[4].copy_(saved_recurrent)
|
||||
actual = run(qkv, g, b, [3, 4], [True, True])
|
||||
torch.testing.assert_close(
|
||||
actual[:, : length - offset], expected[:, suffix], atol=0.003, rtol=0.03
|
||||
)
|
||||
torch.testing.assert_close(recurrent[3], expected_state, atol=0.003, rtol=0.03)
|
||||
|
||||
@@ -367,3 +367,66 @@ def test_cudagraph_capture_batch_stays_decode_only():
|
||||
assert staged is not None
|
||||
assert staged.data_ptr() == builder.non_spec_state_indices_tensor.data_ptr()
|
||||
torch.testing.assert_close(staged, common_attn_metadata.block_table_tensor[:, 0])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_spec", [0, 3])
|
||||
@pytest.mark.parametrize("prefix_match_unit, expected_offset", [(16, 96), (8, 96)])
|
||||
def test_checkpoint_metadata_preserves_non_spec_order(
|
||||
num_spec, prefix_match_unit, expected_offset
|
||||
):
|
||||
"""Only eligible non-spec rows get a checkpoint in their reserved page."""
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
if num_spec > 0:
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=num_spec,
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = "align"
|
||||
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=64,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=num_spec,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
prefill_checkpoint_alignment=16,
|
||||
)
|
||||
builder = GDNAttentionMetadataBuilder(
|
||||
kv_cache_spec=mamba_spec,
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=vllm_config,
|
||||
device=DEVICE,
|
||||
)
|
||||
batch = BatchSpec(
|
||||
seq_lens=[65, 100, 100], query_lens=[3 if num_spec else 1, 100, 99]
|
||||
)
|
||||
common = create_common_attn_metadata(batch, 64, DEVICE, arange_block_indices=True)
|
||||
table = common.block_table_tensor + 1
|
||||
if num_spec:
|
||||
table = torch.cat([table, torch.zeros(3, num_spec, dtype=table.dtype)], dim=1)
|
||||
common = common.replace(block_table_tensor=table)
|
||||
kwargs = {}
|
||||
if num_spec:
|
||||
kwargs = dict(
|
||||
num_decode_draft_tokens_cpu=torch.tensor([2, -1, -1], dtype=torch.int32),
|
||||
num_accepted_tokens=torch.ones(3, dtype=torch.int32),
|
||||
)
|
||||
|
||||
actual = builder.build(0, common, **kwargs)
|
||||
assert actual.checkpoint is not None
|
||||
offsets = [expected_offset, 0] if num_spec else [0, expected_offset, 0]
|
||||
slots = [common.block_table_tensor[1, 0].item(), NULL_BLOCK_ID]
|
||||
if not num_spec:
|
||||
slots.insert(0, NULL_BLOCK_ID)
|
||||
torch.testing.assert_close(
|
||||
actual.checkpoint.checkpoint_offsets, torch.tensor(offsets, dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual.checkpoint.state_indices, torch.tensor(slots, dtype=torch.int32)
|
||||
)
|
||||
|
||||
@@ -76,6 +76,8 @@ class MambaPrefillCheckpointBuilder:
|
||||
m: CommonAttentionMetadata,
|
||||
request_rows: list[int],
|
||||
) -> MambaPrefillCheckpointMetadata | None:
|
||||
if self.vllm_config.cache_config.mamba_cache_mode != "align":
|
||||
return None
|
||||
if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0:
|
||||
return None
|
||||
assert m.seq_lens_cpu_upper_bound is not None
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.3-Flash KDA layer with separate convolutions and a bounded safe gate."""
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
@@ -14,7 +16,14 @@ from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.kda_checkpoint import (
|
||||
FlashKDAPrefillCheckpointExporter,
|
||||
kda_prefill_checkpoint_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
@@ -37,6 +46,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if current_platform.is_rocm():
|
||||
@@ -155,6 +165,16 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
num_heads: int
|
||||
conv_size: int
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> MambaSpec:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
assert isinstance(spec, MambaSpec)
|
||||
alignment = kda_prefill_checkpoint_alignment(self.kda_prefill_backend)
|
||||
return replace(
|
||||
spec,
|
||||
num_prefill_checkpoint_blocks=int(alignment is not None),
|
||||
prefill_checkpoint_alignment=alignment,
|
||||
)
|
||||
|
||||
def get_state_dtype(
|
||||
self,
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
@@ -330,6 +350,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
self._flashkda_buffer_specs: (
|
||||
tuple[tuple[tuple[int, ...], torch.dtype], ...] | None
|
||||
) = None
|
||||
self._checkpoint_exporter: FlashKDAPrefillCheckpointExporter | None = None
|
||||
if self.kda_prefill_backend == "flashkda":
|
||||
import vllm._flashkda_C # noqa: F401
|
||||
|
||||
@@ -339,6 +360,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
max_tokens, self.local_num_heads, max_seqs
|
||||
)
|
||||
self._flashkda_buffer_specs = (
|
||||
(
|
||||
(max_seqs, self.local_num_heads, self.head_dim, self.head_dim),
|
||||
self.get_state_dtype()[1],
|
||||
),
|
||||
(
|
||||
(max_seqs, self.local_num_heads, self.head_dim, self.head_dim),
|
||||
self.get_state_dtype()[1],
|
||||
@@ -351,6 +376,9 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
vllm_config.model_config.dtype,
|
||||
),
|
||||
)
|
||||
self._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(
|
||||
state_len=self.conv_size - 1
|
||||
)
|
||||
|
||||
def _flashkda_prefill(
|
||||
self,
|
||||
@@ -362,6 +390,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
out: torch.Tensor | None,
|
||||
checkpoint: MambaPrefillCheckpointMetadata | None = None,
|
||||
raw_qkv: torch.Tensor | None = None,
|
||||
conv_state: torch.Tensor | None = None,
|
||||
recurrent_state: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused KDA chunked prefill (FlashKDA). Takes the raw gate logits ``g``
|
||||
and raw ``beta`` logits, l2-normalizes q/k in-kernel and applies the
|
||||
@@ -370,10 +402,15 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
attention output into ``out`` (a workspace buffer when ``None``) and
|
||||
returns ``(out, final_state)``."""
|
||||
assert self._flashkda_buffer_specs is not None
|
||||
final_state, workspace, workspace_out = (
|
||||
final_state, checkpoint_state, workspace, workspace_out = (
|
||||
current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs)
|
||||
)
|
||||
final_state = final_state[: initial_state.shape[0]]
|
||||
checkpoint_state = (
|
||||
checkpoint_state[: initial_state.shape[0]]
|
||||
if checkpoint is not None
|
||||
else None
|
||||
)
|
||||
if out is None:
|
||||
out = workspace_out[:, : q.shape[1]]
|
||||
# FlashKDA hardcodes dense q/k/v/g strides; beta may be row-strided.
|
||||
@@ -392,9 +429,23 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
initial_state.contiguous(),
|
||||
final_state,
|
||||
cu_seqlens.contiguous(),
|
||||
None,
|
||||
None,
|
||||
checkpoint_state,
|
||||
checkpoint.checkpoint_offsets.contiguous()
|
||||
if checkpoint is not None
|
||||
else None,
|
||||
)
|
||||
if checkpoint is not None:
|
||||
assert self._checkpoint_exporter is not None
|
||||
assert raw_qkv is not None and conv_state is not None
|
||||
assert recurrent_state is not None
|
||||
self._checkpoint_exporter.export(
|
||||
checkpoint,
|
||||
raw_qkv=raw_qkv,
|
||||
conv_state=conv_state,
|
||||
recurrent_checkpoint=checkpoint_state,
|
||||
recurrent_state=recurrent_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
return out, final_state
|
||||
|
||||
def forward(
|
||||
@@ -565,6 +616,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
q_spec, k_spec, v_spec = qkv_spec.split(self.local_projection_size, dim=-1)
|
||||
|
||||
# --- causal conv1d: non-spec path (prefill or plain decode) ---
|
||||
raw_qkv_ns = qkv_ns
|
||||
q_ns = k_ns = v_ns = None
|
||||
if attn_metadata_narrowed.num_prefills > 0:
|
||||
assert qkv_ns is not None
|
||||
@@ -662,6 +714,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
initial_state=initial_state,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
out=ns_out,
|
||||
checkpoint=attn_metadata_narrowed.checkpoint,
|
||||
raw_qkv=raw_qkv_ns,
|
||||
conv_state=conv_state,
|
||||
recurrent_state=recurrent_state,
|
||||
)
|
||||
else:
|
||||
(
|
||||
|
||||
@@ -18,10 +18,6 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointBuilder,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
@@ -271,7 +267,6 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
|
||||
recoverssm_context: "KDARecoverSSMCommitContext | None" = field(
|
||||
default=None, repr=False, compare=False
|
||||
)
|
||||
checkpoint: MambaPrefillCheckpointMetadata | None = None
|
||||
|
||||
def commit_recoverssm_state(
|
||||
self, num_accepted_tokens: torch.Tensor
|
||||
@@ -319,9 +314,6 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.checkpoint_builder = MambaPrefillCheckpointBuilder(
|
||||
vllm_config, kv_cache_spec
|
||||
)
|
||||
additional_config = vllm_config.additional_config
|
||||
self.use_flashinfer_prefill = (
|
||||
isinstance(additional_config, dict)
|
||||
|
||||
@@ -8,6 +8,10 @@ from typing import Literal
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointBuilder,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
@@ -48,6 +52,7 @@ class GDNAttentionMetadata:
|
||||
num_spec_decode_tokens: int
|
||||
num_actual_tokens: int
|
||||
|
||||
checkpoint: MambaPrefillCheckpointMetadata | None = None
|
||||
has_initial_state: torch.Tensor | None = None
|
||||
|
||||
spec_query_start_loc: torch.Tensor | None = None # shape: [num_spec_decodes + 1,]
|
||||
@@ -97,6 +102,9 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.speculative_config = vllm_config.speculative_config
|
||||
self.checkpoint_builder = MambaPrefillCheckpointBuilder(
|
||||
vllm_config, kv_cache_spec
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
_resolve_gdn_prefill_backend,
|
||||
)
|
||||
@@ -466,6 +474,13 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
else:
|
||||
has_initial_state = None
|
||||
|
||||
checkpoint: MambaPrefillCheckpointMetadata | None = None
|
||||
if num_prefills > 0:
|
||||
request_rows = list(range(m.num_reqs))
|
||||
if spec_sequence_masks_cpu is not None:
|
||||
request_rows = (~spec_sequence_masks_cpu).nonzero().flatten().tolist()
|
||||
checkpoint = self.checkpoint_builder.build(m, request_rows)
|
||||
|
||||
# Function code counted on either presency non-spec decode or spec decode,
|
||||
# but not both.
|
||||
assert not (num_decodes > 0 and num_spec_decodes > 0), (
|
||||
@@ -543,6 +558,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
num_spec_decodes=num_spec_decodes,
|
||||
num_spec_decode_tokens=num_spec_decode_tokens,
|
||||
num_actual_tokens=m.num_actual_tokens,
|
||||
checkpoint=checkpoint,
|
||||
has_initial_state=has_initial_state,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
|
||||
Reference in New Issue
Block a user