[Spec Decode] DSpark (#46995)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Giancarlo Delfin
mgoin
parent
63fcce4de1
commit
f5a8d73377
@@ -472,7 +472,7 @@ class DFlashQwen3Model(nn.Module):
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mapping: torch.Tensor | None = None,
|
||||
context_slot_mapping: torch.Tensor | list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
"""Precompute K/V for context states write them into each layer's KV cache.
|
||||
|
||||
@@ -551,7 +551,13 @@ class DFlashQwen3Model(nn.Module):
|
||||
|
||||
# --- Per-layer cache insert ---
|
||||
all_k_final = all_k_flat.view(L, num_ctx, nkv, hd)
|
||||
per_layer = isinstance(context_slot_mapping, (list, tuple))
|
||||
for i in range(L):
|
||||
slot_mapping = (
|
||||
context_slot_mapping[i] if per_layer else context_slot_mapping
|
||||
)
|
||||
if slot_mapping is None:
|
||||
continue # dummy run: skip cache ops
|
||||
attn = self._attn_layers[i]
|
||||
kv_cache = attn.kv_cache
|
||||
attn.impl.do_kv_cache_update(
|
||||
@@ -559,7 +565,7 @@ class DFlashQwen3Model(nn.Module):
|
||||
all_k_final[i],
|
||||
all_v[i],
|
||||
kv_cache,
|
||||
context_slot_mapping,
|
||||
slot_mapping,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -701,7 +707,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mapping: torch.Tensor | None = None,
|
||||
context_slot_mapping: torch.Tensor | list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
"""Precompute projected + RoPE'd K/V and write to cache."""
|
||||
self.model.precompute_and_store_context_kv(
|
||||
|
||||
Reference in New Issue
Block a user