[Spec Decode] DSpark (#46995)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Benjamin Chislett
2026-07-01 08:30:24 -07:00
committed by GitHub
co-authored by Giancarlo Delfin mgoin
parent 63fcce4de1
commit f5a8d73377
24 changed files with 1821 additions and 94 deletions
+9 -3
View File
@@ -472,7 +472,7 @@ class DFlashQwen3Model(nn.Module):
self,
context_states: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mapping: torch.Tensor | None = None,
context_slot_mapping: torch.Tensor | list[torch.Tensor | None] | None = None,
) -> None:
"""Precompute K/V for context states write them into each layer's KV cache.
@@ -551,7 +551,13 @@ class DFlashQwen3Model(nn.Module):
# --- Per-layer cache insert ---
all_k_final = all_k_flat.view(L, num_ctx, nkv, hd)
per_layer = isinstance(context_slot_mapping, (list, tuple))
for i in range(L):
slot_mapping = (
context_slot_mapping[i] if per_layer else context_slot_mapping
)
if slot_mapping is None:
continue # dummy run: skip cache ops
attn = self._attn_layers[i]
kv_cache = attn.kv_cache
attn.impl.do_kv_cache_update(
@@ -559,7 +565,7 @@ class DFlashQwen3Model(nn.Module):
all_k_final[i],
all_v[i],
kv_cache,
context_slot_mapping,
slot_mapping,
)
def forward(
@@ -701,7 +707,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
self,
context_states: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mapping: torch.Tensor | None = None,
context_slot_mapping: torch.Tensor | list[torch.Tensor | None] | None = None,
) -> None:
"""Precompute projected + RoPE'd K/V and write to cache."""
self.model.precompute_and_store_context_kv(