[Spec Decode] Add top-k DSpark Markov projection (#49969)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-08-04 12:52:53 +00:00
committed by GitHub
co-authored by Andrii Skliar
parent 1eb3694521
commit 5789897aa4
4 changed files with 235 additions and 26 deletions
@@ -77,6 +77,30 @@ class DSparkMarkovHead(nn.Module):
"""Vocab-size transition bias from a Markov embedding ([B, r] -> [B, V])."""
return logits_processor(self.markov_w2, markov_embed)
def apply_bias_gathered(
self,
markov_embed: torch.Tensor,
logits: torch.Tensor,
values: torch.Tensor,
index: torch.Tensor,
scale: float = 1.0,
) -> torch.Tensor:
"""Apply the Markov bias only to selected rows of ``logits``.
The caller initializes ``logits`` to ``-inf`` once for all draft
positions. This method scatters the corrected candidate values into
that dense buffer so the normal sampler sees the truncated proposal.
"""
weight = self.markov_w2.weight[index]
corrected = values.unsqueeze(-1)
corrected.baddbmm_(
weight,
markov_embed.unsqueeze(-1),
beta=1.0,
alpha=scale,
)
return logits.scatter_(1, index, corrected.squeeze(-1))
class Qwen3DSparkModel(DFlashQwen3Model):
"""DFlash Qwen3 backbone + DSpark Markov head."""
@@ -158,6 +182,21 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.bias(markov_embed, self.logits_processor)
def apply_markov_bias_gathered(
self,
markov_embed: torch.Tensor,
logits: torch.Tensor,
values: torch.Tensor,
index: torch.Tensor,
) -> torch.Tensor:
return self.model.markov_head.apply_bias_gathered(
markov_embed,
logits,
values,
index,
self.logits_processor.scale,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
model_weights = {}
includes_embed_tokens = False