[Spec Decode] Add top-k DSpark Markov projection (#49969)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
1eb3694521
commit
5789897aa4
@@ -77,6 +77,30 @@ class DSparkMarkovHead(nn.Module):
|
||||
"""Vocab-size transition bias from a Markov embedding ([B, r] -> [B, V])."""
|
||||
return logits_processor(self.markov_w2, markov_embed)
|
||||
|
||||
def apply_bias_gathered(
|
||||
self,
|
||||
markov_embed: torch.Tensor,
|
||||
logits: torch.Tensor,
|
||||
values: torch.Tensor,
|
||||
index: torch.Tensor,
|
||||
scale: float = 1.0,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the Markov bias only to selected rows of ``logits``.
|
||||
|
||||
The caller initializes ``logits`` to ``-inf`` once for all draft
|
||||
positions. This method scatters the corrected candidate values into
|
||||
that dense buffer so the normal sampler sees the truncated proposal.
|
||||
"""
|
||||
weight = self.markov_w2.weight[index]
|
||||
corrected = values.unsqueeze(-1)
|
||||
corrected.baddbmm_(
|
||||
weight,
|
||||
markov_embed.unsqueeze(-1),
|
||||
beta=1.0,
|
||||
alpha=scale,
|
||||
)
|
||||
return logits.scatter_(1, index, corrected.squeeze(-1))
|
||||
|
||||
|
||||
class Qwen3DSparkModel(DFlashQwen3Model):
|
||||
"""DFlash Qwen3 backbone + DSpark Markov head."""
|
||||
@@ -158,6 +182,21 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
def apply_markov_bias_gathered(
|
||||
self,
|
||||
markov_embed: torch.Tensor,
|
||||
logits: torch.Tensor,
|
||||
values: torch.Tensor,
|
||||
index: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.model.markov_head.apply_bias_gathered(
|
||||
markov_embed,
|
||||
logits,
|
||||
values,
|
||||
index,
|
||||
self.logits_processor.scale,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
model_weights = {}
|
||||
includes_embed_tokens = False
|
||||
|
||||
Reference in New Issue
Block a user