[Spec Decode] Add top-k DSpark Markov projection (#49969)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
1eb3694521
commit
5789897aa4
@@ -0,0 +1,58 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
|
||||
|
||||
def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
|
||||
head = DSparkMarkovHead.__new__(DSparkMarkovHead)
|
||||
nn.Module.__init__(head)
|
||||
head.markov_w2 = nn.Linear(
|
||||
weight.shape[1], weight.shape[0], bias=False, dtype=weight.dtype
|
||||
)
|
||||
head.markov_w2.weight.data.copy_(weight)
|
||||
return head
|
||||
|
||||
|
||||
def test_gathered_markov_bias_overwrites_dense_logits():
|
||||
weight = torch.arange(21, dtype=torch.float32).view(7, 3) / 10
|
||||
markov_embed = torch.tensor([[0.5, -1.0, 0.25], [1.0, 0.5, -0.5]])
|
||||
logits = torch.tensor(
|
||||
[
|
||||
[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7],
|
||||
[0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1],
|
||||
]
|
||||
)
|
||||
values, index = logits.topk(3, dim=-1)
|
||||
values = torch.stack((values, torch.zeros_like(values)), dim=1)[:, 0]
|
||||
expected = values + torch.bmm(weight[index], markov_embed.unsqueeze(-1)).squeeze(-1)
|
||||
logits.fill_(float("-inf"))
|
||||
|
||||
result = _markov_head(weight).apply_bias_gathered(
|
||||
markov_embed, logits, values, index
|
||||
)
|
||||
|
||||
assert result is logits
|
||||
torch.testing.assert_close(result.gather(1, index), expected)
|
||||
selected = torch.zeros_like(result, dtype=torch.bool).scatter_(1, index, True)
|
||||
assert torch.isneginf(result.masked_select(~selected)).all()
|
||||
|
||||
|
||||
def test_gathered_markov_bias_matches_dense_at_full_vocab():
|
||||
weight = torch.arange(15, dtype=torch.float32).view(5, 3) / 10
|
||||
markov_embed = torch.tensor([[0.5, -1.0, 0.25]])
|
||||
logits = torch.tensor([[0.1, 0.4, -0.2, 0.3, 0.0]])
|
||||
original = logits.clone()
|
||||
values, index = logits.topk(logits.shape[-1], dim=-1)
|
||||
scale = 0.5
|
||||
logits.fill_(float("-inf"))
|
||||
|
||||
result = _markov_head(weight).apply_bias_gathered(
|
||||
markov_embed, logits, values, index, scale
|
||||
)
|
||||
|
||||
expected = original + markov_embed @ weight.T * scale
|
||||
torch.testing.assert_close(result, expected)
|
||||
Reference in New Issue
Block a user