[Spec Decode] Add NVFP4 DSpark gathered top-k projection (#55713)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Andrii Skliar
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
af51d0ef03
commit
83fe99399e
@@ -1,10 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptLinearMethod,
|
||||
resolve,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_to_dtype,
|
||||
nvfp4_gathered_bias,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
|
||||
@@ -14,6 +24,8 @@ def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
|
||||
weight.shape[1], weight.shape[0], bias=False, dtype=weight.dtype
|
||||
)
|
||||
head.markov_w2.weight.data.copy_(weight)
|
||||
head.markov_w2._retain_weight_for_gather = False
|
||||
head.markov_w2.is_w4a16_nvfp4 = False
|
||||
return head
|
||||
|
||||
|
||||
@@ -56,3 +68,174 @@ def test_gathered_markov_bias_matches_dense_at_full_vocab():
|
||||
|
||||
expected = original + markov_embed @ weight.T * scale
|
||||
torch.testing.assert_close(result, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cuda_alike_platform", [False, True])
|
||||
def test_gathered_markov_bias_dequantizes_selected_w4a16_rows(
|
||||
monkeypatch, cuda_alike_platform
|
||||
):
|
||||
packed_weight = torch.tensor(
|
||||
[[0x00] * 8, [0x22] * 8, [0xAA] * 8, [0x31] * 8], dtype=torch.uint8
|
||||
)
|
||||
weight_scale = torch.ones((4, 1), dtype=torch.float8_e4m3fn)
|
||||
|
||||
layer = nn.Module()
|
||||
layer.weight = nn.Parameter(packed_weight, requires_grad=False)
|
||||
layer.weight_scale = nn.Parameter(weight_scale, requires_grad=False)
|
||||
layer.weight_scale_2 = nn.Parameter(torch.ones(1), requires_grad=False)
|
||||
layer._retain_weight_for_gather = True
|
||||
|
||||
spec, ctx, fmt = resolve("W4A16_NVFP4", type("Config", (), {"group_size": 16}), "")
|
||||
method = ModelOptLinearMethod(spec, ctx, fmt)
|
||||
|
||||
class RepackingKernel:
|
||||
@staticmethod
|
||||
def process_weights_after_loading(layer):
|
||||
layer.weight = nn.Parameter(
|
||||
torch.zeros((1, 1), dtype=torch.int32), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = nn.Parameter(
|
||||
torch.zeros((1, 1), dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
layer.weight_global_scale = nn.Parameter(
|
||||
torch.zeros(1, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
|
||||
method.kernel = RepackingKernel()
|
||||
method.process_weights_after_loading(layer)
|
||||
assert layer.is_w4a16_nvfp4
|
||||
|
||||
head = DSparkMarkovHead.__new__(DSparkMarkovHead)
|
||||
nn.Module.__init__(head)
|
||||
head.markov_w2 = layer
|
||||
markov_embed = torch.tensor([[1.0] * 16, [0.5, -0.5] * 8], dtype=torch.float32)
|
||||
index = torch.tensor([[1, 3], [2, 0]])
|
||||
values = torch.tensor([[0.25, -0.5], [1.0, 2.0]])
|
||||
original_values = values.clone()
|
||||
logits = torch.full((2, 4), float("-inf"))
|
||||
|
||||
# CPU tensors must use the fallback even on CUDA/ROCm hosts.
|
||||
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: cuda_alike_platform)
|
||||
result = head.apply_bias_gathered(markov_embed, logits, values, index)
|
||||
|
||||
dense_weight = torch.tensor([[0.0] * 16, [1.0] * 16, [-1.0] * 16, [0.5, 1.5] * 8])
|
||||
expected = original_values + torch.bmm(
|
||||
dense_weight[index], markov_embed.unsqueeze(-1)
|
||||
).squeeze(-1)
|
||||
torch.testing.assert_close(result.gather(1, index), expected)
|
||||
torch.testing.assert_close(values, original_values)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_nvfp4_gathered_bias_rejects_non_16_group_size():
|
||||
device = torch.device("cuda")
|
||||
rank = 32
|
||||
vocab_size = 2
|
||||
|
||||
with pytest.raises(ValueError, match="group size of 16"):
|
||||
nvfp4_gathered_bias(
|
||||
torch.zeros((1, rank), dtype=torch.bfloat16, device=device),
|
||||
torch.zeros((vocab_size, rank // 2), dtype=torch.uint8, device=device),
|
||||
torch.ones(
|
||||
(vocab_size, rank // 32), dtype=torch.float8_e4m3fn, device=device
|
||||
),
|
||||
torch.ones(1, dtype=torch.float32, device=device),
|
||||
torch.zeros((1, 1), dtype=torch.bfloat16, device=device),
|
||||
torch.zeros((1, 1), dtype=torch.int64, device=device),
|
||||
torch.full(
|
||||
(1, vocab_size), float("-inf"), dtype=torch.bfloat16, device=device
|
||||
),
|
||||
1.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("batch_size", [1, 2, 4, 8])
|
||||
def test_nvfp4_gathered_markov_bias_kernel_matches_reference(batch_size: int):
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
topk = 256
|
||||
rank = 512
|
||||
vocab_size = 512
|
||||
alpha = 0.75
|
||||
|
||||
packed_weight = torch.randint(
|
||||
0, 256, (vocab_size, rank // 2), dtype=torch.uint8, device=device
|
||||
)
|
||||
weight_scale = torch.full(
|
||||
(vocab_size, rank // 16),
|
||||
0.5,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=device,
|
||||
)
|
||||
global_scale = torch.tensor([0.25], dtype=torch.float32, device=device)
|
||||
|
||||
markov_storage = torch.randn(
|
||||
batch_size, 2, rank, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
values_storage = torch.randn(
|
||||
batch_size, 2, topk, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
index_storage = torch.empty(batch_size, 2, topk, dtype=torch.int64, device=device)
|
||||
for batch_idx in range(batch_size):
|
||||
index_storage[batch_idx, 0] = (
|
||||
torch.arange(topk, device=device) + batch_idx
|
||||
) % vocab_size
|
||||
logits_storage = torch.full(
|
||||
(batch_size, 2, vocab_size),
|
||||
float("-inf"),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
markov_embed = markov_storage[:, 0]
|
||||
values = values_storage[:, 0]
|
||||
index = index_storage[:, 0]
|
||||
logits = logits_storage[:, 0]
|
||||
|
||||
layer = nn.Module()
|
||||
layer.register_buffer("_nvfp4_weight_for_gather", packed_weight)
|
||||
layer.register_buffer("_nvfp4_weight_scale_for_gather", weight_scale)
|
||||
layer.register_buffer("_nvfp4_weight_global_scale_for_gather", global_scale)
|
||||
layer._nvfp4_group_size_for_gather = 16
|
||||
layer._retain_weight_for_gather = True
|
||||
layer.is_w4a16_nvfp4 = True
|
||||
head = DSparkMarkovHead.__new__(DSparkMarkovHead)
|
||||
nn.Module.__init__(head)
|
||||
head.markov_w2 = layer
|
||||
|
||||
flat_index = index.reshape(-1)
|
||||
selected_weight = dequantize_to_dtype(
|
||||
packed_weight.index_select(0, flat_index),
|
||||
weight_scale.index_select(0, flat_index),
|
||||
global_scale,
|
||||
dtype=torch.bfloat16,
|
||||
block_size=16,
|
||||
swizzle=False,
|
||||
).view(batch_size, topk, rank)
|
||||
expected = (
|
||||
values
|
||||
+ torch.bmm(selected_weight, markov_embed.unsqueeze(-1)).squeeze(-1) * alpha
|
||||
)
|
||||
|
||||
result = head.apply_bias_gathered(markov_embed, logits, values, index, alpha)
|
||||
|
||||
torch.testing.assert_close(
|
||||
result.gather(1, index), expected, rtol=1e-2, atol=6.25e-2
|
||||
)
|
||||
selected = torch.zeros_like(result, dtype=torch.bool).scatter_(1, index, True)
|
||||
assert torch.isneginf(result.masked_select(~selected)).all()
|
||||
|
||||
if batch_size == 1:
|
||||
logits.fill_(float("-inf"))
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = head.apply_bias_gathered(
|
||||
markov_embed, logits, values, index, alpha
|
||||
)
|
||||
logits.fill_(float("-inf"))
|
||||
graph.replay()
|
||||
assert captured is logits
|
||||
torch.testing.assert_close(
|
||||
logits.gather(1, index), expected, rtol=1e-2, atol=6.25e-2
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user