[Spec Decode] Add NVFP4 DSpark gathered top-k projection (#55713)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Andrii Skliar
2026-09-09 17:33:46 +00:00
committed by GitHub
co-authored by Andrii Skliar mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent af51d0ef03
commit 83fe99399e
4 changed files with 491 additions and 3 deletions
+183
View File
@@ -1,10 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch import nn
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptLinearMethod,
resolve,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
dequantize_to_dtype,
nvfp4_gathered_bias,
)
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
from vllm.platforms import current_platform
def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
@@ -14,6 +24,8 @@ def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
weight.shape[1], weight.shape[0], bias=False, dtype=weight.dtype
)
head.markov_w2.weight.data.copy_(weight)
head.markov_w2._retain_weight_for_gather = False
head.markov_w2.is_w4a16_nvfp4 = False
return head
@@ -56,3 +68,174 @@ def test_gathered_markov_bias_matches_dense_at_full_vocab():
expected = original + markov_embed @ weight.T * scale
torch.testing.assert_close(result, expected)
@pytest.mark.parametrize("cuda_alike_platform", [False, True])
def test_gathered_markov_bias_dequantizes_selected_w4a16_rows(
monkeypatch, cuda_alike_platform
):
packed_weight = torch.tensor(
[[0x00] * 8, [0x22] * 8, [0xAA] * 8, [0x31] * 8], dtype=torch.uint8
)
weight_scale = torch.ones((4, 1), dtype=torch.float8_e4m3fn)
layer = nn.Module()
layer.weight = nn.Parameter(packed_weight, requires_grad=False)
layer.weight_scale = nn.Parameter(weight_scale, requires_grad=False)
layer.weight_scale_2 = nn.Parameter(torch.ones(1), requires_grad=False)
layer._retain_weight_for_gather = True
spec, ctx, fmt = resolve("W4A16_NVFP4", type("Config", (), {"group_size": 16}), "")
method = ModelOptLinearMethod(spec, ctx, fmt)
class RepackingKernel:
@staticmethod
def process_weights_after_loading(layer):
layer.weight = nn.Parameter(
torch.zeros((1, 1), dtype=torch.int32), requires_grad=False
)
layer.weight_scale = nn.Parameter(
torch.zeros((1, 1), dtype=torch.float32), requires_grad=False
)
layer.weight_global_scale = nn.Parameter(
torch.zeros(1, dtype=torch.float32), requires_grad=False
)
method.kernel = RepackingKernel()
method.process_weights_after_loading(layer)
assert layer.is_w4a16_nvfp4
head = DSparkMarkovHead.__new__(DSparkMarkovHead)
nn.Module.__init__(head)
head.markov_w2 = layer
markov_embed = torch.tensor([[1.0] * 16, [0.5, -0.5] * 8], dtype=torch.float32)
index = torch.tensor([[1, 3], [2, 0]])
values = torch.tensor([[0.25, -0.5], [1.0, 2.0]])
original_values = values.clone()
logits = torch.full((2, 4), float("-inf"))
# CPU tensors must use the fallback even on CUDA/ROCm hosts.
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: cuda_alike_platform)
result = head.apply_bias_gathered(markov_embed, logits, values, index)
dense_weight = torch.tensor([[0.0] * 16, [1.0] * 16, [-1.0] * 16, [0.5, 1.5] * 8])
expected = original_values + torch.bmm(
dense_weight[index], markov_embed.unsqueeze(-1)
).squeeze(-1)
torch.testing.assert_close(result.gather(1, index), expected)
torch.testing.assert_close(values, original_values)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_nvfp4_gathered_bias_rejects_non_16_group_size():
device = torch.device("cuda")
rank = 32
vocab_size = 2
with pytest.raises(ValueError, match="group size of 16"):
nvfp4_gathered_bias(
torch.zeros((1, rank), dtype=torch.bfloat16, device=device),
torch.zeros((vocab_size, rank // 2), dtype=torch.uint8, device=device),
torch.ones(
(vocab_size, rank // 32), dtype=torch.float8_e4m3fn, device=device
),
torch.ones(1, dtype=torch.float32, device=device),
torch.zeros((1, 1), dtype=torch.bfloat16, device=device),
torch.zeros((1, 1), dtype=torch.int64, device=device),
torch.full(
(1, vocab_size), float("-inf"), dtype=torch.bfloat16, device=device
),
1.0,
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("batch_size", [1, 2, 4, 8])
def test_nvfp4_gathered_markov_bias_kernel_matches_reference(batch_size: int):
torch.manual_seed(0)
device = torch.device("cuda")
topk = 256
rank = 512
vocab_size = 512
alpha = 0.75
packed_weight = torch.randint(
0, 256, (vocab_size, rank // 2), dtype=torch.uint8, device=device
)
weight_scale = torch.full(
(vocab_size, rank // 16),
0.5,
dtype=torch.float8_e4m3fn,
device=device,
)
global_scale = torch.tensor([0.25], dtype=torch.float32, device=device)
markov_storage = torch.randn(
batch_size, 2, rank, dtype=torch.bfloat16, device=device
)
values_storage = torch.randn(
batch_size, 2, topk, dtype=torch.bfloat16, device=device
)
index_storage = torch.empty(batch_size, 2, topk, dtype=torch.int64, device=device)
for batch_idx in range(batch_size):
index_storage[batch_idx, 0] = (
torch.arange(topk, device=device) + batch_idx
) % vocab_size
logits_storage = torch.full(
(batch_size, 2, vocab_size),
float("-inf"),
dtype=torch.bfloat16,
device=device,
)
markov_embed = markov_storage[:, 0]
values = values_storage[:, 0]
index = index_storage[:, 0]
logits = logits_storage[:, 0]
layer = nn.Module()
layer.register_buffer("_nvfp4_weight_for_gather", packed_weight)
layer.register_buffer("_nvfp4_weight_scale_for_gather", weight_scale)
layer.register_buffer("_nvfp4_weight_global_scale_for_gather", global_scale)
layer._nvfp4_group_size_for_gather = 16
layer._retain_weight_for_gather = True
layer.is_w4a16_nvfp4 = True
head = DSparkMarkovHead.__new__(DSparkMarkovHead)
nn.Module.__init__(head)
head.markov_w2 = layer
flat_index = index.reshape(-1)
selected_weight = dequantize_to_dtype(
packed_weight.index_select(0, flat_index),
weight_scale.index_select(0, flat_index),
global_scale,
dtype=torch.bfloat16,
block_size=16,
swizzle=False,
).view(batch_size, topk, rank)
expected = (
values
+ torch.bmm(selected_weight, markov_embed.unsqueeze(-1)).squeeze(-1) * alpha
)
result = head.apply_bias_gathered(markov_embed, logits, values, index, alpha)
torch.testing.assert_close(
result.gather(1, index), expected, rtol=1e-2, atol=6.25e-2
)
selected = torch.zeros_like(result, dtype=torch.bool).scatter_(1, index, True)
assert torch.isneginf(result.masked_select(~selected)).all()
if batch_size == 1:
logits.fill_(float("-inf"))
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = head.apply_bias_gathered(
markov_embed, logits, values, index, alpha
)
logits.fill_(float("-inf"))
graph.replay()
assert captured is logits
torch.testing.assert_close(
logits.gather(1, index), expected, rtol=1e-2, atol=6.25e-2
)
@@ -2458,6 +2458,30 @@ class ModelOptLinearMethod(LinearMethodBase):
self.akey.process(layer, ACT)
maybe_fuse_global_scales(layer)
self.fmt.post_process(layer)
layer.is_w4a16_nvfp4 = (
self.spec.weight == kNvfp4Static and self.spec.activation is None
)
if getattr(layer, "_retain_weight_for_gather", False):
if not layer.is_w4a16_nvfp4:
raise NotImplementedError(
"Gathered projection is only supported for ModelOpt W4A16 "
"NVFP4 weights."
)
assert self.ctx.group_size is not None
layer.register_buffer(
"_nvfp4_weight_for_gather", layer.weight.detach(), persistent=False
)
layer.register_buffer(
"_nvfp4_weight_scale_for_gather",
layer.weight_scale.detach(),
persistent=False,
)
layer.register_buffer(
"_nvfp4_weight_global_scale_for_gather",
layer.weight_global_scale.detach(),
persistent=False,
)
layer._nvfp4_group_size_for_gather = self.ctx.group_size
self.kernel.process_weights_after_loading(layer)
def apply(self, layer, x, bias=None):
@@ -7,10 +7,12 @@ import torch
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
__all__ = [
"break_fp4_bytes",
"dequantize_to_dtype",
"nvfp4_gathered_bias",
"ref_nvfp4_quant",
]
@@ -43,6 +45,230 @@ def _e2m1_inline(nibble):
return tl.where(sign == 1, -val, val)
@triton.jit
def _nvfp4_gathered_bias_kernel(
markov_ptr,
packed_weight_ptr,
weight_scale_ptr,
global_scale_ptr,
values_ptr,
index_ptr,
logits_ptr,
stride_markov_b,
stride_markov_k,
stride_weight_v,
stride_weight_k,
stride_scale_v,
stride_scale_k,
stride_values_b,
stride_values_j,
stride_index_b,
stride_index_j,
stride_logits_b,
stride_logits_v,
alpha,
TOPK: tl.constexpr,
K: tl.constexpr,
NUM_BLOCKS: tl.constexpr,
):
"""Apply selected NVFP4 Markov-projection rows to dense logits.
BF16: ``weight[index] -> baddbmm_ -> scatter_``.
NVFP4: ``gather packed W2 row -> dequantize 16-element blocks -> dot with
Markov embedding -> add base value -> scatter corrected logit``.
The fused NVFP4 path is needed because packed weights cannot be indexed as
BF16 rows, while dequantizing the full vocabulary would defeat top-k.
"""
candidate_id = tl.program_id(0)
request_id = candidate_id // TOPK
candidate = candidate_id - request_id * TOPK
vocab_id = tl.load(
index_ptr + request_id * stride_index_b + candidate * stride_index_j
).to(tl.int64)
block_ids = tl.arange(0, NUM_BLOCKS)
byte_ids = tl.arange(0, 8)
block_mask = block_ids * 16 < K
packed_offsets = block_ids[:, None] * 8 + byte_ids[None, :]
packed = tl.load(
packed_weight_ptr
+ vocab_id * stride_weight_v
+ packed_offsets * stride_weight_k,
mask=block_mask[:, None],
other=0,
)
low = _e2m1_inline(packed & 0x0F)
high = _e2m1_inline((packed >> 4) & 0x0F)
decoded = tl.interleave(low, high)
raw_weight_scale = tl.load(
weight_scale_ptr + vocab_id * stride_scale_v + block_ids * stride_scale_k,
mask=block_mask,
other=0,
)
weight_scale = tl.cast(
raw_weight_scale,
tl.float8e4nv,
bitcast=True,
).to(tl.float32)
global_scale = tl.load(global_scale_ptr).to(tl.float32)
element_ids = tl.arange(0, 16)
k_offsets = block_ids[:, None] * 16 + element_ids[None, :]
k_mask = k_offsets < K
markov = tl.load(
markov_ptr + request_id * stride_markov_b + k_offsets * stride_markov_k,
mask=k_mask,
other=0.0,
)
dequant_weight = (decoded * weight_scale[:, None] * global_scale).to(tl.bfloat16)
products = dequant_weight.to(tl.float32) * markov.to(tl.float32)
block_sums = tl.sum(products, axis=1)
dot = tl.sum(block_sums, axis=0)
base_value = tl.load(
values_ptr + request_id * stride_values_b + candidate * stride_values_j
).to(tl.float32)
corrected = base_value + alpha * dot
tl.store(
logits_ptr + request_id * stride_logits_b + vocab_id * stride_logits_v,
corrected,
)
def _nvfp4_gathered_bias_impl(
markov_embed: torch.Tensor,
packed_weight: torch.Tensor,
weight_scale: torch.Tensor,
global_scale: torch.Tensor,
values: torch.Tensor,
index: torch.Tensor,
logits: torch.Tensor,
alpha: float,
) -> None:
if index.numel() == 0:
return
assert markov_embed.dtype == torch.bfloat16
assert values.dtype == torch.bfloat16
assert logits.dtype == torch.bfloat16
assert packed_weight.dtype == torch.uint8
assert weight_scale.dtype == torch.float8_e4m3fn
assert index.dtype == torch.int64
assert markov_embed.is_cuda
assert all(
tensor.device == markov_embed.device
for tensor in (
packed_weight,
weight_scale,
global_scale,
values,
index,
logits,
)
)
assert markov_embed.ndim == 2
assert packed_weight.ndim == 2
assert weight_scale.ndim == 2
assert values.ndim == 2
assert index.ndim == 2
assert logits.ndim == 2
assert global_scale.numel() == 1
assert values.shape == index.shape
assert logits.shape[0] == index.shape[0] == markov_embed.shape[0]
k = markov_embed.shape[-1]
if k % 16 != 0 or packed_weight.shape[-1] * 2 != k:
raise ValueError(
"nvfp4_gathered_bias requires k to be divisible by 16 and "
f"packed_weight width to equal k / 2; got k={k} and "
f"packed_weight width={packed_weight.shape[-1]}"
)
if weight_scale.shape[-1] != k // 16:
raise ValueError(
"nvfp4_gathered_bias only supports an NVFP4 group size of 16; "
f"got {weight_scale.shape[-1]} scale blocks for k={k}"
)
topk = index.shape[1]
num_blocks = triton.next_power_of_2(k // 16)
scale_bytes = weight_scale.view(torch.uint8)
_nvfp4_gathered_bias_kernel[(index.numel(),)](
markov_embed,
packed_weight,
scale_bytes,
global_scale,
values,
index,
logits,
markov_embed.stride(0),
markov_embed.stride(1),
packed_weight.stride(0),
packed_weight.stride(1),
scale_bytes.stride(0),
scale_bytes.stride(1),
values.stride(0),
values.stride(1),
index.stride(0),
index.stride(1),
logits.stride(0),
logits.stride(1),
alpha,
TOPK=topk,
K=k,
NUM_BLOCKS=num_blocks,
num_warps=4,
num_stages=2,
)
def _nvfp4_gathered_bias_fake(
markov_embed: torch.Tensor,
packed_weight: torch.Tensor,
weight_scale: torch.Tensor,
global_scale: torch.Tensor,
values: torch.Tensor,
index: torch.Tensor,
logits: torch.Tensor,
alpha: float,
) -> None:
return None
direct_register_custom_op(
op_name="nvfp4_gathered_bias",
op_func=_nvfp4_gathered_bias_impl,
mutates_args=["logits"],
fake_impl=_nvfp4_gathered_bias_fake,
)
def nvfp4_gathered_bias(
markov_embed: torch.Tensor,
packed_weight: torch.Tensor,
weight_scale: torch.Tensor,
global_scale: torch.Tensor,
values: torch.Tensor,
index: torch.Tensor,
logits: torch.Tensor,
alpha: float,
) -> None:
torch.ops.vllm.nvfp4_gathered_bias(
markov_embed,
packed_weight,
weight_scale,
global_scale,
values,
index,
logits,
alpha,
)
@triton.jit
def _dequantize_nvfp4_kernel(
fp4_ptr,
@@ -347,7 +573,7 @@ def dequantize_to_dtype(
# Two fp4 values are packed into one uint8.
assert tensor_fp4.dtype == torch.uint8
if not swizzle and current_platform.is_cuda_alike():
if not swizzle and tensor_fp4.is_cuda and current_platform.is_cuda_alike():
return _triton_dequantize_nvfp4(
tensor_fp4, tensor_sf, global_scale, dtype, block_size
)
+57 -2
View File
@@ -25,9 +25,14 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
dequantize_to_dtype,
nvfp4_gathered_bias,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
)
from vllm.platforms import current_platform
from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model
from .utils import (
@@ -60,6 +65,7 @@ class DSparkMarkovHead(nn.Module):
markov_rank: int,
prefix: str,
quant_config: QuantizationConfig | None = None,
retain_weight_for_gather: bool = False,
) -> None:
super().__init__()
self.markov_w1 = nn.Embedding(vocab_size, markov_rank)
@@ -71,6 +77,8 @@ class DSparkMarkovHead(nn.Module):
prefix=maybe_prefix(prefix, "markov_w2"),
disable_tp=True,
)
self.markov_w2._retain_weight_for_gather = retain_weight_for_gather
self.markov_w2.is_w4a16_nvfp4 = False
def embed(self, token_ids: torch.Tensor) -> torch.Tensor:
"""r-dim Markov embedding of ``token_ids`` ([B] -> [B, r])."""
@@ -98,8 +106,52 @@ class DSparkMarkovHead(nn.Module):
positions. This method scatters the corrected candidate values into
that dense buffer so the normal sampler sees the truncated proposal.
"""
weight = self.markov_w2.weight[index]
corrected = values.unsqueeze(-1)
if self.markov_w2._retain_weight_for_gather and self.markov_w2.is_w4a16_nvfp4:
# Use the fast Triton kernel for W4A16 NVFP4 weights with BF16
# activations on CUDA. FP16 activations, non-CUDA accelerators, and
# CPU use the slower dequantize/scatter fallback below.
if (
current_platform.is_cuda()
and markov_embed.is_cuda
and markov_embed.dtype == torch.bfloat16
and values.dtype == torch.bfloat16
and logits.dtype == torch.bfloat16
):
nvfp4_gathered_bias(
markov_embed,
self.markov_w2._nvfp4_weight_for_gather,
self.markov_w2._nvfp4_weight_scale_for_gather,
self.markov_w2._nvfp4_weight_global_scale_for_gather,
values,
index,
logits,
scale,
)
return logits
flat_index = index.reshape(-1)
packed_weight = self.markov_w2._nvfp4_weight_for_gather.index_select(
0, flat_index
)
weight_scale = self.markov_w2._nvfp4_weight_scale_for_gather.index_select(
0, flat_index
)
weight = dequantize_to_dtype(
packed_weight,
weight_scale,
self.markov_w2._nvfp4_weight_global_scale_for_gather,
dtype=markov_embed.dtype,
block_size=self.markov_w2._nvfp4_group_size_for_gather,
swizzle=False,
).view(*index.shape, -1)
else:
weight = self.markov_w2.weight[index]
if weight.dtype != markov_embed.dtype:
raise RuntimeError(
"Gathered Markov projection requires unquantized weights or "
"retained ModelOpt W4A16 NVFP4 weights."
)
corrected = values.unsqueeze(-1).clone()
corrected.baddbmm_(
weight,
markov_embed.unsqueeze(-1),
@@ -160,6 +212,9 @@ class Qwen3DSparkModel(DFlashQwen3Model):
config.markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
quant_config=self.quant_config,
retain_weight_for_gather=(
getattr(config, "dspark_draft_topk", None) is not None
),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(config, "enable_confidence_head", False):