[Spec Decode] Add NVFP4 DSpark gathered top-k projection (#55713)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Andrii Skliar
2026-09-09 17:33:46 +00:00
committed by GitHub
co-authored by Andrii Skliar mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent af51d0ef03
commit 83fe99399e
4 changed files with 491 additions and 3 deletions
+57 -2
View File
@@ -25,9 +25,14 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
dequantize_to_dtype,
nvfp4_gathered_bias,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
)
from vllm.platforms import current_platform
from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model
from .utils import (
@@ -60,6 +65,7 @@ class DSparkMarkovHead(nn.Module):
markov_rank: int,
prefix: str,
quant_config: QuantizationConfig | None = None,
retain_weight_for_gather: bool = False,
) -> None:
super().__init__()
self.markov_w1 = nn.Embedding(vocab_size, markov_rank)
@@ -71,6 +77,8 @@ class DSparkMarkovHead(nn.Module):
prefix=maybe_prefix(prefix, "markov_w2"),
disable_tp=True,
)
self.markov_w2._retain_weight_for_gather = retain_weight_for_gather
self.markov_w2.is_w4a16_nvfp4 = False
def embed(self, token_ids: torch.Tensor) -> torch.Tensor:
"""r-dim Markov embedding of ``token_ids`` ([B] -> [B, r])."""
@@ -98,8 +106,52 @@ class DSparkMarkovHead(nn.Module):
positions. This method scatters the corrected candidate values into
that dense buffer so the normal sampler sees the truncated proposal.
"""
weight = self.markov_w2.weight[index]
corrected = values.unsqueeze(-1)
if self.markov_w2._retain_weight_for_gather and self.markov_w2.is_w4a16_nvfp4:
# Use the fast Triton kernel for W4A16 NVFP4 weights with BF16
# activations on CUDA. FP16 activations, non-CUDA accelerators, and
# CPU use the slower dequantize/scatter fallback below.
if (
current_platform.is_cuda()
and markov_embed.is_cuda
and markov_embed.dtype == torch.bfloat16
and values.dtype == torch.bfloat16
and logits.dtype == torch.bfloat16
):
nvfp4_gathered_bias(
markov_embed,
self.markov_w2._nvfp4_weight_for_gather,
self.markov_w2._nvfp4_weight_scale_for_gather,
self.markov_w2._nvfp4_weight_global_scale_for_gather,
values,
index,
logits,
scale,
)
return logits
flat_index = index.reshape(-1)
packed_weight = self.markov_w2._nvfp4_weight_for_gather.index_select(
0, flat_index
)
weight_scale = self.markov_w2._nvfp4_weight_scale_for_gather.index_select(
0, flat_index
)
weight = dequantize_to_dtype(
packed_weight,
weight_scale,
self.markov_w2._nvfp4_weight_global_scale_for_gather,
dtype=markov_embed.dtype,
block_size=self.markov_w2._nvfp4_group_size_for_gather,
swizzle=False,
).view(*index.shape, -1)
else:
weight = self.markov_w2.weight[index]
if weight.dtype != markov_embed.dtype:
raise RuntimeError(
"Gathered Markov projection requires unquantized weights or "
"retained ModelOpt W4A16 NVFP4 weights."
)
corrected = values.unsqueeze(-1).clone()
corrected.baddbmm_(
weight,
markov_embed.unsqueeze(-1),
@@ -160,6 +212,9 @@ class Qwen3DSparkModel(DFlashQwen3Model):
config.markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
quant_config=self.quant_config,
retain_weight_for_gather=(
getattr(config, "dspark_draft_topk", None) is not None
),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(config, "enable_confidence_head", False):