[Spec Decode] Add NVFP4 DSpark gathered top-k projection (#55713)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Andrii Skliar
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
af51d0ef03
commit
83fe99399e
@@ -25,9 +25,14 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_to_dtype,
|
||||
nvfp4_gathered_bias,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model
|
||||
from .utils import (
|
||||
@@ -60,6 +65,7 @@ class DSparkMarkovHead(nn.Module):
|
||||
markov_rank: int,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
retain_weight_for_gather: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.markov_w1 = nn.Embedding(vocab_size, markov_rank)
|
||||
@@ -71,6 +77,8 @@ class DSparkMarkovHead(nn.Module):
|
||||
prefix=maybe_prefix(prefix, "markov_w2"),
|
||||
disable_tp=True,
|
||||
)
|
||||
self.markov_w2._retain_weight_for_gather = retain_weight_for_gather
|
||||
self.markov_w2.is_w4a16_nvfp4 = False
|
||||
|
||||
def embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""r-dim Markov embedding of ``token_ids`` ([B] -> [B, r])."""
|
||||
@@ -98,8 +106,52 @@ class DSparkMarkovHead(nn.Module):
|
||||
positions. This method scatters the corrected candidate values into
|
||||
that dense buffer so the normal sampler sees the truncated proposal.
|
||||
"""
|
||||
weight = self.markov_w2.weight[index]
|
||||
corrected = values.unsqueeze(-1)
|
||||
if self.markov_w2._retain_weight_for_gather and self.markov_w2.is_w4a16_nvfp4:
|
||||
# Use the fast Triton kernel for W4A16 NVFP4 weights with BF16
|
||||
# activations on CUDA. FP16 activations, non-CUDA accelerators, and
|
||||
# CPU use the slower dequantize/scatter fallback below.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and markov_embed.is_cuda
|
||||
and markov_embed.dtype == torch.bfloat16
|
||||
and values.dtype == torch.bfloat16
|
||||
and logits.dtype == torch.bfloat16
|
||||
):
|
||||
nvfp4_gathered_bias(
|
||||
markov_embed,
|
||||
self.markov_w2._nvfp4_weight_for_gather,
|
||||
self.markov_w2._nvfp4_weight_scale_for_gather,
|
||||
self.markov_w2._nvfp4_weight_global_scale_for_gather,
|
||||
values,
|
||||
index,
|
||||
logits,
|
||||
scale,
|
||||
)
|
||||
return logits
|
||||
|
||||
flat_index = index.reshape(-1)
|
||||
packed_weight = self.markov_w2._nvfp4_weight_for_gather.index_select(
|
||||
0, flat_index
|
||||
)
|
||||
weight_scale = self.markov_w2._nvfp4_weight_scale_for_gather.index_select(
|
||||
0, flat_index
|
||||
)
|
||||
weight = dequantize_to_dtype(
|
||||
packed_weight,
|
||||
weight_scale,
|
||||
self.markov_w2._nvfp4_weight_global_scale_for_gather,
|
||||
dtype=markov_embed.dtype,
|
||||
block_size=self.markov_w2._nvfp4_group_size_for_gather,
|
||||
swizzle=False,
|
||||
).view(*index.shape, -1)
|
||||
else:
|
||||
weight = self.markov_w2.weight[index]
|
||||
if weight.dtype != markov_embed.dtype:
|
||||
raise RuntimeError(
|
||||
"Gathered Markov projection requires unquantized weights or "
|
||||
"retained ModelOpt W4A16 NVFP4 weights."
|
||||
)
|
||||
corrected = values.unsqueeze(-1).clone()
|
||||
corrected.baddbmm_(
|
||||
weight,
|
||||
markov_embed.unsqueeze(-1),
|
||||
@@ -160,6 +212,9 @@ class Qwen3DSparkModel(DFlashQwen3Model):
|
||||
config.markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
quant_config=self.quant_config,
|
||||
retain_weight_for_gather=(
|
||||
getattr(config, "dspark_draft_topk", None) is not None
|
||||
),
|
||||
)
|
||||
self.confidence_head: DSparkConfidenceHead | None = None
|
||||
if getattr(config, "enable_confidence_head", False):
|
||||
|
||||
Reference in New Issue
Block a user