diff --git a/tests/v1/spec_decode/test_dspark_topk.py b/tests/v1/spec_decode/test_dspark_topk.py index a33483d6db..2de1ff052a 100644 --- a/tests/v1/spec_decode/test_dspark_topk.py +++ b/tests/v1/spec_decode/test_dspark_topk.py @@ -1,10 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest import torch from torch import nn +from vllm.model_executor.layers.quantization.modelopt import ( + ModelOptLinearMethod, + resolve, +) +from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( + dequantize_to_dtype, + nvfp4_gathered_bias, +) from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead +from vllm.platforms import current_platform def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead: @@ -14,6 +24,8 @@ def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead: weight.shape[1], weight.shape[0], bias=False, dtype=weight.dtype ) head.markov_w2.weight.data.copy_(weight) + head.markov_w2._retain_weight_for_gather = False + head.markov_w2.is_w4a16_nvfp4 = False return head @@ -56,3 +68,174 @@ def test_gathered_markov_bias_matches_dense_at_full_vocab(): expected = original + markov_embed @ weight.T * scale torch.testing.assert_close(result, expected) + + +@pytest.mark.parametrize("cuda_alike_platform", [False, True]) +def test_gathered_markov_bias_dequantizes_selected_w4a16_rows( + monkeypatch, cuda_alike_platform +): + packed_weight = torch.tensor( + [[0x00] * 8, [0x22] * 8, [0xAA] * 8, [0x31] * 8], dtype=torch.uint8 + ) + weight_scale = torch.ones((4, 1), dtype=torch.float8_e4m3fn) + + layer = nn.Module() + layer.weight = nn.Parameter(packed_weight, requires_grad=False) + layer.weight_scale = nn.Parameter(weight_scale, requires_grad=False) + layer.weight_scale_2 = nn.Parameter(torch.ones(1), requires_grad=False) + layer._retain_weight_for_gather = True + + spec, ctx, fmt = resolve("W4A16_NVFP4", type("Config", (), {"group_size": 16}), "") + method = ModelOptLinearMethod(spec, ctx, fmt) + + class RepackingKernel: + @staticmethod + def process_weights_after_loading(layer): + layer.weight = nn.Parameter( + torch.zeros((1, 1), dtype=torch.int32), requires_grad=False + ) + layer.weight_scale = nn.Parameter( + torch.zeros((1, 1), dtype=torch.float32), requires_grad=False + ) + layer.weight_global_scale = nn.Parameter( + torch.zeros(1, dtype=torch.float32), requires_grad=False + ) + + method.kernel = RepackingKernel() + method.process_weights_after_loading(layer) + assert layer.is_w4a16_nvfp4 + + head = DSparkMarkovHead.__new__(DSparkMarkovHead) + nn.Module.__init__(head) + head.markov_w2 = layer + markov_embed = torch.tensor([[1.0] * 16, [0.5, -0.5] * 8], dtype=torch.float32) + index = torch.tensor([[1, 3], [2, 0]]) + values = torch.tensor([[0.25, -0.5], [1.0, 2.0]]) + original_values = values.clone() + logits = torch.full((2, 4), float("-inf")) + + # CPU tensors must use the fallback even on CUDA/ROCm hosts. + monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: cuda_alike_platform) + result = head.apply_bias_gathered(markov_embed, logits, values, index) + + dense_weight = torch.tensor([[0.0] * 16, [1.0] * 16, [-1.0] * 16, [0.5, 1.5] * 8]) + expected = original_values + torch.bmm( + dense_weight[index], markov_embed.unsqueeze(-1) + ).squeeze(-1) + torch.testing.assert_close(result.gather(1, index), expected) + torch.testing.assert_close(values, original_values) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_nvfp4_gathered_bias_rejects_non_16_group_size(): + device = torch.device("cuda") + rank = 32 + vocab_size = 2 + + with pytest.raises(ValueError, match="group size of 16"): + nvfp4_gathered_bias( + torch.zeros((1, rank), dtype=torch.bfloat16, device=device), + torch.zeros((vocab_size, rank // 2), dtype=torch.uint8, device=device), + torch.ones( + (vocab_size, rank // 32), dtype=torch.float8_e4m3fn, device=device + ), + torch.ones(1, dtype=torch.float32, device=device), + torch.zeros((1, 1), dtype=torch.bfloat16, device=device), + torch.zeros((1, 1), dtype=torch.int64, device=device), + torch.full( + (1, vocab_size), float("-inf"), dtype=torch.bfloat16, device=device + ), + 1.0, + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8]) +def test_nvfp4_gathered_markov_bias_kernel_matches_reference(batch_size: int): + torch.manual_seed(0) + device = torch.device("cuda") + topk = 256 + rank = 512 + vocab_size = 512 + alpha = 0.75 + + packed_weight = torch.randint( + 0, 256, (vocab_size, rank // 2), dtype=torch.uint8, device=device + ) + weight_scale = torch.full( + (vocab_size, rank // 16), + 0.5, + dtype=torch.float8_e4m3fn, + device=device, + ) + global_scale = torch.tensor([0.25], dtype=torch.float32, device=device) + + markov_storage = torch.randn( + batch_size, 2, rank, dtype=torch.bfloat16, device=device + ) + values_storage = torch.randn( + batch_size, 2, topk, dtype=torch.bfloat16, device=device + ) + index_storage = torch.empty(batch_size, 2, topk, dtype=torch.int64, device=device) + for batch_idx in range(batch_size): + index_storage[batch_idx, 0] = ( + torch.arange(topk, device=device) + batch_idx + ) % vocab_size + logits_storage = torch.full( + (batch_size, 2, vocab_size), + float("-inf"), + dtype=torch.bfloat16, + device=device, + ) + + markov_embed = markov_storage[:, 0] + values = values_storage[:, 0] + index = index_storage[:, 0] + logits = logits_storage[:, 0] + + layer = nn.Module() + layer.register_buffer("_nvfp4_weight_for_gather", packed_weight) + layer.register_buffer("_nvfp4_weight_scale_for_gather", weight_scale) + layer.register_buffer("_nvfp4_weight_global_scale_for_gather", global_scale) + layer._nvfp4_group_size_for_gather = 16 + layer._retain_weight_for_gather = True + layer.is_w4a16_nvfp4 = True + head = DSparkMarkovHead.__new__(DSparkMarkovHead) + nn.Module.__init__(head) + head.markov_w2 = layer + + flat_index = index.reshape(-1) + selected_weight = dequantize_to_dtype( + packed_weight.index_select(0, flat_index), + weight_scale.index_select(0, flat_index), + global_scale, + dtype=torch.bfloat16, + block_size=16, + swizzle=False, + ).view(batch_size, topk, rank) + expected = ( + values + + torch.bmm(selected_weight, markov_embed.unsqueeze(-1)).squeeze(-1) * alpha + ) + + result = head.apply_bias_gathered(markov_embed, logits, values, index, alpha) + + torch.testing.assert_close( + result.gather(1, index), expected, rtol=1e-2, atol=6.25e-2 + ) + selected = torch.zeros_like(result, dtype=torch.bool).scatter_(1, index, True) + assert torch.isneginf(result.masked_select(~selected)).all() + + if batch_size == 1: + logits.fill_(float("-inf")) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = head.apply_bias_gathered( + markov_embed, logits, values, index, alpha + ) + logits.fill_(float("-inf")) + graph.replay() + assert captured is logits + torch.testing.assert_close( + logits.gather(1, index), expected, rtol=1e-2, atol=6.25e-2 + ) diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 2abe60d7ad..59292ec990 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -2458,6 +2458,30 @@ class ModelOptLinearMethod(LinearMethodBase): self.akey.process(layer, ACT) maybe_fuse_global_scales(layer) self.fmt.post_process(layer) + layer.is_w4a16_nvfp4 = ( + self.spec.weight == kNvfp4Static and self.spec.activation is None + ) + if getattr(layer, "_retain_weight_for_gather", False): + if not layer.is_w4a16_nvfp4: + raise NotImplementedError( + "Gathered projection is only supported for ModelOpt W4A16 " + "NVFP4 weights." + ) + assert self.ctx.group_size is not None + layer.register_buffer( + "_nvfp4_weight_for_gather", layer.weight.detach(), persistent=False + ) + layer.register_buffer( + "_nvfp4_weight_scale_for_gather", + layer.weight_scale.detach(), + persistent=False, + ) + layer.register_buffer( + "_nvfp4_weight_global_scale_for_gather", + layer.weight_global_scale.detach(), + persistent=False, + ) + layer._nvfp4_group_size_for_gather = self.ctx.group_size self.kernel.process_weights_after_loading(layer) def apply(self, layer, x, bias=None): diff --git a/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py b/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py index cb76bbcd5c..a5272dd9cb 100644 --- a/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py +++ b/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py @@ -7,10 +7,12 @@ import torch from vllm.platforms import current_platform from vllm.scalar_type import scalar_types from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op __all__ = [ "break_fp4_bytes", "dequantize_to_dtype", + "nvfp4_gathered_bias", "ref_nvfp4_quant", ] @@ -43,6 +45,230 @@ def _e2m1_inline(nibble): return tl.where(sign == 1, -val, val) +@triton.jit +def _nvfp4_gathered_bias_kernel( + markov_ptr, + packed_weight_ptr, + weight_scale_ptr, + global_scale_ptr, + values_ptr, + index_ptr, + logits_ptr, + stride_markov_b, + stride_markov_k, + stride_weight_v, + stride_weight_k, + stride_scale_v, + stride_scale_k, + stride_values_b, + stride_values_j, + stride_index_b, + stride_index_j, + stride_logits_b, + stride_logits_v, + alpha, + TOPK: tl.constexpr, + K: tl.constexpr, + NUM_BLOCKS: tl.constexpr, +): + """Apply selected NVFP4 Markov-projection rows to dense logits. + + BF16: ``weight[index] -> baddbmm_ -> scatter_``. + NVFP4: ``gather packed W2 row -> dequantize 16-element blocks -> dot with + Markov embedding -> add base value -> scatter corrected logit``. + + The fused NVFP4 path is needed because packed weights cannot be indexed as + BF16 rows, while dequantizing the full vocabulary would defeat top-k. + """ + candidate_id = tl.program_id(0) + request_id = candidate_id // TOPK + candidate = candidate_id - request_id * TOPK + + vocab_id = tl.load( + index_ptr + request_id * stride_index_b + candidate * stride_index_j + ).to(tl.int64) + + block_ids = tl.arange(0, NUM_BLOCKS) + byte_ids = tl.arange(0, 8) + block_mask = block_ids * 16 < K + + packed_offsets = block_ids[:, None] * 8 + byte_ids[None, :] + packed = tl.load( + packed_weight_ptr + + vocab_id * stride_weight_v + + packed_offsets * stride_weight_k, + mask=block_mask[:, None], + other=0, + ) + low = _e2m1_inline(packed & 0x0F) + high = _e2m1_inline((packed >> 4) & 0x0F) + decoded = tl.interleave(low, high) + + raw_weight_scale = tl.load( + weight_scale_ptr + vocab_id * stride_scale_v + block_ids * stride_scale_k, + mask=block_mask, + other=0, + ) + weight_scale = tl.cast( + raw_weight_scale, + tl.float8e4nv, + bitcast=True, + ).to(tl.float32) + global_scale = tl.load(global_scale_ptr).to(tl.float32) + + element_ids = tl.arange(0, 16) + k_offsets = block_ids[:, None] * 16 + element_ids[None, :] + k_mask = k_offsets < K + markov = tl.load( + markov_ptr + request_id * stride_markov_b + k_offsets * stride_markov_k, + mask=k_mask, + other=0.0, + ) + + dequant_weight = (decoded * weight_scale[:, None] * global_scale).to(tl.bfloat16) + products = dequant_weight.to(tl.float32) * markov.to(tl.float32) + block_sums = tl.sum(products, axis=1) + dot = tl.sum(block_sums, axis=0) + + base_value = tl.load( + values_ptr + request_id * stride_values_b + candidate * stride_values_j + ).to(tl.float32) + corrected = base_value + alpha * dot + tl.store( + logits_ptr + request_id * stride_logits_b + vocab_id * stride_logits_v, + corrected, + ) + + +def _nvfp4_gathered_bias_impl( + markov_embed: torch.Tensor, + packed_weight: torch.Tensor, + weight_scale: torch.Tensor, + global_scale: torch.Tensor, + values: torch.Tensor, + index: torch.Tensor, + logits: torch.Tensor, + alpha: float, +) -> None: + if index.numel() == 0: + return + + assert markov_embed.dtype == torch.bfloat16 + assert values.dtype == torch.bfloat16 + assert logits.dtype == torch.bfloat16 + assert packed_weight.dtype == torch.uint8 + assert weight_scale.dtype == torch.float8_e4m3fn + assert index.dtype == torch.int64 + assert markov_embed.is_cuda + assert all( + tensor.device == markov_embed.device + for tensor in ( + packed_weight, + weight_scale, + global_scale, + values, + index, + logits, + ) + ) + assert markov_embed.ndim == 2 + assert packed_weight.ndim == 2 + assert weight_scale.ndim == 2 + assert values.ndim == 2 + assert index.ndim == 2 + assert logits.ndim == 2 + assert global_scale.numel() == 1 + assert values.shape == index.shape + assert logits.shape[0] == index.shape[0] == markov_embed.shape[0] + + k = markov_embed.shape[-1] + if k % 16 != 0 or packed_weight.shape[-1] * 2 != k: + raise ValueError( + "nvfp4_gathered_bias requires k to be divisible by 16 and " + f"packed_weight width to equal k / 2; got k={k} and " + f"packed_weight width={packed_weight.shape[-1]}" + ) + if weight_scale.shape[-1] != k // 16: + raise ValueError( + "nvfp4_gathered_bias only supports an NVFP4 group size of 16; " + f"got {weight_scale.shape[-1]} scale blocks for k={k}" + ) + + topk = index.shape[1] + num_blocks = triton.next_power_of_2(k // 16) + scale_bytes = weight_scale.view(torch.uint8) + _nvfp4_gathered_bias_kernel[(index.numel(),)]( + markov_embed, + packed_weight, + scale_bytes, + global_scale, + values, + index, + logits, + markov_embed.stride(0), + markov_embed.stride(1), + packed_weight.stride(0), + packed_weight.stride(1), + scale_bytes.stride(0), + scale_bytes.stride(1), + values.stride(0), + values.stride(1), + index.stride(0), + index.stride(1), + logits.stride(0), + logits.stride(1), + alpha, + TOPK=topk, + K=k, + NUM_BLOCKS=num_blocks, + num_warps=4, + num_stages=2, + ) + + +def _nvfp4_gathered_bias_fake( + markov_embed: torch.Tensor, + packed_weight: torch.Tensor, + weight_scale: torch.Tensor, + global_scale: torch.Tensor, + values: torch.Tensor, + index: torch.Tensor, + logits: torch.Tensor, + alpha: float, +) -> None: + return None + + +direct_register_custom_op( + op_name="nvfp4_gathered_bias", + op_func=_nvfp4_gathered_bias_impl, + mutates_args=["logits"], + fake_impl=_nvfp4_gathered_bias_fake, +) + + +def nvfp4_gathered_bias( + markov_embed: torch.Tensor, + packed_weight: torch.Tensor, + weight_scale: torch.Tensor, + global_scale: torch.Tensor, + values: torch.Tensor, + index: torch.Tensor, + logits: torch.Tensor, + alpha: float, +) -> None: + torch.ops.vllm.nvfp4_gathered_bias( + markov_embed, + packed_weight, + weight_scale, + global_scale, + values, + index, + logits, + alpha, + ) + + @triton.jit def _dequantize_nvfp4_kernel( fp4_ptr, @@ -347,7 +573,7 @@ def dequantize_to_dtype( # Two fp4 values are packed into one uint8. assert tensor_fp4.dtype == torch.uint8 - if not swizzle and current_platform.is_cuda_alike(): + if not swizzle and tensor_fp4.is_cuda and current_platform.is_cuda_alike(): return _triton_dequantize_nvfp4( tensor_fp4, tensor_sf, global_scale, dtype, block_size ) diff --git a/vllm/model_executor/models/qwen3_dspark.py b/vllm/model_executor/models/qwen3_dspark.py index 7c37c6db36..c2c252280d 100644 --- a/vllm/model_executor/models/qwen3_dspark.py +++ b/vllm/model_executor/models/qwen3_dspark.py @@ -25,9 +25,14 @@ from vllm.logger import init_logger from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( + dequantize_to_dtype, + nvfp4_gathered_bias, +) from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, ) +from vllm.platforms import current_platform from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model from .utils import ( @@ -60,6 +65,7 @@ class DSparkMarkovHead(nn.Module): markov_rank: int, prefix: str, quant_config: QuantizationConfig | None = None, + retain_weight_for_gather: bool = False, ) -> None: super().__init__() self.markov_w1 = nn.Embedding(vocab_size, markov_rank) @@ -71,6 +77,8 @@ class DSparkMarkovHead(nn.Module): prefix=maybe_prefix(prefix, "markov_w2"), disable_tp=True, ) + self.markov_w2._retain_weight_for_gather = retain_weight_for_gather + self.markov_w2.is_w4a16_nvfp4 = False def embed(self, token_ids: torch.Tensor) -> torch.Tensor: """r-dim Markov embedding of ``token_ids`` ([B] -> [B, r]).""" @@ -98,8 +106,52 @@ class DSparkMarkovHead(nn.Module): positions. This method scatters the corrected candidate values into that dense buffer so the normal sampler sees the truncated proposal. """ - weight = self.markov_w2.weight[index] - corrected = values.unsqueeze(-1) + if self.markov_w2._retain_weight_for_gather and self.markov_w2.is_w4a16_nvfp4: + # Use the fast Triton kernel for W4A16 NVFP4 weights with BF16 + # activations on CUDA. FP16 activations, non-CUDA accelerators, and + # CPU use the slower dequantize/scatter fallback below. + if ( + current_platform.is_cuda() + and markov_embed.is_cuda + and markov_embed.dtype == torch.bfloat16 + and values.dtype == torch.bfloat16 + and logits.dtype == torch.bfloat16 + ): + nvfp4_gathered_bias( + markov_embed, + self.markov_w2._nvfp4_weight_for_gather, + self.markov_w2._nvfp4_weight_scale_for_gather, + self.markov_w2._nvfp4_weight_global_scale_for_gather, + values, + index, + logits, + scale, + ) + return logits + + flat_index = index.reshape(-1) + packed_weight = self.markov_w2._nvfp4_weight_for_gather.index_select( + 0, flat_index + ) + weight_scale = self.markov_w2._nvfp4_weight_scale_for_gather.index_select( + 0, flat_index + ) + weight = dequantize_to_dtype( + packed_weight, + weight_scale, + self.markov_w2._nvfp4_weight_global_scale_for_gather, + dtype=markov_embed.dtype, + block_size=self.markov_w2._nvfp4_group_size_for_gather, + swizzle=False, + ).view(*index.shape, -1) + else: + weight = self.markov_w2.weight[index] + if weight.dtype != markov_embed.dtype: + raise RuntimeError( + "Gathered Markov projection requires unquantized weights or " + "retained ModelOpt W4A16 NVFP4 weights." + ) + corrected = values.unsqueeze(-1).clone() corrected.baddbmm_( weight, markov_embed.unsqueeze(-1), @@ -160,6 +212,9 @@ class Qwen3DSparkModel(DFlashQwen3Model): config.markov_rank, prefix=maybe_prefix(prefix, "markov_head"), quant_config=self.quant_config, + retain_weight_for_gather=( + getattr(config, "dspark_draft_topk", None) is not None + ), ) self.confidence_head: DSparkConfidenceHead | None = None if getattr(config, "enable_confidence_head", False):