[Spec Decode] Add NVFP4 DSpark gathered top-k projection (#55713)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Andrii Skliar
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
af51d0ef03
commit
83fe99399e
@@ -1,10 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptLinearMethod,
|
||||
resolve,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_to_dtype,
|
||||
nvfp4_gathered_bias,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
|
||||
@@ -14,6 +24,8 @@ def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
|
||||
weight.shape[1], weight.shape[0], bias=False, dtype=weight.dtype
|
||||
)
|
||||
head.markov_w2.weight.data.copy_(weight)
|
||||
head.markov_w2._retain_weight_for_gather = False
|
||||
head.markov_w2.is_w4a16_nvfp4 = False
|
||||
return head
|
||||
|
||||
|
||||
@@ -56,3 +68,174 @@ def test_gathered_markov_bias_matches_dense_at_full_vocab():
|
||||
|
||||
expected = original + markov_embed @ weight.T * scale
|
||||
torch.testing.assert_close(result, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cuda_alike_platform", [False, True])
|
||||
def test_gathered_markov_bias_dequantizes_selected_w4a16_rows(
|
||||
monkeypatch, cuda_alike_platform
|
||||
):
|
||||
packed_weight = torch.tensor(
|
||||
[[0x00] * 8, [0x22] * 8, [0xAA] * 8, [0x31] * 8], dtype=torch.uint8
|
||||
)
|
||||
weight_scale = torch.ones((4, 1), dtype=torch.float8_e4m3fn)
|
||||
|
||||
layer = nn.Module()
|
||||
layer.weight = nn.Parameter(packed_weight, requires_grad=False)
|
||||
layer.weight_scale = nn.Parameter(weight_scale, requires_grad=False)
|
||||
layer.weight_scale_2 = nn.Parameter(torch.ones(1), requires_grad=False)
|
||||
layer._retain_weight_for_gather = True
|
||||
|
||||
spec, ctx, fmt = resolve("W4A16_NVFP4", type("Config", (), {"group_size": 16}), "")
|
||||
method = ModelOptLinearMethod(spec, ctx, fmt)
|
||||
|
||||
class RepackingKernel:
|
||||
@staticmethod
|
||||
def process_weights_after_loading(layer):
|
||||
layer.weight = nn.Parameter(
|
||||
torch.zeros((1, 1), dtype=torch.int32), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = nn.Parameter(
|
||||
torch.zeros((1, 1), dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
layer.weight_global_scale = nn.Parameter(
|
||||
torch.zeros(1, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
|
||||
method.kernel = RepackingKernel()
|
||||
method.process_weights_after_loading(layer)
|
||||
assert layer.is_w4a16_nvfp4
|
||||
|
||||
head = DSparkMarkovHead.__new__(DSparkMarkovHead)
|
||||
nn.Module.__init__(head)
|
||||
head.markov_w2 = layer
|
||||
markov_embed = torch.tensor([[1.0] * 16, [0.5, -0.5] * 8], dtype=torch.float32)
|
||||
index = torch.tensor([[1, 3], [2, 0]])
|
||||
values = torch.tensor([[0.25, -0.5], [1.0, 2.0]])
|
||||
original_values = values.clone()
|
||||
logits = torch.full((2, 4), float("-inf"))
|
||||
|
||||
# CPU tensors must use the fallback even on CUDA/ROCm hosts.
|
||||
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: cuda_alike_platform)
|
||||
result = head.apply_bias_gathered(markov_embed, logits, values, index)
|
||||
|
||||
dense_weight = torch.tensor([[0.0] * 16, [1.0] * 16, [-1.0] * 16, [0.5, 1.5] * 8])
|
||||
expected = original_values + torch.bmm(
|
||||
dense_weight[index], markov_embed.unsqueeze(-1)
|
||||
).squeeze(-1)
|
||||
torch.testing.assert_close(result.gather(1, index), expected)
|
||||
torch.testing.assert_close(values, original_values)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_nvfp4_gathered_bias_rejects_non_16_group_size():
|
||||
device = torch.device("cuda")
|
||||
rank = 32
|
||||
vocab_size = 2
|
||||
|
||||
with pytest.raises(ValueError, match="group size of 16"):
|
||||
nvfp4_gathered_bias(
|
||||
torch.zeros((1, rank), dtype=torch.bfloat16, device=device),
|
||||
torch.zeros((vocab_size, rank // 2), dtype=torch.uint8, device=device),
|
||||
torch.ones(
|
||||
(vocab_size, rank // 32), dtype=torch.float8_e4m3fn, device=device
|
||||
),
|
||||
torch.ones(1, dtype=torch.float32, device=device),
|
||||
torch.zeros((1, 1), dtype=torch.bfloat16, device=device),
|
||||
torch.zeros((1, 1), dtype=torch.int64, device=device),
|
||||
torch.full(
|
||||
(1, vocab_size), float("-inf"), dtype=torch.bfloat16, device=device
|
||||
),
|
||||
1.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("batch_size", [1, 2, 4, 8])
|
||||
def test_nvfp4_gathered_markov_bias_kernel_matches_reference(batch_size: int):
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
topk = 256
|
||||
rank = 512
|
||||
vocab_size = 512
|
||||
alpha = 0.75
|
||||
|
||||
packed_weight = torch.randint(
|
||||
0, 256, (vocab_size, rank // 2), dtype=torch.uint8, device=device
|
||||
)
|
||||
weight_scale = torch.full(
|
||||
(vocab_size, rank // 16),
|
||||
0.5,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=device,
|
||||
)
|
||||
global_scale = torch.tensor([0.25], dtype=torch.float32, device=device)
|
||||
|
||||
markov_storage = torch.randn(
|
||||
batch_size, 2, rank, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
values_storage = torch.randn(
|
||||
batch_size, 2, topk, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
index_storage = torch.empty(batch_size, 2, topk, dtype=torch.int64, device=device)
|
||||
for batch_idx in range(batch_size):
|
||||
index_storage[batch_idx, 0] = (
|
||||
torch.arange(topk, device=device) + batch_idx
|
||||
) % vocab_size
|
||||
logits_storage = torch.full(
|
||||
(batch_size, 2, vocab_size),
|
||||
float("-inf"),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
markov_embed = markov_storage[:, 0]
|
||||
values = values_storage[:, 0]
|
||||
index = index_storage[:, 0]
|
||||
logits = logits_storage[:, 0]
|
||||
|
||||
layer = nn.Module()
|
||||
layer.register_buffer("_nvfp4_weight_for_gather", packed_weight)
|
||||
layer.register_buffer("_nvfp4_weight_scale_for_gather", weight_scale)
|
||||
layer.register_buffer("_nvfp4_weight_global_scale_for_gather", global_scale)
|
||||
layer._nvfp4_group_size_for_gather = 16
|
||||
layer._retain_weight_for_gather = True
|
||||
layer.is_w4a16_nvfp4 = True
|
||||
head = DSparkMarkovHead.__new__(DSparkMarkovHead)
|
||||
nn.Module.__init__(head)
|
||||
head.markov_w2 = layer
|
||||
|
||||
flat_index = index.reshape(-1)
|
||||
selected_weight = dequantize_to_dtype(
|
||||
packed_weight.index_select(0, flat_index),
|
||||
weight_scale.index_select(0, flat_index),
|
||||
global_scale,
|
||||
dtype=torch.bfloat16,
|
||||
block_size=16,
|
||||
swizzle=False,
|
||||
).view(batch_size, topk, rank)
|
||||
expected = (
|
||||
values
|
||||
+ torch.bmm(selected_weight, markov_embed.unsqueeze(-1)).squeeze(-1) * alpha
|
||||
)
|
||||
|
||||
result = head.apply_bias_gathered(markov_embed, logits, values, index, alpha)
|
||||
|
||||
torch.testing.assert_close(
|
||||
result.gather(1, index), expected, rtol=1e-2, atol=6.25e-2
|
||||
)
|
||||
selected = torch.zeros_like(result, dtype=torch.bool).scatter_(1, index, True)
|
||||
assert torch.isneginf(result.masked_select(~selected)).all()
|
||||
|
||||
if batch_size == 1:
|
||||
logits.fill_(float("-inf"))
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = head.apply_bias_gathered(
|
||||
markov_embed, logits, values, index, alpha
|
||||
)
|
||||
logits.fill_(float("-inf"))
|
||||
graph.replay()
|
||||
assert captured is logits
|
||||
torch.testing.assert_close(
|
||||
logits.gather(1, index), expected, rtol=1e-2, atol=6.25e-2
|
||||
)
|
||||
|
||||
@@ -2458,6 +2458,30 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
self.akey.process(layer, ACT)
|
||||
maybe_fuse_global_scales(layer)
|
||||
self.fmt.post_process(layer)
|
||||
layer.is_w4a16_nvfp4 = (
|
||||
self.spec.weight == kNvfp4Static and self.spec.activation is None
|
||||
)
|
||||
if getattr(layer, "_retain_weight_for_gather", False):
|
||||
if not layer.is_w4a16_nvfp4:
|
||||
raise NotImplementedError(
|
||||
"Gathered projection is only supported for ModelOpt W4A16 "
|
||||
"NVFP4 weights."
|
||||
)
|
||||
assert self.ctx.group_size is not None
|
||||
layer.register_buffer(
|
||||
"_nvfp4_weight_for_gather", layer.weight.detach(), persistent=False
|
||||
)
|
||||
layer.register_buffer(
|
||||
"_nvfp4_weight_scale_for_gather",
|
||||
layer.weight_scale.detach(),
|
||||
persistent=False,
|
||||
)
|
||||
layer.register_buffer(
|
||||
"_nvfp4_weight_global_scale_for_gather",
|
||||
layer.weight_global_scale.detach(),
|
||||
persistent=False,
|
||||
)
|
||||
layer._nvfp4_group_size_for_gather = self.ctx.group_size
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply(self, layer, x, bias=None):
|
||||
|
||||
@@ -7,10 +7,12 @@ import torch
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
__all__ = [
|
||||
"break_fp4_bytes",
|
||||
"dequantize_to_dtype",
|
||||
"nvfp4_gathered_bias",
|
||||
"ref_nvfp4_quant",
|
||||
]
|
||||
|
||||
@@ -43,6 +45,230 @@ def _e2m1_inline(nibble):
|
||||
return tl.where(sign == 1, -val, val)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _nvfp4_gathered_bias_kernel(
|
||||
markov_ptr,
|
||||
packed_weight_ptr,
|
||||
weight_scale_ptr,
|
||||
global_scale_ptr,
|
||||
values_ptr,
|
||||
index_ptr,
|
||||
logits_ptr,
|
||||
stride_markov_b,
|
||||
stride_markov_k,
|
||||
stride_weight_v,
|
||||
stride_weight_k,
|
||||
stride_scale_v,
|
||||
stride_scale_k,
|
||||
stride_values_b,
|
||||
stride_values_j,
|
||||
stride_index_b,
|
||||
stride_index_j,
|
||||
stride_logits_b,
|
||||
stride_logits_v,
|
||||
alpha,
|
||||
TOPK: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
NUM_BLOCKS: tl.constexpr,
|
||||
):
|
||||
"""Apply selected NVFP4 Markov-projection rows to dense logits.
|
||||
|
||||
BF16: ``weight[index] -> baddbmm_ -> scatter_``.
|
||||
NVFP4: ``gather packed W2 row -> dequantize 16-element blocks -> dot with
|
||||
Markov embedding -> add base value -> scatter corrected logit``.
|
||||
|
||||
The fused NVFP4 path is needed because packed weights cannot be indexed as
|
||||
BF16 rows, while dequantizing the full vocabulary would defeat top-k.
|
||||
"""
|
||||
candidate_id = tl.program_id(0)
|
||||
request_id = candidate_id // TOPK
|
||||
candidate = candidate_id - request_id * TOPK
|
||||
|
||||
vocab_id = tl.load(
|
||||
index_ptr + request_id * stride_index_b + candidate * stride_index_j
|
||||
).to(tl.int64)
|
||||
|
||||
block_ids = tl.arange(0, NUM_BLOCKS)
|
||||
byte_ids = tl.arange(0, 8)
|
||||
block_mask = block_ids * 16 < K
|
||||
|
||||
packed_offsets = block_ids[:, None] * 8 + byte_ids[None, :]
|
||||
packed = tl.load(
|
||||
packed_weight_ptr
|
||||
+ vocab_id * stride_weight_v
|
||||
+ packed_offsets * stride_weight_k,
|
||||
mask=block_mask[:, None],
|
||||
other=0,
|
||||
)
|
||||
low = _e2m1_inline(packed & 0x0F)
|
||||
high = _e2m1_inline((packed >> 4) & 0x0F)
|
||||
decoded = tl.interleave(low, high)
|
||||
|
||||
raw_weight_scale = tl.load(
|
||||
weight_scale_ptr + vocab_id * stride_scale_v + block_ids * stride_scale_k,
|
||||
mask=block_mask,
|
||||
other=0,
|
||||
)
|
||||
weight_scale = tl.cast(
|
||||
raw_weight_scale,
|
||||
tl.float8e4nv,
|
||||
bitcast=True,
|
||||
).to(tl.float32)
|
||||
global_scale = tl.load(global_scale_ptr).to(tl.float32)
|
||||
|
||||
element_ids = tl.arange(0, 16)
|
||||
k_offsets = block_ids[:, None] * 16 + element_ids[None, :]
|
||||
k_mask = k_offsets < K
|
||||
markov = tl.load(
|
||||
markov_ptr + request_id * stride_markov_b + k_offsets * stride_markov_k,
|
||||
mask=k_mask,
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
dequant_weight = (decoded * weight_scale[:, None] * global_scale).to(tl.bfloat16)
|
||||
products = dequant_weight.to(tl.float32) * markov.to(tl.float32)
|
||||
block_sums = tl.sum(products, axis=1)
|
||||
dot = tl.sum(block_sums, axis=0)
|
||||
|
||||
base_value = tl.load(
|
||||
values_ptr + request_id * stride_values_b + candidate * stride_values_j
|
||||
).to(tl.float32)
|
||||
corrected = base_value + alpha * dot
|
||||
tl.store(
|
||||
logits_ptr + request_id * stride_logits_b + vocab_id * stride_logits_v,
|
||||
corrected,
|
||||
)
|
||||
|
||||
|
||||
def _nvfp4_gathered_bias_impl(
|
||||
markov_embed: torch.Tensor,
|
||||
packed_weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
values: torch.Tensor,
|
||||
index: torch.Tensor,
|
||||
logits: torch.Tensor,
|
||||
alpha: float,
|
||||
) -> None:
|
||||
if index.numel() == 0:
|
||||
return
|
||||
|
||||
assert markov_embed.dtype == torch.bfloat16
|
||||
assert values.dtype == torch.bfloat16
|
||||
assert logits.dtype == torch.bfloat16
|
||||
assert packed_weight.dtype == torch.uint8
|
||||
assert weight_scale.dtype == torch.float8_e4m3fn
|
||||
assert index.dtype == torch.int64
|
||||
assert markov_embed.is_cuda
|
||||
assert all(
|
||||
tensor.device == markov_embed.device
|
||||
for tensor in (
|
||||
packed_weight,
|
||||
weight_scale,
|
||||
global_scale,
|
||||
values,
|
||||
index,
|
||||
logits,
|
||||
)
|
||||
)
|
||||
assert markov_embed.ndim == 2
|
||||
assert packed_weight.ndim == 2
|
||||
assert weight_scale.ndim == 2
|
||||
assert values.ndim == 2
|
||||
assert index.ndim == 2
|
||||
assert logits.ndim == 2
|
||||
assert global_scale.numel() == 1
|
||||
assert values.shape == index.shape
|
||||
assert logits.shape[0] == index.shape[0] == markov_embed.shape[0]
|
||||
|
||||
k = markov_embed.shape[-1]
|
||||
if k % 16 != 0 or packed_weight.shape[-1] * 2 != k:
|
||||
raise ValueError(
|
||||
"nvfp4_gathered_bias requires k to be divisible by 16 and "
|
||||
f"packed_weight width to equal k / 2; got k={k} and "
|
||||
f"packed_weight width={packed_weight.shape[-1]}"
|
||||
)
|
||||
if weight_scale.shape[-1] != k // 16:
|
||||
raise ValueError(
|
||||
"nvfp4_gathered_bias only supports an NVFP4 group size of 16; "
|
||||
f"got {weight_scale.shape[-1]} scale blocks for k={k}"
|
||||
)
|
||||
|
||||
topk = index.shape[1]
|
||||
num_blocks = triton.next_power_of_2(k // 16)
|
||||
scale_bytes = weight_scale.view(torch.uint8)
|
||||
_nvfp4_gathered_bias_kernel[(index.numel(),)](
|
||||
markov_embed,
|
||||
packed_weight,
|
||||
scale_bytes,
|
||||
global_scale,
|
||||
values,
|
||||
index,
|
||||
logits,
|
||||
markov_embed.stride(0),
|
||||
markov_embed.stride(1),
|
||||
packed_weight.stride(0),
|
||||
packed_weight.stride(1),
|
||||
scale_bytes.stride(0),
|
||||
scale_bytes.stride(1),
|
||||
values.stride(0),
|
||||
values.stride(1),
|
||||
index.stride(0),
|
||||
index.stride(1),
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
alpha,
|
||||
TOPK=topk,
|
||||
K=k,
|
||||
NUM_BLOCKS=num_blocks,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
|
||||
def _nvfp4_gathered_bias_fake(
|
||||
markov_embed: torch.Tensor,
|
||||
packed_weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
values: torch.Tensor,
|
||||
index: torch.Tensor,
|
||||
logits: torch.Tensor,
|
||||
alpha: float,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="nvfp4_gathered_bias",
|
||||
op_func=_nvfp4_gathered_bias_impl,
|
||||
mutates_args=["logits"],
|
||||
fake_impl=_nvfp4_gathered_bias_fake,
|
||||
)
|
||||
|
||||
|
||||
def nvfp4_gathered_bias(
|
||||
markov_embed: torch.Tensor,
|
||||
packed_weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
values: torch.Tensor,
|
||||
index: torch.Tensor,
|
||||
logits: torch.Tensor,
|
||||
alpha: float,
|
||||
) -> None:
|
||||
torch.ops.vllm.nvfp4_gathered_bias(
|
||||
markov_embed,
|
||||
packed_weight,
|
||||
weight_scale,
|
||||
global_scale,
|
||||
values,
|
||||
index,
|
||||
logits,
|
||||
alpha,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dequantize_nvfp4_kernel(
|
||||
fp4_ptr,
|
||||
@@ -347,7 +573,7 @@ def dequantize_to_dtype(
|
||||
# Two fp4 values are packed into one uint8.
|
||||
assert tensor_fp4.dtype == torch.uint8
|
||||
|
||||
if not swizzle and current_platform.is_cuda_alike():
|
||||
if not swizzle and tensor_fp4.is_cuda and current_platform.is_cuda_alike():
|
||||
return _triton_dequantize_nvfp4(
|
||||
tensor_fp4, tensor_sf, global_scale, dtype, block_size
|
||||
)
|
||||
|
||||
@@ -25,9 +25,14 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_to_dtype,
|
||||
nvfp4_gathered_bias,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model
|
||||
from .utils import (
|
||||
@@ -60,6 +65,7 @@ class DSparkMarkovHead(nn.Module):
|
||||
markov_rank: int,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
retain_weight_for_gather: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.markov_w1 = nn.Embedding(vocab_size, markov_rank)
|
||||
@@ -71,6 +77,8 @@ class DSparkMarkovHead(nn.Module):
|
||||
prefix=maybe_prefix(prefix, "markov_w2"),
|
||||
disable_tp=True,
|
||||
)
|
||||
self.markov_w2._retain_weight_for_gather = retain_weight_for_gather
|
||||
self.markov_w2.is_w4a16_nvfp4 = False
|
||||
|
||||
def embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""r-dim Markov embedding of ``token_ids`` ([B] -> [B, r])."""
|
||||
@@ -98,8 +106,52 @@ class DSparkMarkovHead(nn.Module):
|
||||
positions. This method scatters the corrected candidate values into
|
||||
that dense buffer so the normal sampler sees the truncated proposal.
|
||||
"""
|
||||
weight = self.markov_w2.weight[index]
|
||||
corrected = values.unsqueeze(-1)
|
||||
if self.markov_w2._retain_weight_for_gather and self.markov_w2.is_w4a16_nvfp4:
|
||||
# Use the fast Triton kernel for W4A16 NVFP4 weights with BF16
|
||||
# activations on CUDA. FP16 activations, non-CUDA accelerators, and
|
||||
# CPU use the slower dequantize/scatter fallback below.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and markov_embed.is_cuda
|
||||
and markov_embed.dtype == torch.bfloat16
|
||||
and values.dtype == torch.bfloat16
|
||||
and logits.dtype == torch.bfloat16
|
||||
):
|
||||
nvfp4_gathered_bias(
|
||||
markov_embed,
|
||||
self.markov_w2._nvfp4_weight_for_gather,
|
||||
self.markov_w2._nvfp4_weight_scale_for_gather,
|
||||
self.markov_w2._nvfp4_weight_global_scale_for_gather,
|
||||
values,
|
||||
index,
|
||||
logits,
|
||||
scale,
|
||||
)
|
||||
return logits
|
||||
|
||||
flat_index = index.reshape(-1)
|
||||
packed_weight = self.markov_w2._nvfp4_weight_for_gather.index_select(
|
||||
0, flat_index
|
||||
)
|
||||
weight_scale = self.markov_w2._nvfp4_weight_scale_for_gather.index_select(
|
||||
0, flat_index
|
||||
)
|
||||
weight = dequantize_to_dtype(
|
||||
packed_weight,
|
||||
weight_scale,
|
||||
self.markov_w2._nvfp4_weight_global_scale_for_gather,
|
||||
dtype=markov_embed.dtype,
|
||||
block_size=self.markov_w2._nvfp4_group_size_for_gather,
|
||||
swizzle=False,
|
||||
).view(*index.shape, -1)
|
||||
else:
|
||||
weight = self.markov_w2.weight[index]
|
||||
if weight.dtype != markov_embed.dtype:
|
||||
raise RuntimeError(
|
||||
"Gathered Markov projection requires unquantized weights or "
|
||||
"retained ModelOpt W4A16 NVFP4 weights."
|
||||
)
|
||||
corrected = values.unsqueeze(-1).clone()
|
||||
corrected.baddbmm_(
|
||||
weight,
|
||||
markov_embed.unsqueeze(-1),
|
||||
@@ -160,6 +212,9 @@ class Qwen3DSparkModel(DFlashQwen3Model):
|
||||
config.markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
quant_config=self.quant_config,
|
||||
retain_weight_for_gather=(
|
||||
getattr(config, "dspark_draft_topk", None) is not None
|
||||
),
|
||||
)
|
||||
self.confidence_head: DSparkConfidenceHead | None = None
|
||||
if getattr(config, "enable_confidence_head", False):
|
||||
|
||||
Reference in New Issue
Block a user