Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -115,7 +115,9 @@ def test_b12x_backend_registration_priority_and_selection(
|
||||
assert names.index(before) < names.index(kernel_cls.__name__) < names.index(after)
|
||||
|
||||
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
|
||||
monkeypatch.setattr(linear_mod, "_get_linear_backend", lambda: "b12x")
|
||||
monkeypatch.setattr(
|
||||
linear_mod, "_get_linear_backend", lambda *, quantization: "b12x"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kernel_cls,
|
||||
"is_supported",
|
||||
@@ -784,7 +786,9 @@ def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
|
||||
import vllm.model_executor.kernels.linear as linear_mod
|
||||
|
||||
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
|
||||
monkeypatch.setattr(linear_mod, "_get_linear_backend", lambda: "b12x")
|
||||
monkeypatch.setattr(
|
||||
linear_mod, "_get_linear_backend", lambda *, quantization: "b12x"
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
linear_mod._POSSIBLE_NVFP4_KERNELS,
|
||||
PlatformEnum.CUDA,
|
||||
|
||||
@@ -450,3 +450,547 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding(
|
||||
keep = valid & (block >= 0) & ((cols - ks[:, None]) // block_size == block)
|
||||
reference = torch.where(keep, 3.0, -torch.inf)
|
||||
torch.testing.assert_close(logits, reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
# --- DeepGEMM sparse-MQA indexer path (V4.1 two-level candidate filtering) ---
|
||||
|
||||
|
||||
def _make_candidates(lens: torch.Tensor, num_candidates: int, cbk: int):
|
||||
"""Production-faithful candidates: top min(avail, K) blocks per row —
|
||||
full coverage for lens <= K*cbk, a fixed-size subset beyond that."""
|
||||
avail = ((lens.cpu() + cbk - 1) // cbk).tolist()
|
||||
rows = []
|
||||
for a in avail:
|
||||
n = min(a, num_candidates)
|
||||
picks = torch.randperm(a)[:n].sort().values[:num_candidates].tolist()
|
||||
rows.append(picks + [-1] * (num_candidates - len(picks)))
|
||||
return torch.tensor(rows, dtype=torch.int32)
|
||||
|
||||
|
||||
def _candidate_token_mask(
|
||||
candidates: torch.Tensor, cbk: int, total: int, ks: torch.Tensor | None = None
|
||||
):
|
||||
"""[rows, total] bool mask of the candidate-covered token positions.
|
||||
|
||||
``ks`` converts request-local candidate blocks to absolute positions
|
||||
(prefill's packed workspace); pass None when already request-local.
|
||||
"""
|
||||
rows = candidates.shape[0]
|
||||
device = candidates.device
|
||||
# Sentinel block keeps -1 padding from scattering onto real columns.
|
||||
width = max(total, (candidates.shape[1] + 1) * cbk)
|
||||
if ks is not None:
|
||||
width += int(ks.max()) + cbk
|
||||
in_cand = torch.zeros(rows, width, dtype=torch.bool, device=device)
|
||||
block_id = torch.where(candidates >= 0, candidates.long(), candidates.shape[1])
|
||||
tok = block_id.unsqueeze(2) * cbk + torch.arange(cbk, device=device)
|
||||
if ks is not None:
|
||||
tok = tok + ks.unsqueeze(1).unsqueeze(2)
|
||||
keep = (candidates >= 0).unsqueeze(2).expand(-1, -1, cbk)
|
||||
in_cand.scatter_(1, tok.flatten(1), keep.flatten(1))
|
||||
return in_cand[:, :total]
|
||||
|
||||
|
||||
def _reference_topk(masked: torch.Tensor, topk: int) -> torch.Tensor:
|
||||
"""Plain ``torch.topk`` over -inf-masked logits; slots past a row's valid
|
||||
count are -1 (the indexer's "no token" sentinel)."""
|
||||
values, indices = masked.topk(topk, dim=1)
|
||||
return torch.where(values > float("-inf"), indices.int(), -1)
|
||||
|
||||
|
||||
def _assert_same_selection(
|
||||
logits: torch.Tensor,
|
||||
got: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
row_starts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Both top-k index sets must pick the same multiset of logit values per
|
||||
row (-1 padding must match too). Indices themselves may differ: the
|
||||
top-k kernels break ties among equal values in nondeterministic order."""
|
||||
for r in range(logits.shape[0]):
|
||||
base = int(row_starts[r]) if row_starts is not None else 0
|
||||
g, e = got[r][got[r] >= 0], expected[r][expected[r] >= 0]
|
||||
assert g.numel() == e.numel(), f"row {r}: {g.numel()} vs {e.numel()} selected"
|
||||
vg = logits[r][(base + g).long()].sort().values
|
||||
ve = logits[r][(base + e).long()].sort().values
|
||||
assert torch.equal(vg, ve), f"row {r}: selected values differ"
|
||||
|
||||
|
||||
def _sparse_scratch(rows: int, num_sparse: int, topk: int) -> dict:
|
||||
"""Caller-owned buffers the sparse path writes into (the metadata builder
|
||||
owns these in production)."""
|
||||
i32 = dict(dtype=torch.int32, device="cuda")
|
||||
return dict(
|
||||
sparse_indices=torch.zeros(rows, num_sparse, **i32),
|
||||
end=torch.zeros(rows, **i32),
|
||||
col_indices=torch.zeros(rows, topk, **i32),
|
||||
)
|
||||
|
||||
|
||||
def _expand_candidates_reference(
|
||||
candidates: torch.Tensor, ks: torch.Tensor, ke: torch.Tensor, cbk: int, sbk: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""PyTorch reference for `candidate_blocks_to_sparse_indices`: expand
|
||||
each candidate block into ``cbk // sbk`` sparse blocks, keep those inside
|
||||
``[0, ke - ks)``, sort ascending, anchor at ``ks // sbk``, pad by
|
||||
repeating the last valid block, and count the valid sparse columns."""
|
||||
ratio = cbk // sbk
|
||||
rows, num_candidates = candidates.shape
|
||||
num_sparse = num_candidates * ratio
|
||||
big = torch.iinfo(torch.int32).max
|
||||
offsets = torch.arange(ratio, device=candidates.device, dtype=torch.int32)
|
||||
blocks = (candidates.unsqueeze(-1) * ratio + offsets).flatten(1)
|
||||
max_block = ((ke - ks).clamp(min=0) + sbk - 1) // sbk
|
||||
valid = (blocks >= 0) & (blocks < max_block.unsqueeze(1))
|
||||
sorted_blocks, _ = torch.where(valid, blocks, big).sort(dim=1)
|
||||
n_valid = (sorted_blocks != big).sum(dim=1, dtype=torch.int32)
|
||||
base = (ks // sbk).unsqueeze(1)
|
||||
kernel_blocks = torch.where(sorted_blocks != big, sorted_blocks, 0) + base
|
||||
last_slot = (n_valid - 1).clamp(min=0).unsqueeze(1).long()
|
||||
last_block = kernel_blocks.gather(1, last_slot)
|
||||
pad = torch.arange(num_sparse, device=candidates.device) >= n_valid.unsqueeze(1)
|
||||
sparse_indices = torch.where(pad, last_block, kernel_blocks)
|
||||
last_start = last_block.squeeze(1) * sbk + ks % sbk
|
||||
last_fill = (ke - last_start).clamp(min=1, max=sbk)
|
||||
end = torch.where(n_valid > 0, (n_valid - 1) * sbk + last_fill, 0)
|
||||
return sparse_indices.int(), end.int()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel")
|
||||
def test_candidate_blocks_to_sparse_indices_math():
|
||||
"""Candidate->sparse-block expansion: ks anchoring, range filtering,
|
||||
repeat-last padding, and the per-row valid column count (``end``)."""
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
candidate_blocks_to_sparse_indices,
|
||||
)
|
||||
|
||||
# Rows cover: exactly one block, an unaligned ks with a partial tail
|
||||
# block, an empty row, duplicates / -1 padding / out-of-range garbage
|
||||
# (warmup runs touch uninitialized buffers), and int32-overflow junk.
|
||||
ks = torch.tensor([64, 13, 100, 8, 0], dtype=torch.int32, device="cuda")
|
||||
ke = torch.tensor([72, 30, 100, 40, 1000], dtype=torch.int32, device="cuda")
|
||||
candidates = torch.tensor(
|
||||
[
|
||||
[0, -1, -1, -1],
|
||||
[2, 0, -1, -1],
|
||||
[3, -1, -1, -1],
|
||||
[1, 1, 7, 0],
|
||||
[2**30, -(2**30), 0, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
indices, end = candidate_blocks_to_sparse_indices(
|
||||
candidates, ks, ke, candidate_block_size=8, sparse_block_kv=8
|
||||
)
|
||||
assert indices.tolist() == [
|
||||
[8, 8, 8, 8],
|
||||
[1, 3, 3, 3],
|
||||
[12, 12, 12, 12],
|
||||
[1, 2, 2, 2],
|
||||
[0, 0, 0, 0],
|
||||
]
|
||||
# end: row 1's last block covers [29, 37) but ke = 30 -> one valid token;
|
||||
# row 3 keeps the duplicated block (production candidates are unique).
|
||||
assert end.tolist() == [8, 9, 0, 24, 8]
|
||||
|
||||
# Wider candidate blocks expand into multiple sparse blocks.
|
||||
idx2, end2 = candidate_blocks_to_sparse_indices(
|
||||
torch.tensor([[1, 0]], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([48], dtype=torch.int32, device="cuda"),
|
||||
candidate_block_size=16,
|
||||
sparse_block_kv=8,
|
||||
)
|
||||
assert idx2.tolist() == [[0, 1, 2, 3]]
|
||||
assert end2.tolist() == [32]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel")
|
||||
@pytest.mark.parametrize("cbk,sbk", [(8, 8), (16, 8), (16, 16)])
|
||||
def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int):
|
||||
"""The Triton expansion equals the PyTorch reference on production-shaped
|
||||
random inputs, including strided (column-sliced) candidate rows and
|
||||
strided output buffers."""
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
candidate_blocks_to_sparse_indices,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rows, num_candidates = 41, 2048
|
||||
ratio = cbk // sbk
|
||||
lens = torch.randint(1, 40000, (rows,), dtype=torch.int32)
|
||||
lens[0], lens[1] = 1, 3
|
||||
ks = torch.zeros(rows, dtype=torch.int32)
|
||||
ks[1:] = lens.cumsum(0)[:-1]
|
||||
ks, ke = ks.cuda(), (ks + lens).cuda()
|
||||
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
|
||||
# Row 2 gets out-of-range garbage past the padding, as a CUDA-graph
|
||||
# warm-up on uninitialized buffers would.
|
||||
candidates[2, -3:] = torch.tensor([2**30, -(2**29), 10**6], dtype=torch.int32)
|
||||
|
||||
ref_indices, ref_end = _expand_candidates_reference(candidates, ks, ke, cbk, sbk)
|
||||
|
||||
cand_buf = torch.full((rows, num_candidates + 64), -7, dtype=torch.int32).cuda()
|
||||
cand_buf[:, :num_candidates] = candidates
|
||||
si_buf = torch.zeros(rows, num_candidates * ratio + 32, dtype=torch.int32).cuda()
|
||||
end_buf = torch.zeros(rows * 2, dtype=torch.int32, device="cuda")
|
||||
ks_buf = torch.zeros(rows * 2, dtype=torch.int32, device="cuda")
|
||||
ks_buf[::2] = ks
|
||||
got_indices, got_end = candidate_blocks_to_sparse_indices(
|
||||
cand_buf[:, :num_candidates],
|
||||
ks_buf[::2],
|
||||
ke,
|
||||
cbk,
|
||||
sbk,
|
||||
out=(si_buf[:, : num_candidates * ratio], end_buf[::2]),
|
||||
)
|
||||
assert torch.equal(got_indices, ref_indices)
|
||||
assert torch.equal(got_end, ref_end)
|
||||
assert torch.equal(
|
||||
cand_buf[:, num_candidates:], torch.full_like(cand_buf[:, num_candidates:], -7)
|
||||
)
|
||||
assert not si_buf[:, num_candidates * ratio :].any()
|
||||
assert not end_buf[1::2].any()
|
||||
|
||||
|
||||
def _skip_unless_sm100_sparse_kernels():
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10:
|
||||
pytest.skip("SM100 required")
|
||||
deep_gemm = pytest.importorskip("deep_gemm")
|
||||
if not hasattr(deep_gemm, "fp8_fp4_sparse_mqa_logits"):
|
||||
pytest.skip("DeepGEMM >= 2.8 required for sparse MQA logits")
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
has_deep_select,
|
||||
)
|
||||
|
||||
if not has_deep_select():
|
||||
pytest.skip("DeepSelect extension (vllm._deepselect_C) required")
|
||||
return deep_gemm
|
||||
|
||||
|
||||
def _quant_fp4(x: torch.Tensor):
|
||||
from deep_gemm.utils import per_token_cast_to_fp4
|
||||
|
||||
return per_token_cast_to_fp4(x, use_ue8m0=True, gran_k=32, use_packed_ue8m0=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capture", [False, True])
|
||||
def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
"""The sparse-MQA candidate path selects exactly the same logit values as
|
||||
``torch.topk`` over dense bf16 logits masked to the candidates (the two
|
||||
DeepGEMM kernels agree bitwise on candidate columns), incl. under CUDA
|
||||
graph capture, and a later layer reusing the step's expansion/schedule
|
||||
selects the same values."""
|
||||
deep_gemm = _skip_unless_sm100_sparse_kernels()
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
sparse_mqa_logits_prefill_chunk,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rows, num_heads, head_dim = 37, 32, 128
|
||||
cbk, topk, num_candidates = 8, 512, 2048 # V4.1-Flash production values
|
||||
lens = torch.randint(1, 30000, (rows,), dtype=torch.int32)
|
||||
lens[0], lens[1], lens[2] = 1, 5, 2048 # single token, sub-block, long
|
||||
ks = torch.zeros(rows, dtype=torch.int32)
|
||||
ks[1:] = lens.cumsum(0)[:-1]
|
||||
ks, ke, lens = ks.cuda(), (ks + lens).cuda(), lens.cuda()
|
||||
total = int(lens.sum())
|
||||
q_fp, q_sf = _quant_fp4(
|
||||
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
k_fp, k_sf = _quant_fp4(
|
||||
torch.randn(total, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
|
||||
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
|
||||
|
||||
# Reference: dense bf16 logits masked to the candidates, then torch.topk.
|
||||
dense = deep_gemm.fp8_fp4_mqa_logits(
|
||||
(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
q_sf.view(rows, num_heads),
|
||||
),
|
||||
(k_fp.view(torch.int8), k_sf.view(total)),
|
||||
weights,
|
||||
ks,
|
||||
ke,
|
||||
clean_logits=False,
|
||||
logits_dtype=torch.bfloat16,
|
||||
)
|
||||
cols = torch.arange(total, device="cuda")
|
||||
in_range = (cols - ks.unsqueeze(1) >= 0) & (
|
||||
cols - ks.unsqueeze(1) < lens.unsqueeze(1)
|
||||
)
|
||||
masked = dense.masked_fill(
|
||||
~(in_range & _candidate_token_mask(candidates, cbk, total, ks)),
|
||||
float("-inf"),
|
||||
).float()
|
||||
ref_abs = _reference_topk(masked, topk) # packed-workspace positions
|
||||
ref = torch.where(ref_abs >= 0, ref_abs - ks.unsqueeze(1), -1)
|
||||
|
||||
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
|
||||
scratch = _sparse_scratch(rows, num_candidates, topk)
|
||||
|
||||
def run_sparse(kernel_metadata=None):
|
||||
return sparse_mqa_logits_prefill_chunk(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
q_sf.view(rows, num_heads),
|
||||
k_fp.view(torch.int8),
|
||||
k_sf.view(total),
|
||||
weights,
|
||||
ks,
|
||||
ke,
|
||||
candidates,
|
||||
cbk,
|
||||
cbk,
|
||||
topk,
|
||||
out,
|
||||
kernel_metadata=kernel_metadata,
|
||||
**scratch,
|
||||
)
|
||||
|
||||
if capture:
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
run_sparse() # warm up JIT + the DeepGEMM workspace pre-capture
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
run_sparse()
|
||||
out.fill_(-1)
|
||||
graph.replay()
|
||||
else:
|
||||
kernel_metadata = run_sparse()
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
out.fill_(-1)
|
||||
run_sparse(kernel_metadata)
|
||||
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
|
||||
|
||||
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
|
||||
"""MXFP4 paged cache in DeepGEMM's fused layout: per page, all tokens'
|
||||
fp4 data first, then their packed UE8M0 scales."""
|
||||
device = req_lens.device
|
||||
pages_per = (req_lens + page_kv - 1) // page_kv
|
||||
num_pages, max_pages = int(pages_per.sum()), int(pages_per.max())
|
||||
kv_cache = torch.zeros(
|
||||
num_pages, page_kv, 1, head_dim // 2 + 4, dtype=torch.uint8, device=device
|
||||
)
|
||||
block_table = torch.zeros(
|
||||
len(req_lens), max_pages, dtype=torch.int32, device=device
|
||||
)
|
||||
page_pool = torch.randperm(num_pages, device=device)
|
||||
offset = 0
|
||||
for b in range(len(req_lens)):
|
||||
n = int(pages_per[b])
|
||||
block_table[b, :n] = page_pool[offset : offset + n].to(torch.int32)
|
||||
offset += n
|
||||
k_fp, k_sf = _quant_fp4(
|
||||
torch.randn(int(req_lens[b]), head_dim, device=device, dtype=torch.bfloat16)
|
||||
)
|
||||
rows68 = torch.zeros(n * page_kv, 68, dtype=torch.uint8, device=device)
|
||||
rows68[: int(req_lens[b]), : head_dim // 2] = k_fp.view(torch.uint8)
|
||||
rows68[: int(req_lens[b]), head_dim // 2 :] = k_sf.view(torch.uint8).view(-1, 4)
|
||||
by_page = rows68.view(n, page_kv, 68)
|
||||
split = torch.cat(
|
||||
[by_page[..., :64].reshape(n, -1), by_page[..., 64:].reshape(n, -1)], dim=1
|
||||
).view(n, page_kv, 1, head_dim // 2 + 4)
|
||||
kv_cache[block_table[b, :n].long()] = split
|
||||
return kv_cache, block_table
|
||||
|
||||
|
||||
@pytest.mark.parametrize("next_n", [1, 2])
|
||||
@pytest.mark.parametrize("capture", [False, True])
|
||||
def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool):
|
||||
"""Paged sparse-MQA decode selects exactly the values ``torch.topk`` picks
|
||||
from the dense bf16 masked logits, incl. MTP rows (next_n > 1 flattened to
|
||||
one query row each), CUDA graph capture/replay and schedule reuse by a
|
||||
later layer."""
|
||||
deep_gemm = _skip_unless_sm100_sparse_kernels()
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
sparse_mqa_logits_paged_decode,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
batch, num_heads, head_dim, page_kv = 9, 32, 128, 128
|
||||
cbk, topk, num_candidates = 8, 512, 2048
|
||||
rows = batch * next_n
|
||||
req_lens = torch.randint(1, 30000, (batch,), dtype=torch.int32, device="cuda")
|
||||
req_lens[0], req_lens[1] = 1, 30000 # tiny row + a filtered (subset) row
|
||||
# Row (b, j) sees L_b - next_n + j + 1 tokens (native MTP layout).
|
||||
row_lens = (
|
||||
(req_lens.unsqueeze(1) - next_n + torch.arange(1, next_n + 1, device="cuda"))
|
||||
.clamp(min=0)
|
||||
.to(torch.int32)
|
||||
)
|
||||
kv_cache, block_table = _build_fp4_paged_cache(req_lens, page_kv, head_dim)
|
||||
assert kv_cache.stride(0) % 512 == 0 # the paged sparse kernel requires it
|
||||
|
||||
q_fp, q_sf = _quant_fp4(
|
||||
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
q = q_fp.view(torch.int8).view(batch, next_n, num_heads, head_dim // 2)
|
||||
q_scale = q_sf.view(batch, next_n, num_heads)
|
||||
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
|
||||
candidates = _make_candidates(row_lens.flatten(), num_candidates, cbk).cuda()
|
||||
|
||||
def reference_topk():
|
||||
context_lens_2d = row_lens.view(batch, next_n)
|
||||
sched = deep_gemm.get_paged_mqa_logits_metadata(
|
||||
context_lens_2d, page_kv, deep_gemm.get_num_sms(), None
|
||||
)
|
||||
logits = deep_gemm.fp8_fp4_paged_mqa_logits(
|
||||
(q, q_scale),
|
||||
kv_cache,
|
||||
weights,
|
||||
context_lens_2d,
|
||||
block_table,
|
||||
sched,
|
||||
int(req_lens.max()),
|
||||
False,
|
||||
torch.bfloat16,
|
||||
None,
|
||||
)
|
||||
lens_flat = row_lens.flatten()
|
||||
cols = torch.arange(logits.shape[1], device="cuda")
|
||||
masked = logits.masked_fill(
|
||||
~(
|
||||
(cols.unsqueeze(0) < lens_flat.unsqueeze(1))
|
||||
& _candidate_token_mask(candidates, cbk, logits.shape[1])
|
||||
),
|
||||
float("-inf"),
|
||||
).float()
|
||||
return _reference_topk(masked, topk), logits
|
||||
|
||||
expected, dense_logits = reference_topk()
|
||||
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
|
||||
|
||||
# The metadata builder flattens every query to one row: per-row context
|
||||
# lengths, an expanded block table and a row -> request map.
|
||||
context_lens = row_lens.flatten().contiguous()
|
||||
row_block_table = block_table.repeat_interleave(next_n, dim=0)
|
||||
row_indices = torch.arange(batch, dtype=torch.int32, device="cuda")
|
||||
row_indices = row_indices.repeat_interleave(next_n)
|
||||
row_ks = torch.zeros(rows, dtype=torch.int32, device="cuda")
|
||||
scratch = _sparse_scratch(rows, num_candidates, topk)
|
||||
|
||||
def run_sparse(o: torch.Tensor, kernel_metadata=None):
|
||||
return sparse_mqa_logits_paged_decode(
|
||||
q.view(rows, 1, num_heads, head_dim // 2),
|
||||
q_scale.view(rows, 1, num_heads),
|
||||
kv_cache,
|
||||
weights,
|
||||
context_lens,
|
||||
row_block_table,
|
||||
row_indices,
|
||||
candidates,
|
||||
cbk,
|
||||
cbk,
|
||||
topk,
|
||||
o,
|
||||
row_ks=row_ks,
|
||||
kernel_metadata=kernel_metadata,
|
||||
**scratch,
|
||||
)
|
||||
|
||||
if capture:
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
run_sparse(out)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
run_sparse(out)
|
||||
out.fill_(-1)
|
||||
graph.replay()
|
||||
else:
|
||||
kernel_metadata = run_sparse(out)
|
||||
_assert_same_selection(dense_logits, out, expected)
|
||||
out.fill_(-1)
|
||||
run_sparse(out, kernel_metadata)
|
||||
|
||||
_assert_same_selection(dense_logits, out, expected)
|
||||
|
||||
|
||||
def test_sparse_mqa_logits_accuracy_vs_torch():
|
||||
"""Sparse-kernel numerics vs a PyTorch fp32 reference on the dequantized
|
||||
MXFP4 inputs."""
|
||||
_skip_unless_sm100_sparse_kernels()
|
||||
from deep_gemm.utils import cast_back_from_fp4
|
||||
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
candidate_blocks_to_sparse_indices,
|
||||
)
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_sparse_mqa_logits,
|
||||
get_sparse_mqa_logits_metadata,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rows, num_heads, head_dim = 17, 32, 128
|
||||
sbk, num_candidates = 8, 2048
|
||||
lens = torch.randint(1, 1500, (rows,), dtype=torch.int32)
|
||||
lens[0] = 1
|
||||
ks = torch.zeros(rows, dtype=torch.int32)
|
||||
ks[1:] = lens.cumsum(0)[:-1]
|
||||
ks, ke, lens = ks.cuda(), (ks + lens).cuda(), lens.cuda()
|
||||
total = int(lens.sum())
|
||||
q_fp, q_sf = _quant_fp4(
|
||||
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
k_fp, k_sf = _quant_fp4(
|
||||
torch.randn(total, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
|
||||
candidates = _make_candidates(lens, num_candidates, sbk).cuda()
|
||||
|
||||
sparse_idx, end = candidate_blocks_to_sparse_indices(candidates, ks, ke, sbk, sbk)
|
||||
metadata = get_sparse_mqa_logits_metadata(
|
||||
ks, ke, total, sparse_idx, torch.int8, sbk
|
||||
)
|
||||
logits = fp8_fp4_sparse_mqa_logits(
|
||||
(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
q_sf.view(rows, num_heads),
|
||||
),
|
||||
(k_fp.view(torch.int8), k_sf.view(total)),
|
||||
weights,
|
||||
metadata,
|
||||
sparse_idx.shape[1],
|
||||
sbk,
|
||||
)
|
||||
|
||||
# PyTorch fp32 reference on the dequantized inputs.
|
||||
q_deq = cast_back_from_fp4(q_fp, q_sf, gran_k=32, use_packed_ue8m0=True)
|
||||
k_deq = cast_back_from_fp4(k_fp, k_sf, gran_k=32, use_packed_ue8m0=True)
|
||||
dots = torch.einsum(
|
||||
"rhd,td->rht",
|
||||
q_deq.view(rows, num_heads, head_dim).float(),
|
||||
k_deq.float(),
|
||||
)
|
||||
ref = (dots.relu() * weights.float().unsqueeze(-1)).sum(1) # [rows, total]
|
||||
|
||||
# Compare on valid sparse columns (absolute packed-workspace positions).
|
||||
width = sparse_idx.shape[1] * sbk
|
||||
cols = torch.arange(width, device="cuda")
|
||||
valid = cols.unsqueeze(0) < end.unsqueeze(1).long()
|
||||
pos = (
|
||||
sparse_idx.repeat_interleave(sbk, dim=1).long() * sbk
|
||||
+ (ks % sbk).unsqueeze(1)
|
||||
+ (cols % sbk).unsqueeze(0)
|
||||
).clamp(min=0, max=total - 1)
|
||||
abs_err = (logits.float() - ref.gather(1, pos)).abs()[valid]
|
||||
row_scale = ref.abs().amax(dim=1)
|
||||
norm_err = ((logits.float() - ref.gather(1, pos)).abs() / row_scale.unsqueeze(1))[
|
||||
valid
|
||||
]
|
||||
print(
|
||||
f"sparse vs torch fp32: max_abs={abs_err.max().item():.4f}; "
|
||||
f"rowmax-normalized p50={norm_err.median().item():.5f} "
|
||||
f"p99={norm_err.quantile(0.99).item():.5f} max={norm_err.max().item():.5f}"
|
||||
)
|
||||
# bf16-level numerics: far below the value gaps that change top-k selection.
|
||||
assert norm_err.quantile(0.99).item() < 1e-2
|
||||
assert norm_err.max().item() < 2e-2
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
import importlib.machinery
|
||||
import inspect
|
||||
import sys
|
||||
import types
|
||||
from unittest.mock import Mock
|
||||
from weakref import WeakKeyDictionary, ref
|
||||
|
||||
@@ -817,6 +820,110 @@ def test_layerwise_reload_updates_loaded_non_persistent_buffers(monkeypatch):
|
||||
assert "0.scale" not in model.state_dict()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hpc_rope_norm(monkeypatch, default_vllm_config):
|
||||
"""Import HpcRopeNorm with the external ``hpc`` package stubbed out."""
|
||||
if "hpc" not in sys.modules:
|
||||
stub = types.ModuleType("hpc")
|
||||
stub.__spec__ = importlib.machinery.ModuleSpec("hpc", loader=None)
|
||||
stub.QuantType = types.SimpleNamespace( # type: ignore[attr-defined]
|
||||
QPERTOKEN_PERHEAD_KPERTENSOR_VPERTENSOR=types.SimpleNamespace(value=0)
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hpc", stub)
|
||||
from vllm.model_executor.layers.hpc import rope_norm
|
||||
|
||||
monkeypatch.setattr(rope_norm, "_hpc_rope_norm_instances", {})
|
||||
return rope_norm
|
||||
|
||||
|
||||
def test_hpc_rope_norm_kernel_sees_refit_norm_weights(monkeypatch, hpc_rope_norm):
|
||||
"""The fused HPC kernel is handed the live QK-norm weights after a refit.
|
||||
|
||||
Drives the production ``_forward_impl`` with a recording ``hpc`` stub. The
|
||||
Q/K norm weights it receives must be the model's own float32 parameters,
|
||||
so a layerwise reload that rewrites them in place (same storage) is what
|
||||
the kernel sees. Previously the kernel read separate mirrors that no
|
||||
reload path refreshed.
|
||||
"""
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
head_dim, num_heads, num_kv_heads, block_size = 128, 8, 1, 4
|
||||
layer = torch.nn.Module()
|
||||
layer.q_norm = RMSNorm(head_dim, 1e-6, dtype=torch.float32)
|
||||
layer.k_norm = RMSNorm(head_dim, 1e-6, dtype=torch.float32)
|
||||
layer.hpc_rope_norm = hpc_rope_norm.HpcRopeNorm(
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
cos_sin_cache=torch.ones(16, head_dim),
|
||||
use_qk_norm=True,
|
||||
fallback_qnorm=layer.q_norm,
|
||||
fallback_knorm=layer.k_norm,
|
||||
kv_cache_dtype="auto",
|
||||
layer_name="hpc_test_layer",
|
||||
)
|
||||
model = torch.nn.Sequential(layer)
|
||||
rnorm = layer.hpc_rope_norm
|
||||
|
||||
calls: list[dict] = []
|
||||
|
||||
def record(*args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(sys.modules["hpc"], "rope_norm_store_kv", record, raising=False)
|
||||
|
||||
def kernel_norm_weights():
|
||||
q_size, kv_size = num_heads * head_dim, num_kv_heads * head_dim
|
||||
qkv = torch.zeros(1, q_size + 2 * kv_size, dtype=torch.bfloat16)
|
||||
kv_cache = torch.zeros(
|
||||
2, num_kv_heads, block_size, 2 * head_dim, dtype=torch.bfloat16
|
||||
)
|
||||
attn_layer = types.SimpleNamespace(
|
||||
_k_scale=torch.ones(1), _v_scale=torch.ones(1)
|
||||
)
|
||||
attn_metadata = types.SimpleNamespace(
|
||||
num_actual_tokens=1,
|
||||
num_decodes=1,
|
||||
num_decode_tokens=1,
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
max_query_len=1,
|
||||
decode_query_len=1,
|
||||
qo_indptr=None,
|
||||
qo_indptr_decode=None,
|
||||
slot_mapping=torch.tensor([4]),
|
||||
seq_lens=torch.tensor([1]),
|
||||
block_table_tensor=torch.tensor([[1]]),
|
||||
hpc_kv_written=False,
|
||||
)
|
||||
output = torch.zeros(1, q_size, dtype=torch.bfloat16)
|
||||
rnorm._forward_impl(qkv, kv_cache, attn_metadata, attn_layer, output)
|
||||
return calls[-1]["q_norm_weight"], calls[-1]["k_norm_weight"]
|
||||
|
||||
def loaded(value):
|
||||
return torch.full((head_dim,), value, dtype=torch.bfloat16)
|
||||
|
||||
default_weight_loader(layer.q_norm.weight, loaded(0.5))
|
||||
default_weight_loader(layer.k_norm.weight, loaded(0.25))
|
||||
q, k = kernel_norm_weights()
|
||||
assert torch.equal(q, loaded(0.5).float())
|
||||
assert torch.equal(k, loaded(0.25).float())
|
||||
q_ptr, k_ptr = q.data_ptr(), k.data_ptr()
|
||||
|
||||
record_metadata_for_reloading(model)
|
||||
initialize_layerwise_reload(model)
|
||||
layer.q_norm.weight.weight_loader(layer.q_norm.weight, loaded(2.0))
|
||||
layer.k_norm.weight.weight_loader(layer.k_norm.weight, loaded(3.0))
|
||||
finalize_layerwise_reload(model, model_config=None)
|
||||
|
||||
q, k = kernel_norm_weights()
|
||||
assert q.dtype == k.dtype == torch.float32
|
||||
assert torch.equal(q, loaded(2.0).float())
|
||||
assert torch.equal(k, loaded(3.0).float())
|
||||
assert (q.data_ptr(), k.data_ptr()) == (q_ptr, k_ptr)
|
||||
assert not hasattr(rnorm, "qnorm_weight")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
|
||||
identical outputs.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -21,7 +22,6 @@ import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import get_socket_path
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DAEMON_TIMEOUT_S = 600
|
||||
@@ -79,9 +79,7 @@ class WeightCacheDaemon:
|
||||
# Poll for the socket files rather than a log line: model loading can
|
||||
# pull in JIT compilers that swap the process's stderr and swallow
|
||||
# everything logged afterwards, making log-based readiness flaky.
|
||||
expected = [
|
||||
get_socket_path(gpu_id, self.socket_dir) for gpu_id in range(self.tp_size)
|
||||
]
|
||||
pattern = os.path.join(self.socket_dir, "vllm_weight_cache_*.sock")
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
if self._proc.poll() is not None:
|
||||
@@ -89,7 +87,7 @@ class WeightCacheDaemon:
|
||||
f"Weight cache daemon exited with {self._proc.returncode}:\n"
|
||||
f"{self._logs()}"
|
||||
)
|
||||
if all(os.path.exists(path) for path in expected):
|
||||
if len(glob.glob(pattern)) >= self.tp_size:
|
||||
return
|
||||
time.sleep(1.0)
|
||||
raise TimeoutError(
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def test_nemotron_h_lm_head_receives_quant_config():
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
|
||||
@@ -32,3 +34,28 @@ def test_nemotron_h_lm_head_receives_quant_config():
|
||||
MockLMHead.assert_called_once()
|
||||
call_kwargs = MockLMHead.call_args.kwargs
|
||||
assert call_kwargs["quant_config"] is mock_quant_config
|
||||
|
||||
|
||||
def test_relu2_fp8_fusion_uses_registry():
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHMLP
|
||||
|
||||
projected = torch.empty((1, 1), dtype=torch.bfloat16)
|
||||
fused = Mock()
|
||||
act_fn = Mock()
|
||||
down_proj = Mock(side_effect=lambda x: (x, None))
|
||||
|
||||
mlp = NemotronHMLP.__new__(NemotronHMLP)
|
||||
torch.nn.Module.__init__(mlp)
|
||||
mlp.up_proj = Mock(return_value=(projected, None))
|
||||
mlp.down_proj = down_proj
|
||||
mlp.act_fn = act_fn
|
||||
|
||||
with patch(
|
||||
"vllm.model_executor.models.nemotron_h.maybe_fused_act_quant",
|
||||
return_value=fused,
|
||||
) as maybe_fused:
|
||||
result = mlp(Mock())
|
||||
|
||||
maybe_fused.assert_called_once_with(act_fn, projected, down_proj)
|
||||
act_fn.assert_not_called()
|
||||
assert result is fused
|
||||
|
||||
@@ -10,6 +10,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
|
||||
|
||||
@@ -293,6 +294,25 @@ def test_qwen3_omni_text_model_collects_post_deepstack_aux_hidden_states():
|
||||
torch.testing.assert_close(aux_hidden_states[0], torch.tensor([[15.0]]))
|
||||
|
||||
|
||||
def _dspark_vocab_stub(input_vocab_size: int, draft_vocab_size: int):
|
||||
"""A Qwen3 DSpark model with only what `load_weights` reads."""
|
||||
from vllm.model_executor.models.qwen3_dspark import Qwen3DSparkForCausalLM
|
||||
|
||||
model = Qwen3DSparkForCausalLM.__new__(Qwen3DSparkForCausalLM)
|
||||
nn.Module.__init__(model)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"config",
|
||||
SimpleNamespace(
|
||||
vocab_size=input_vocab_size,
|
||||
draft_vocab_size=draft_vocab_size,
|
||||
),
|
||||
)
|
||||
object.__setattr__(model, "target_vocab_size", 100)
|
||||
object.__setattr__(model, "model", Mock(confidence_head=None))
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
("input_vocab_size", "draft_vocab_size", "weights", "error"),
|
||||
@@ -311,24 +331,25 @@ def test_qwen3_omni_text_model_collects_post_deepstack_aux_hidden_states():
|
||||
def test_qwen3_dspark_rejects_incomplete_vocab_weights(
|
||||
input_vocab_size, draft_vocab_size, weights, error
|
||||
):
|
||||
from vllm.model_executor.models.qwen3_dspark import Qwen3DSparkForCausalLM
|
||||
|
||||
model = Qwen3DSparkForCausalLM.__new__(Qwen3DSparkForCausalLM)
|
||||
nn.Module.__init__(model)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"config",
|
||||
SimpleNamespace(
|
||||
vocab_size=input_vocab_size,
|
||||
draft_vocab_size=draft_vocab_size,
|
||||
),
|
||||
)
|
||||
object.__setattr__(model, "target_vocab_size", 100)
|
||||
model = _dspark_vocab_stub(input_vocab_size, draft_vocab_size)
|
||||
|
||||
with pytest.raises(ValueError, match=error):
|
||||
model.load_weights(weights)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("draft_vocab_size", [100, 101])
|
||||
def test_qwen3_dspark_accepts_draft_vocab_at_least_target(draft_vocab_size):
|
||||
"""A draft vocab >= the target's maps identically, so needs no lm_head/d2t.
|
||||
|
||||
Guards the padded case (draft > target): the draft's unembedding is padded
|
||||
to its physical size, which is not a reduced vocabulary.
|
||||
"""
|
||||
model = _dspark_vocab_stub(100, draft_vocab_size)
|
||||
|
||||
model.load_weights([])
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark import utils as dspark_utils
|
||||
@@ -351,7 +372,7 @@ def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
|
||||
attention_backend=None,
|
||||
kv_cache_dtype=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
||||
parallel_config=ParallelConfig(),
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
cache_config=SimpleNamespace(),
|
||||
load_config=SimpleNamespace(),
|
||||
|
||||
Reference in New Issue
Block a user