[nv] add pcp support in dsv3.2 (#52046)
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -292,6 +292,81 @@ def test_fused_norm_rope_no_indexer(num_tokens: int):
|
||||
assert (topk == 7).all(), "topk buffer should be untouched on shared layer"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_indexer", [False, True])
|
||||
def test_fused_norm_rope_materializes_pcp_cache_inputs(has_indexer: bool):
|
||||
"""PCP gets local normalized/rotated K rows without direct cache writes."""
|
||||
torch.manual_seed(6)
|
||||
dev = "cuda"
|
||||
num_tokens = 17
|
||||
max_pos = 8192
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
|
||||
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
|
||||
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
ik = (
|
||||
torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
|
||||
if has_indexer
|
||||
else None
|
||||
)
|
||||
ikw = (
|
||||
torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
if has_indexer
|
||||
else None
|
||||
)
|
||||
ikb = (
|
||||
torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
if has_indexer
|
||||
else None
|
||||
)
|
||||
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev)
|
||||
idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) if has_indexer else None
|
||||
q_out = torch.empty_like(q_c)
|
||||
kv_out = torch.empty_like(kv_c)
|
||||
kpe_out = torch.empty_like(k_pe)
|
||||
ik_out = torch.empty_like(ik) if ik is not None else None
|
||||
topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32)
|
||||
|
||||
actual_q = K.fused_norm_rope(
|
||||
pos,
|
||||
q_c,
|
||||
qw,
|
||||
EPS,
|
||||
kv_c,
|
||||
kvw,
|
||||
EPS,
|
||||
k_pe,
|
||||
mla_cos_sin,
|
||||
ik,
|
||||
ikw,
|
||||
ikb,
|
||||
EPS,
|
||||
idx_cos_sin,
|
||||
topk,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=True,
|
||||
q_c_out=q_out,
|
||||
kv_c_out=kv_out,
|
||||
k_pe_out=kpe_out,
|
||||
index_k_out=ik_out,
|
||||
)
|
||||
|
||||
assert actual_q.data_ptr() == q_out.data_ptr()
|
||||
assert_bf16(actual_q, rms_norm(q_c, qw), "PCP q norm")
|
||||
assert_bf16(kv_out, rms_norm(kv_c, kvw), "PCP kv norm")
|
||||
assert_bf16(
|
||||
kpe_out,
|
||||
rope(k_pe.float(), pos, mla_cos_sin, interleave=True),
|
||||
"PCP k_pe RoPE",
|
||||
)
|
||||
if has_indexer:
|
||||
assert ik is not None and ikw is not None and ikb is not None
|
||||
assert ik_out is not None and idx_cos_sin is not None
|
||||
ik_ref = rope(layer_norm(ik, ikw, ikb), pos, idx_cos_sin, interleave=True)
|
||||
assert_bf16(ik_out, ik_ref, "PCP indexer-K")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
|
||||
def test_fused_norm_rope_ds_mla(num_tokens: int):
|
||||
"""fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100).
|
||||
@@ -405,19 +480,29 @@ def test_fused_norm_rope_supports_large_token_count():
|
||||
# ── fused_q ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096])
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "num_q_heads"),
|
||||
[
|
||||
(1, NUM_HEADS),
|
||||
(4, NUM_HEADS),
|
||||
(17, NUM_HEADS),
|
||||
(512, NUM_HEADS),
|
||||
(4096, NUM_HEADS),
|
||||
(17, 64),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("index_interleave", [True, False])
|
||||
def test_fused_q(num_tokens: int, index_interleave: bool):
|
||||
def test_fused_q(num_tokens: int, num_q_heads: int, index_interleave: bool):
|
||||
torch.manual_seed(2)
|
||||
dev = "cuda"
|
||||
max_pos = 8192
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
|
||||
|
||||
q_pe = torch.randn(
|
||||
num_tokens, NUM_HEADS, ROPE_DIM, device=dev, dtype=torch.bfloat16
|
||||
num_tokens, num_q_heads, ROPE_DIM, device=dev, dtype=torch.bfloat16
|
||||
)
|
||||
ql_nope = torch.randn(
|
||||
num_tokens, NUM_HEADS, KV_LORA, device=dev, dtype=torch.bfloat16
|
||||
num_tokens, num_q_heads, KV_LORA, device=dev, dtype=torch.bfloat16
|
||||
)
|
||||
index_q = torch.randn(
|
||||
num_tokens, INDEX_HEADS, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16
|
||||
@@ -449,7 +534,7 @@ def test_fused_q(num_tokens: int, index_interleave: bool):
|
||||
mqa_nope_ref = (ql_nope.float() / s).to(FP8)
|
||||
qpe_ref = rope(
|
||||
q_pe.float(),
|
||||
pos.unsqueeze(-1).expand(num_tokens, NUM_HEADS),
|
||||
pos.unsqueeze(-1).expand(num_tokens, num_q_heads),
|
||||
q_cos_sin,
|
||||
interleave=True,
|
||||
)
|
||||
@@ -484,7 +569,6 @@ def test_fused_q_no_indexer(num_tokens: int):
|
||||
)
|
||||
q_scale = torch.tensor([0.5], device=dev, dtype=torch.float32)
|
||||
q_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev)
|
||||
|
||||
_, _, mqa = K.fused_q(
|
||||
pos,
|
||||
q_pe,
|
||||
|
||||
@@ -177,3 +177,48 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
# K cache is always updated before the scoring decision.
|
||||
torch.testing.assert_close(observed["k"], k[: slot_mapping.numel()])
|
||||
assert observed["slots"] is slot_mapping
|
||||
|
||||
|
||||
def test_skipped_k_cache_insert_accepts_no_k(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
indexer_metadata = make_indexer_metadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
slot_mapping=torch.empty(0, dtype=torch.long),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_indexer,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(
|
||||
attn_metadata={INDEXER_LAYER: indexer_metadata},
|
||||
cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_indexer.current_platform, "fp8_dtype", lambda: torch.float16
|
||||
)
|
||||
|
||||
topk_indices = torch.full((1, 2048), 17, dtype=torch.int32)
|
||||
result = sparse_indexer.sparse_attn_indexer(
|
||||
torch.empty(1, 1),
|
||||
INDEXER_LAYER,
|
||||
torch.empty(1),
|
||||
torch.empty(1, 1),
|
||||
None,
|
||||
None,
|
||||
torch.empty(1, 1),
|
||||
128,
|
||||
"ue8m0",
|
||||
2048,
|
||||
4,
|
||||
4096,
|
||||
4096,
|
||||
topk_indices,
|
||||
True,
|
||||
False,
|
||||
"",
|
||||
)
|
||||
|
||||
assert result is topk_indices
|
||||
assert torch.all(topk_indices == -1)
|
||||
|
||||
Reference in New Issue
Block a user