[Perf][DSpark] Add KV-only context insertion across V4.1 cache formats (#56441)

Signed-off-by: 0z5a <0z5a@users.noreply.github.com>
Signed-off-by: Yongye Zhu <yongye@inferact.ai>
Co-authored-by: 0z5a <0z5a@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
0z5a
2026-09-15 21:34:55 -07:00
committed by GitHub
co-authored by 0z5a Codex Yongye Zhu Claude Opus 5
parent a31ec3a68b
commit 6ecd97f1b7
5 changed files with 234 additions and 61 deletions
+116
View File
@@ -42,6 +42,122 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize(
"cache_dtype,kv_mxfp8",
[
(torch.uint8, False),
(torch.uint8, True),
(torch.bfloat16, False),
(torch.float8_e4m3fn, False),
],
)
@pytest.mark.parametrize("num_tokens", [1, 17, 1023, 1024])
@pytest.mark.parametrize("use_graph", [False, True])
def test_dspark_context_kv_matches_query_insert(
cache_dtype, kv_mxfp8, num_tokens, use_graph
):
"""KV-only insertion must preserve every cache byte, including graph replay."""
from vllm.models.deepseek_v41.nvidia.dspark import _insert_context_kv
torch.manual_seed(42)
block_size = 256
num_blocks = math.ceil((num_tokens + 7) / block_size)
row_size = (528 if kv_mxfp8 else 584) if cache_dtype == torch.uint8 else 512
page_stride = math.ceil(block_size * row_size / 576) * 576 + 576
backing = torch.full((num_blocks, page_stride), 3, device="cuda").to(cache_dtype)
cache = backing.as_strided(
(num_blocks, block_size, row_size), (page_stride, row_size, 1)
)
reference_backing = backing.clone()
reference = reference_backing.as_strided(cache.shape, cache.stride())
kv = torch.randn(num_tokens + 3, 512, device="cuda", dtype=torch.bfloat16)
positions = torch.arange(num_tokens + 3, device="cuda") + 7
angles = torch.randn(num_tokens + 16, 32, device="cuda")
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
slots = torch.randperm(num_blocks * block_size, device="cuda")[:num_tokens]
slots[::5] = -1
scale = torch.tensor([0.7], device="cuda")
# Only what _insert_context_kv actually reads: the record width comes off
# the cache tensor, so no per-record width attribute is stubbed here.
attn = SimpleNamespace(
swa_cache_layer=SimpleNamespace(kv_cache=cache, block_size=block_size),
kv_mxfp8=kv_mxfp8,
rotary_emb=SimpleNamespace(cos_sin_cache=cos_sin),
_flashinfer_fp8_kv_scale=scale,
)
def legacy_insert():
q = torch.zeros(kv.shape[0], 8, 512, dtype=kv.dtype, device="cuda")
if cache_dtype == torch.uint8:
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q,
kv,
reference.view(num_blocks, -1),
slots,
positions,
cos_sin,
8,
1e-20,
block_size,
True,
kv_mxfp8,
)
elif cache_dtype == torch.bfloat16:
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
q,
kv,
reference,
slots,
positions,
cos_sin,
1e-20,
block_size,
)
else:
q_fp8 = torch.empty_like(q, dtype=cache_dtype)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
q,
kv,
q_fp8,
reference,
slots,
positions,
cos_sin,
scale,
scale,
1e-20,
block_size,
)
def insert():
_insert_context_kv(attn, kv, positions, slots)
insert()
graph = None
if use_graph:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
insert()
for iteration in range(3):
kv.normal_()
positions.add_(1)
if iteration == 2:
slots.fill_(-1)
backing.view(torch.uint8).fill_(165)
reference_backing.copy_(backing)
before = kv.clone()
legacy_insert()
insert() if graph is None else graph.replay()
torch.testing.assert_close(
backing.view(torch.uint8),
reference_backing.view(torch.uint8),
rtol=0,
atol=0,
)
torch.testing.assert_close(kv, before, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="graph capture coverage"
)