[ROCm][Perf] Enable layer-aware CSA2 multi-stream overlap for DeepSeek-V4.1-Flash (#57407)

Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
Shanshan Shen
2026-09-27 23:48:01 +08:00
committed by GitHub
parent 924707f1bf
commit 44af287ebe
4 changed files with 496 additions and 20 deletions
+266
View File
@@ -2075,3 +2075,269 @@ def test_fused_kv_insert_split(num_tokens: int, kv_block_size: int):
# RoPE (last 64): stored as bf16. The kernel recomputes the rotation, so it
# is bf16-close to the reference rather than bit-exact (cf. test_cutedsl).
torch.testing.assert_close(recovered[:, NOPE_DIM:], ref[:, NOPE_DIM:])
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage")
def test_v41_rocm_csa2_full_pipeline():
"""_forward_csa2_full: one fork, then K write and q-side after the join.
The default chain must run first on the current stream, the compressor
chain on aux 0 and the indexer weights projection on aux 1; the join must
publish both cache writes before _produce_k / forward_q run.
"""
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
torch.manual_seed(43)
num_tokens = 19
hidden_states = torch.randn(num_tokens, 1024, device="cuda")
positions = torch.arange(7, 26, device="cuda")
qr_kv = torch.zeros(num_tokens, 1536, device="cuda")
qr = torch.zeros(num_tokens, 4, device="cuda")
kv = torch.zeros(num_tokens, 512, device="cuda")
q = torch.zeros(num_tokens, 512, device="cuda")
latent = torch.ones(num_tokens, 512, dtype=torch.bfloat16, device="cuda")
weights = torch.randn(num_tokens, 32, device="cuda")
index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda")
index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda")
index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda")
main_cache = torch.full((1, 128, 584), 165, dtype=torch.uint8, device="cuda")
index_cache = torch.full((1, 128, 132), 165, dtype=torch.uint8, device="cuda")
order: list[str] = []
streams: dict[str, torch.cuda.Stream] = {}
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
torch.nn.Module.__init__(compressor)
compressor.fused_wkv_wgate = SimpleNamespace(
weight=torch.randn(1024, 1024, device="cuda")
)
def compressor_forward(kv_score, positions_):
order.append("compressor")
streams["compressor"] = torch.cuda.current_stream()
return latent
def compressor_insert(latent_, positions_, rotary_emb):
order.append("insert_cache")
streams["insert_cache"] = torch.cuda.current_stream()
main_cache.fill_(5)
compressor.forward = compressor_forward
compressor.insert_cache = compressor_insert
def weights_proj(hidden_states_):
order.append("weights_proj")
streams["weights_proj"] = torch.cuda.current_stream()
return weights, None
def produce_k(latent_, positions_, rotary_emb):
order.append("produce_k")
streams["produce_k"] = torch.cuda.current_stream()
index_cache.fill_(7)
def forward_q(qr_, qr_scale, indexer_weights, positions_, rotary_emb):
order.append("forward_q")
streams["forward_q"] = torch.cuda.current_stream()
return index_q, index_q_scale, index_weights_out
indexer = SimpleNamespace(
weights_proj=weights_proj,
_produce_k=produce_k,
forward_q=forward_q,
)
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda"))
aux_streams = [torch.cuda.Stream(), torch.cuda.Stream()]
def wq_b(qr_, qr_scale):
order.append("wq_b")
streams["wq_b"] = torch.cuda.current_stream()
return q
def qnorm_insert(q_, kv_, positions_, attn_metadata):
order.append("qnorm_insert")
streams["qnorm_insert"] = torch.cuda.current_stream()
return q_
attention = SimpleNamespace(
compressor=compressor,
indexer=indexer,
aux_stream_list=aux_streams,
ln_events=[torch.cuda.Event() for _ in range(4)],
rotary_emb=rotary,
indexer_rotary_emb=rotary,
n_local_heads=1,
head_dim=512,
_fused_wqa_wkv_gemm=lambda hidden_states_: qr_kv,
_split_qkv_and_norm=lambda qr_kv_: (qr, None, kv),
_wq_b_proj=wq_b,
_fused_qnorm_rope_kv_insert=qnorm_insert,
)
context = ForwardContext({}, {}, {})
with override_forward_context(context):
result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_full(
attention, hidden_states, positions
)
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
assert torch.equal(q_out, q)
assert torch.equal(kv_out, kv)
assert torch.equal(index_q_out, index_q)
assert torch.equal(index_q_scale_out, index_q_scale)
assert torch.equal(index_weights_out_out, index_weights_out)
# Join publishes the aux-0 cache insert and orders the post-join K write.
assert torch.equal(main_cache, torch.full_like(main_cache, 5))
assert torch.equal(index_cache, torch.full_like(index_cache, 7))
assert order == [
"wq_b",
"qnorm_insert",
"compressor",
"insert_cache",
"weights_proj",
"produce_k",
"forward_q",
]
current = torch.cuda.current_stream()
for name in ("wq_b", "qnorm_insert", "produce_k", "forward_q"):
assert streams[name] == current, name
for name in ("compressor", "insert_cache"):
assert streams[name] == aux_streams[0], name
assert streams["weights_proj"] == aux_streams[1]
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage")
def test_v41_rocm_csa2_reindex_pipeline():
"""_forward_csa2_reindex: serial projections, then SWA q path vs q-side."""
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
torch.manual_seed(45)
num_tokens = 19
hidden_states = torch.randn(num_tokens, 1024, device="cuda")
positions = torch.arange(7, 26, device="cuda")
qr_kv = torch.zeros(num_tokens, 1536, device="cuda")
qr = torch.zeros(num_tokens, 4, device="cuda")
kv = torch.zeros(num_tokens, 512, device="cuda")
q = torch.zeros(num_tokens, 512, device="cuda")
indexer_weights = torch.randn(num_tokens, 32, device="cuda")
index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda")
index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda")
index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda")
order: list[str] = []
streams: dict[str, torch.cuda.Stream] = {}
def forward_q(qr_, qr_scale, indexer_weights_, positions_, rotary_emb):
order.append("forward_q")
streams["forward_q"] = torch.cuda.current_stream()
return index_q, index_q_scale, index_weights_out
indexer = SimpleNamespace(forward_q=forward_q)
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda"))
aux0 = torch.cuda.Stream()
def wq_b(qr_, qr_scale):
order.append("wq_b")
streams["wq_b"] = torch.cuda.current_stream()
return q
def qnorm_insert(q_, kv_, positions_, attn_metadata):
order.append("qnorm_insert")
streams["qnorm_insert"] = torch.cuda.current_stream()
return q_
attention = SimpleNamespace(
indexer=indexer,
aux_stream_list=[aux0],
ln_events=[torch.cuda.Event() for _ in range(2)],
indexer_rotary_emb=rotary,
n_local_heads=1,
head_dim=512,
_run_parallel_input_projections=lambda hidden_states_: (
qr_kv,
None,
indexer_weights,
),
_split_qkv_and_norm=lambda qr_kv_: (qr, None, kv),
_wq_b_proj=wq_b,
_fused_qnorm_rope_kv_insert=qnorm_insert,
)
context = ForwardContext({}, {}, {})
with override_forward_context(context):
result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_reindex(
attention, hidden_states, positions
)
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
assert torch.equal(q_out, q)
assert torch.equal(kv_out, kv)
assert torch.equal(index_q_out, index_q)
assert torch.equal(index_q_scale_out, index_q_scale)
assert torch.equal(index_weights_out_out, index_weights_out)
assert order == ["wq_b", "qnorm_insert", "forward_q"]
current = torch.cuda.current_stream()
for name in ("wq_b", "qnorm_insert"):
assert streams[name] == current, name
assert streams["forward_q"] == aux0
@pytest.mark.parametrize("quant_tuple", [False, True])
def test_v41_indexer_forward_q(quant_tuple):
"""forward_q runs wq_b then the fused RoPE/quant and unpacks both shapes."""
from unittest.mock import patch
from vllm.models.deepseek_v41.attention import DeepseekV4Indexer
num_tokens, n_head, head_dim = 5, 32, 128
qr = torch.randn(num_tokens, 1280)
q = torch.randn(num_tokens, n_head * head_dim)
positions = torch.arange(num_tokens)
indexer_weights = torch.randn(num_tokens, n_head)
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64))
fake = SimpleNamespace(
n_head=n_head,
head_dim=head_dim,
softmax_scale=head_dim**-0.5,
use_fp4_kv=False,
indexer_weights_dtype=torch.float32,
_wq_b_proj=lambda qr_, qr_scale: q,
)
q_quant = torch.randn(num_tokens, n_head, head_dim)
q_scale = torch.randn(num_tokens, n_head)
weights_out = torch.randn(num_tokens, n_head)
if quant_tuple:
fused_return = ((q_quant, q_scale), weights_out)
expected = (q_quant, q_scale, weights_out)
else:
fused_return = (q_quant, weights_out)
expected = (q_quant, None, weights_out)
with patch(
"vllm.models.deepseek_v41.attention.fused_indexer_q_rope_quant",
return_value=fused_return,
) as mock_fused:
result = DeepseekV4Indexer.forward_q(
fake, qr, None, indexer_weights, positions, rotary
)
assert torch.equal(result[0], expected[0])
assert (result[1] is None) == (expected[1] is None)
if expected[1] is not None:
assert torch.equal(result[1], expected[1])
assert torch.equal(result[2], expected[2])
args = mock_fused.call_args.args
assert args[0] is positions
torch.testing.assert_close(args[1], q.view(-1, n_head, head_dim))
assert args[2] is rotary.cos_sin_cache
assert args[3] is indexer_weights
assert args[4] == head_dim**-0.5
assert args[5] == n_head**-0.5
assert mock_fused.call_args.kwargs == {
"use_fp4": False,
"weights_out_dtype": torch.float32,
}