[ROCm][Perf] Enable layer-aware CSA2 multi-stream overlap for DeepSeek-V4.1-Flash (#57407)
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
@@ -2075,3 +2075,269 @@ def test_fused_kv_insert_split(num_tokens: int, kv_block_size: int):
|
||||
# RoPE (last 64): stored as bf16. The kernel recomputes the rotation, so it
|
||||
# is bf16-close to the reference rather than bit-exact (cf. test_cutedsl).
|
||||
torch.testing.assert_close(recovered[:, NOPE_DIM:], ref[:, NOPE_DIM:])
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage")
|
||||
def test_v41_rocm_csa2_full_pipeline():
|
||||
"""_forward_csa2_full: one fork, then K write and q-side after the join.
|
||||
|
||||
The default chain must run first on the current stream, the compressor
|
||||
chain on aux 0 and the indexer weights projection on aux 1; the join must
|
||||
publish both cache writes before _produce_k / forward_q run.
|
||||
"""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
|
||||
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
|
||||
|
||||
torch.manual_seed(43)
|
||||
num_tokens = 19
|
||||
hidden_states = torch.randn(num_tokens, 1024, device="cuda")
|
||||
positions = torch.arange(7, 26, device="cuda")
|
||||
qr_kv = torch.zeros(num_tokens, 1536, device="cuda")
|
||||
qr = torch.zeros(num_tokens, 4, device="cuda")
|
||||
kv = torch.zeros(num_tokens, 512, device="cuda")
|
||||
q = torch.zeros(num_tokens, 512, device="cuda")
|
||||
latent = torch.ones(num_tokens, 512, dtype=torch.bfloat16, device="cuda")
|
||||
weights = torch.randn(num_tokens, 32, device="cuda")
|
||||
index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda")
|
||||
index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda")
|
||||
index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda")
|
||||
main_cache = torch.full((1, 128, 584), 165, dtype=torch.uint8, device="cuda")
|
||||
index_cache = torch.full((1, 128, 132), 165, dtype=torch.uint8, device="cuda")
|
||||
|
||||
order: list[str] = []
|
||||
streams: dict[str, torch.cuda.Stream] = {}
|
||||
|
||||
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
|
||||
torch.nn.Module.__init__(compressor)
|
||||
compressor.fused_wkv_wgate = SimpleNamespace(
|
||||
weight=torch.randn(1024, 1024, device="cuda")
|
||||
)
|
||||
|
||||
def compressor_forward(kv_score, positions_):
|
||||
order.append("compressor")
|
||||
streams["compressor"] = torch.cuda.current_stream()
|
||||
return latent
|
||||
|
||||
def compressor_insert(latent_, positions_, rotary_emb):
|
||||
order.append("insert_cache")
|
||||
streams["insert_cache"] = torch.cuda.current_stream()
|
||||
main_cache.fill_(5)
|
||||
|
||||
compressor.forward = compressor_forward
|
||||
compressor.insert_cache = compressor_insert
|
||||
|
||||
def weights_proj(hidden_states_):
|
||||
order.append("weights_proj")
|
||||
streams["weights_proj"] = torch.cuda.current_stream()
|
||||
return weights, None
|
||||
|
||||
def produce_k(latent_, positions_, rotary_emb):
|
||||
order.append("produce_k")
|
||||
streams["produce_k"] = torch.cuda.current_stream()
|
||||
index_cache.fill_(7)
|
||||
|
||||
def forward_q(qr_, qr_scale, indexer_weights, positions_, rotary_emb):
|
||||
order.append("forward_q")
|
||||
streams["forward_q"] = torch.cuda.current_stream()
|
||||
return index_q, index_q_scale, index_weights_out
|
||||
|
||||
indexer = SimpleNamespace(
|
||||
weights_proj=weights_proj,
|
||||
_produce_k=produce_k,
|
||||
forward_q=forward_q,
|
||||
)
|
||||
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda"))
|
||||
aux_streams = [torch.cuda.Stream(), torch.cuda.Stream()]
|
||||
|
||||
def wq_b(qr_, qr_scale):
|
||||
order.append("wq_b")
|
||||
streams["wq_b"] = torch.cuda.current_stream()
|
||||
return q
|
||||
|
||||
def qnorm_insert(q_, kv_, positions_, attn_metadata):
|
||||
order.append("qnorm_insert")
|
||||
streams["qnorm_insert"] = torch.cuda.current_stream()
|
||||
return q_
|
||||
|
||||
attention = SimpleNamespace(
|
||||
compressor=compressor,
|
||||
indexer=indexer,
|
||||
aux_stream_list=aux_streams,
|
||||
ln_events=[torch.cuda.Event() for _ in range(4)],
|
||||
rotary_emb=rotary,
|
||||
indexer_rotary_emb=rotary,
|
||||
n_local_heads=1,
|
||||
head_dim=512,
|
||||
_fused_wqa_wkv_gemm=lambda hidden_states_: qr_kv,
|
||||
_split_qkv_and_norm=lambda qr_kv_: (qr, None, kv),
|
||||
_wq_b_proj=wq_b,
|
||||
_fused_qnorm_rope_kv_insert=qnorm_insert,
|
||||
)
|
||||
|
||||
context = ForwardContext({}, {}, {})
|
||||
with override_forward_context(context):
|
||||
result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_full(
|
||||
attention, hidden_states, positions
|
||||
)
|
||||
|
||||
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
|
||||
assert torch.equal(q_out, q)
|
||||
assert torch.equal(kv_out, kv)
|
||||
assert torch.equal(index_q_out, index_q)
|
||||
assert torch.equal(index_q_scale_out, index_q_scale)
|
||||
assert torch.equal(index_weights_out_out, index_weights_out)
|
||||
# Join publishes the aux-0 cache insert and orders the post-join K write.
|
||||
assert torch.equal(main_cache, torch.full_like(main_cache, 5))
|
||||
assert torch.equal(index_cache, torch.full_like(index_cache, 7))
|
||||
|
||||
assert order == [
|
||||
"wq_b",
|
||||
"qnorm_insert",
|
||||
"compressor",
|
||||
"insert_cache",
|
||||
"weights_proj",
|
||||
"produce_k",
|
||||
"forward_q",
|
||||
]
|
||||
current = torch.cuda.current_stream()
|
||||
for name in ("wq_b", "qnorm_insert", "produce_k", "forward_q"):
|
||||
assert streams[name] == current, name
|
||||
for name in ("compressor", "insert_cache"):
|
||||
assert streams[name] == aux_streams[0], name
|
||||
assert streams["weights_proj"] == aux_streams[1]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage")
|
||||
def test_v41_rocm_csa2_reindex_pipeline():
|
||||
"""_forward_csa2_reindex: serial projections, then SWA q path vs q-side."""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
|
||||
|
||||
torch.manual_seed(45)
|
||||
num_tokens = 19
|
||||
hidden_states = torch.randn(num_tokens, 1024, device="cuda")
|
||||
positions = torch.arange(7, 26, device="cuda")
|
||||
qr_kv = torch.zeros(num_tokens, 1536, device="cuda")
|
||||
qr = torch.zeros(num_tokens, 4, device="cuda")
|
||||
kv = torch.zeros(num_tokens, 512, device="cuda")
|
||||
q = torch.zeros(num_tokens, 512, device="cuda")
|
||||
indexer_weights = torch.randn(num_tokens, 32, device="cuda")
|
||||
index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda")
|
||||
index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda")
|
||||
index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda")
|
||||
|
||||
order: list[str] = []
|
||||
streams: dict[str, torch.cuda.Stream] = {}
|
||||
|
||||
def forward_q(qr_, qr_scale, indexer_weights_, positions_, rotary_emb):
|
||||
order.append("forward_q")
|
||||
streams["forward_q"] = torch.cuda.current_stream()
|
||||
return index_q, index_q_scale, index_weights_out
|
||||
|
||||
indexer = SimpleNamespace(forward_q=forward_q)
|
||||
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda"))
|
||||
aux0 = torch.cuda.Stream()
|
||||
|
||||
def wq_b(qr_, qr_scale):
|
||||
order.append("wq_b")
|
||||
streams["wq_b"] = torch.cuda.current_stream()
|
||||
return q
|
||||
|
||||
def qnorm_insert(q_, kv_, positions_, attn_metadata):
|
||||
order.append("qnorm_insert")
|
||||
streams["qnorm_insert"] = torch.cuda.current_stream()
|
||||
return q_
|
||||
|
||||
attention = SimpleNamespace(
|
||||
indexer=indexer,
|
||||
aux_stream_list=[aux0],
|
||||
ln_events=[torch.cuda.Event() for _ in range(2)],
|
||||
indexer_rotary_emb=rotary,
|
||||
n_local_heads=1,
|
||||
head_dim=512,
|
||||
_run_parallel_input_projections=lambda hidden_states_: (
|
||||
qr_kv,
|
||||
None,
|
||||
indexer_weights,
|
||||
),
|
||||
_split_qkv_and_norm=lambda qr_kv_: (qr, None, kv),
|
||||
_wq_b_proj=wq_b,
|
||||
_fused_qnorm_rope_kv_insert=qnorm_insert,
|
||||
)
|
||||
|
||||
context = ForwardContext({}, {}, {})
|
||||
with override_forward_context(context):
|
||||
result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_reindex(
|
||||
attention, hidden_states, positions
|
||||
)
|
||||
|
||||
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
|
||||
assert torch.equal(q_out, q)
|
||||
assert torch.equal(kv_out, kv)
|
||||
assert torch.equal(index_q_out, index_q)
|
||||
assert torch.equal(index_q_scale_out, index_q_scale)
|
||||
assert torch.equal(index_weights_out_out, index_weights_out)
|
||||
|
||||
assert order == ["wq_b", "qnorm_insert", "forward_q"]
|
||||
current = torch.cuda.current_stream()
|
||||
for name in ("wq_b", "qnorm_insert"):
|
||||
assert streams[name] == current, name
|
||||
assert streams["forward_q"] == aux0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quant_tuple", [False, True])
|
||||
def test_v41_indexer_forward_q(quant_tuple):
|
||||
"""forward_q runs wq_b then the fused RoPE/quant and unpacks both shapes."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Indexer
|
||||
|
||||
num_tokens, n_head, head_dim = 5, 32, 128
|
||||
qr = torch.randn(num_tokens, 1280)
|
||||
q = torch.randn(num_tokens, n_head * head_dim)
|
||||
positions = torch.arange(num_tokens)
|
||||
indexer_weights = torch.randn(num_tokens, n_head)
|
||||
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64))
|
||||
fake = SimpleNamespace(
|
||||
n_head=n_head,
|
||||
head_dim=head_dim,
|
||||
softmax_scale=head_dim**-0.5,
|
||||
use_fp4_kv=False,
|
||||
indexer_weights_dtype=torch.float32,
|
||||
_wq_b_proj=lambda qr_, qr_scale: q,
|
||||
)
|
||||
q_quant = torch.randn(num_tokens, n_head, head_dim)
|
||||
q_scale = torch.randn(num_tokens, n_head)
|
||||
weights_out = torch.randn(num_tokens, n_head)
|
||||
if quant_tuple:
|
||||
fused_return = ((q_quant, q_scale), weights_out)
|
||||
expected = (q_quant, q_scale, weights_out)
|
||||
else:
|
||||
fused_return = (q_quant, weights_out)
|
||||
expected = (q_quant, None, weights_out)
|
||||
|
||||
with patch(
|
||||
"vllm.models.deepseek_v41.attention.fused_indexer_q_rope_quant",
|
||||
return_value=fused_return,
|
||||
) as mock_fused:
|
||||
result = DeepseekV4Indexer.forward_q(
|
||||
fake, qr, None, indexer_weights, positions, rotary
|
||||
)
|
||||
|
||||
assert torch.equal(result[0], expected[0])
|
||||
assert (result[1] is None) == (expected[1] is None)
|
||||
if expected[1] is not None:
|
||||
assert torch.equal(result[1], expected[1])
|
||||
assert torch.equal(result[2], expected[2])
|
||||
args = mock_fused.call_args.args
|
||||
assert args[0] is positions
|
||||
torch.testing.assert_close(args[1], q.view(-1, n_head, head_dim))
|
||||
assert args[2] is rotary.cos_sin_cache
|
||||
assert args[3] is indexer_weights
|
||||
assert args[4] == head_dim**-0.5
|
||||
assert args[5] == n_head**-0.5
|
||||
assert mock_fused.call_args.kwargs == {
|
||||
"use_fp4": False,
|
||||
"weights_out_dtype": torch.float32,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user