[ROCm][MLA] Keep DCP prefill context in FP8 through AllGather (#59070)
Signed-off-by: whx-sjtu <xiaowang990929@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Code <noreply@anthropic.com> Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Code
Fangzhou Ai
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Simon Danielsson
parent
98f7211162
commit
c6566bb9f6
@@ -33,6 +33,26 @@ def _require_aiter():
|
||||
pytest.skip("aiter is required on supported ROCm hardware for this test")
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_gather_kv_b_proj_op_schema() -> None:
|
||||
"""Validate fake tensors and mutation of both expanded K/V outputs."""
|
||||
_require_aiter()
|
||||
from vllm._aiter_ops import rocm_aiter_ops # noqa: F401
|
||||
|
||||
rows, num_heads = 32, 12
|
||||
cache = torch.randn(rows, 1, 576, device="cuda").to(current_platform.fp8_dtype())
|
||||
scale = torch.tensor([0.07], device="cuda")
|
||||
cu_seq_lens = torch.tensor([0, 13, rows], dtype=torch.int32, device="cuda")
|
||||
indices = torch.arange(rows, dtype=torch.int32, device="cuda").flip(0)
|
||||
weight = torch.randn(num_heads * 256, 512, dtype=torch.bfloat16, device="cuda")
|
||||
k = torch.empty(rows, num_heads, 192, dtype=torch.bfloat16, device="cuda")
|
||||
v = torch.empty(rows, num_heads, 128, dtype=torch.bfloat16, device="cuda")
|
||||
opcheck(
|
||||
torch.ops.vllm.rocm_aiter_gather_kv_b_proj,
|
||||
(cache, scale, cu_seq_lens, indices, cu_seq_lens, weight, k, v),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_fwd_op_schema() -> None:
|
||||
"""Opcheck validates registration, schema, fake-tensor, and ``o`` aliasing.
|
||||
|
||||
Reference in New Issue
Block a user