[ROCm][Bugfix] Pass per-sequence context lengths to the AITER paged MQA-logits kernels (#59420)
Signed-off-by: Sriram Kumar <sriramkumar.kishorekumar@amd.com>
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm() or not is_aiter_found_and_supported():
|
||||
pytest.skip("Requires ROCm with AITER.", allow_module_level=True)
|
||||
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import _per_sequence_context_lens
|
||||
|
||||
|
||||
def test_per_row_table_uses_last_column():
|
||||
seq_lens = torch.tensor([8192, 1500, 3000, 700], dtype=torch.int32)
|
||||
next_n = 3
|
||||
offsets = torch.arange(next_n, dtype=torch.int32) - next_n + 1
|
||||
per_row = seq_lens.unsqueeze(1) + offsets
|
||||
|
||||
out = _per_sequence_context_lens(per_row)
|
||||
|
||||
assert out.shape == (4,)
|
||||
assert out.is_contiguous()
|
||||
assert torch.equal(out, seq_lens)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(4,), (4, 1)])
|
||||
def test_per_sequence_input_is_unchanged(shape):
|
||||
context_lens = torch.arange(1, 5, dtype=torch.int32).reshape(shape)
|
||||
assert _per_sequence_context_lens(context_lens) is context_lens
|
||||
@@ -707,6 +707,14 @@ def paged_mqa_logits_module():
|
||||
return None
|
||||
|
||||
|
||||
def _per_sequence_context_lens(context_lens: torch.Tensor) -> torch.Tensor:
|
||||
# AITER takes one length per sequence: the last column of a (B, next_n) table.
|
||||
# TODO: remove once AITER accepts (B, next_n) context_lens (ROCm/aiter#6153).
|
||||
if context_lens.dim() == 2 and context_lens.shape[1] > 1:
|
||||
return context_lens[:, -1].contiguous()
|
||||
return context_lens
|
||||
|
||||
|
||||
def rocm_fp8_paged_mqa_logits(
|
||||
q_fp8: torch.Tensor,
|
||||
kv_cache_fp8: torch.Tensor,
|
||||
@@ -726,8 +734,8 @@ def rocm_fp8_paged_mqa_logits(
|
||||
kv_cache_fp8: Paged KV-cache in packed FP8+scale layout with shape
|
||||
[num_blocks, block_size, 1, D+4], dtype `torch.uint8`.
|
||||
weights: Tensor of shape [B * next_n, H], dtype `torch.float32`.
|
||||
context_lens: Tensor of shape [B], dtype int32; effective context length
|
||||
for each batch element.
|
||||
context_lens: Tensor of shape [B] or [B, next_n], dtype int32; effective
|
||||
context length for each batch element, or for each of its Q rows.
|
||||
block_tables: Tensor of shape [B, max_blocks], dtype int32; maps logical
|
||||
block indices to physical blocks in the paged cache.
|
||||
schedule_metadata: Returned by `get_paged_mqa_logits_metadata`;
|
||||
@@ -779,7 +787,7 @@ def rocm_fp8_paged_mqa_logits(
|
||||
kv_cache_fp8,
|
||||
weights,
|
||||
out_logits,
|
||||
context_lens,
|
||||
_per_sequence_context_lens(context_lens),
|
||||
block_tables,
|
||||
max_model_len,
|
||||
ChunkK=256,
|
||||
@@ -801,7 +809,7 @@ def rocm_fp8_paged_mqa_logits(
|
||||
kv_cache_fp8,
|
||||
weights,
|
||||
out_qk,
|
||||
context_lens,
|
||||
_per_sequence_context_lens(context_lens),
|
||||
block_tables,
|
||||
max_model_len,
|
||||
ChunkQ=heads,
|
||||
|
||||
Reference in New Issue
Block a user