diff --git a/tests/v1/attention/test_rocm_paged_mqa_context_lens.py b/tests/v1/attention/test_rocm_paged_mqa_context_lens.py new file mode 100644 index 0000000000..3122cc915a --- /dev/null +++ b/tests/v1/attention/test_rocm_paged_mqa_context_lens.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm._aiter_ops import is_aiter_found_and_supported +from vllm.platforms import current_platform + +if not current_platform.is_rocm() or not is_aiter_found_and_supported(): + pytest.skip("Requires ROCm with AITER.", allow_module_level=True) + +from vllm.v1.attention.ops.rocm_aiter_mla_sparse import _per_sequence_context_lens + + +def test_per_row_table_uses_last_column(): + seq_lens = torch.tensor([8192, 1500, 3000, 700], dtype=torch.int32) + next_n = 3 + offsets = torch.arange(next_n, dtype=torch.int32) - next_n + 1 + per_row = seq_lens.unsqueeze(1) + offsets + + out = _per_sequence_context_lens(per_row) + + assert out.shape == (4,) + assert out.is_contiguous() + assert torch.equal(out, seq_lens) + + +@pytest.mark.parametrize("shape", [(4,), (4, 1)]) +def test_per_sequence_input_is_unchanged(shape): + context_lens = torch.arange(1, 5, dtype=torch.int32).reshape(shape) + assert _per_sequence_context_lens(context_lens) is context_lens diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 90a73c101c..6a93d21511 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -707,6 +707,14 @@ def paged_mqa_logits_module(): return None +def _per_sequence_context_lens(context_lens: torch.Tensor) -> torch.Tensor: + # AITER takes one length per sequence: the last column of a (B, next_n) table. + # TODO: remove once AITER accepts (B, next_n) context_lens (ROCm/aiter#6153). + if context_lens.dim() == 2 and context_lens.shape[1] > 1: + return context_lens[:, -1].contiguous() + return context_lens + + def rocm_fp8_paged_mqa_logits( q_fp8: torch.Tensor, kv_cache_fp8: torch.Tensor, @@ -726,8 +734,8 @@ def rocm_fp8_paged_mqa_logits( kv_cache_fp8: Paged KV-cache in packed FP8+scale layout with shape [num_blocks, block_size, 1, D+4], dtype `torch.uint8`. weights: Tensor of shape [B * next_n, H], dtype `torch.float32`. - context_lens: Tensor of shape [B], dtype int32; effective context length - for each batch element. + context_lens: Tensor of shape [B] or [B, next_n], dtype int32; effective + context length for each batch element, or for each of its Q rows. block_tables: Tensor of shape [B, max_blocks], dtype int32; maps logical block indices to physical blocks in the paged cache. schedule_metadata: Returned by `get_paged_mqa_logits_metadata`; @@ -779,7 +787,7 @@ def rocm_fp8_paged_mqa_logits( kv_cache_fp8, weights, out_logits, - context_lens, + _per_sequence_context_lens(context_lens), block_tables, max_model_len, ChunkK=256, @@ -801,7 +809,7 @@ def rocm_fp8_paged_mqa_logits( kv_cache_fp8, weights, out_qk, - context_lens, + _per_sequence_context_lens(context_lens), block_tables, max_model_len, ChunkQ=heads,