From 0a309b5ee9480d0d78e7f00946a418ffd7e3baf1 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 10 May 2026 05:03:00 -0500 Subject: [PATCH] [ROCm] Cap Triton paged attention block size to fix ROCm shared memory OOM (#38502) Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 15 ++++---- vllm/v1/attention/backends/rocm_attn.py | 12 ++++--- .../ops/chunked_prefill_paged_decode.py | 35 ++++++++++++++----- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 1f52149e4d..c5369950fb 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1803,9 +1803,10 @@ steps: - tests/models/multimodal/generation - tests/models/multimodal/test_mapping.py commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py - - pytest -v -s models/multimodal/test_mapping.py + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@rocm-7.0-v2.3.0' + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' + - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - label: Multi-Modal Models (Extended Generation 2) # TBD timeout_in_minutes: 180 @@ -1817,8 +1818,10 @@ steps: - vllm/ - tests/models/multimodal/generation commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@rocm-7.0-v2.3.0' + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' + - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' + - label: Multi-Modal Models (Extended Generation 3) # TBD timeout_in_minutes: 180 @@ -3043,7 +3046,7 @@ steps: - vllm/ - tests/models/language/generation commands: - - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@rocm-7.0-v2.3.0' - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' diff --git a/vllm/v1/attention/backends/rocm_attn.py b/vllm/v1/attention/backends/rocm_attn.py index a238ff4ad5..d533268e21 100644 --- a/vllm/v1/attention/backends/rocm_attn.py +++ b/vllm/v1/attention/backends/rocm_attn.py @@ -29,6 +29,7 @@ from vllm.v1.attention.backend import ( ) from vllm.v1.attention.ops.chunked_prefill_paged_decode import ( chunked_prefill_paged_decode, + has_native_kv_cache_layout, ) from vllm.v1.attention.ops.paged_attn import PagedAttention from vllm.v1.attention.ops.triton_reshape_and_cache_flash import ( @@ -468,9 +469,10 @@ class RocmAttentionImpl(AttentionImpl): # Get the actual block_size from value_cache # value_cache shape: [num_blocks, num_heads, head_size, block_size] block_size = value_cache.shape[3] + has_native_layout = has_native_kv_cache_layout(key_cache, value_cache) - if block_size in (16, 32): - # Normal 16, 32, use vLLM native HIP C++ logic + if block_size in (16, 32) and has_native_layout: + # Normal 16, 32 with contiguous blocks: use vLLM native HIP C++ logic. PagedAttention.write_to_paged_cache( key, value, @@ -482,8 +484,10 @@ class RocmAttentionImpl(AttentionImpl): layer._v_scale, ) else: - # Case B: Non-standard blocks (e.g., 64, 128, 544 in Qwen3Next or Qwen3.5 ), - # force using our modified Triton logic + # Non-standard blocks and hybrid attention/Mamba layouts need the + # stride-aware Triton writer. The native reshape_and_cache kernel + # assumes contiguous block storage and writes to the wrong hybrid + # cache blocks. triton_reshape_and_cache_flash( key, value, diff --git a/vllm/v1/attention/ops/chunked_prefill_paged_decode.py b/vllm/v1/attention/ops/chunked_prefill_paged_decode.py index ea1f075ef6..77eb3ac60b 100644 --- a/vllm/v1/attention/ops/chunked_prefill_paged_decode.py +++ b/vllm/v1/attention/ops/chunked_prefill_paged_decode.py @@ -21,6 +21,22 @@ logger = init_logger(__name__) float8_info = torch.finfo(current_platform.fp8_dtype()) +def has_native_kv_cache_layout( + key_cache: torch.Tensor, + value_cache: torch.Tensor, +) -> bool: + """Return whether KV cache blocks can use the native ROCm pairing. + + The native reshape_and_cache writer assumes packed blocks. If cache update + needs reshape_and_cache_flash for a stride-padded hybrid layout, decode + should use the matching Triton path too. + """ + return ( + key_cache.stride(0) == key_cache.shape[1:].numel() + and value_cache.stride(0) == value_cache.shape[1:].numel() + ) + + @triton.jit def cdiv_fn(x, y): return (x + y - 1) // y @@ -346,14 +362,12 @@ def chunked_prefill_paged_decode( alibi_slopes, sinks, ) - # Triton is only forced when encountering a non-standard block - # like Qwen3 with a size of 544. - # 1. Check if block_size is a power of 2 (16, 32, 64...) - # 2. If it's a power of 2, we trust the vLLM's native use_custom decision. - # 3. If it's not a power of 2 (such as Qwen3's 544), - # then our Triton path is forced. + has_native_layout = has_native_kv_cache_layout(key_cache, value_cache) + # Force Triton for non-standard blocks like Qwen3's 544 and for + # stride-padded hybrid layouts. The latter use reshape_and_cache_flash + # during cache update, so keep decode on the matching stride-aware path. is_pow2 = block_size > 0 and (block_size & (block_size - 1) == 0) - if not is_pow2: + if not is_pow2 or not has_native_layout: use_custom = False if use_custom: @@ -404,7 +418,12 @@ def chunked_prefill_paged_decode( real_block_size = value_cache.shape[3] # The standard model directly uses the original block_size. # Non-standard 544 uses 32 to accommodate integer division logic. - TRITON_BLOCK_SIZE = block_size if is_pow2 else 32 + # Cap at 128 to avoid exceeding GPU shared memory limits + # (e.g. hybrid Mamba models inflate block_size to 2048). + # The kernel handles TRITON_BLOCK_SIZE != PHYSICAL_BLOCK_SIZE + # via the l_block_idx/internal_offsets addressing logic. + MAX_TRITON_BLOCK_SIZE = 128 + TRITON_BLOCK_SIZE = min(block_size, MAX_TRITON_BLOCK_SIZE) if is_pow2 else 32 if is_block_table_ptr: # Using the physical base address of tensors kv_element_size = key_cache.element_size()