From 548597367e66dcc90e24576c08e0e25f88c9ee6c Mon Sep 17 00:00:00 2001 From: Mikko Tukiainen Date: Wed, 7 Oct 2026 17:44:22 +0300 Subject: [PATCH] [ROCm][Perf] Add AITER FlyDSL GDN prefill backend (#57560) Signed-off-by: Mikko Tukiainen Co-authored-by: Cursor --- tests/v1/attention/test_gdn_aiter_flydsl.py | 199 ++++++++++++++++++ vllm/_aiter_ops.py | 113 ++++++++++ vllm/engine/arg_utils.py | 6 +- .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 99 ++++++++- vllm/v1/attention/backends/gdn_attn.py | 57 ++++- 5 files changed, 462 insertions(+), 12 deletions(-) create mode 100644 tests/v1/attention/test_gdn_aiter_flydsl.py diff --git a/tests/v1/attention/test_gdn_aiter_flydsl.py b/tests/v1/attention/test_gdn_aiter_flydsl.py new file mode 100644 index 0000000000..3de625e936 --- /dev/null +++ b/tests/v1/attention/test_gdn_aiter_flydsl.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from vllm._aiter_ops import rocm_aiter_ops +from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn +from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( + ChunkGatedDeltaRule, + _resolve_gdn_prefill_backend, +) +from vllm.platforms import current_platform + + +def _make_config( + *, + backend: str = "aiter_flydsl", + head_k_dim: int | None = 128, + head_v_dim: int | None = 128, + dtype: torch.dtype = torch.bfloat16, +): + return SimpleNamespace( + additional_config={"gdn_prefill_backend": backend}, + model_config=SimpleNamespace( + dtype=dtype, + hf_text_config=SimpleNamespace( + linear_key_head_dim=head_k_dim, + linear_value_head_dim=head_v_dim, + ), + ), + ) + + +def _resolve_on_rocm(config, *, kernels_available: bool = True): + with ( + patch.object( + qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=True + ), + patch.object( + qwen_gdn_linear_attn.rocm_aiter_ops, + "is_gdn_flydsl_prefill_available", + return_value=kernels_available, + ), + ): + return _resolve_gdn_prefill_backend(config) + + +@pytest.mark.parametrize( + "head_k_dim,head_v_dim,dtype,expected", + [ + (128, 128, torch.bfloat16, "aiter_flydsl"), + (64, 128, torch.bfloat16, "triton"), + (128, 64, torch.bfloat16, "triton"), + (128, 128, torch.float16, "triton"), + ], +) +def test_resolve_aiter_flydsl_gdn_prefill_backend( + head_k_dim: int, + head_v_dim: int, + dtype: torch.dtype, + expected: str, +): + config = _make_config(head_k_dim=head_k_dim, head_v_dim=head_v_dim, dtype=dtype) + + requested, active = _resolve_on_rocm(config) + + assert requested == "aiter_flydsl" + assert active == expected + + +def test_kda_style_model_cannot_select_flydsl(): + """Kimi K3 KDA reports head dims through linear_attn_config, not these. + + Its builder overrides _build_chunk_metadata, which the AITER path skips, + so this pins the reason the two never meet: the resolver turns the model + down before the builder is ever constructed. + """ + config = _make_config(head_k_dim=None, head_v_dim=None) + + assert _resolve_on_rocm(config) == ("aiter_flydsl", "triton") + + +@pytest.mark.parametrize("on_rocm", [True, False], ids=["kernels_missing", "not_rocm"]) +def test_explicit_aiter_flydsl_fails_closed(on_rocm: bool): + """An explicit request that cannot be honoured is a configuration error. + + Falling back silently would report the run as healthy while the requested + kernels were never used. + """ + config = _make_config() + if on_rocm: + with pytest.raises(RuntimeError, match="not importable"): + _resolve_on_rocm(config, kernels_available=False) + return + with ( + patch.object( + qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=False + ), + patch.object( + qwen_gdn_linear_attn.current_platform, "is_cuda", return_value=True + ), + pytest.raises(RuntimeError, match="aiter_flydsl"), + ): + _resolve_gdn_prefill_backend(config) + + +def _flydsl_prefill_unavailable() -> bool: + return ( + not current_platform.is_rocm() + or not rocm_aiter_ops.is_gdn_flydsl_prefill_available() + ) + + +@pytest.mark.skipif( + _flydsl_prefill_unavailable(), + reason="needs ROCm with the AITER FlyDSL GDN prefill kernels installed", +) +@pytest.mark.parametrize( + "state_dtype", [torch.float32, torch.bfloat16], ids=["fp32_state", "bf16_state"] +) +def test_aiter_flydsl_prefill_matches_triton_reference(state_dtype: torch.dtype): + """Run both prefill paths on device and compare. + + The reference is the same AITER entry point with the FlyDSL prepare and K5 + kernels switched off, so the only thing that differs between the two calls + is the kernels this backend exists to select. The state dtype follows + --mamba-ssm-cache-dtype, which may be float32 or bfloat16. + """ + from aiter.ops.triton.gated_delta_net import chunk_gated_delta_rule_opt_vk + + torch.manual_seed(0) + device = torch.device("cuda") + seq_lens = [70, 130] + total_tokens = sum(seq_lens) + num_k_heads, num_v_heads, head_dim = 2, 4, 128 + + # use_qk_l2norm_in_kernel=False matches the prefill path, where the model + # has already normalised q and k. Feeding raw normals instead lets the + # delta rule run away to NaN, which says nothing about either kernel. + q = torch.nn.functional.normalize( + torch.randn(1, total_tokens, num_k_heads, head_dim, device=device), dim=-1 + ).to(torch.bfloat16) + k = torch.nn.functional.normalize( + torch.randn(1, total_tokens, num_k_heads, head_dim, device=device), dim=-1 + ).to(torch.bfloat16) + v = torch.randn( + 1, total_tokens, num_v_heads, head_dim, dtype=torch.bfloat16, device=device + ) + # g is a log-domain decay, so keep it negative or the recurrence diverges + # and the comparison stops saying anything about the kernels. + g = -torch.rand(1, total_tokens, num_v_heads, dtype=torch.float32, device=device) + beta = torch.rand(1, total_tokens, num_v_heads, dtype=torch.float32, device=device) + cu_seqlens = torch.tensor([0, 70, 200], dtype=torch.int32, device=device) + initial_state = torch.randn( + len(seq_lens), + num_v_heads, + head_dim, + head_dim, + device=device, + ).to(state_dtype) + + shared = { + "q": q, + "k": k, + "v": v, + "g": g, + "beta": beta, + "output_final_state": True, + "cu_seqlens": cu_seqlens, + "use_qk_l2norm_in_kernel": False, + } + + reference_out, reference_state = chunk_gated_delta_rule_opt_vk( + initial_state=initial_state.clone(), + state_dtype=state_dtype, + use_chunk_flydsl=False, + use_prepare_flydsl=False, + **shared, + ) + flydsl_out, flydsl_state = ChunkGatedDeltaRule.forward_aiter_flydsl( + None, + initial_state=initial_state.clone(), + aiter_prefill_metadata=rocm_aiter_ops.build_gdn_flydsl_prefill_metadata( + seq_lens, cu_seqlens=cu_seqlens + ), + **shared, + ) + + assert flydsl_state.dtype == state_dtype + torch.testing.assert_close( + flydsl_out.float(), reference_out.float(), rtol=2e-2, atol=2e-2 + ) + torch.testing.assert_close( + flydsl_state.float(), reference_state.float(), rtol=2e-2, atol=2e-2 + ) diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 941f66e5b7..44b89aeaf1 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1997,6 +1997,8 @@ class rocm_aiter_ops: # Check if the env variable is set _AITER_ENABLED = envs.VLLM_ROCM_USE_AITER + # Chunk length the AITER FlyDSL K5 prefill kernels are compiled for. + GDN_FLYDSL_CHUNK_SIZE = 64 _CUSTOM_ALL_REDUCE_ENABLED = envs.VLLM_ROCM_USE_AITER_CUSTOM_AR _LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR _FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE @@ -2467,6 +2469,23 @@ class rocm_aiter_ops: except (ImportError, ModuleNotFoundError): return False + @staticmethod + def _gdn_flydsl_prefill_kernels_importable() -> bool: + try: + from aiter.ops.flydsl.linear_attention_prefill_kernels import ( # noqa: F401 + chunk_gated_delta_rule_fwd_h_flydsl_opt, + gdn_prepare_flydsl_supported, + gdn_prepare_fwd_flydsl, + ) + from aiter.ops.triton.gated_delta_net import ( # noqa: F401 + build_gated_delta_rule_prefill_metadata, + chunk_gated_delta_rule_opt_vk, + ) + + return True + except (ImportError, ModuleNotFoundError): + return False + @classmethod @if_aiter_supported def are_gdn_triton_kernels_available(cls) -> bool: @@ -2478,6 +2497,100 @@ class rocm_aiter_ops: """ return cls._AITER_ENABLED and cls._gdn_triton_kernels_importable() + @classmethod + @functools.cache + def is_gdn_flydsl_prefill_available(cls) -> bool: + """Whether the opt-in AITER FlyDSL GDN prefill path can be used. + + Selecting the backend is an explicit opt-in in itself, but it still + runs AITER kernels, so VLLM_ROCM_USE_AITER remains the one switch that + turns all of them off. + """ + return ( + cls._AITER_ENABLED + and is_aiter_found_and_supported() + and cls._gdn_flydsl_prefill_kernels_importable() + ) + + @classmethod + def build_gdn_flydsl_prefill_metadata( + cls, + seq_lens_cpu: list[int], + cu_seqlens: torch.Tensor, + ) -> object: + """Build the reusable varlen metadata FlyDSL GDN prefill runs against. + + The K5 recurrence is compiled for a fixed 64-token chunk, so the chunk + size is a property of the AITER kernel rather than of FLA, and is not + the caller's to choose. + """ + from aiter.ops.triton.gated_delta_net import ( + build_gated_delta_rule_prefill_metadata, + ) + + return build_gated_delta_rule_prefill_metadata( + seq_lens_cpu, + cu_seqlens=cu_seqlens, + chunk_size=cls.GDN_FLYDSL_CHUNK_SIZE, + ) + + @classmethod + def gdn_flydsl_prefill( + cls, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + initial_state: torch.Tensor, + output_final_state: bool, + cu_seqlens: torch.Tensor | None, + aiter_prefill_metadata: object | None, + use_qk_l2norm_in_kernel: bool, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Run GDN prefill through AITER's VK path with the FlyDSL kernels. + + Every argument AITER takes is named here rather than forwarded as + ``**kwargs``, so a rename or reordering on the AITER side fails at this + call instead of silently binding to the wrong parameter. + """ + from aiter.ops.flydsl.linear_attention_prefill_kernels import ( + gdn_prepare_flydsl_supported, + ) + from aiter.ops.triton.gated_delta_net import chunk_gated_delta_rule_opt_vk + + if cu_seqlens is not None and aiter_prefill_metadata is None: + raise RuntimeError( + "AITER FlyDSL GDN prefill requires reusable varlen prefill metadata." + ) + if not gdn_prepare_flydsl_supported(k, v): + raise RuntimeError( + "AITER FlyDSL GDN prepare does not support the runtime input shape, " + "dtype, or device." + ) + logger.info_once( + "Dispatching AITER FlyDSL GDN prefill (prepare=flydsl, chunk=flydsl)." + ) + return chunk_gated_delta_rule_opt_vk( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=initial_state, + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + use_chunk_flydsl=True, + use_prepare_flydsl=True, + state_dtype=initial_state.dtype, + prefill_metadata=aiter_prefill_metadata, + # The caller stages the state densely and writes it back, so the + # kernel gets a plain [N, H, V, K] buffer and never an index into + # the pool; there is correspondingly nothing to update in place. + inplace_final_state=False, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + ) + @classmethod def is_rdna_gdn_triton_kernels_available(cls) -> bool: """RDNA4 (gfx12) analog of are_gdn_triton_kernels_available().""" diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 527b8956d0..800d1b2f2c 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -839,7 +839,9 @@ class EngineArgs: ) fail_on_environ_validation: bool = False - gdn_prefill_backend: Literal["flashinfer", "triton", "cutedsl"] | None = None + gdn_prefill_backend: ( + Literal["flashinfer", "triton", "cutedsl", "aiter_flydsl"] | None + ) = None kda_prefill_backend: ( Literal["auto", "triton", "flashkda", "flashinfer", "fused"] | None ) = None @@ -1890,7 +1892,7 @@ class EngineArgs: parser.add_argument( "--gdn-prefill-backend", dest="gdn_prefill_backend", - choices=["flashinfer", "triton", "cutedsl"], + choices=["flashinfer", "triton", "cutedsl", "aiter_flydsl"], default=None, help="Select GDN prefill backend.", ) diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 7b164f1dd3..1e78838640 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -92,7 +92,7 @@ FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16) def _resolve_gdn_prefill_backend( vllm_config: VllmConfig, -) -> tuple[str, Literal["triton", "flashinfer", "cutedsl"]]: +) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]: """Resolve GDN prefill backend. FlashInfer's GDN prefill kernel is chosen when: @@ -106,6 +106,11 @@ def _resolve_gdn_prefill_backend( In-tree CuteDSL GDN prefill kernel is chosen when: * "cutedsl" is requested; (opt-in only) * Blackwell (SM10.x) with ``head_k_dim == 128``; + + AITER FlyDSL GDN prefill kernels are chosen when: + * "aiter_flydsl" is requested; (opt-in only) + * ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels; + * the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``. """ additional_config = vllm_config.additional_config backend_cfg = ( @@ -115,12 +120,50 @@ def _resolve_gdn_prefill_backend( ) backend = str(backend_cfg).strip().lower() - if not current_platform.is_cuda(): - return backend, "triton" - head_k_dim = getattr( vllm_config.model_config.hf_text_config, "linear_key_head_dim", None ) + head_v_dim = getattr( + vllm_config.model_config.hf_text_config, "linear_value_head_dim", None + ) + + if current_platform.is_rocm(): + if backend == "aiter_flydsl": + if not rocm_aiter_ops.is_gdn_flydsl_prefill_available(): + raise RuntimeError( + "GDN prefill backend 'aiter_flydsl' was requested but the " + "AITER FlyDSL prefill kernels are not importable. Check " + "that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or " + "newer GPU, and that the installed AITER exports them." + ) + if head_k_dim != 128 or head_v_dim != 128: + logger.warning_once( + "GDN prefill backend 'aiter_flydsl' was requested but " + "linear head dims are K=%s V=%s; FlyDSL requires 128/128. " + "Falling back to Triton/FLA.", + head_k_dim, + head_v_dim, + ) + return backend, "triton" + if vllm_config.model_config.dtype != torch.bfloat16: + logger.warning_once( + "GDN prefill backend 'aiter_flydsl' was requested but " + "model dtype is %s; FlyDSL requires bfloat16. " + "Falling back to Triton/FLA.", + vllm_config.model_config.dtype, + ) + return backend, "triton" + return backend, "aiter_flydsl" + return backend, "triton" + + if backend == "aiter_flydsl": + raise RuntimeError( + "GDN prefill backend 'aiter_flydsl' was requested but it is a ROCm " + f"AITER backend and the current platform is {current_platform.device_name}." + ) + + if not current_platform.is_cuda(): + return backend, "triton" supports_flashinfer = False supports_cutedsl = False @@ -172,6 +215,7 @@ def _log_gdn_backend_decision( chosen = { "flashinfer": "FlashInfer", "cutedsl": "CuteDSL", + "aiter_flydsl": "AITER FlyDSL", "triton": "Triton/FLA", }[active_backend] logger.info_once( @@ -259,6 +303,8 @@ class ChunkGatedDeltaRule(CustomOp): self._forward_method = self.forward_cuda elif active_backend == "cutedsl": self._forward_method = self.forward_cutedsl + elif active_backend == "aiter_flydsl": + self._forward_method = self.forward_aiter_flydsl else: self._forward_method = self.forward_native @@ -276,6 +322,7 @@ class ChunkGatedDeltaRule(CustomOp): chunk_offsets: torch.Tensor | None = None, use_qk_l2norm_in_kernel: bool = True, core_attn_out: torch.Tensor | None = None, + aiter_prefill_metadata: object | None = None, ): o, final_state = fi_chunk_gated_delta_rule( q=q, @@ -308,6 +355,7 @@ class ChunkGatedDeltaRule(CustomOp): chunk_offsets: torch.Tensor | None = None, use_qk_l2norm_in_kernel: bool = True, core_attn_out: torch.Tensor | None = None, + aiter_prefill_metadata: object | None = None, ): return fla_chunk_gated_delta_rule( q=q, @@ -338,6 +386,7 @@ class ChunkGatedDeltaRule(CustomOp): chunk_offsets: torch.Tensor | None = None, use_qk_l2norm_in_kernel: bool = True, core_attn_out: torch.Tensor | None = None, + aiter_prefill_metadata: object | None = None, ): from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import ( chunk_gated_delta_rule_cutedsl, @@ -367,6 +416,39 @@ class ChunkGatedDeltaRule(CustomOp): final_state = None return o, final_state + def forward_aiter_flydsl( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + initial_state: torch.Tensor, + output_final_state: bool, + cu_seqlens: torch.Tensor | None = None, + chunk_indices: torch.Tensor | None = None, + chunk_offsets: torch.Tensor | None = None, + use_qk_l2norm_in_kernel: bool = True, + core_attn_out: torch.Tensor | None = None, + aiter_prefill_metadata: object | None = None, + ): + o, final_state = rocm_aiter_ops.gdn_flydsl_prefill( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=initial_state, + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + aiter_prefill_metadata=aiter_prefill_metadata, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + ) + if core_attn_out is not None: + o_flat = o.reshape(-1) + core_attn_out.reshape(-1)[: o_flat.numel()].copy_(o_flat) + return o, final_state + @PluggableLayer.register("qwen_gated_delta_net_attention") class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): @@ -1132,12 +1214,18 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): # CuteDSL kernels require metadata chunk_indices = None chunk_offsets = None + aiter_prefill_metadata = None if self.gdn_prefill_backend == "cutedsl": from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import ( prepare_metadata_cutedsl, ) chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, T) + elif self.gdn_prefill_backend == "aiter_flydsl": + aiter_prefill_metadata = rocm_aiter_ops.build_gdn_flydsl_prefill_metadata( + [T], + cu_seqlens=cu_seqlens, + ) try: self.chunk_gated_delta_rule( @@ -1152,6 +1240,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): chunk_indices=chunk_indices, chunk_offsets=chunk_offsets, use_qk_l2norm_in_kernel=False, + aiter_prefill_metadata=aiter_prefill_metadata, ) except Exception: logger.warning( @@ -1182,6 +1271,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): cu_seqlens, chunk_indices, chunk_offsets, + aiter_prefill_metadata, ) torch.accelerator.empty_cache() @@ -1523,6 +1613,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): chunk_indices=attn_metadata.chunk_indices, chunk_offsets=attn_metadata.chunk_offsets, use_qk_l2norm_in_kernel=False, + aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata, ) # Init cache ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype) diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index 95f6a808a5..08369adcb1 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -7,6 +7,7 @@ from typing import Literal import torch +from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig from vllm.model_executor.layers.mamba.checkpoint import ( MambaPrefillCheckpointBuilder, @@ -79,6 +80,7 @@ class GDNAttentionMetadata: prefill_query_start_loc: torch.Tensor | None = None prefill_state_indices: torch.Tensor | None = None prefill_has_initial_state: torch.Tensor | None = None + aiter_prefill_metadata: object | None = None # The following attributes are for triton implementation of causal_conv1d nums_dict: dict | None = None @@ -109,8 +111,11 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] _resolve_gdn_prefill_backend, ) - self.gdn_prefill_backend: Literal["triton", "flashinfer", "cutedsl"] + self.gdn_prefill_backend: Literal[ + "triton", "flashinfer", "cutedsl", "aiter_flydsl" + ] _, self.gdn_prefill_backend = _resolve_gdn_prefill_backend(vllm_config) + self._check_chunk_metadata_override(type(self), self.gdn_prefill_backend) if self.speculative_config: assert self.speculative_config.num_speculative_tokens is not None @@ -186,6 +191,32 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] device=device, ) + @staticmethod + def _check_chunk_metadata_override( + builder_cls: type["GDNAttentionMetadataBuilder"], backend: str + ) -> None: + """Reject a subclass whose chunk metadata the AITER path would skip. + + AITER brings its own varlen prefill metadata and never calls + ``_build_chunk_metadata``, so a builder that overrides it would lose + that override without a word. No in-tree subclass can get here -- + ``_resolve_gdn_prefill_backend`` only selects this backend for models + with 128-dim GDN key and value heads, which the KDA builders are not -- + but a future one should be told rather than quietly ignored. + """ + if backend != "aiter_flydsl": + return + if ( + builder_cls._build_chunk_metadata + is GDNAttentionMetadataBuilder._build_chunk_metadata + ): + return + raise RuntimeError( + f"{builder_cls.__name__} builds its own FLA chunk metadata, which " + "the 'aiter_flydsl' GDN prefill backend does not use. Select a " + "different gdn_prefill_backend for this model." + ) + def _build_chunk_metadata( self, prefill_query_start_loc: torch.Tensor, @@ -431,6 +462,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] prefill_query_start_loc: torch.Tensor | None = None prefill_state_indices: torch.Tensor | None = None prefill_has_initial_state: torch.Tensor | None = None + aiter_prefill_metadata: object | None = None if num_prefills > 0: # In a mixed non-spec batch, decodes are peeled off to the recurrent # kernel (decode-first front slice), so build chunk metadata from the @@ -452,11 +484,23 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] prefill_query_start_loc_cpu = non_spec_query_start_loc_cpu prefill_state_indices = non_spec_state_indices_tensor - chunk_indices, chunk_offsets = self._build_chunk_metadata( - prefill_query_start_loc, - prefill_query_start_loc_cpu, - query_start_loc.device, - ) + if self.gdn_prefill_backend == "aiter_flydsl": + # AITER carries its own reusable varlen metadata and has no use + # for FLA's chunk indices, so it replaces them rather than + # extending what _build_chunk_metadata returns. + assert prefill_query_start_loc_cpu is not None + aiter_prefill_metadata = ( + rocm_aiter_ops.build_gdn_flydsl_prefill_metadata( + torch.diff(prefill_query_start_loc_cpu).tolist(), + cu_seqlens=prefill_query_start_loc, + ) + ) + else: + chunk_indices, chunk_offsets = self._build_chunk_metadata( + prefill_query_start_loc, + prefill_query_start_loc_cpu, + query_start_loc.device, + ) if num_prefills > 0: context_lens_tensor = m.compute_num_computed_tokens() @@ -568,6 +612,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] prefill_query_start_loc=prefill_query_start_loc, prefill_state_indices=prefill_state_indices, prefill_has_initial_state=prefill_has_initial_state, + aiter_prefill_metadata=aiter_prefill_metadata, spec_query_start_loc=spec_query_start_loc, non_spec_query_start_loc=non_spec_query_start_loc, spec_state_indices_tensor=spec_state_indices_tensor,