[ROCm][Perf] Add AITER FlyDSL GDN prefill backend (#57560)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Mikko Tukiainen
2026-10-07 09:44:22 -05:00
committed by GitHub
co-authored by Cursor
parent f6e3944e31
commit 548597367e
5 changed files with 462 additions and 12 deletions
+199
View File
@@ -0,0 +1,199 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
ChunkGatedDeltaRule,
_resolve_gdn_prefill_backend,
)
from vllm.platforms import current_platform
def _make_config(
*,
backend: str = "aiter_flydsl",
head_k_dim: int | None = 128,
head_v_dim: int | None = 128,
dtype: torch.dtype = torch.bfloat16,
):
return SimpleNamespace(
additional_config={"gdn_prefill_backend": backend},
model_config=SimpleNamespace(
dtype=dtype,
hf_text_config=SimpleNamespace(
linear_key_head_dim=head_k_dim,
linear_value_head_dim=head_v_dim,
),
),
)
def _resolve_on_rocm(config, *, kernels_available: bool = True):
with (
patch.object(
qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=True
),
patch.object(
qwen_gdn_linear_attn.rocm_aiter_ops,
"is_gdn_flydsl_prefill_available",
return_value=kernels_available,
),
):
return _resolve_gdn_prefill_backend(config)
@pytest.mark.parametrize(
"head_k_dim,head_v_dim,dtype,expected",
[
(128, 128, torch.bfloat16, "aiter_flydsl"),
(64, 128, torch.bfloat16, "triton"),
(128, 64, torch.bfloat16, "triton"),
(128, 128, torch.float16, "triton"),
],
)
def test_resolve_aiter_flydsl_gdn_prefill_backend(
head_k_dim: int,
head_v_dim: int,
dtype: torch.dtype,
expected: str,
):
config = _make_config(head_k_dim=head_k_dim, head_v_dim=head_v_dim, dtype=dtype)
requested, active = _resolve_on_rocm(config)
assert requested == "aiter_flydsl"
assert active == expected
def test_kda_style_model_cannot_select_flydsl():
"""Kimi K3 KDA reports head dims through linear_attn_config, not these.
Its builder overrides _build_chunk_metadata, which the AITER path skips,
so this pins the reason the two never meet: the resolver turns the model
down before the builder is ever constructed.
"""
config = _make_config(head_k_dim=None, head_v_dim=None)
assert _resolve_on_rocm(config) == ("aiter_flydsl", "triton")
@pytest.mark.parametrize("on_rocm", [True, False], ids=["kernels_missing", "not_rocm"])
def test_explicit_aiter_flydsl_fails_closed(on_rocm: bool):
"""An explicit request that cannot be honoured is a configuration error.
Falling back silently would report the run as healthy while the requested
kernels were never used.
"""
config = _make_config()
if on_rocm:
with pytest.raises(RuntimeError, match="not importable"):
_resolve_on_rocm(config, kernels_available=False)
return
with (
patch.object(
qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=False
),
patch.object(
qwen_gdn_linear_attn.current_platform, "is_cuda", return_value=True
),
pytest.raises(RuntimeError, match="aiter_flydsl"),
):
_resolve_gdn_prefill_backend(config)
def _flydsl_prefill_unavailable() -> bool:
return (
not current_platform.is_rocm()
or not rocm_aiter_ops.is_gdn_flydsl_prefill_available()
)
@pytest.mark.skipif(
_flydsl_prefill_unavailable(),
reason="needs ROCm with the AITER FlyDSL GDN prefill kernels installed",
)
@pytest.mark.parametrize(
"state_dtype", [torch.float32, torch.bfloat16], ids=["fp32_state", "bf16_state"]
)
def test_aiter_flydsl_prefill_matches_triton_reference(state_dtype: torch.dtype):
"""Run both prefill paths on device and compare.
The reference is the same AITER entry point with the FlyDSL prepare and K5
kernels switched off, so the only thing that differs between the two calls
is the kernels this backend exists to select. The state dtype follows
--mamba-ssm-cache-dtype, which may be float32 or bfloat16.
"""
from aiter.ops.triton.gated_delta_net import chunk_gated_delta_rule_opt_vk
torch.manual_seed(0)
device = torch.device("cuda")
seq_lens = [70, 130]
total_tokens = sum(seq_lens)
num_k_heads, num_v_heads, head_dim = 2, 4, 128
# use_qk_l2norm_in_kernel=False matches the prefill path, where the model
# has already normalised q and k. Feeding raw normals instead lets the
# delta rule run away to NaN, which says nothing about either kernel.
q = torch.nn.functional.normalize(
torch.randn(1, total_tokens, num_k_heads, head_dim, device=device), dim=-1
).to(torch.bfloat16)
k = torch.nn.functional.normalize(
torch.randn(1, total_tokens, num_k_heads, head_dim, device=device), dim=-1
).to(torch.bfloat16)
v = torch.randn(
1, total_tokens, num_v_heads, head_dim, dtype=torch.bfloat16, device=device
)
# g is a log-domain decay, so keep it negative or the recurrence diverges
# and the comparison stops saying anything about the kernels.
g = -torch.rand(1, total_tokens, num_v_heads, dtype=torch.float32, device=device)
beta = torch.rand(1, total_tokens, num_v_heads, dtype=torch.float32, device=device)
cu_seqlens = torch.tensor([0, 70, 200], dtype=torch.int32, device=device)
initial_state = torch.randn(
len(seq_lens),
num_v_heads,
head_dim,
head_dim,
device=device,
).to(state_dtype)
shared = {
"q": q,
"k": k,
"v": v,
"g": g,
"beta": beta,
"output_final_state": True,
"cu_seqlens": cu_seqlens,
"use_qk_l2norm_in_kernel": False,
}
reference_out, reference_state = chunk_gated_delta_rule_opt_vk(
initial_state=initial_state.clone(),
state_dtype=state_dtype,
use_chunk_flydsl=False,
use_prepare_flydsl=False,
**shared,
)
flydsl_out, flydsl_state = ChunkGatedDeltaRule.forward_aiter_flydsl(
None,
initial_state=initial_state.clone(),
aiter_prefill_metadata=rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
seq_lens, cu_seqlens=cu_seqlens
),
**shared,
)
assert flydsl_state.dtype == state_dtype
torch.testing.assert_close(
flydsl_out.float(), reference_out.float(), rtol=2e-2, atol=2e-2
)
torch.testing.assert_close(
flydsl_state.float(), reference_state.float(), rtol=2e-2, atol=2e-2
)
+113
View File
@@ -1997,6 +1997,8 @@ class rocm_aiter_ops:
# Check if the env variable is set
_AITER_ENABLED = envs.VLLM_ROCM_USE_AITER
# Chunk length the AITER FlyDSL K5 prefill kernels are compiled for.
GDN_FLYDSL_CHUNK_SIZE = 64
_CUSTOM_ALL_REDUCE_ENABLED = envs.VLLM_ROCM_USE_AITER_CUSTOM_AR
_LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR
_FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE
@@ -2467,6 +2469,23 @@ class rocm_aiter_ops:
except (ImportError, ModuleNotFoundError):
return False
@staticmethod
def _gdn_flydsl_prefill_kernels_importable() -> bool:
try:
from aiter.ops.flydsl.linear_attention_prefill_kernels import ( # noqa: F401
chunk_gated_delta_rule_fwd_h_flydsl_opt,
gdn_prepare_flydsl_supported,
gdn_prepare_fwd_flydsl,
)
from aiter.ops.triton.gated_delta_net import ( # noqa: F401
build_gated_delta_rule_prefill_metadata,
chunk_gated_delta_rule_opt_vk,
)
return True
except (ImportError, ModuleNotFoundError):
return False
@classmethod
@if_aiter_supported
def are_gdn_triton_kernels_available(cls) -> bool:
@@ -2478,6 +2497,100 @@ class rocm_aiter_ops:
"""
return cls._AITER_ENABLED and cls._gdn_triton_kernels_importable()
@classmethod
@functools.cache
def is_gdn_flydsl_prefill_available(cls) -> bool:
"""Whether the opt-in AITER FlyDSL GDN prefill path can be used.
Selecting the backend is an explicit opt-in in itself, but it still
runs AITER kernels, so VLLM_ROCM_USE_AITER remains the one switch that
turns all of them off.
"""
return (
cls._AITER_ENABLED
and is_aiter_found_and_supported()
and cls._gdn_flydsl_prefill_kernels_importable()
)
@classmethod
def build_gdn_flydsl_prefill_metadata(
cls,
seq_lens_cpu: list[int],
cu_seqlens: torch.Tensor,
) -> object:
"""Build the reusable varlen metadata FlyDSL GDN prefill runs against.
The K5 recurrence is compiled for a fixed 64-token chunk, so the chunk
size is a property of the AITER kernel rather than of FLA, and is not
the caller's to choose.
"""
from aiter.ops.triton.gated_delta_net import (
build_gated_delta_rule_prefill_metadata,
)
return build_gated_delta_rule_prefill_metadata(
seq_lens_cpu,
cu_seqlens=cu_seqlens,
chunk_size=cls.GDN_FLYDSL_CHUNK_SIZE,
)
@classmethod
def gdn_flydsl_prefill(
cls,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
initial_state: torch.Tensor,
output_final_state: bool,
cu_seqlens: torch.Tensor | None,
aiter_prefill_metadata: object | None,
use_qk_l2norm_in_kernel: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run GDN prefill through AITER's VK path with the FlyDSL kernels.
Every argument AITER takes is named here rather than forwarded as
``**kwargs``, so a rename or reordering on the AITER side fails at this
call instead of silently binding to the wrong parameter.
"""
from aiter.ops.flydsl.linear_attention_prefill_kernels import (
gdn_prepare_flydsl_supported,
)
from aiter.ops.triton.gated_delta_net import chunk_gated_delta_rule_opt_vk
if cu_seqlens is not None and aiter_prefill_metadata is None:
raise RuntimeError(
"AITER FlyDSL GDN prefill requires reusable varlen prefill metadata."
)
if not gdn_prepare_flydsl_supported(k, v):
raise RuntimeError(
"AITER FlyDSL GDN prepare does not support the runtime input shape, "
"dtype, or device."
)
logger.info_once(
"Dispatching AITER FlyDSL GDN prefill (prepare=flydsl, chunk=flydsl)."
)
return chunk_gated_delta_rule_opt_vk(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=output_final_state,
cu_seqlens=cu_seqlens,
use_chunk_flydsl=True,
use_prepare_flydsl=True,
state_dtype=initial_state.dtype,
prefill_metadata=aiter_prefill_metadata,
# The caller stages the state densely and writes it back, so the
# kernel gets a plain [N, H, V, K] buffer and never an index into
# the pool; there is correspondingly nothing to update in place.
inplace_final_state=False,
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
)
@classmethod
def is_rdna_gdn_triton_kernels_available(cls) -> bool:
"""RDNA4 (gfx12) analog of are_gdn_triton_kernels_available()."""
+4 -2
View File
@@ -839,7 +839,9 @@ class EngineArgs:
)
fail_on_environ_validation: bool = False
gdn_prefill_backend: Literal["flashinfer", "triton", "cutedsl"] | None = None
gdn_prefill_backend: (
Literal["flashinfer", "triton", "cutedsl", "aiter_flydsl"] | None
) = None
kda_prefill_backend: (
Literal["auto", "triton", "flashkda", "flashinfer", "fused"] | None
) = None
@@ -1890,7 +1892,7 @@ class EngineArgs:
parser.add_argument(
"--gdn-prefill-backend",
dest="gdn_prefill_backend",
choices=["flashinfer", "triton", "cutedsl"],
choices=["flashinfer", "triton", "cutedsl", "aiter_flydsl"],
default=None,
help="Select GDN prefill backend.",
)
@@ -92,7 +92,7 @@ FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
def _resolve_gdn_prefill_backend(
vllm_config: VllmConfig,
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl"]]:
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
"""Resolve GDN prefill backend.
FlashInfer's GDN prefill kernel is chosen when:
@@ -106,6 +106,11 @@ def _resolve_gdn_prefill_backend(
In-tree CuteDSL GDN prefill kernel is chosen when:
* "cutedsl" is requested; (opt-in only)
* Blackwell (SM10.x) with ``head_k_dim == 128``;
AITER FlyDSL GDN prefill kernels are chosen when:
* "aiter_flydsl" is requested; (opt-in only)
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
"""
additional_config = vllm_config.additional_config
backend_cfg = (
@@ -115,12 +120,50 @@ def _resolve_gdn_prefill_backend(
)
backend = str(backend_cfg).strip().lower()
if not current_platform.is_cuda():
return backend, "triton"
head_k_dim = getattr(
vllm_config.model_config.hf_text_config, "linear_key_head_dim", None
)
head_v_dim = getattr(
vllm_config.model_config.hf_text_config, "linear_value_head_dim", None
)
if current_platform.is_rocm():
if backend == "aiter_flydsl":
if not rocm_aiter_ops.is_gdn_flydsl_prefill_available():
raise RuntimeError(
"GDN prefill backend 'aiter_flydsl' was requested but the "
"AITER FlyDSL prefill kernels are not importable. Check "
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
"newer GPU, and that the installed AITER exports them."
)
if head_k_dim != 128 or head_v_dim != 128:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
"Falling back to Triton/FLA.",
head_k_dim,
head_v_dim,
)
return backend, "triton"
if vllm_config.model_config.dtype != torch.bfloat16:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"model dtype is %s; FlyDSL requires bfloat16. "
"Falling back to Triton/FLA.",
vllm_config.model_config.dtype,
)
return backend, "triton"
return backend, "aiter_flydsl"
return backend, "triton"
if backend == "aiter_flydsl":
raise RuntimeError(
"GDN prefill backend 'aiter_flydsl' was requested but it is a ROCm "
f"AITER backend and the current platform is {current_platform.device_name}."
)
if not current_platform.is_cuda():
return backend, "triton"
supports_flashinfer = False
supports_cutedsl = False
@@ -172,6 +215,7 @@ def _log_gdn_backend_decision(
chosen = {
"flashinfer": "FlashInfer",
"cutedsl": "CuteDSL",
"aiter_flydsl": "AITER FlyDSL",
"triton": "Triton/FLA",
}[active_backend]
logger.info_once(
@@ -259,6 +303,8 @@ class ChunkGatedDeltaRule(CustomOp):
self._forward_method = self.forward_cuda
elif active_backend == "cutedsl":
self._forward_method = self.forward_cutedsl
elif active_backend == "aiter_flydsl":
self._forward_method = self.forward_aiter_flydsl
else:
self._forward_method = self.forward_native
@@ -276,6 +322,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
o, final_state = fi_chunk_gated_delta_rule(
q=q,
@@ -308,6 +355,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
return fla_chunk_gated_delta_rule(
q=q,
@@ -338,6 +386,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
chunk_gated_delta_rule_cutedsl,
@@ -367,6 +416,39 @@ class ChunkGatedDeltaRule(CustomOp):
final_state = None
return o, final_state
def forward_aiter_flydsl(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
initial_state: torch.Tensor,
output_final_state: bool,
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
o, final_state = rocm_aiter_ops.gdn_flydsl_prefill(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=output_final_state,
cu_seqlens=cu_seqlens,
aiter_prefill_metadata=aiter_prefill_metadata,
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
)
if core_attn_out is not None:
o_flat = o.reshape(-1)
core_attn_out.reshape(-1)[: o_flat.numel()].copy_(o_flat)
return o, final_state
@PluggableLayer.register("qwen_gated_delta_net_attention")
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
@@ -1132,12 +1214,18 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
# CuteDSL kernels require metadata
chunk_indices = None
chunk_offsets = None
aiter_prefill_metadata = None
if self.gdn_prefill_backend == "cutedsl":
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
prepare_metadata_cutedsl,
)
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, T)
elif self.gdn_prefill_backend == "aiter_flydsl":
aiter_prefill_metadata = rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
[T],
cu_seqlens=cu_seqlens,
)
try:
self.chunk_gated_delta_rule(
@@ -1152,6 +1240,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=aiter_prefill_metadata,
)
except Exception:
logger.warning(
@@ -1182,6 +1271,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
cu_seqlens,
chunk_indices,
chunk_offsets,
aiter_prefill_metadata,
)
torch.accelerator.empty_cache()
@@ -1523,6 +1613,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
)
# Init cache
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
+51 -6
View File
@@ -7,6 +7,7 @@ from typing import Literal
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointBuilder,
@@ -79,6 +80,7 @@ class GDNAttentionMetadata:
prefill_query_start_loc: torch.Tensor | None = None
prefill_state_indices: torch.Tensor | None = None
prefill_has_initial_state: torch.Tensor | None = None
aiter_prefill_metadata: object | None = None
# The following attributes are for triton implementation of causal_conv1d
nums_dict: dict | None = None
@@ -109,8 +111,11 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
_resolve_gdn_prefill_backend,
)
self.gdn_prefill_backend: Literal["triton", "flashinfer", "cutedsl"]
self.gdn_prefill_backend: Literal[
"triton", "flashinfer", "cutedsl", "aiter_flydsl"
]
_, self.gdn_prefill_backend = _resolve_gdn_prefill_backend(vllm_config)
self._check_chunk_metadata_override(type(self), self.gdn_prefill_backend)
if self.speculative_config:
assert self.speculative_config.num_speculative_tokens is not None
@@ -186,6 +191,32 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
device=device,
)
@staticmethod
def _check_chunk_metadata_override(
builder_cls: type["GDNAttentionMetadataBuilder"], backend: str
) -> None:
"""Reject a subclass whose chunk metadata the AITER path would skip.
AITER brings its own varlen prefill metadata and never calls
``_build_chunk_metadata``, so a builder that overrides it would lose
that override without a word. No in-tree subclass can get here --
``_resolve_gdn_prefill_backend`` only selects this backend for models
with 128-dim GDN key and value heads, which the KDA builders are not --
but a future one should be told rather than quietly ignored.
"""
if backend != "aiter_flydsl":
return
if (
builder_cls._build_chunk_metadata
is GDNAttentionMetadataBuilder._build_chunk_metadata
):
return
raise RuntimeError(
f"{builder_cls.__name__} builds its own FLA chunk metadata, which "
"the 'aiter_flydsl' GDN prefill backend does not use. Select a "
"different gdn_prefill_backend for this model."
)
def _build_chunk_metadata(
self,
prefill_query_start_loc: torch.Tensor,
@@ -431,6 +462,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
prefill_query_start_loc: torch.Tensor | None = None
prefill_state_indices: torch.Tensor | None = None
prefill_has_initial_state: torch.Tensor | None = None
aiter_prefill_metadata: object | None = None
if num_prefills > 0:
# In a mixed non-spec batch, decodes are peeled off to the recurrent
# kernel (decode-first front slice), so build chunk metadata from the
@@ -452,11 +484,23 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
prefill_query_start_loc_cpu = non_spec_query_start_loc_cpu
prefill_state_indices = non_spec_state_indices_tensor
chunk_indices, chunk_offsets = self._build_chunk_metadata(
prefill_query_start_loc,
prefill_query_start_loc_cpu,
query_start_loc.device,
)
if self.gdn_prefill_backend == "aiter_flydsl":
# AITER carries its own reusable varlen metadata and has no use
# for FLA's chunk indices, so it replaces them rather than
# extending what _build_chunk_metadata returns.
assert prefill_query_start_loc_cpu is not None
aiter_prefill_metadata = (
rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
torch.diff(prefill_query_start_loc_cpu).tolist(),
cu_seqlens=prefill_query_start_loc,
)
)
else:
chunk_indices, chunk_offsets = self._build_chunk_metadata(
prefill_query_start_loc,
prefill_query_start_loc_cpu,
query_start_loc.device,
)
if num_prefills > 0:
context_lens_tensor = m.compute_num_computed_tokens()
@@ -568,6 +612,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
prefill_query_start_loc=prefill_query_start_loc,
prefill_state_indices=prefill_state_indices,
prefill_has_initial_state=prefill_has_initial_state,
aiter_prefill_metadata=aiter_prefill_metadata,
spec_query_start_loc=spec_query_start_loc,
non_spec_query_start_loc=non_spec_query_start_loc,
spec_state_indices_tensor=spec_state_indices_tensor,