[ROCm][Perf] Add AITER FlyDSL GDN prefill backend (#57560)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
f6e3944e31
commit
548597367e
@@ -0,0 +1,199 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
ChunkGatedDeltaRule,
|
||||
_resolve_gdn_prefill_backend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _make_config(
|
||||
*,
|
||||
backend: str = "aiter_flydsl",
|
||||
head_k_dim: int | None = 128,
|
||||
head_v_dim: int | None = 128,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
additional_config={"gdn_prefill_backend": backend},
|
||||
model_config=SimpleNamespace(
|
||||
dtype=dtype,
|
||||
hf_text_config=SimpleNamespace(
|
||||
linear_key_head_dim=head_k_dim,
|
||||
linear_value_head_dim=head_v_dim,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _resolve_on_rocm(config, *, kernels_available: bool = True):
|
||||
with (
|
||||
patch.object(
|
||||
qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=True
|
||||
),
|
||||
patch.object(
|
||||
qwen_gdn_linear_attn.rocm_aiter_ops,
|
||||
"is_gdn_flydsl_prefill_available",
|
||||
return_value=kernels_available,
|
||||
),
|
||||
):
|
||||
return _resolve_gdn_prefill_backend(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"head_k_dim,head_v_dim,dtype,expected",
|
||||
[
|
||||
(128, 128, torch.bfloat16, "aiter_flydsl"),
|
||||
(64, 128, torch.bfloat16, "triton"),
|
||||
(128, 64, torch.bfloat16, "triton"),
|
||||
(128, 128, torch.float16, "triton"),
|
||||
],
|
||||
)
|
||||
def test_resolve_aiter_flydsl_gdn_prefill_backend(
|
||||
head_k_dim: int,
|
||||
head_v_dim: int,
|
||||
dtype: torch.dtype,
|
||||
expected: str,
|
||||
):
|
||||
config = _make_config(head_k_dim=head_k_dim, head_v_dim=head_v_dim, dtype=dtype)
|
||||
|
||||
requested, active = _resolve_on_rocm(config)
|
||||
|
||||
assert requested == "aiter_flydsl"
|
||||
assert active == expected
|
||||
|
||||
|
||||
def test_kda_style_model_cannot_select_flydsl():
|
||||
"""Kimi K3 KDA reports head dims through linear_attn_config, not these.
|
||||
|
||||
Its builder overrides _build_chunk_metadata, which the AITER path skips,
|
||||
so this pins the reason the two never meet: the resolver turns the model
|
||||
down before the builder is ever constructed.
|
||||
"""
|
||||
config = _make_config(head_k_dim=None, head_v_dim=None)
|
||||
|
||||
assert _resolve_on_rocm(config) == ("aiter_flydsl", "triton")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("on_rocm", [True, False], ids=["kernels_missing", "not_rocm"])
|
||||
def test_explicit_aiter_flydsl_fails_closed(on_rocm: bool):
|
||||
"""An explicit request that cannot be honoured is a configuration error.
|
||||
|
||||
Falling back silently would report the run as healthy while the requested
|
||||
kernels were never used.
|
||||
"""
|
||||
config = _make_config()
|
||||
if on_rocm:
|
||||
with pytest.raises(RuntimeError, match="not importable"):
|
||||
_resolve_on_rocm(config, kernels_available=False)
|
||||
return
|
||||
with (
|
||||
patch.object(
|
||||
qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=False
|
||||
),
|
||||
patch.object(
|
||||
qwen_gdn_linear_attn.current_platform, "is_cuda", return_value=True
|
||||
),
|
||||
pytest.raises(RuntimeError, match="aiter_flydsl"),
|
||||
):
|
||||
_resolve_gdn_prefill_backend(config)
|
||||
|
||||
|
||||
def _flydsl_prefill_unavailable() -> bool:
|
||||
return (
|
||||
not current_platform.is_rocm()
|
||||
or not rocm_aiter_ops.is_gdn_flydsl_prefill_available()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_flydsl_prefill_unavailable(),
|
||||
reason="needs ROCm with the AITER FlyDSL GDN prefill kernels installed",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"state_dtype", [torch.float32, torch.bfloat16], ids=["fp32_state", "bf16_state"]
|
||||
)
|
||||
def test_aiter_flydsl_prefill_matches_triton_reference(state_dtype: torch.dtype):
|
||||
"""Run both prefill paths on device and compare.
|
||||
|
||||
The reference is the same AITER entry point with the FlyDSL prepare and K5
|
||||
kernels switched off, so the only thing that differs between the two calls
|
||||
is the kernels this backend exists to select. The state dtype follows
|
||||
--mamba-ssm-cache-dtype, which may be float32 or bfloat16.
|
||||
"""
|
||||
from aiter.ops.triton.gated_delta_net import chunk_gated_delta_rule_opt_vk
|
||||
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
seq_lens = [70, 130]
|
||||
total_tokens = sum(seq_lens)
|
||||
num_k_heads, num_v_heads, head_dim = 2, 4, 128
|
||||
|
||||
# use_qk_l2norm_in_kernel=False matches the prefill path, where the model
|
||||
# has already normalised q and k. Feeding raw normals instead lets the
|
||||
# delta rule run away to NaN, which says nothing about either kernel.
|
||||
q = torch.nn.functional.normalize(
|
||||
torch.randn(1, total_tokens, num_k_heads, head_dim, device=device), dim=-1
|
||||
).to(torch.bfloat16)
|
||||
k = torch.nn.functional.normalize(
|
||||
torch.randn(1, total_tokens, num_k_heads, head_dim, device=device), dim=-1
|
||||
).to(torch.bfloat16)
|
||||
v = torch.randn(
|
||||
1, total_tokens, num_v_heads, head_dim, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
# g is a log-domain decay, so keep it negative or the recurrence diverges
|
||||
# and the comparison stops saying anything about the kernels.
|
||||
g = -torch.rand(1, total_tokens, num_v_heads, dtype=torch.float32, device=device)
|
||||
beta = torch.rand(1, total_tokens, num_v_heads, dtype=torch.float32, device=device)
|
||||
cu_seqlens = torch.tensor([0, 70, 200], dtype=torch.int32, device=device)
|
||||
initial_state = torch.randn(
|
||||
len(seq_lens),
|
||||
num_v_heads,
|
||||
head_dim,
|
||||
head_dim,
|
||||
device=device,
|
||||
).to(state_dtype)
|
||||
|
||||
shared = {
|
||||
"q": q,
|
||||
"k": k,
|
||||
"v": v,
|
||||
"g": g,
|
||||
"beta": beta,
|
||||
"output_final_state": True,
|
||||
"cu_seqlens": cu_seqlens,
|
||||
"use_qk_l2norm_in_kernel": False,
|
||||
}
|
||||
|
||||
reference_out, reference_state = chunk_gated_delta_rule_opt_vk(
|
||||
initial_state=initial_state.clone(),
|
||||
state_dtype=state_dtype,
|
||||
use_chunk_flydsl=False,
|
||||
use_prepare_flydsl=False,
|
||||
**shared,
|
||||
)
|
||||
flydsl_out, flydsl_state = ChunkGatedDeltaRule.forward_aiter_flydsl(
|
||||
None,
|
||||
initial_state=initial_state.clone(),
|
||||
aiter_prefill_metadata=rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
|
||||
seq_lens, cu_seqlens=cu_seqlens
|
||||
),
|
||||
**shared,
|
||||
)
|
||||
|
||||
assert flydsl_state.dtype == state_dtype
|
||||
torch.testing.assert_close(
|
||||
flydsl_out.float(), reference_out.float(), rtol=2e-2, atol=2e-2
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
flydsl_state.float(), reference_state.float(), rtol=2e-2, atol=2e-2
|
||||
)
|
||||
@@ -1997,6 +1997,8 @@ class rocm_aiter_ops:
|
||||
|
||||
# Check if the env variable is set
|
||||
_AITER_ENABLED = envs.VLLM_ROCM_USE_AITER
|
||||
# Chunk length the AITER FlyDSL K5 prefill kernels are compiled for.
|
||||
GDN_FLYDSL_CHUNK_SIZE = 64
|
||||
_CUSTOM_ALL_REDUCE_ENABLED = envs.VLLM_ROCM_USE_AITER_CUSTOM_AR
|
||||
_LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR
|
||||
_FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE
|
||||
@@ -2467,6 +2469,23 @@ class rocm_aiter_ops:
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _gdn_flydsl_prefill_kernels_importable() -> bool:
|
||||
try:
|
||||
from aiter.ops.flydsl.linear_attention_prefill_kernels import ( # noqa: F401
|
||||
chunk_gated_delta_rule_fwd_h_flydsl_opt,
|
||||
gdn_prepare_flydsl_supported,
|
||||
gdn_prepare_fwd_flydsl,
|
||||
)
|
||||
from aiter.ops.triton.gated_delta_net import ( # noqa: F401
|
||||
build_gated_delta_rule_prefill_metadata,
|
||||
chunk_gated_delta_rule_opt_vk,
|
||||
)
|
||||
|
||||
return True
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def are_gdn_triton_kernels_available(cls) -> bool:
|
||||
@@ -2478,6 +2497,100 @@ class rocm_aiter_ops:
|
||||
"""
|
||||
return cls._AITER_ENABLED and cls._gdn_triton_kernels_importable()
|
||||
|
||||
@classmethod
|
||||
@functools.cache
|
||||
def is_gdn_flydsl_prefill_available(cls) -> bool:
|
||||
"""Whether the opt-in AITER FlyDSL GDN prefill path can be used.
|
||||
|
||||
Selecting the backend is an explicit opt-in in itself, but it still
|
||||
runs AITER kernels, so VLLM_ROCM_USE_AITER remains the one switch that
|
||||
turns all of them off.
|
||||
"""
|
||||
return (
|
||||
cls._AITER_ENABLED
|
||||
and is_aiter_found_and_supported()
|
||||
and cls._gdn_flydsl_prefill_kernels_importable()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_gdn_flydsl_prefill_metadata(
|
||||
cls,
|
||||
seq_lens_cpu: list[int],
|
||||
cu_seqlens: torch.Tensor,
|
||||
) -> object:
|
||||
"""Build the reusable varlen metadata FlyDSL GDN prefill runs against.
|
||||
|
||||
The K5 recurrence is compiled for a fixed 64-token chunk, so the chunk
|
||||
size is a property of the AITER kernel rather than of FLA, and is not
|
||||
the caller's to choose.
|
||||
"""
|
||||
from aiter.ops.triton.gated_delta_net import (
|
||||
build_gated_delta_rule_prefill_metadata,
|
||||
)
|
||||
|
||||
return build_gated_delta_rule_prefill_metadata(
|
||||
seq_lens_cpu,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_size=cls.GDN_FLYDSL_CHUNK_SIZE,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def gdn_flydsl_prefill(
|
||||
cls,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None,
|
||||
aiter_prefill_metadata: object | None,
|
||||
use_qk_l2norm_in_kernel: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run GDN prefill through AITER's VK path with the FlyDSL kernels.
|
||||
|
||||
Every argument AITER takes is named here rather than forwarded as
|
||||
``**kwargs``, so a rename or reordering on the AITER side fails at this
|
||||
call instead of silently binding to the wrong parameter.
|
||||
"""
|
||||
from aiter.ops.flydsl.linear_attention_prefill_kernels import (
|
||||
gdn_prepare_flydsl_supported,
|
||||
)
|
||||
from aiter.ops.triton.gated_delta_net import chunk_gated_delta_rule_opt_vk
|
||||
|
||||
if cu_seqlens is not None and aiter_prefill_metadata is None:
|
||||
raise RuntimeError(
|
||||
"AITER FlyDSL GDN prefill requires reusable varlen prefill metadata."
|
||||
)
|
||||
if not gdn_prepare_flydsl_supported(k, v):
|
||||
raise RuntimeError(
|
||||
"AITER FlyDSL GDN prepare does not support the runtime input shape, "
|
||||
"dtype, or device."
|
||||
)
|
||||
logger.info_once(
|
||||
"Dispatching AITER FlyDSL GDN prefill (prepare=flydsl, chunk=flydsl)."
|
||||
)
|
||||
return chunk_gated_delta_rule_opt_vk(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_chunk_flydsl=True,
|
||||
use_prepare_flydsl=True,
|
||||
state_dtype=initial_state.dtype,
|
||||
prefill_metadata=aiter_prefill_metadata,
|
||||
# The caller stages the state densely and writes it back, so the
|
||||
# kernel gets a plain [N, H, V, K] buffer and never an index into
|
||||
# the pool; there is correspondingly nothing to update in place.
|
||||
inplace_final_state=False,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def is_rdna_gdn_triton_kernels_available(cls) -> bool:
|
||||
"""RDNA4 (gfx12) analog of are_gdn_triton_kernels_available()."""
|
||||
|
||||
@@ -839,7 +839,9 @@ class EngineArgs:
|
||||
)
|
||||
|
||||
fail_on_environ_validation: bool = False
|
||||
gdn_prefill_backend: Literal["flashinfer", "triton", "cutedsl"] | None = None
|
||||
gdn_prefill_backend: (
|
||||
Literal["flashinfer", "triton", "cutedsl", "aiter_flydsl"] | None
|
||||
) = None
|
||||
kda_prefill_backend: (
|
||||
Literal["auto", "triton", "flashkda", "flashinfer", "fused"] | None
|
||||
) = None
|
||||
@@ -1890,7 +1892,7 @@ class EngineArgs:
|
||||
parser.add_argument(
|
||||
"--gdn-prefill-backend",
|
||||
dest="gdn_prefill_backend",
|
||||
choices=["flashinfer", "triton", "cutedsl"],
|
||||
choices=["flashinfer", "triton", "cutedsl", "aiter_flydsl"],
|
||||
default=None,
|
||||
help="Select GDN prefill backend.",
|
||||
)
|
||||
|
||||
@@ -92,7 +92,7 @@ FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
|
||||
|
||||
def _resolve_gdn_prefill_backend(
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl"]]:
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
|
||||
"""Resolve GDN prefill backend.
|
||||
|
||||
FlashInfer's GDN prefill kernel is chosen when:
|
||||
@@ -106,6 +106,11 @@ def _resolve_gdn_prefill_backend(
|
||||
In-tree CuteDSL GDN prefill kernel is chosen when:
|
||||
* "cutedsl" is requested; (opt-in only)
|
||||
* Blackwell (SM10.x) with ``head_k_dim == 128``;
|
||||
|
||||
AITER FlyDSL GDN prefill kernels are chosen when:
|
||||
* "aiter_flydsl" is requested; (opt-in only)
|
||||
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
|
||||
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
|
||||
"""
|
||||
additional_config = vllm_config.additional_config
|
||||
backend_cfg = (
|
||||
@@ -115,12 +120,50 @@ def _resolve_gdn_prefill_backend(
|
||||
)
|
||||
backend = str(backend_cfg).strip().lower()
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
return backend, "triton"
|
||||
|
||||
head_k_dim = getattr(
|
||||
vllm_config.model_config.hf_text_config, "linear_key_head_dim", None
|
||||
)
|
||||
head_v_dim = getattr(
|
||||
vllm_config.model_config.hf_text_config, "linear_value_head_dim", None
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
if backend == "aiter_flydsl":
|
||||
if not rocm_aiter_ops.is_gdn_flydsl_prefill_available():
|
||||
raise RuntimeError(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but the "
|
||||
"AITER FlyDSL prefill kernels are not importable. Check "
|
||||
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
|
||||
"newer GPU, and that the installed AITER exports them."
|
||||
)
|
||||
if head_k_dim != 128 or head_v_dim != 128:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
|
||||
"Falling back to Triton/FLA.",
|
||||
head_k_dim,
|
||||
head_v_dim,
|
||||
)
|
||||
return backend, "triton"
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"model dtype is %s; FlyDSL requires bfloat16. "
|
||||
"Falling back to Triton/FLA.",
|
||||
vllm_config.model_config.dtype,
|
||||
)
|
||||
return backend, "triton"
|
||||
return backend, "aiter_flydsl"
|
||||
return backend, "triton"
|
||||
|
||||
if backend == "aiter_flydsl":
|
||||
raise RuntimeError(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but it is a ROCm "
|
||||
f"AITER backend and the current platform is {current_platform.device_name}."
|
||||
)
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
return backend, "triton"
|
||||
|
||||
supports_flashinfer = False
|
||||
supports_cutedsl = False
|
||||
@@ -172,6 +215,7 @@ def _log_gdn_backend_decision(
|
||||
chosen = {
|
||||
"flashinfer": "FlashInfer",
|
||||
"cutedsl": "CuteDSL",
|
||||
"aiter_flydsl": "AITER FlyDSL",
|
||||
"triton": "Triton/FLA",
|
||||
}[active_backend]
|
||||
logger.info_once(
|
||||
@@ -259,6 +303,8 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
self._forward_method = self.forward_cuda
|
||||
elif active_backend == "cutedsl":
|
||||
self._forward_method = self.forward_cutedsl
|
||||
elif active_backend == "aiter_flydsl":
|
||||
self._forward_method = self.forward_aiter_flydsl
|
||||
else:
|
||||
self._forward_method = self.forward_native
|
||||
|
||||
@@ -276,6 +322,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
o, final_state = fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
@@ -308,6 +355,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
return fla_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
@@ -338,6 +386,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
|
||||
chunk_gated_delta_rule_cutedsl,
|
||||
@@ -367,6 +416,39 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
final_state = None
|
||||
return o, final_state
|
||||
|
||||
def forward_aiter_flydsl(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
o, final_state = rocm_aiter_ops.gdn_flydsl_prefill(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
)
|
||||
if core_attn_out is not None:
|
||||
o_flat = o.reshape(-1)
|
||||
core_attn_out.reshape(-1)[: o_flat.numel()].copy_(o_flat)
|
||||
return o, final_state
|
||||
|
||||
|
||||
@PluggableLayer.register("qwen_gated_delta_net_attention")
|
||||
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
@@ -1132,12 +1214,18 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
# CuteDSL kernels require metadata
|
||||
chunk_indices = None
|
||||
chunk_offsets = None
|
||||
aiter_prefill_metadata = None
|
||||
if self.gdn_prefill_backend == "cutedsl":
|
||||
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
|
||||
prepare_metadata_cutedsl,
|
||||
)
|
||||
|
||||
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, T)
|
||||
elif self.gdn_prefill_backend == "aiter_flydsl":
|
||||
aiter_prefill_metadata = rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
|
||||
[T],
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
|
||||
try:
|
||||
self.chunk_gated_delta_rule(
|
||||
@@ -1152,6 +1240,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
@@ -1182,6 +1271,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
chunk_offsets,
|
||||
aiter_prefill_metadata,
|
||||
)
|
||||
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1523,6 +1613,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
chunk_indices=attn_metadata.chunk_indices,
|
||||
chunk_offsets=attn_metadata.chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
|
||||
)
|
||||
# Init cache
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Literal
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointBuilder,
|
||||
@@ -79,6 +80,7 @@ class GDNAttentionMetadata:
|
||||
prefill_query_start_loc: torch.Tensor | None = None
|
||||
prefill_state_indices: torch.Tensor | None = None
|
||||
prefill_has_initial_state: torch.Tensor | None = None
|
||||
aiter_prefill_metadata: object | None = None
|
||||
|
||||
# The following attributes are for triton implementation of causal_conv1d
|
||||
nums_dict: dict | None = None
|
||||
@@ -109,8 +111,11 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
_resolve_gdn_prefill_backend,
|
||||
)
|
||||
|
||||
self.gdn_prefill_backend: Literal["triton", "flashinfer", "cutedsl"]
|
||||
self.gdn_prefill_backend: Literal[
|
||||
"triton", "flashinfer", "cutedsl", "aiter_flydsl"
|
||||
]
|
||||
_, self.gdn_prefill_backend = _resolve_gdn_prefill_backend(vllm_config)
|
||||
self._check_chunk_metadata_override(type(self), self.gdn_prefill_backend)
|
||||
|
||||
if self.speculative_config:
|
||||
assert self.speculative_config.num_speculative_tokens is not None
|
||||
@@ -186,6 +191,32 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
device=device,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _check_chunk_metadata_override(
|
||||
builder_cls: type["GDNAttentionMetadataBuilder"], backend: str
|
||||
) -> None:
|
||||
"""Reject a subclass whose chunk metadata the AITER path would skip.
|
||||
|
||||
AITER brings its own varlen prefill metadata and never calls
|
||||
``_build_chunk_metadata``, so a builder that overrides it would lose
|
||||
that override without a word. No in-tree subclass can get here --
|
||||
``_resolve_gdn_prefill_backend`` only selects this backend for models
|
||||
with 128-dim GDN key and value heads, which the KDA builders are not --
|
||||
but a future one should be told rather than quietly ignored.
|
||||
"""
|
||||
if backend != "aiter_flydsl":
|
||||
return
|
||||
if (
|
||||
builder_cls._build_chunk_metadata
|
||||
is GDNAttentionMetadataBuilder._build_chunk_metadata
|
||||
):
|
||||
return
|
||||
raise RuntimeError(
|
||||
f"{builder_cls.__name__} builds its own FLA chunk metadata, which "
|
||||
"the 'aiter_flydsl' GDN prefill backend does not use. Select a "
|
||||
"different gdn_prefill_backend for this model."
|
||||
)
|
||||
|
||||
def _build_chunk_metadata(
|
||||
self,
|
||||
prefill_query_start_loc: torch.Tensor,
|
||||
@@ -431,6 +462,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
prefill_query_start_loc: torch.Tensor | None = None
|
||||
prefill_state_indices: torch.Tensor | None = None
|
||||
prefill_has_initial_state: torch.Tensor | None = None
|
||||
aiter_prefill_metadata: object | None = None
|
||||
if num_prefills > 0:
|
||||
# In a mixed non-spec batch, decodes are peeled off to the recurrent
|
||||
# kernel (decode-first front slice), so build chunk metadata from the
|
||||
@@ -452,11 +484,23 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
prefill_query_start_loc_cpu = non_spec_query_start_loc_cpu
|
||||
prefill_state_indices = non_spec_state_indices_tensor
|
||||
|
||||
chunk_indices, chunk_offsets = self._build_chunk_metadata(
|
||||
prefill_query_start_loc,
|
||||
prefill_query_start_loc_cpu,
|
||||
query_start_loc.device,
|
||||
)
|
||||
if self.gdn_prefill_backend == "aiter_flydsl":
|
||||
# AITER carries its own reusable varlen metadata and has no use
|
||||
# for FLA's chunk indices, so it replaces them rather than
|
||||
# extending what _build_chunk_metadata returns.
|
||||
assert prefill_query_start_loc_cpu is not None
|
||||
aiter_prefill_metadata = (
|
||||
rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
|
||||
torch.diff(prefill_query_start_loc_cpu).tolist(),
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
)
|
||||
)
|
||||
else:
|
||||
chunk_indices, chunk_offsets = self._build_chunk_metadata(
|
||||
prefill_query_start_loc,
|
||||
prefill_query_start_loc_cpu,
|
||||
query_start_loc.device,
|
||||
)
|
||||
|
||||
if num_prefills > 0:
|
||||
context_lens_tensor = m.compute_num_computed_tokens()
|
||||
@@ -568,6 +612,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
prefill_query_start_loc=prefill_query_start_loc,
|
||||
prefill_state_indices=prefill_state_indices,
|
||||
prefill_has_initial_state=prefill_has_initial_state,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
spec_query_start_loc=spec_query_start_loc,
|
||||
non_spec_query_start_loc=non_spec_query_start_loc,
|
||||
spec_state_indices_tensor=spec_state_indices_tensor,
|
||||
|
||||
Reference in New Issue
Block a user