[ROCm][Spec Decode] Add Aiter MLA decode support non-causal draft block (#55966)
Signed-off-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com> Co-authored-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com> Co-authored-by: John Qin <zhengsheng1997@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com>
This commit is contained in:
co-authored by
Poovaiah Palangappa
John Qin
Cursor
Hongxia Yang
parent
241e9391ed
commit
35dc273072
+26
-1
@@ -583,10 +583,11 @@ def _rocm_aiter_mla_decode_fwd_impl(
|
||||
reduce_indptr: torch.Tensor | None = None,
|
||||
reduce_final_map: torch.Tensor | None = None,
|
||||
reduce_partial_map: torch.Tensor | None = None,
|
||||
causal: bool = True,
|
||||
) -> None:
|
||||
from aiter.mla import mla_decode_fwd
|
||||
|
||||
kwargs: dict[str, float | torch.Tensor | None] = {
|
||||
kwargs: dict[str, float | torch.Tensor | None | bool] = {
|
||||
"sm_scale": sm_scale,
|
||||
"logit_cap": logit_cap,
|
||||
}
|
||||
@@ -596,6 +597,8 @@ def _rocm_aiter_mla_decode_fwd_impl(
|
||||
kwargs["q_scale"] = q_scale
|
||||
kwargs["kv_scale"] = kv_scale
|
||||
|
||||
kwargs["causal"] = causal
|
||||
|
||||
if work_meta_data is not None:
|
||||
assert work_indptr is not None, (
|
||||
"work_indptr must be provided with work_meta_data"
|
||||
@@ -2036,6 +2039,26 @@ class rocm_aiter_ops:
|
||||
def is_mla_enabled(cls) -> bool:
|
||||
return cls._AITER_ENABLED and cls._MLA_ENABLED
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def mla_decode_supports_non_causal(cls) -> bool:
|
||||
"""Whether installed aiter.mla.mla_decode_fwd accepts `causal`.
|
||||
|
||||
Added in aiter v0.1.20. A missing argument is a hard error rather
|
||||
than a boolean the caller can use to pick another MLA backend.
|
||||
"""
|
||||
import inspect
|
||||
|
||||
from aiter.mla import mla_decode_fwd
|
||||
|
||||
if "causal" not in inspect.signature(mla_decode_fwd).parameters:
|
||||
raise RuntimeError(
|
||||
"ROCM_AITER_MLA requires aiter.mla.mla_decode_fwd(..., causal=). "
|
||||
"The installed aiter is causal-only; upgrade aiter rather than "
|
||||
"falling back to another MLA backend."
|
||||
)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_mha_enabled(cls) -> bool:
|
||||
@@ -2858,6 +2881,7 @@ class rocm_aiter_ops:
|
||||
reduce_indptr: torch.Tensor | None = None,
|
||||
reduce_final_map: torch.Tensor | None = None,
|
||||
reduce_partial_map: torch.Tensor | None = None,
|
||||
causal: bool = True,
|
||||
):
|
||||
torch.ops.vllm.rocm_aiter_mla_decode_fwd(
|
||||
q,
|
||||
@@ -2878,6 +2902,7 @@ class rocm_aiter_ops:
|
||||
reduce_indptr=reduce_indptr,
|
||||
reduce_final_map=reduce_final_map,
|
||||
reduce_partial_map=reduce_partial_map,
|
||||
causal=causal,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user