[ROCm][MLA] Use asm decode for non-divisor small head counts (#50578)
Signed-off-by: vanshbhatia-amd <210711135+vanshbhatia-amd@users.noreply.github.com> Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com> Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com> Co-authored-by: vanshbhatia-amd <210711135+vanshbhatia-amd@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Markus Hartikainen <markus.hartikainen@amd.com> Co-authored-by: Liuyinfeng01 <yinfeliu@amd.com>
This commit is contained in:
co-authored by
vanshbhatia-amd
Cursor
Markus Hartikainen
Liuyinfeng01
parent
7b9f2dad89
commit
d8eabdbfbe
@@ -133,6 +133,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4: bool = False
|
||||
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
|
||||
VLLM_ROCM_USE_AITER_MLA: bool = True
|
||||
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
|
||||
VLLM_ROCM_USE_AITER_MHA: bool = True
|
||||
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
|
||||
@@ -1237,6 +1238,20 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ROCM_USE_AITER_MLA": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MLA", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Small-head (<16) AITER MLA decode kernel selection. Small head counts
|
||||
# (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through
|
||||
# the Gluon small-head kernel or through the padded persistent-scheduling
|
||||
# (PS) ASM kernel. "auto" (default) keeps Gluon for head counts that divide
|
||||
# 16 where a Gluon build exists (gfx950/CDNA4) and otherwise uses the padded
|
||||
# PS ASM decode; "gluon" forces the Gluon path wherever a build exists;
|
||||
# "asm" forces the padded PS ASM decode. On gfx942/CDNA3 there is no Gluon
|
||||
# build, so the ASM path is always used regardless of this setting.
|
||||
"VLLM_ROCM_AITER_MLA_ASM_PADDING": env_with_choices(
|
||||
"VLLM_ROCM_AITER_MLA_ASM_PADDING",
|
||||
"auto",
|
||||
["auto", "gluon", "asm"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
# Whether to use aiter mha ops.
|
||||
# By default is enabled.
|
||||
"VLLM_ROCM_USE_AITER_MHA": lambda: (
|
||||
|
||||
Reference in New Issue
Block a user