[ROCm][Perf] Kimi-K3 Fused kernels for KDA prefill (#52606)
Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -764,7 +764,7 @@ class EngineArgs:
|
||||
|
||||
fail_on_environ_validation: bool = False
|
||||
gdn_prefill_backend: Literal["flashinfer", "triton", "cutedsl"] | None = None
|
||||
kda_prefill_backend: Literal["auto", "triton", "flashkda"] | None = None
|
||||
kda_prefill_backend: Literal["auto", "triton", "flashkda", "fused"] | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
# support `EngineArgs(compilation_config={...})`
|
||||
@@ -1720,9 +1720,10 @@ class EngineArgs:
|
||||
parser.add_argument(
|
||||
"--kda-prefill-backend",
|
||||
dest="kda_prefill_backend",
|
||||
choices=["auto", "triton", "flashkda"],
|
||||
choices=["auto", "triton", "flashkda", "fused"],
|
||||
default=None,
|
||||
help="Select KDA prefill backend.",
|
||||
help="Select KDA prefill backend. 'flashkda' is CUDA-only and "
|
||||
"'fused' is ROCm-only; 'auto' picks a supported backend.",
|
||||
)
|
||||
return parser
|
||||
|
||||
@@ -2523,6 +2524,17 @@ class EngineArgs:
|
||||
if self.gdn_prefill_backend is not None:
|
||||
self.additional_config["gdn_prefill_backend"] = self.gdn_prefill_backend
|
||||
if self.kda_prefill_backend is not None:
|
||||
if (
|
||||
self.kda_prefill_backend == "flashkda"
|
||||
and not current_platform.is_cuda()
|
||||
):
|
||||
raise ValueError(
|
||||
"--kda-prefill-backend=flashkda is only available on CUDA."
|
||||
)
|
||||
if self.kda_prefill_backend == "fused" and not current_platform.is_rocm():
|
||||
raise ValueError(
|
||||
"--kda-prefill-backend=fused is only available on ROCm."
|
||||
)
|
||||
self.additional_config["kda_prefill_backend"] = self.kda_prefill_backend
|
||||
|
||||
config = VllmConfig(
|
||||
|
||||
Reference in New Issue
Block a user