[Perf] Integrate FlashInfer KDA kernels (#55364)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
@@ -772,7 +772,10 @@ class EngineArgs:
|
||||
|
||||
fail_on_environ_validation: bool = False
|
||||
gdn_prefill_backend: Literal["flashinfer", "triton", "cutedsl"] | None = None
|
||||
kda_prefill_backend: Literal["auto", "triton", "flashkda"] | None = None
|
||||
kda_prefill_backend: Literal["auto", "triton", "flashkda", "flashinfer"] | None = (
|
||||
None
|
||||
)
|
||||
kda_decode_backend: Literal["auto", "native", "flashinfer", "triton"] | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
# support `EngineArgs(compilation_config={...})`
|
||||
@@ -1747,10 +1750,17 @@ class EngineArgs:
|
||||
parser.add_argument(
|
||||
"--kda-prefill-backend",
|
||||
dest="kda_prefill_backend",
|
||||
choices=["auto", "triton", "flashkda"],
|
||||
choices=["auto", "triton", "flashkda", "flashinfer"],
|
||||
default=None,
|
||||
help="Select KDA prefill backend.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--kda-decode-backend",
|
||||
dest="kda_decode_backend",
|
||||
choices=["auto", "native", "flashinfer", "triton"],
|
||||
default=None,
|
||||
help="Select KDA decode backend.",
|
||||
)
|
||||
return parser
|
||||
|
||||
@classmethod
|
||||
@@ -2567,6 +2577,8 @@ class EngineArgs:
|
||||
self.additional_config["gdn_prefill_backend"] = self.gdn_prefill_backend
|
||||
if self.kda_prefill_backend is not None:
|
||||
self.additional_config["kda_prefill_backend"] = self.kda_prefill_backend
|
||||
if self.kda_decode_backend is not None:
|
||||
self.additional_config["kda_decode_backend"] = self.kda_decode_backend
|
||||
|
||||
config = VllmConfig(
|
||||
model_config=model_config,
|
||||
|
||||
Reference in New Issue
Block a user