Reduce the kernel overhead when num of active loras is smaller than max loras. Multiple cuda graphs are captured for each num of active-loras. (#32005)
Signed-off-by: Yu Gong <yu3.gong@gmail.com>
This commit is contained in:
+13
-1
@@ -47,6 +47,14 @@ class BatchDescriptor(NamedTuple):
|
||||
"""
|
||||
Whether this batch has active LoRA adapters.
|
||||
"""
|
||||
num_active_loras: int = 0
|
||||
"""
|
||||
Number of distinct active LoRA adapters in this batch.
|
||||
When cudagraph_specialize_lora_count is enabled, separate CUDA graphs
|
||||
are captured for each num_active_loras value. This allows kernels
|
||||
(like fused_moe_lora) whose grid size depends on num_active_loras
|
||||
to be properly captured.
|
||||
"""
|
||||
|
||||
def relax_for_mixed_batch_cudagraphs(self) -> "BatchDescriptor":
|
||||
"""
|
||||
@@ -54,7 +62,11 @@ class BatchDescriptor(NamedTuple):
|
||||
with PIECEWISE cudagraphs (or mixed prefill-decode FA cudagraphs).
|
||||
"""
|
||||
return BatchDescriptor(
|
||||
self.num_tokens, num_reqs=None, uniform=False, has_lora=self.has_lora
|
||||
self.num_tokens,
|
||||
num_reqs=None,
|
||||
uniform=False,
|
||||
has_lora=self.has_lora,
|
||||
num_active_loras=self.num_active_loras,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user