[LoRA] LoRA cuda graph specialization (#25914)

Signed-off-by: Andy Lo <andy@mistral.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Andy Lo
2025-10-20 04:21:09 +00:00
committed by GitHub
co-authored by Jee Jee Li
parent f32bf7582e
commit b63f2143f8
9 changed files with 122 additions and 34 deletions
+7 -1
View File
@@ -40,13 +40,19 @@ class BatchDescriptor(NamedTuple):
False can also be used for an uniform decode batch to dispatch to the
cudagraph supporting non-uniform batches.
"""
has_lora: bool = False
"""
Whether this batch has active LoRA adapters.
"""
@property
def non_uniform(self) -> "BatchDescriptor":
"""
Return a non-uniform version of current batch descriptor.
"""
return BatchDescriptor(self.num_tokens, uniform_decode=False)
return BatchDescriptor(
self.num_tokens, uniform_decode=False, has_lora=self.has_lora
)
def _compute_sp_num_tokens(