[LoRA] LoRA cuda graph specialization (#25914)
Signed-off-by: Andy Lo <andy@mistral.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -40,13 +40,19 @@ class BatchDescriptor(NamedTuple):
|
||||
False can also be used for an uniform decode batch to dispatch to the
|
||||
cudagraph supporting non-uniform batches.
|
||||
"""
|
||||
has_lora: bool = False
|
||||
"""
|
||||
Whether this batch has active LoRA adapters.
|
||||
"""
|
||||
|
||||
@property
|
||||
def non_uniform(self) -> "BatchDescriptor":
|
||||
"""
|
||||
Return a non-uniform version of current batch descriptor.
|
||||
"""
|
||||
return BatchDescriptor(self.num_tokens, uniform_decode=False)
|
||||
return BatchDescriptor(
|
||||
self.num_tokens, uniform_decode=False, has_lora=self.has_lora
|
||||
)
|
||||
|
||||
|
||||
def _compute_sp_num_tokens(
|
||||
|
||||
Reference in New Issue
Block a user