Release prompt_embeds tensor when its InputBatch slot is freed (#57988)
Signed-off-by: khushali9 <khushali.desai9@gmail.com>
This commit is contained in:
@@ -541,6 +541,7 @@ class InputBatch:
|
||||
self.req_output_token_ids[req_index] = None
|
||||
self.spec_token_ids[req_index].clear()
|
||||
self.block_table.clear_row(req_index)
|
||||
self.req_prompt_embeds.pop(req_index, None)
|
||||
|
||||
# LoRA
|
||||
lora_id = self.request_lora_mapping[req_index]
|
||||
|
||||
Reference in New Issue
Block a user