[V1] Make v1 more testable (#9888)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
This commit is contained in:
Joe Runde
2024-11-06 11:57:35 -08:00
committed by GitHub
parent 87bd7e0515
commit d58268c56a
75 changed files with 243 additions and 165 deletions
+6
View File
@@ -155,6 +155,12 @@ class LLMEngine:
# GPU and CPU blocks, which are profiled in the distributed executor.
self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
def __del__(self):
# Small hack- implicit clean up of resources on garbage collect
# TODO: this should probably be explicitly invoked when we're done with
# the engine
self.terminate_detokenizer()
def _initialize_kv_caches(self) -> None:
num_gpu_blocks, _ = self.model_executor.determine_num_available_blocks(
)