Use CuPy for CUDA graphs (#2811)

This commit is contained in:
Woosuk Kwon
2024-02-13 11:32:06 -08:00
committed by GitHub
parent ea356004d4
commit a463c333dd
8 changed files with 246 additions and 22 deletions
+1 -1
View File
@@ -283,7 +283,7 @@ class LLMEngine:
is_driver_worker=True,
)
self._run_workers("init_model")
self._run_workers("init_model", cupy_port=get_open_port())
self._run_workers(
"load_model",
max_concurrent_workers=self.parallel_config.