[CI/Build] vLLM cache directory for images (#6444)

This commit is contained in:
Cyrus Leung
2024-07-15 23:12:25 -07:00
committed by GitHub
parent 37d776606f
commit d97011512e
13 changed files with 123 additions and 140 deletions
+1 -2
View File
@@ -98,8 +98,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
# Use persistent cache to avoid XLA recompilation.
# NOTE(woosuk): This does not completely eliminate the recompilation
# overhead because dynamo does not cache the compiled results.
xr.initialize_cache(os.path.expanduser(envs.VLLM_XLA_CACHE_PATH),
readonly=False)
xr.initialize_cache(envs.VLLM_XLA_CACHE_PATH, readonly=False)
def load_model(self):
self.model_runner.load_model()