[Bugfix] Set worker runtime threads before profiling and compilation (#55891)

Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Andreas Karatzas
2026-09-22 21:12:20 -05:00
committed by GitHub
co-authored by Codex Nick Hill
parent e581e14002
commit 94f4170df3
2 changed files with 42 additions and 4 deletions
+5 -4
View File
@@ -550,6 +550,11 @@ class Worker(WorkerBase):
self.model_runner.get_model(),
)
# Preserve parallel weight loading, then use serving's thread count
# for profiling and compilation so Dynamo's global-state guards remain
# valid when requests arrive.
set_torch_threads_for_runtime()
def update_config(self, overrides: dict[str, Any]) -> None:
self.model_runner.update_config(overrides)
@@ -978,10 +983,6 @@ class Worker(WorkerBase):
# gate so subsequent `execute_model` / `sample_tokens` calls enforce it.
enable_gpu_sync_check()
# Startup is done; steady-state serving gets no benefit from torch
# intra-op parallelism.
set_torch_threads_for_runtime()
return CompilationTimes(
language_model=self.compilation_config.compilation_time,
encoder=self.compilation_config.encoder_compilation_time,