[Bugfix] Set worker runtime threads before profiling and compilation (#55891)
Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Codex
Nick Hill
parent
e581e14002
commit
94f4170df3
@@ -550,6 +550,11 @@ class Worker(WorkerBase):
|
||||
self.model_runner.get_model(),
|
||||
)
|
||||
|
||||
# Preserve parallel weight loading, then use serving's thread count
|
||||
# for profiling and compilation so Dynamo's global-state guards remain
|
||||
# valid when requests arrive.
|
||||
set_torch_threads_for_runtime()
|
||||
|
||||
def update_config(self, overrides: dict[str, Any]) -> None:
|
||||
self.model_runner.update_config(overrides)
|
||||
|
||||
@@ -978,10 +983,6 @@ class Worker(WorkerBase):
|
||||
# gate so subsequent `execute_model` / `sample_tokens` calls enforce it.
|
||||
enable_gpu_sync_check()
|
||||
|
||||
# Startup is done; steady-state serving gets no benefit from torch
|
||||
# intra-op parallelism.
|
||||
set_torch_threads_for_runtime()
|
||||
|
||||
return CompilationTimes(
|
||||
language_model=self.compilation_config.compilation_time,
|
||||
encoder=self.compilation_config.encoder_compilation_time,
|
||||
|
||||
Reference in New Issue
Block a user