[V1] Move OOM check into sampler run (#14728)

Signed-off-by: Roger Wang <ywang@roblox.com>
Co-authored-by: Simon Mo <simon.mo@hey.com>
This commit is contained in:
Roger Wang
2025-03-13 20:40:23 -07:00
committed by GitHub
co-authored by Simon Mo
parent 2a602b055a
commit ad19c8a003
2 changed files with 17 additions and 18 deletions
+12 -3
View File
@@ -1288,9 +1288,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
allowed_token_ids_mask=None,
bad_words_token_ids={},
)
sampler_output = self.model.sample(logits=logits,
sampling_metadata=dummy_metadata)
try:
sampler_output = self.model.sample(
logits=logits, sampling_metadata=dummy_metadata)
except RuntimeError as e:
if 'out of memory' in str(e):
raise RuntimeError(
"CUDA out of memory occurred when warming up sampler with "
f"{num_reqs} dummy requests. Please try lowering "
"`max_num_seqs` or `gpu_memory_utilization` when "
"initializing the engine.") from e
else:
raise e
return sampler_output
def profile_run(self) -> None: