[Perf] Warmup forward_native sampler kernel (#41375)

Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Artem Perevedentsev
2026-05-01 18:04:11 +04:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent 4f7bde572a
commit c3e64696cd
+20
View File
@@ -5701,6 +5701,26 @@ class GPUModelRunner(
sampler_output = self.sampler(
logits=logits, sampling_metadata=dummy_metadata
)
# Also warm forward_native (taken when generators dict is non-empty),
# but skip the extra call in 'processed_logits' / 'processed_logprobs'
# modes — there TopKTopPSampler binds forward = forward_native at
# init time, so the warmup call is redundant and only inflates peak
# memory during profile_run.
# No .clone() of logits: warmup output is discarded, so any in-place
# mutation by forward_native does not affect correctness.
if self.sampler.logprobs_mode not in (
"processed_logits",
"processed_logprobs",
):
self.sampler(
logits=logits,
sampling_metadata=replace(
dummy_metadata,
generators={
0: torch.Generator(device=self.device).manual_seed(0)
},
),
)
except RuntimeError as e:
if "out of memory" in str(e):
raise RuntimeError(