[Core] Warm up runner-owned Triton kernels before the first request (#49903)

This commit is contained in:
Nick Hill
2026-07-28 07:58:02 -07:00
committed by GitHub
parent 1e81853afc
commit 0d0504b54c
7 changed files with 183 additions and 189 deletions
+85 -36
View File
@@ -157,13 +157,10 @@ def warmup_kernels(
worker_execute_model: Callable[[SchedulerOutput], Any],
worker_sample_tokens: Callable[[GrammarOutput | None], Any],
) -> None:
"""Run two execute_model + sample_tokens iterations to JIT compile
triton kernels. We must call the provided worker's execute_model for
pipeline parallel coordination.
"""Run scheduler-realistic prefill and decode steps to JIT compile kernels.
The first iteration simulates a prefill with requests of
decode_query_len + 1 prompt tokens each. The second iteration simulates
a decode step with all requests generating decode_query_len tokens.
We must call the provided worker's execute_model for pipeline parallel
coordination.
"""
num_spec_steps = model_runner.num_speculative_steps
decode_query_len = model_runner.decode_query_len
@@ -172,8 +169,13 @@ def warmup_kernels(
# a uniform decode batch.
prompt_len = decode_query_len + 1
prompt_token_ids = list(range(prompt_len))
# After prefill, decode generates decode_query_len tokens.
decode_len = prompt_len + decode_query_len
# Upper bound on the decode steps built in `decode_steps` below.
num_decode_steps = 1
if not model_runner.is_pooling_model:
num_decode_steps = 5 if num_spec_steps > 0 else 3
# Size the block allocation for the worst case: every request advancing
# decode_query_len tokens on every decode step.
decode_len = prompt_len + num_decode_steps * decode_query_len
kv_cache_groups = model_runner.kv_cache_config.kv_cache_groups
num_kv_cache_groups = len(kv_cache_groups)
@@ -208,9 +210,6 @@ def warmup_kernels(
kv_cache_specs = [g.kv_cache_spec for g in kv_cache_groups]
prefill_block_counts = [_warmup_block_count(prompt_len, s) for s in kv_cache_specs]
decode_block_counts = [_warmup_block_count(decode_len, s) for s in kv_cache_specs]
decode_block_deltas = [
d - p for d, p in zip(decode_block_counts, prefill_block_counts)
]
max_blocks_per_req = sum(decode_block_counts)
num_reqs = min(
@@ -243,6 +242,11 @@ def warmup_kernels(
nonlocal next_block_id
return list(range(next_block_id, next_block_id := next_block_id + num_blocks))
# The KV-block zeroing kernel is driven by the scheduler's
# new_block_ids_to_zero, so none of the steps below reach it.
if model_runner.kv_block_zeroer is not None:
model_runner.kv_block_zeroer.warmup(model_runner.kv_cache_config.num_blocks)
# Step 1: Prefill all requests with 1 + decode_query_len prompt tokens each.
new_reqs = [
NewRequestData.from_request(
@@ -287,33 +291,78 @@ def warmup_kernels(
worker_sample_tokens(grammar_output)
# Step 2: Decode all requests with decode_query_len tokens each.
cached_req_data = CachedRequestData.make_empty()
cached_req_data.req_ids = list(req_ids)
cached_req_data.num_computed_tokens = [prompt_len] * num_reqs
cached_req_data.num_output_tokens = [1] * num_reqs
new_block = any(decode_block_deltas)
cached_req_data.new_block_ids = [
tuple(_alloc_blocks(n) for n in decode_block_deltas) if new_block else None
for _ in range(num_reqs)
# Per-request state carried across the decode steps.
req_computed = [prompt_len] * num_reqs
req_blocks = [list(prefill_block_counts) for _ in range(num_reqs)]
def _run_decode_step(indices: list[int], spec_flags: list[bool]) -> None:
"""Decode `indices`, spec-decoding the ones flagged in `spec_flags`."""
cached_req_data = CachedRequestData.make_empty()
cached_req_data.req_ids = [req_ids[i] for i in indices]
cached_req_data.num_computed_tokens = [req_computed[i] for i in indices]
cached_req_data.num_output_tokens = [1] * len(indices)
cached_req_data.new_block_ids = []
step_num_scheduled_tokens: dict[str, int] = {}
step_spec_tokens: dict[str, list[int]] = {}
for i, use_spec in zip(indices, spec_flags):
num_tokens = decode_query_len if use_spec else 1
after = req_computed[i] + num_tokens
deltas = [
_warmup_block_count(after, spec) - held
for spec, held in zip(kv_cache_specs, req_blocks[i])
]
cached_req_data.new_block_ids.append(
tuple(_alloc_blocks(n) for n in deltas) if any(deltas) else None
)
req_blocks[i] = [
held + delta for held, delta in zip(req_blocks[i], deltas)
]
step_num_scheduled_tokens[req_ids[i]] = num_tokens
if use_spec:
step_spec_tokens[req_ids[i]] = [0] * num_spec_steps
decode_output = SchedulerOutput.make_empty()
decode_output.scheduled_cached_reqs = cached_req_data
decode_output.num_scheduled_tokens = step_num_scheduled_tokens
decode_output.scheduled_spec_decode_tokens = step_spec_tokens
decode_output.total_num_scheduled_tokens = sum(
step_num_scheduled_tokens.values()
)
decode_output.num_common_prefix_blocks = [0] * num_kv_cache_groups
worker_execute_model(decode_output)
worker_sample_tokens(None)
for i, use_spec in zip(indices, spec_flags):
req_computed[i] += decode_query_len if use_spec else 1
all_indices = list(range(num_reqs))
use_spec_decode = num_spec_steps > 0
# Decode steps to warm, as (request indices, per-request spec flag).
# Under spec decoding the scheduler drops requests the drafter proposed
# nothing for, so warm each batch shape with and without draft tokens.
decode_steps: list[tuple[list[int], list[bool]]] = [
(all_indices, [use_spec_decode] * num_reqs),
]
if num_reqs >= 2:
# Mixed spec / non-spec: GDN and KDA reclassify the non-spec decode
# as a prefill and split the batch into spec/non-spec token indices.
decode_steps.append(([0, 1], [use_spec_decode, False]))
if use_spec_decode:
# Exercise the model paths that split a batch by whether each
# request received draft tokens.
decode_steps.append(([0, 1], [False, False]))
if num_reqs > 1:
decode_steps.append(([0], [use_spec_decode]))
if use_spec_decode:
decode_steps.append(([0], [False]))
elif use_spec_decode:
decode_steps.append(([0], [False]))
decode_output = SchedulerOutput.make_empty()
decode_output.scheduled_cached_reqs = cached_req_data
decode_output.num_scheduled_tokens = {
req_id: decode_query_len for req_id in req_ids
}
if num_spec_steps > 0:
decode_output.scheduled_spec_decode_tokens = {
req_id: [0] * num_spec_steps for req_id in req_ids
}
decode_output.total_num_scheduled_tokens = sum(
decode_output.num_scheduled_tokens.values()
)
decode_output.num_common_prefix_blocks = [0] * num_kv_cache_groups
worker_execute_model(decode_output)
worker_sample_tokens(None)
for step_indices, step_spec_flags in decode_steps:
_run_decode_step(step_indices, step_spec_flags)
# Clean up - process finish_req_ids.
cleanup_output = SchedulerOutput.make_empty()