[Core] Warm up runner-owned Triton kernels before the first request (#49903)
This commit is contained in:
@@ -157,13 +157,10 @@ def warmup_kernels(
|
||||
worker_execute_model: Callable[[SchedulerOutput], Any],
|
||||
worker_sample_tokens: Callable[[GrammarOutput | None], Any],
|
||||
) -> None:
|
||||
"""Run two execute_model + sample_tokens iterations to JIT compile
|
||||
triton kernels. We must call the provided worker's execute_model for
|
||||
pipeline parallel coordination.
|
||||
"""Run scheduler-realistic prefill and decode steps to JIT compile kernels.
|
||||
|
||||
The first iteration simulates a prefill with requests of
|
||||
decode_query_len + 1 prompt tokens each. The second iteration simulates
|
||||
a decode step with all requests generating decode_query_len tokens.
|
||||
We must call the provided worker's execute_model for pipeline parallel
|
||||
coordination.
|
||||
"""
|
||||
num_spec_steps = model_runner.num_speculative_steps
|
||||
decode_query_len = model_runner.decode_query_len
|
||||
@@ -172,8 +169,13 @@ def warmup_kernels(
|
||||
# a uniform decode batch.
|
||||
prompt_len = decode_query_len + 1
|
||||
prompt_token_ids = list(range(prompt_len))
|
||||
# After prefill, decode generates decode_query_len tokens.
|
||||
decode_len = prompt_len + decode_query_len
|
||||
# Upper bound on the decode steps built in `decode_steps` below.
|
||||
num_decode_steps = 1
|
||||
if not model_runner.is_pooling_model:
|
||||
num_decode_steps = 5 if num_spec_steps > 0 else 3
|
||||
# Size the block allocation for the worst case: every request advancing
|
||||
# decode_query_len tokens on every decode step.
|
||||
decode_len = prompt_len + num_decode_steps * decode_query_len
|
||||
|
||||
kv_cache_groups = model_runner.kv_cache_config.kv_cache_groups
|
||||
num_kv_cache_groups = len(kv_cache_groups)
|
||||
@@ -208,9 +210,6 @@ def warmup_kernels(
|
||||
kv_cache_specs = [g.kv_cache_spec for g in kv_cache_groups]
|
||||
prefill_block_counts = [_warmup_block_count(prompt_len, s) for s in kv_cache_specs]
|
||||
decode_block_counts = [_warmup_block_count(decode_len, s) for s in kv_cache_specs]
|
||||
decode_block_deltas = [
|
||||
d - p for d, p in zip(decode_block_counts, prefill_block_counts)
|
||||
]
|
||||
max_blocks_per_req = sum(decode_block_counts)
|
||||
|
||||
num_reqs = min(
|
||||
@@ -243,6 +242,11 @@ def warmup_kernels(
|
||||
nonlocal next_block_id
|
||||
return list(range(next_block_id, next_block_id := next_block_id + num_blocks))
|
||||
|
||||
# The KV-block zeroing kernel is driven by the scheduler's
|
||||
# new_block_ids_to_zero, so none of the steps below reach it.
|
||||
if model_runner.kv_block_zeroer is not None:
|
||||
model_runner.kv_block_zeroer.warmup(model_runner.kv_cache_config.num_blocks)
|
||||
|
||||
# Step 1: Prefill all requests with 1 + decode_query_len prompt tokens each.
|
||||
new_reqs = [
|
||||
NewRequestData.from_request(
|
||||
@@ -287,33 +291,78 @@ def warmup_kernels(
|
||||
|
||||
worker_sample_tokens(grammar_output)
|
||||
|
||||
# Step 2: Decode all requests with decode_query_len tokens each.
|
||||
cached_req_data = CachedRequestData.make_empty()
|
||||
cached_req_data.req_ids = list(req_ids)
|
||||
cached_req_data.num_computed_tokens = [prompt_len] * num_reqs
|
||||
cached_req_data.num_output_tokens = [1] * num_reqs
|
||||
new_block = any(decode_block_deltas)
|
||||
cached_req_data.new_block_ids = [
|
||||
tuple(_alloc_blocks(n) for n in decode_block_deltas) if new_block else None
|
||||
for _ in range(num_reqs)
|
||||
# Per-request state carried across the decode steps.
|
||||
req_computed = [prompt_len] * num_reqs
|
||||
req_blocks = [list(prefill_block_counts) for _ in range(num_reqs)]
|
||||
|
||||
def _run_decode_step(indices: list[int], spec_flags: list[bool]) -> None:
|
||||
"""Decode `indices`, spec-decoding the ones flagged in `spec_flags`."""
|
||||
cached_req_data = CachedRequestData.make_empty()
|
||||
cached_req_data.req_ids = [req_ids[i] for i in indices]
|
||||
cached_req_data.num_computed_tokens = [req_computed[i] for i in indices]
|
||||
cached_req_data.num_output_tokens = [1] * len(indices)
|
||||
cached_req_data.new_block_ids = []
|
||||
|
||||
step_num_scheduled_tokens: dict[str, int] = {}
|
||||
step_spec_tokens: dict[str, list[int]] = {}
|
||||
for i, use_spec in zip(indices, spec_flags):
|
||||
num_tokens = decode_query_len if use_spec else 1
|
||||
after = req_computed[i] + num_tokens
|
||||
deltas = [
|
||||
_warmup_block_count(after, spec) - held
|
||||
for spec, held in zip(kv_cache_specs, req_blocks[i])
|
||||
]
|
||||
cached_req_data.new_block_ids.append(
|
||||
tuple(_alloc_blocks(n) for n in deltas) if any(deltas) else None
|
||||
)
|
||||
req_blocks[i] = [
|
||||
held + delta for held, delta in zip(req_blocks[i], deltas)
|
||||
]
|
||||
step_num_scheduled_tokens[req_ids[i]] = num_tokens
|
||||
if use_spec:
|
||||
step_spec_tokens[req_ids[i]] = [0] * num_spec_steps
|
||||
|
||||
decode_output = SchedulerOutput.make_empty()
|
||||
decode_output.scheduled_cached_reqs = cached_req_data
|
||||
decode_output.num_scheduled_tokens = step_num_scheduled_tokens
|
||||
decode_output.scheduled_spec_decode_tokens = step_spec_tokens
|
||||
decode_output.total_num_scheduled_tokens = sum(
|
||||
step_num_scheduled_tokens.values()
|
||||
)
|
||||
decode_output.num_common_prefix_blocks = [0] * num_kv_cache_groups
|
||||
|
||||
worker_execute_model(decode_output)
|
||||
worker_sample_tokens(None)
|
||||
|
||||
for i, use_spec in zip(indices, spec_flags):
|
||||
req_computed[i] += decode_query_len if use_spec else 1
|
||||
|
||||
all_indices = list(range(num_reqs))
|
||||
use_spec_decode = num_spec_steps > 0
|
||||
|
||||
# Decode steps to warm, as (request indices, per-request spec flag).
|
||||
# Under spec decoding the scheduler drops requests the drafter proposed
|
||||
# nothing for, so warm each batch shape with and without draft tokens.
|
||||
decode_steps: list[tuple[list[int], list[bool]]] = [
|
||||
(all_indices, [use_spec_decode] * num_reqs),
|
||||
]
|
||||
if num_reqs >= 2:
|
||||
# Mixed spec / non-spec: GDN and KDA reclassify the non-spec decode
|
||||
# as a prefill and split the batch into spec/non-spec token indices.
|
||||
decode_steps.append(([0, 1], [use_spec_decode, False]))
|
||||
if use_spec_decode:
|
||||
# Exercise the model paths that split a batch by whether each
|
||||
# request received draft tokens.
|
||||
decode_steps.append(([0, 1], [False, False]))
|
||||
if num_reqs > 1:
|
||||
decode_steps.append(([0], [use_spec_decode]))
|
||||
if use_spec_decode:
|
||||
decode_steps.append(([0], [False]))
|
||||
elif use_spec_decode:
|
||||
decode_steps.append(([0], [False]))
|
||||
|
||||
decode_output = SchedulerOutput.make_empty()
|
||||
decode_output.scheduled_cached_reqs = cached_req_data
|
||||
decode_output.num_scheduled_tokens = {
|
||||
req_id: decode_query_len for req_id in req_ids
|
||||
}
|
||||
if num_spec_steps > 0:
|
||||
decode_output.scheduled_spec_decode_tokens = {
|
||||
req_id: [0] * num_spec_steps for req_id in req_ids
|
||||
}
|
||||
decode_output.total_num_scheduled_tokens = sum(
|
||||
decode_output.num_scheduled_tokens.values()
|
||||
)
|
||||
decode_output.num_common_prefix_blocks = [0] * num_kv_cache_groups
|
||||
|
||||
worker_execute_model(decode_output)
|
||||
worker_sample_tokens(None)
|
||||
for step_indices, step_spec_flags in decode_steps:
|
||||
_run_decode_step(step_indices, step_spec_flags)
|
||||
|
||||
# Clean up - process finish_req_ids.
|
||||
cleanup_output = SchedulerOutput.make_empty()
|
||||
|
||||
Reference in New Issue
Block a user