[Bugfix] Defer adaptive verification until after kernel warmup (#55455)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -88,6 +88,7 @@ def _make_runner(
|
||||
"""Stub model runner exposing only what the warmup entry points read."""
|
||||
return SimpleNamespace(
|
||||
num_speculative_steps=num_spec_steps,
|
||||
adaptive_verification=None,
|
||||
decode_query_len=num_spec_steps + 1,
|
||||
is_pooling_model=False,
|
||||
is_encoder_decoder=False,
|
||||
|
||||
@@ -6,6 +6,7 @@ from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.v1.worker.gpu import warmup
|
||||
from vllm.v1.worker.gpu.warmup import run_mixed_prefill_decode_warmup
|
||||
|
||||
|
||||
@@ -28,3 +29,24 @@ def test_mixed_warmup_skipped_for_single_seq(max_num_reqs):
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail_warmup", [False, True])
|
||||
def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_warmup):
|
||||
"""Startup must warm fixed drafts before calibration and retain its manager."""
|
||||
manager = SimpleNamespace(cost_tables=None)
|
||||
runner = SimpleNamespace(adaptive_verification=manager)
|
||||
|
||||
def run_steps(model_runner, execute, sample):
|
||||
assert model_runner.adaptive_verification is None
|
||||
if fail_warmup:
|
||||
raise RuntimeError("warmup failed")
|
||||
|
||||
monkeypatch.setattr(warmup, "_warmup_kernels", run_steps)
|
||||
if fail_warmup:
|
||||
with pytest.raises(RuntimeError, match="warmup failed"):
|
||||
warmup.warmup_kernels(runner, _fail, _fail)
|
||||
else:
|
||||
warmup.warmup_kernels(runner, _fail, _fail)
|
||||
assert runner.adaptive_verification is manager
|
||||
assert manager.cost_tables is None
|
||||
|
||||
@@ -220,6 +220,21 @@ def warmup_kernels(
|
||||
We must call the provided worker's execute_model for pipeline parallel
|
||||
coordination.
|
||||
"""
|
||||
# Adaptive costs are calibrated during capture, after this warmup. Exercise
|
||||
# fixed draft counts here, then restore the manager for capture and serving.
|
||||
adaptive_verification = model_runner.adaptive_verification
|
||||
model_runner.adaptive_verification = None
|
||||
try:
|
||||
_warmup_kernels(model_runner, worker_execute_model, worker_sample_tokens)
|
||||
finally:
|
||||
model_runner.adaptive_verification = adaptive_verification
|
||||
|
||||
|
||||
def _warmup_kernels(
|
||||
model_runner: GPUModelRunner,
|
||||
worker_execute_model: Callable[[SchedulerOutput], Any],
|
||||
worker_sample_tokens: Callable[[GrammarOutput | None], Any],
|
||||
) -> None:
|
||||
if model_runner.vllm_config.is_mm_encoder_only:
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user