[Spec Decode][MRV2] Warm up top-k/top-p and greedy rejection kernels for speculative decoding (#59260)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ae94e9c9e8
commit
c1407a7e86
@@ -76,7 +76,7 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
|
||||
rejection_sampler=rejection_sampler,
|
||||
)
|
||||
|
||||
def run_steps(model_runner, execute, sample):
|
||||
def run_steps(model_runner, execute, sample, greedy=False):
|
||||
assert model_runner.adaptive_verification is None
|
||||
assert not model_runner.rejection_sampler.enable_adaptive_verification
|
||||
if fail_warmup:
|
||||
|
||||
@@ -458,9 +458,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.vllm_config, self.req_states, self.is_pooling_model, config_processors
|
||||
)
|
||||
if self.is_last_pp_rank and not self.is_pooling_model:
|
||||
# V2 bypasses TopKTopPSampler, which registers native warmups.
|
||||
# ROCm only: on CUDA these warmups add ~2 min to every engine start.
|
||||
if current_platform.is_rocm():
|
||||
# Speculative verification uses native top-k/top-p masking even
|
||||
# when target-only sampling can use FlashInfer's fused sampler.
|
||||
if current_platform.is_rocm() or (
|
||||
current_platform.is_cuda() and self.speculative_config is not None
|
||||
):
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import (
|
||||
register_top_k_top_p_warmups,
|
||||
)
|
||||
|
||||
@@ -238,6 +238,12 @@ def warmup_kernels(
|
||||
rejection_sampler.enable_adaptive_verification = False
|
||||
try:
|
||||
_warmup_kernels(model_runner, worker_execute_model, worker_sample_tokens)
|
||||
if rejection_sampler is not None:
|
||||
# Greedy batches skip logits processing, so rejection kernels
|
||||
# also see model-dtype logits instead of FP32.
|
||||
_warmup_kernels(
|
||||
model_runner, worker_execute_model, worker_sample_tokens, greedy=True
|
||||
)
|
||||
finally:
|
||||
model_runner.adaptive_verification = adaptive_verification
|
||||
if adaptive_sampling:
|
||||
@@ -249,6 +255,7 @@ def _warmup_kernels(
|
||||
model_runner: GPUModelRunner,
|
||||
worker_execute_model: Callable[[SchedulerOutput], Any],
|
||||
worker_sample_tokens: Callable[[GrammarOutput | None], Any],
|
||||
greedy: bool = False,
|
||||
) -> None:
|
||||
if model_runner.vllm_config.is_mm_encoder_only:
|
||||
return
|
||||
@@ -321,7 +328,11 @@ def _warmup_kernels(
|
||||
pooling_params = PoolingParams(task=pooling_task)
|
||||
pooling_params.verify(model_runner.model_config)
|
||||
else:
|
||||
sampling_params = SamplingParams.for_sampler_warmup()
|
||||
sampling_params = (
|
||||
SamplingParams(temperature=0)
|
||||
if greedy
|
||||
else SamplingParams.for_sampler_warmup()
|
||||
)
|
||||
pooling_params = None
|
||||
|
||||
# Assign distinct block IDs per request per group. 0 null block, start from 1.
|
||||
|
||||
Reference in New Issue
Block a user