[ModelRunner V2] Warmup cross-attn properly in encoder-decoder case (#47308)

This commit is contained in:
Nick Hill
2026-07-01 12:36:48 -07:00
committed by GitHub
parent 5fd442187c
commit 8cfeb84dba
+27 -2
View File
@@ -10,6 +10,7 @@ import torch
from vllm import PoolingParams, SamplingParams
from vllm.logger import init_logger
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.utils.math_utils import cdiv
from vllm.v1.core.sched.output import (
CachedRequestData,
@@ -17,7 +18,7 @@ from vllm.v1.core.sched.output import (
NewRequestData,
SchedulerOutput,
)
from vllm.v1.kv_cache_interface import MambaSpec
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
from vllm.v1.request import Request
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
@@ -177,8 +178,26 @@ def warmup_kernels(
kv_cache_groups = model_runner.kv_cache_config.kv_cache_groups
num_kv_cache_groups = len(kv_cache_groups)
# Encoder-decoder models: give each warmup request a dummy encoder input so
# cross-attention warms up over a realistic, non-empty key sequence.
# The dummy mm_feature is registered in the encoder cache and only its encoder
# length is read (not the inputs themselves); the encoder itself is not scheduled.
max_encoder_len = getattr(model_runner.model_state, "max_encoder_len", 0)
warmup_mm_features: list[MultiModalFeatureSpec] = []
if model_runner.is_encoder_decoder and max_encoder_len:
warmup_mm_features = [
MultiModalFeatureSpec(
data=None,
modality="",
identifier="_warmup_encoder",
mm_position=PlaceholderRange(offset=0, length=max_encoder_len),
)
]
# Compute per-request block counts for each KV cache group.
def _warmup_block_count(num_tokens: int, spec: Any) -> int:
if isinstance(spec, CrossAttentionSpec):
num_tokens = max_encoder_len
num_blocks = cdiv(num_tokens, spec.block_size)
if isinstance(spec, MambaSpec) and spec.mamba_cache_mode == "align":
# Align mode reserves extra blocks beyond the token range for the
@@ -222,7 +241,13 @@ def warmup_kernels(
# Step 1: Prefill all requests with 1 + decode_query_len prompt tokens each.
new_reqs = [
NewRequestData.from_request(
Request(req_ids[i], prompt_token_ids, sampling_params, pooling_params),
Request(
req_ids[i],
prompt_token_ids,
sampling_params,
pooling_params,
mm_features=warmup_mm_features,
),
block_ids=tuple(_alloc_blocks(n) for n in prefill_block_counts),
prefill_token_ids=prompt_token_ids,
)