[Bugfix][MRV2] Fix FlashInfer attention and ReplaySSM warmups on MRV2 (#61013)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.worker.gpu.warmup import run_mixed_prefill_decode_warmup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def mixed_batch_attention_warmup(worker: "Worker", num_tokens: int = 16) -> None:
|
||||
"""Run a mixed prefill+decode batch to warm up both attention paths."""
|
||||
num_tokens = min(
|
||||
num_tokens,
|
||||
worker.scheduler_config.max_num_batched_tokens,
|
||||
worker.model_config.max_model_len,
|
||||
)
|
||||
if not worker.use_v2_model_runner:
|
||||
worker.model_runner._dummy_run(
|
||||
num_tokens=num_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
return
|
||||
|
||||
if not run_mixed_prefill_decode_warmup(
|
||||
cast("GPUModelRunner", worker.model_runner),
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
num_tokens,
|
||||
):
|
||||
logger.info_once(
|
||||
"Skipping mixed prefill+decode attention warmup; attention kernels "
|
||||
"will be loaded on the first request."
|
||||
)
|
||||
@@ -15,6 +15,9 @@ import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.attention_warmup import (
|
||||
mixed_batch_attention_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.b12x_warmup import b12x_warmup
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
@@ -286,15 +289,7 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
)
|
||||
):
|
||||
logger.info_once("Warming up FlashInfer attention.")
|
||||
# Warmup with mixed batch containing both prefill and decode tokens
|
||||
# This is to warm up both prefill and decode attention kernels
|
||||
worker.model_runner._dummy_run(
|
||||
num_tokens=16,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
mixed_batch_attention_warmup(worker)
|
||||
|
||||
|
||||
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
|
||||
|
||||
@@ -4,6 +4,9 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.attention_warmup import (
|
||||
mixed_batch_attention_warmup,
|
||||
)
|
||||
from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tracing import instrument
|
||||
@@ -34,10 +37,4 @@ def minimax_m3_msa_warmup(worker: "Worker") -> None:
|
||||
logger.info("Warming up MiniMax M3 MSA kernels.")
|
||||
|
||||
# Cover sparse prefill through the normal model path.
|
||||
worker.model_runner._dummy_run(
|
||||
num_tokens=16,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
mixed_batch_attention_warmup(worker)
|
||||
|
||||
Reference in New Issue
Block a user