[Perf] Warmup FlashInfer attention during startup (#23439)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Signed-off-by: Luka Govedič <lgovedic@redhat.com> Co-authored-by: Luka Govedič <lgovedic@redhat.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Matthew Bonanni <mbonanni001@gmail.com>
This commit is contained in:
co-authored by
Luka Govedič
Luka Govedič
Matthew Bonanni
parent
b5e383cd8b
commit
fba7856581
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
@@ -19,6 +20,8 @@ if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def kernel_warmup(worker: "Worker"):
|
||||
# Deep GEMM warmup
|
||||
@@ -30,10 +33,33 @@ def kernel_warmup(worker: "Worker"):
|
||||
max_tokens = worker.scheduler_config.max_num_batched_tokens
|
||||
deep_gemm_warmup(model, max_tokens)
|
||||
|
||||
# FlashInfer autotune for Blackwell (SM 10.0) GPUs
|
||||
# FlashInfer kernel autotune for Blackwell (SM 10.0) GPUs
|
||||
if has_flashinfer() and current_platform.is_device_capability(100):
|
||||
flashinfer_autotune(worker.model_runner)
|
||||
|
||||
# FlashInfer attention warmup
|
||||
# Only warmup if the model has FlashInfer attention groups
|
||||
# and is not a pooling model
|
||||
def _is_flashinfer_backend(backend):
|
||||
try:
|
||||
return backend.get_name() == "FLASHINFER_VLLM_V1"
|
||||
except NotImplementedError:
|
||||
return False
|
||||
|
||||
if not worker.model_runner.is_pooling_model and all(
|
||||
_is_flashinfer_backend(group.backend)
|
||||
for groups in worker.model_runner.attn_groups for group in groups):
|
||||
logger.info("Warming up FlashInfer attention.")
|
||||
# Warmup with mixed batch containing both prefill and decode tokens
|
||||
# This is to warm up both prefill and decode attention kernels
|
||||
worker.model_runner._dummy_run(
|
||||
num_tokens=16,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
|
||||
|
||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user