[Perf][Feat] Add generic cuteDSL LL BF16 router (GEMM) (#42562)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
parent
31be872f55
commit
0762f2afeb
@@ -42,6 +42,31 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_LL_BF16_WARMUP_MODEL_SHAPES: tuple[tuple[int, int], ...] = (
|
||||
(7168, 256), # DSV3
|
||||
(7168, 384), # DSV4-Pro
|
||||
(14400, 256), # DSV4-Flash
|
||||
)
|
||||
_LL_BF16_WARMUP_M_RANGE = range(1, 17)
|
||||
|
||||
|
||||
def _warmup_ll_bf16_router_gemm() -> None:
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
|
||||
is_available as is_ll_bf16_gemm_available,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
|
||||
ll_bf16_gemm_kernel,
|
||||
)
|
||||
|
||||
if not is_ll_bf16_gemm_available():
|
||||
return
|
||||
|
||||
logger.info("Warming up ll_bf16 router GEMM kernels.")
|
||||
ll_bf16_gemm_kernel.warmup(
|
||||
shapes=_LL_BF16_WARMUP_MODEL_SHAPES,
|
||||
m_values=_LL_BF16_WARMUP_M_RANGE,
|
||||
)
|
||||
|
||||
|
||||
def kernel_warmup(worker: "Worker"):
|
||||
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
|
||||
@@ -94,6 +119,9 @@ def kernel_warmup(worker: "Worker"):
|
||||
elif has_flashinfer() and current_platform.has_device_capability(90):
|
||||
flashinfer_autotune(worker.model_runner)
|
||||
|
||||
if current_platform.has_device_capability(90):
|
||||
_warmup_ll_bf16_router_gemm()
|
||||
|
||||
# FlashInfer attention warmup
|
||||
# Only warmup if the model has FlashInfer attention groups
|
||||
# and is not a pooling model
|
||||
|
||||
Reference in New Issue
Block a user