[Bugfix] Fix gpt-oss w4a8 DP/EP on B200 (#26729)
Signed-off-by: Varun Sundar Rabindranath <vsundarr@redhat.com> Co-authored-by: Varun Sundar Rabindranath <vsundarr@redhat.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
Michael Goin
parent
f95da13c3d
commit
5ff5d94e77
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.platforms import current_platform
|
||||
@@ -24,6 +25,20 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def flashinfer_autotune_supported(vllm_config: VllmConfig) -> bool:
|
||||
"""
|
||||
Record known issues with vllm + flashinfer autotune here. Return True if
|
||||
and only if flashinfer autotune will run through without issues.
|
||||
"""
|
||||
return not (
|
||||
vllm_config.parallel_config.data_parallel_size > 1
|
||||
and (
|
||||
envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16
|
||||
or envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def kernel_warmup(worker: "Worker"):
|
||||
# Deep GEMM warmup
|
||||
do_deep_gemm_warmup = (
|
||||
@@ -37,7 +52,11 @@ def kernel_warmup(worker: "Worker"):
|
||||
deep_gemm_warmup(model, max_tokens)
|
||||
|
||||
# FlashInfer autotune for Hopper (SM 9.0) and Blackwell (SM 10.0) GPUs
|
||||
if has_flashinfer() and current_platform.has_device_capability(90):
|
||||
if (
|
||||
has_flashinfer()
|
||||
and current_platform.has_device_capability(90)
|
||||
and flashinfer_autotune_supported(worker.vllm_config)
|
||||
):
|
||||
flashinfer_autotune(worker.model_runner)
|
||||
|
||||
# FlashInfer attention warmup
|
||||
|
||||
Reference in New Issue
Block a user