Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+68
-58
@@ -4,6 +4,7 @@
|
||||
|
||||
Users of vLLM should always import **only** these wrappers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
@@ -44,7 +45,8 @@ def _missing(*_: Any, **__: Any) -> NoReturn:
|
||||
raise RuntimeError(
|
||||
"FlashInfer backend is not available. Please install the package "
|
||||
"to enable FlashInfer kernels: "
|
||||
"https://github.com/flashinfer-ai/flashinfer")
|
||||
"https://github.com/flashinfer-ai/flashinfer"
|
||||
)
|
||||
|
||||
|
||||
def _get_submodule(module_name: str) -> Any | None:
|
||||
@@ -56,9 +58,9 @@ def _get_submodule(module_name: str) -> Any | None:
|
||||
|
||||
|
||||
# General lazy import wrapper
|
||||
def _lazy_import_wrapper(module_name: str,
|
||||
attr_name: str,
|
||||
fallback_fn: Callable[..., Any] = _missing):
|
||||
def _lazy_import_wrapper(
|
||||
module_name: str, attr_name: str, fallback_fn: Callable[..., Any] = _missing
|
||||
):
|
||||
"""Create a lazy import wrapper for a specific function."""
|
||||
|
||||
@functools.cache
|
||||
@@ -79,29 +81,34 @@ def _lazy_import_wrapper(module_name: str,
|
||||
|
||||
# Create lazy wrappers for each function
|
||||
flashinfer_trtllm_fp8_block_scale_moe = _lazy_import_wrapper(
|
||||
"flashinfer.fused_moe", "trtllm_fp8_block_scale_moe")
|
||||
"flashinfer.fused_moe", "trtllm_fp8_block_scale_moe"
|
||||
)
|
||||
flashinfer_trtllm_fp8_per_tensor_scale_moe = _lazy_import_wrapper(
|
||||
"flashinfer.fused_moe", "trtllm_fp8_per_tensor_scale_moe")
|
||||
flashinfer_cutlass_fused_moe = _lazy_import_wrapper("flashinfer.fused_moe",
|
||||
"cutlass_fused_moe")
|
||||
"flashinfer.fused_moe", "trtllm_fp8_per_tensor_scale_moe"
|
||||
)
|
||||
flashinfer_cutlass_fused_moe = _lazy_import_wrapper(
|
||||
"flashinfer.fused_moe", "cutlass_fused_moe"
|
||||
)
|
||||
fp4_quantize = _lazy_import_wrapper("flashinfer", "fp4_quantize")
|
||||
nvfp4_block_scale_interleave = _lazy_import_wrapper(
|
||||
"flashinfer", "nvfp4_block_scale_interleave")
|
||||
"flashinfer", "nvfp4_block_scale_interleave"
|
||||
)
|
||||
trtllm_fp4_block_scale_moe = _lazy_import_wrapper(
|
||||
"flashinfer", "trtllm_fp4_block_scale_moe")
|
||||
"flashinfer", "trtllm_fp4_block_scale_moe"
|
||||
)
|
||||
|
||||
# Special case for autotune since it returns a context manager
|
||||
autotune = _lazy_import_wrapper(
|
||||
"flashinfer.autotuner",
|
||||
"autotune",
|
||||
fallback_fn=lambda *args, **kwargs: contextlib.nullcontext())
|
||||
fallback_fn=lambda *args, **kwargs: contextlib.nullcontext(),
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_comm() -> bool:
|
||||
"""Return ``True`` if FlashInfer comm module is available."""
|
||||
return has_flashinfer() and importlib.util.find_spec(
|
||||
"flashinfer.comm") is not None
|
||||
return has_flashinfer() and importlib.util.find_spec("flashinfer.comm") is not None
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -128,8 +135,10 @@ def has_flashinfer_all2all() -> bool:
|
||||
@functools.cache
|
||||
def has_flashinfer_moe() -> bool:
|
||||
"""Return ``True`` if FlashInfer MoE module is available."""
|
||||
return has_flashinfer() and importlib.util.find_spec(
|
||||
"flashinfer.fused_moe") is not None
|
||||
return (
|
||||
has_flashinfer()
|
||||
and importlib.util.find_spec("flashinfer.fused_moe") is not None
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -174,7 +183,8 @@ def has_nvidia_artifactory() -> bool:
|
||||
else:
|
||||
logger.warning_once(
|
||||
"NVIDIA artifactory returned failed status code: %d",
|
||||
response.status_code)
|
||||
response.status_code,
|
||||
)
|
||||
return accessible
|
||||
except Exception as e:
|
||||
logger.warning_once("Failed to connect to NVIDIA artifactory: %s", e)
|
||||
@@ -188,8 +198,7 @@ def supports_trtllm_attention() -> bool:
|
||||
NVIDIA artifactory is accessible
|
||||
"""
|
||||
# Requires SM100 and NVIDIA artifactory to be accessible to download cubins
|
||||
return current_platform.is_device_capability(
|
||||
100) and has_nvidia_artifactory()
|
||||
return current_platform.is_device_capability(100) and has_nvidia_artifactory()
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -238,7 +247,8 @@ def use_trtllm_attention(
|
||||
if force_use_trtllm:
|
||||
logger.warning_once(
|
||||
"TRTLLM attention is not supported on this platform, "
|
||||
"but VLLM_USE_TRTLLM_ATTENTION is set to 1")
|
||||
"but VLLM_USE_TRTLLM_ATTENTION is set to 1"
|
||||
)
|
||||
return False
|
||||
|
||||
# The combination of query and key heads is not supported
|
||||
@@ -252,8 +262,7 @@ def use_trtllm_attention(
|
||||
|
||||
if has_spec and not is_prefill:
|
||||
# Speculative decoding requires TRTLLM attention for decodes
|
||||
logger.info_once(
|
||||
"Using TRTLLM attention (enabled for speculative decoding).")
|
||||
logger.info_once("Using TRTLLM attention (enabled for speculative decoding).")
|
||||
return True
|
||||
|
||||
# Must use TRTLLM attention if query is FP8 quantized
|
||||
@@ -261,28 +270,28 @@ def use_trtllm_attention(
|
||||
if has_sinks:
|
||||
raise RuntimeError(
|
||||
"TRTLLM FP8-qkv kernel is not supported for attention sinks. "
|
||||
"Use kv_cache_dtype=auto for now.")
|
||||
"Use kv_cache_dtype=auto for now."
|
||||
)
|
||||
logger.info_once("Using TRTLLM attention (query is quantized).")
|
||||
return True
|
||||
|
||||
# If sinks are being used, we must use TRTLLM attention as it's
|
||||
# the only backend that supports them
|
||||
if has_sinks:
|
||||
logger.info_once(
|
||||
"Using TRTLLM attention (required for attention sinks).")
|
||||
logger.info_once("Using TRTLLM attention (required for attention sinks).")
|
||||
return True
|
||||
|
||||
if force_use_trtllm is None:
|
||||
# Environment variable not set - use auto-detection
|
||||
use_trtllm = (num_tokens <= 256 and max_seq_len <= 131072
|
||||
and kv_cache_dtype == "auto")
|
||||
use_trtllm = (
|
||||
num_tokens <= 256 and max_seq_len <= 131072 and kv_cache_dtype == "auto"
|
||||
)
|
||||
if use_trtllm:
|
||||
logger.warning_once("Using TRTLLM attention (auto-detected).")
|
||||
return use_trtllm
|
||||
|
||||
# Environment variable is set to 1 - respect it
|
||||
logger.info_once(
|
||||
"Using TRTLLM attention (VLLM_USE_TRTLLM_ATTENTION is set to 1)")
|
||||
logger.info_once("Using TRTLLM attention (VLLM_USE_TRTLLM_ATTENTION is set to 1)")
|
||||
return True
|
||||
|
||||
|
||||
@@ -303,16 +312,14 @@ if has_flashinfer():
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
from flashinfer import mm_fp4 as flashinfer_mm_fp4_
|
||||
return flashinfer_mm_fp4_(A,
|
||||
B,
|
||||
A_scale,
|
||||
B_scale,
|
||||
g_scale,
|
||||
dtype,
|
||||
block_size=16,
|
||||
backend=backend)
|
||||
|
||||
@torch.library.register_fake("vllm::flashinfer_mm_fp4", )
|
||||
return flashinfer_mm_fp4_(
|
||||
A, B, A_scale, B_scale, g_scale, dtype, block_size=16, backend=backend
|
||||
)
|
||||
|
||||
@torch.library.register_fake(
|
||||
"vllm::flashinfer_mm_fp4",
|
||||
)
|
||||
def flashinfer_mm_fp4_fake(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
@@ -322,10 +329,7 @@ if has_flashinfer():
|
||||
dtype: torch.dtype,
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(A.shape[0],
|
||||
B.shape[1],
|
||||
dtype=dtype,
|
||||
device=A.device)
|
||||
return torch.empty(A.shape[0], B.shape[1], dtype=dtype, device=A.device)
|
||||
|
||||
@torch.library.custom_op(
|
||||
"vllm::bmm_fp8",
|
||||
@@ -341,9 +345,12 @@ if has_flashinfer():
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
from flashinfer import bmm_fp8 as bmm_fp8_
|
||||
|
||||
return bmm_fp8_(A, B, A_scale, B_scale, dtype, None, backend)
|
||||
|
||||
@torch.library.register_fake("vllm::bmm_fp8", )
|
||||
@torch.library.register_fake(
|
||||
"vllm::bmm_fp8",
|
||||
)
|
||||
def bmm_fp8_fake(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
@@ -352,18 +359,20 @@ if has_flashinfer():
|
||||
dtype: torch.dtype,
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(A.shape[0],
|
||||
A.shape[1],
|
||||
B.shape[2],
|
||||
dtype=dtype,
|
||||
device=A.device)
|
||||
return torch.empty(
|
||||
A.shape[0], A.shape[1], B.shape[2], dtype=dtype, device=A.device
|
||||
)
|
||||
|
||||
|
||||
def flashinfer_scaled_fp4_mm(a: torch.Tensor, b: torch.Tensor,
|
||||
block_scale_a: torch.Tensor,
|
||||
block_scale_b: torch.Tensor, alpha: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
backend: str) -> torch.Tensor:
|
||||
def flashinfer_scaled_fp4_mm(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
block_scale_a: torch.Tensor,
|
||||
block_scale_b: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
assert a.ndim == 2 and b.ndim == 2
|
||||
assert block_scale_a.ndim == 2 and block_scale_b.ndim == 2
|
||||
assert a.stride(-1) == 1 and b.stride(-1) == 1
|
||||
@@ -387,12 +396,13 @@ def flashinfer_scaled_fp4_mm(a: torch.Tensor, b: torch.Tensor,
|
||||
|
||||
|
||||
def flashinfer_scaled_fp8_mm(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
assert a.ndim == 2 and b.ndim == 2
|
||||
assert a.shape[1] == b.shape[0]
|
||||
assert scale_a.numel() == 1 and scale_b.numel() == 1
|
||||
|
||||
Reference in New Issue
Block a user