Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+431
-354
File diff suppressed because it is too large
Load Diff
+69
-46
@@ -4,6 +4,7 @@
|
||||
|
||||
Users of vLLM should always import **only** these wrappers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
@@ -26,9 +27,14 @@ def is_deep_gemm_supported() -> bool:
|
||||
"""
|
||||
is_supported_arch = current_platform.is_cuda() and (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability(100))
|
||||
return (envs.VLLM_USE_DEEP_GEMM and has_deep_gemm() and is_supported_arch
|
||||
and not envs.VLLM_USE_FLASHINFER_MOE_FP8)
|
||||
or current_platform.is_device_capability(100)
|
||||
)
|
||||
return (
|
||||
envs.VLLM_USE_DEEP_GEMM
|
||||
and has_deep_gemm()
|
||||
and is_supported_arch
|
||||
and not envs.VLLM_USE_FLASHINFER_MOE_FP8
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -38,7 +44,8 @@ def is_deep_gemm_e8m0_used() -> bool:
|
||||
"""
|
||||
if not is_deep_gemm_supported():
|
||||
logger.debug_once(
|
||||
"DeepGEMM E8M0 disabled: DeepGEMM not supported on this system.")
|
||||
"DeepGEMM E8M0 disabled: DeepGEMM not supported on this system."
|
||||
)
|
||||
return False
|
||||
|
||||
_lazy_init()
|
||||
@@ -51,13 +58,14 @@ def is_deep_gemm_e8m0_used() -> bool:
|
||||
logger.info_once("DeepGEMM E8M0 disabled: FlashInfer MOE is enabled.")
|
||||
return False
|
||||
|
||||
if current_platform.is_device_capability(100) and \
|
||||
envs.VLLM_USE_DEEP_GEMM_E8M0:
|
||||
if current_platform.is_device_capability(100) and envs.VLLM_USE_DEEP_GEMM_E8M0:
|
||||
logger.info_once("DeepGEMM E8M0 enabled on Blackwell GPU.")
|
||||
return True
|
||||
|
||||
if current_platform.is_device_capability(90) and \
|
||||
envs.VLLM_USE_DEEP_GEMM_E8M0_HOPPER:
|
||||
if (
|
||||
current_platform.is_device_capability(90)
|
||||
and envs.VLLM_USE_DEEP_GEMM_E8M0_HOPPER
|
||||
):
|
||||
logger.info_once("DeepGEMM E8M0 enabled on Hopper GPU.")
|
||||
return True
|
||||
|
||||
@@ -69,7 +77,8 @@ def _missing(*_: Any, **__: Any) -> NoReturn:
|
||||
"""Placeholder for unavailable DeepGEMM backend."""
|
||||
raise RuntimeError(
|
||||
"DeepGEMM backend is not available or outdated. Please install or "
|
||||
"update the `deep_gemm` to a newer version to enable FP8 kernels.")
|
||||
"update the `deep_gemm` to a newer version to enable FP8 kernels."
|
||||
)
|
||||
|
||||
|
||||
_fp8_gemm_nt_impl: Callable[..., Any] | None = None
|
||||
@@ -89,21 +98,25 @@ def _lazy_init() -> None:
|
||||
global _get_mn_major_tma_aligned_tensor_impl
|
||||
|
||||
# fast path
|
||||
if (_fp8_gemm_nt_impl is not None or _grouped_impl is not None
|
||||
or _grouped_masked_impl is not None
|
||||
or _fp8_mqa_logits_impl is not None
|
||||
or _fp8_paged_mqa_logits_impl is not None
|
||||
or _get_paged_mqa_logits_metadata_impl is not None):
|
||||
if (
|
||||
_fp8_gemm_nt_impl is not None
|
||||
or _grouped_impl is not None
|
||||
or _grouped_masked_impl is not None
|
||||
or _fp8_mqa_logits_impl is not None
|
||||
or _fp8_paged_mqa_logits_impl is not None
|
||||
or _get_paged_mqa_logits_metadata_impl is not None
|
||||
):
|
||||
return
|
||||
|
||||
if not has_deep_gemm():
|
||||
return
|
||||
|
||||
# Set up deep_gemm cache path
|
||||
DEEP_GEMM_JIT_CACHE_ENV_NAME = 'DG_JIT_CACHE_DIR'
|
||||
DEEP_GEMM_JIT_CACHE_ENV_NAME = "DG_JIT_CACHE_DIR"
|
||||
if not os.environ.get(DEEP_GEMM_JIT_CACHE_ENV_NAME, None):
|
||||
os.environ[DEEP_GEMM_JIT_CACHE_ENV_NAME] = os.path.join(
|
||||
envs.VLLM_CACHE_ROOT, "deep_gemm")
|
||||
envs.VLLM_CACHE_ROOT, "deep_gemm"
|
||||
)
|
||||
|
||||
_dg = importlib.import_module("deep_gemm")
|
||||
|
||||
@@ -113,9 +126,11 @@ def _lazy_init() -> None:
|
||||
_fp8_mqa_logits_impl = getattr(_dg, "fp8_mqa_logits", None)
|
||||
_fp8_paged_mqa_logits_impl = getattr(_dg, "fp8_paged_mqa_logits", None)
|
||||
_get_paged_mqa_logits_metadata_impl = getattr(
|
||||
_dg, "get_paged_mqa_logits_metadata", None)
|
||||
_dg, "get_paged_mqa_logits_metadata", None
|
||||
)
|
||||
_get_mn_major_tma_aligned_tensor_impl = getattr(
|
||||
_dg, "get_mn_major_tma_aligned_tensor", None)
|
||||
_dg, "get_mn_major_tma_aligned_tensor", None
|
||||
)
|
||||
|
||||
|
||||
def get_num_sms() -> int:
|
||||
@@ -148,9 +163,9 @@ def m_grouped_fp8_gemm_nt_contiguous(*args, **kwargs):
|
||||
_lazy_init()
|
||||
if _grouped_impl is None:
|
||||
return _missing(*args, **kwargs)
|
||||
return _grouped_impl(*args,
|
||||
disable_ue8m0_cast=not is_deep_gemm_e8m0_used(),
|
||||
**kwargs)
|
||||
return _grouped_impl(
|
||||
*args, disable_ue8m0_cast=not is_deep_gemm_e8m0_used(), **kwargs
|
||||
)
|
||||
|
||||
|
||||
def fp8_m_grouped_gemm_nt_masked(*args, **kwargs):
|
||||
@@ -158,7 +173,8 @@ def fp8_m_grouped_gemm_nt_masked(*args, **kwargs):
|
||||
if _grouped_masked_impl is None:
|
||||
return _missing(*args, **kwargs)
|
||||
return _grouped_masked_impl(
|
||||
*args, disable_ue8m0_cast=not is_deep_gemm_e8m0_used(), **kwargs)
|
||||
*args, disable_ue8m0_cast=not is_deep_gemm_e8m0_used(), **kwargs
|
||||
)
|
||||
|
||||
|
||||
def fp8_mqa_logits(
|
||||
@@ -191,8 +207,9 @@ def fp8_mqa_logits(
|
||||
return _fp8_mqa_logits_impl(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke)
|
||||
|
||||
|
||||
def get_paged_mqa_logits_metadata(context_lens: torch.Tensor, block_size: int,
|
||||
num_sms: int) -> torch.Tensor:
|
||||
def get_paged_mqa_logits_metadata(
|
||||
context_lens: torch.Tensor, block_size: int, num_sms: int
|
||||
) -> torch.Tensor:
|
||||
"""Build scheduling metadata for paged MQA logits.
|
||||
|
||||
Args:
|
||||
@@ -208,8 +225,7 @@ def get_paged_mqa_logits_metadata(context_lens: torch.Tensor, block_size: int,
|
||||
_lazy_init()
|
||||
if _get_paged_mqa_logits_metadata_impl is None:
|
||||
return _missing()
|
||||
return _get_paged_mqa_logits_metadata_impl(context_lens, block_size,
|
||||
num_sms)
|
||||
return _get_paged_mqa_logits_metadata_impl(context_lens, block_size, num_sms)
|
||||
|
||||
|
||||
def fp8_paged_mqa_logits(
|
||||
@@ -245,14 +261,16 @@ def fp8_paged_mqa_logits(
|
||||
_lazy_init()
|
||||
if _fp8_paged_mqa_logits_impl is None:
|
||||
return _missing()
|
||||
return _fp8_paged_mqa_logits_impl(q_fp8,
|
||||
kv_cache_fp8,
|
||||
weights,
|
||||
context_lens,
|
||||
block_tables,
|
||||
schedule_metadata,
|
||||
max_model_len,
|
||||
clean_logits=True)
|
||||
return _fp8_paged_mqa_logits_impl(
|
||||
q_fp8,
|
||||
kv_cache_fp8,
|
||||
weights,
|
||||
context_lens,
|
||||
block_tables,
|
||||
schedule_metadata,
|
||||
max_model_len,
|
||||
clean_logits=True,
|
||||
)
|
||||
|
||||
|
||||
def _ceil_to_ue8m0(x: torch.Tensor):
|
||||
@@ -269,15 +287,14 @@ DEFAULT_BLOCK_SIZE = [128, 128]
|
||||
# Taken from https://github.com/deepseek-ai/DeepGEMM/blob/dd6ed14acbc7445dcef224248a77ab4d22b5f240/deep_gemm/utils/math.py#L38
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
def per_block_cast_to_fp8(
|
||||
x: torch.Tensor,
|
||||
block_size: list[int] = DEFAULT_BLOCK_SIZE,
|
||||
use_ue8m0: bool = False) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
x: torch.Tensor, block_size: list[int] = DEFAULT_BLOCK_SIZE, use_ue8m0: bool = False
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert x.dim() == 2
|
||||
m, n = x.shape
|
||||
block_m, block_n = block_size
|
||||
x_padded = torch.zeros((_align(m, block_m), _align(n, block_n)),
|
||||
dtype=x.dtype,
|
||||
device=x.device)
|
||||
x_padded = torch.zeros(
|
||||
(_align(m, block_m), _align(n, block_n)), dtype=x.dtype, device=x.device
|
||||
)
|
||||
x_padded[:m, :n] = x
|
||||
x_view = x_padded.view(-1, block_m, x_padded.size(1) // block_n, block_n)
|
||||
x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4)
|
||||
@@ -285,7 +302,8 @@ def per_block_cast_to_fp8(
|
||||
sf = _ceil_to_ue8m0(sf) if use_ue8m0 else sf
|
||||
x_scaled = (x_view * (1.0 / sf)).to(torch.float8_e4m3fn)
|
||||
return x_scaled.view_as(x_padded)[:m, :n].contiguous(), sf.view(
|
||||
x_view.size(0), x_view.size(2))
|
||||
x_view.size(0), x_view.size(2)
|
||||
)
|
||||
|
||||
|
||||
def calc_diff(x: torch.Tensor, y: torch.Tensor):
|
||||
@@ -305,13 +323,18 @@ def calc_diff(x: torch.Tensor, y: torch.Tensor):
|
||||
|
||||
|
||||
def should_use_deepgemm_for_fp8_linear(
|
||||
output_dtype: torch.dtype,
|
||||
weight: torch.Tensor,
|
||||
supports_deep_gemm: Optional[bool] = None):
|
||||
output_dtype: torch.dtype,
|
||||
weight: torch.Tensor,
|
||||
supports_deep_gemm: Optional[bool] = None,
|
||||
):
|
||||
if supports_deep_gemm is None:
|
||||
supports_deep_gemm = is_deep_gemm_supported()
|
||||
return (supports_deep_gemm and output_dtype == torch.bfloat16
|
||||
and weight.shape[0] % 128 == 0 and weight.shape[1] % 128 == 0)
|
||||
return (
|
||||
supports_deep_gemm
|
||||
and output_dtype == torch.bfloat16
|
||||
and weight.shape[0] % 128 == 0
|
||||
and weight.shape[1] % 128 == 0
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
+68
-58
@@ -4,6 +4,7 @@
|
||||
|
||||
Users of vLLM should always import **only** these wrappers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
@@ -44,7 +45,8 @@ def _missing(*_: Any, **__: Any) -> NoReturn:
|
||||
raise RuntimeError(
|
||||
"FlashInfer backend is not available. Please install the package "
|
||||
"to enable FlashInfer kernels: "
|
||||
"https://github.com/flashinfer-ai/flashinfer")
|
||||
"https://github.com/flashinfer-ai/flashinfer"
|
||||
)
|
||||
|
||||
|
||||
def _get_submodule(module_name: str) -> Any | None:
|
||||
@@ -56,9 +58,9 @@ def _get_submodule(module_name: str) -> Any | None:
|
||||
|
||||
|
||||
# General lazy import wrapper
|
||||
def _lazy_import_wrapper(module_name: str,
|
||||
attr_name: str,
|
||||
fallback_fn: Callable[..., Any] = _missing):
|
||||
def _lazy_import_wrapper(
|
||||
module_name: str, attr_name: str, fallback_fn: Callable[..., Any] = _missing
|
||||
):
|
||||
"""Create a lazy import wrapper for a specific function."""
|
||||
|
||||
@functools.cache
|
||||
@@ -79,29 +81,34 @@ def _lazy_import_wrapper(module_name: str,
|
||||
|
||||
# Create lazy wrappers for each function
|
||||
flashinfer_trtllm_fp8_block_scale_moe = _lazy_import_wrapper(
|
||||
"flashinfer.fused_moe", "trtllm_fp8_block_scale_moe")
|
||||
"flashinfer.fused_moe", "trtllm_fp8_block_scale_moe"
|
||||
)
|
||||
flashinfer_trtllm_fp8_per_tensor_scale_moe = _lazy_import_wrapper(
|
||||
"flashinfer.fused_moe", "trtllm_fp8_per_tensor_scale_moe")
|
||||
flashinfer_cutlass_fused_moe = _lazy_import_wrapper("flashinfer.fused_moe",
|
||||
"cutlass_fused_moe")
|
||||
"flashinfer.fused_moe", "trtllm_fp8_per_tensor_scale_moe"
|
||||
)
|
||||
flashinfer_cutlass_fused_moe = _lazy_import_wrapper(
|
||||
"flashinfer.fused_moe", "cutlass_fused_moe"
|
||||
)
|
||||
fp4_quantize = _lazy_import_wrapper("flashinfer", "fp4_quantize")
|
||||
nvfp4_block_scale_interleave = _lazy_import_wrapper(
|
||||
"flashinfer", "nvfp4_block_scale_interleave")
|
||||
"flashinfer", "nvfp4_block_scale_interleave"
|
||||
)
|
||||
trtllm_fp4_block_scale_moe = _lazy_import_wrapper(
|
||||
"flashinfer", "trtllm_fp4_block_scale_moe")
|
||||
"flashinfer", "trtllm_fp4_block_scale_moe"
|
||||
)
|
||||
|
||||
# Special case for autotune since it returns a context manager
|
||||
autotune = _lazy_import_wrapper(
|
||||
"flashinfer.autotuner",
|
||||
"autotune",
|
||||
fallback_fn=lambda *args, **kwargs: contextlib.nullcontext())
|
||||
fallback_fn=lambda *args, **kwargs: contextlib.nullcontext(),
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_comm() -> bool:
|
||||
"""Return ``True`` if FlashInfer comm module is available."""
|
||||
return has_flashinfer() and importlib.util.find_spec(
|
||||
"flashinfer.comm") is not None
|
||||
return has_flashinfer() and importlib.util.find_spec("flashinfer.comm") is not None
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -128,8 +135,10 @@ def has_flashinfer_all2all() -> bool:
|
||||
@functools.cache
|
||||
def has_flashinfer_moe() -> bool:
|
||||
"""Return ``True`` if FlashInfer MoE module is available."""
|
||||
return has_flashinfer() and importlib.util.find_spec(
|
||||
"flashinfer.fused_moe") is not None
|
||||
return (
|
||||
has_flashinfer()
|
||||
and importlib.util.find_spec("flashinfer.fused_moe") is not None
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -174,7 +183,8 @@ def has_nvidia_artifactory() -> bool:
|
||||
else:
|
||||
logger.warning_once(
|
||||
"NVIDIA artifactory returned failed status code: %d",
|
||||
response.status_code)
|
||||
response.status_code,
|
||||
)
|
||||
return accessible
|
||||
except Exception as e:
|
||||
logger.warning_once("Failed to connect to NVIDIA artifactory: %s", e)
|
||||
@@ -188,8 +198,7 @@ def supports_trtllm_attention() -> bool:
|
||||
NVIDIA artifactory is accessible
|
||||
"""
|
||||
# Requires SM100 and NVIDIA artifactory to be accessible to download cubins
|
||||
return current_platform.is_device_capability(
|
||||
100) and has_nvidia_artifactory()
|
||||
return current_platform.is_device_capability(100) and has_nvidia_artifactory()
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -238,7 +247,8 @@ def use_trtllm_attention(
|
||||
if force_use_trtllm:
|
||||
logger.warning_once(
|
||||
"TRTLLM attention is not supported on this platform, "
|
||||
"but VLLM_USE_TRTLLM_ATTENTION is set to 1")
|
||||
"but VLLM_USE_TRTLLM_ATTENTION is set to 1"
|
||||
)
|
||||
return False
|
||||
|
||||
# The combination of query and key heads is not supported
|
||||
@@ -252,8 +262,7 @@ def use_trtllm_attention(
|
||||
|
||||
if has_spec and not is_prefill:
|
||||
# Speculative decoding requires TRTLLM attention for decodes
|
||||
logger.info_once(
|
||||
"Using TRTLLM attention (enabled for speculative decoding).")
|
||||
logger.info_once("Using TRTLLM attention (enabled for speculative decoding).")
|
||||
return True
|
||||
|
||||
# Must use TRTLLM attention if query is FP8 quantized
|
||||
@@ -261,28 +270,28 @@ def use_trtllm_attention(
|
||||
if has_sinks:
|
||||
raise RuntimeError(
|
||||
"TRTLLM FP8-qkv kernel is not supported for attention sinks. "
|
||||
"Use kv_cache_dtype=auto for now.")
|
||||
"Use kv_cache_dtype=auto for now."
|
||||
)
|
||||
logger.info_once("Using TRTLLM attention (query is quantized).")
|
||||
return True
|
||||
|
||||
# If sinks are being used, we must use TRTLLM attention as it's
|
||||
# the only backend that supports them
|
||||
if has_sinks:
|
||||
logger.info_once(
|
||||
"Using TRTLLM attention (required for attention sinks).")
|
||||
logger.info_once("Using TRTLLM attention (required for attention sinks).")
|
||||
return True
|
||||
|
||||
if force_use_trtllm is None:
|
||||
# Environment variable not set - use auto-detection
|
||||
use_trtllm = (num_tokens <= 256 and max_seq_len <= 131072
|
||||
and kv_cache_dtype == "auto")
|
||||
use_trtllm = (
|
||||
num_tokens <= 256 and max_seq_len <= 131072 and kv_cache_dtype == "auto"
|
||||
)
|
||||
if use_trtllm:
|
||||
logger.warning_once("Using TRTLLM attention (auto-detected).")
|
||||
return use_trtllm
|
||||
|
||||
# Environment variable is set to 1 - respect it
|
||||
logger.info_once(
|
||||
"Using TRTLLM attention (VLLM_USE_TRTLLM_ATTENTION is set to 1)")
|
||||
logger.info_once("Using TRTLLM attention (VLLM_USE_TRTLLM_ATTENTION is set to 1)")
|
||||
return True
|
||||
|
||||
|
||||
@@ -303,16 +312,14 @@ if has_flashinfer():
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
from flashinfer import mm_fp4 as flashinfer_mm_fp4_
|
||||
return flashinfer_mm_fp4_(A,
|
||||
B,
|
||||
A_scale,
|
||||
B_scale,
|
||||
g_scale,
|
||||
dtype,
|
||||
block_size=16,
|
||||
backend=backend)
|
||||
|
||||
@torch.library.register_fake("vllm::flashinfer_mm_fp4", )
|
||||
return flashinfer_mm_fp4_(
|
||||
A, B, A_scale, B_scale, g_scale, dtype, block_size=16, backend=backend
|
||||
)
|
||||
|
||||
@torch.library.register_fake(
|
||||
"vllm::flashinfer_mm_fp4",
|
||||
)
|
||||
def flashinfer_mm_fp4_fake(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
@@ -322,10 +329,7 @@ if has_flashinfer():
|
||||
dtype: torch.dtype,
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(A.shape[0],
|
||||
B.shape[1],
|
||||
dtype=dtype,
|
||||
device=A.device)
|
||||
return torch.empty(A.shape[0], B.shape[1], dtype=dtype, device=A.device)
|
||||
|
||||
@torch.library.custom_op(
|
||||
"vllm::bmm_fp8",
|
||||
@@ -341,9 +345,12 @@ if has_flashinfer():
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
from flashinfer import bmm_fp8 as bmm_fp8_
|
||||
|
||||
return bmm_fp8_(A, B, A_scale, B_scale, dtype, None, backend)
|
||||
|
||||
@torch.library.register_fake("vllm::bmm_fp8", )
|
||||
@torch.library.register_fake(
|
||||
"vllm::bmm_fp8",
|
||||
)
|
||||
def bmm_fp8_fake(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
@@ -352,18 +359,20 @@ if has_flashinfer():
|
||||
dtype: torch.dtype,
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(A.shape[0],
|
||||
A.shape[1],
|
||||
B.shape[2],
|
||||
dtype=dtype,
|
||||
device=A.device)
|
||||
return torch.empty(
|
||||
A.shape[0], A.shape[1], B.shape[2], dtype=dtype, device=A.device
|
||||
)
|
||||
|
||||
|
||||
def flashinfer_scaled_fp4_mm(a: torch.Tensor, b: torch.Tensor,
|
||||
block_scale_a: torch.Tensor,
|
||||
block_scale_b: torch.Tensor, alpha: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
backend: str) -> torch.Tensor:
|
||||
def flashinfer_scaled_fp4_mm(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
block_scale_a: torch.Tensor,
|
||||
block_scale_b: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
assert a.ndim == 2 and b.ndim == 2
|
||||
assert block_scale_a.ndim == 2 and block_scale_b.ndim == 2
|
||||
assert a.stride(-1) == 1 and b.stride(-1) == 1
|
||||
@@ -387,12 +396,13 @@ def flashinfer_scaled_fp4_mm(a: torch.Tensor, b: torch.Tensor,
|
||||
|
||||
|
||||
def flashinfer_scaled_fp8_mm(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
assert a.ndim == 2 and b.ndim == 2
|
||||
assert a.shape[1] == b.shape[0]
|
||||
assert scale_a.numel() == 1 and scale_b.numel() == 1
|
||||
|
||||
+10
-6
@@ -36,8 +36,7 @@ class GCDebugConfig:
|
||||
self.top_objects = json_conf.get("top_objects", -1)
|
||||
except Exception:
|
||||
self.enabled = False
|
||||
logger.error("Failed to parse VLLM_GC_DEBUG(%s)",
|
||||
VLLM_GC_DEBUG)
|
||||
logger.error("Failed to parse VLLM_GC_DEBUG(%s)", VLLM_GC_DEBUG)
|
||||
logger.info("GC Debug Config. %s", str(self))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
@@ -70,7 +69,8 @@ class GCDebugger:
|
||||
# and top collected objects
|
||||
self.start_time_ns = time.monotonic_ns()
|
||||
self.gc_top_collected_objects = _compute_top_gc_collected_objects(
|
||||
gc.get_objects(generation), self.config.top_objects)
|
||||
gc.get_objects(generation), self.config.top_objects
|
||||
)
|
||||
elif phase == "stop":
|
||||
# After GC finished, Record GC elapsed time and
|
||||
# optionally top collected objects
|
||||
@@ -81,8 +81,11 @@ class GCDebugger:
|
||||
elpased_ms,
|
||||
str(info.get("collected", "?")),
|
||||
generation,
|
||||
(f" Top collected objects: \n{self.gc_top_collected_objects}"
|
||||
if self.gc_top_collected_objects else ""),
|
||||
(
|
||||
f" Top collected objects: \n{self.gc_top_collected_objects}"
|
||||
if self.gc_top_collected_objects
|
||||
else ""
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -125,4 +128,5 @@ def _compute_top_gc_collected_objects(objects: list[Any], top: int) -> str:
|
||||
object_types = [_compute_detailed_type(o) for o in objects]
|
||||
return "\n".join(
|
||||
f"{count:>5}:{object_type}"
|
||||
for object_type, count in Counter(object_types).most_common(top))
|
||||
for object_type, count in Counter(object_types).most_common(top)
|
||||
)
|
||||
|
||||
+14
-24
@@ -52,40 +52,35 @@ def json_iter_leaves(value: JSONTree[_T]) -> Iterable[_T]:
|
||||
def json_map_leaves(
|
||||
func: Callable[["torch.Tensor"], "torch.Tensor"],
|
||||
value: "BatchedTensorInputs",
|
||||
) -> "BatchedTensorInputs":
|
||||
...
|
||||
) -> "BatchedTensorInputs": ...
|
||||
|
||||
|
||||
@overload
|
||||
def json_map_leaves(
|
||||
func: Callable[[_T], _U],
|
||||
value: Union[_T, dict[str, _T]],
|
||||
) -> Union[_U, dict[str, _U]]:
|
||||
...
|
||||
) -> Union[_U, dict[str, _U]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def json_map_leaves(
|
||||
func: Callable[[_T], _U],
|
||||
value: Union[_T, list[_T]],
|
||||
) -> Union[_U, list[_U]]:
|
||||
...
|
||||
) -> Union[_U, list[_U]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def json_map_leaves(
|
||||
func: Callable[[_T], _U],
|
||||
value: Union[_T, tuple[_T, ...]],
|
||||
) -> Union[_U, tuple[_U, ...]]:
|
||||
...
|
||||
) -> Union[_U, tuple[_U, ...]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def json_map_leaves(
|
||||
func: Callable[[_T], _U],
|
||||
value: JSONTree[_T],
|
||||
) -> JSONTree[_U]:
|
||||
...
|
||||
) -> JSONTree[_U]: ...
|
||||
|
||||
|
||||
def json_map_leaves(
|
||||
@@ -111,8 +106,7 @@ def json_reduce_leaves(
|
||||
func: Callable[[_T, _T], _T],
|
||||
value: Union[_T, dict[str, _T]],
|
||||
/,
|
||||
) -> _T:
|
||||
...
|
||||
) -> _T: ...
|
||||
|
||||
|
||||
@overload
|
||||
@@ -120,8 +114,7 @@ def json_reduce_leaves(
|
||||
func: Callable[[_T, _T], _T],
|
||||
value: Union[_T, list[_T]],
|
||||
/,
|
||||
) -> _T:
|
||||
...
|
||||
) -> _T: ...
|
||||
|
||||
|
||||
@overload
|
||||
@@ -129,8 +122,7 @@ def json_reduce_leaves(
|
||||
func: Callable[[_T, _T], _T],
|
||||
value: Union[_T, tuple[_T, ...]],
|
||||
/,
|
||||
) -> _T:
|
||||
...
|
||||
) -> _T: ...
|
||||
|
||||
|
||||
@overload
|
||||
@@ -138,8 +130,7 @@ def json_reduce_leaves(
|
||||
func: Callable[[_T, _T], _T],
|
||||
value: JSONTree[_T],
|
||||
/,
|
||||
) -> _T:
|
||||
...
|
||||
) -> _T: ...
|
||||
|
||||
|
||||
@overload
|
||||
@@ -148,15 +139,14 @@ def json_reduce_leaves(
|
||||
value: JSONTree[_T],
|
||||
initial: _U,
|
||||
/,
|
||||
) -> _U:
|
||||
...
|
||||
) -> _U: ...
|
||||
|
||||
|
||||
def json_reduce_leaves(
|
||||
func: Callable[..., Union[_T, _U]],
|
||||
value: _JSONTree[_T],
|
||||
initial: _U = cast(_U, ...), # noqa: B008
|
||||
/,
|
||||
func: Callable[..., Union[_T, _U]],
|
||||
value: _JSONTree[_T],
|
||||
initial: _U = cast(_U, ...), # noqa: B008
|
||||
/,
|
||||
) -> Union[_T, _U]:
|
||||
"""
|
||||
Apply a function of two arguments cumulatively to each leaf in a
|
||||
|
||||
+48
-39
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import (Annotated, Any, Optional, Union, get_args, get_origin,
|
||||
get_type_hints)
|
||||
from typing import Annotated, Any, Optional, Union, get_args, get_origin, get_type_hints
|
||||
|
||||
import torch
|
||||
|
||||
@@ -11,7 +10,6 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TensorShape:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*dims: Union[int, str],
|
||||
@@ -37,8 +35,7 @@ class TensorShape:
|
||||
for dim in self.dims:
|
||||
if isinstance(dim, str):
|
||||
if dim in self.dynamic_dims:
|
||||
dim_strs.append(
|
||||
f"{dim}*") # Mark dynamic dimensions with *
|
||||
dim_strs.append(f"{dim}*") # Mark dynamic dimensions with *
|
||||
else:
|
||||
dim_strs.append(dim)
|
||||
else:
|
||||
@@ -47,7 +44,6 @@ class TensorShape:
|
||||
|
||||
|
||||
class TensorSchema:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -101,12 +97,12 @@ class TensorSchema:
|
||||
return str(list(idxs))
|
||||
|
||||
def _validate_field(
|
||||
self,
|
||||
value: object,
|
||||
field_name: str,
|
||||
expected_shape: tuple[Union[int, str], ...],
|
||||
dynamic_dims: set[str],
|
||||
leading_idxs: tuple[int, ...] = (),
|
||||
self,
|
||||
value: object,
|
||||
field_name: str,
|
||||
expected_shape: tuple[Union[int, str], ...],
|
||||
dynamic_dims: set[str],
|
||||
leading_idxs: tuple[int, ...] = (),
|
||||
) -> tuple[int, ...]:
|
||||
"""Validate a field and return the actual shape."""
|
||||
if isinstance(value, (int, float)):
|
||||
@@ -118,11 +114,13 @@ class TensorSchema:
|
||||
raise TypeError(
|
||||
f"{field_name}{self._fmt_indexer(leading_idxs)} is not "
|
||||
f"one of the expected types: int, float, Tensor, list, tuple. "
|
||||
f"Got: {type(value)}")
|
||||
f"Got: {type(value)}"
|
||||
)
|
||||
|
||||
if len(value) == 0:
|
||||
raise ValueError(f"{field_name}{self._fmt_indexer(leading_idxs)} "
|
||||
f"is an empty sequence")
|
||||
raise ValueError(
|
||||
f"{field_name}{self._fmt_indexer(leading_idxs)} is an empty sequence"
|
||||
)
|
||||
|
||||
# Ensure all tensors in the list have the same
|
||||
# shape, besides dynamic dimensions
|
||||
@@ -132,25 +130,26 @@ class TensorSchema:
|
||||
field_name,
|
||||
expected_shape[1:],
|
||||
dynamic_dims,
|
||||
leading_idxs=leading_idxs + (i, ),
|
||||
leading_idxs=leading_idxs + (i,),
|
||||
)
|
||||
|
||||
if i == 0:
|
||||
first_shape = shape
|
||||
elif not self._match_shape_with_dynamic(
|
||||
shape,
|
||||
first_shape,
|
||||
expected_shape,
|
||||
dynamic_dims,
|
||||
shape,
|
||||
first_shape,
|
||||
expected_shape,
|
||||
dynamic_dims,
|
||||
):
|
||||
raise ValueError(
|
||||
f"{field_name}{self._fmt_indexer(leading_idxs)} "
|
||||
f"contains inconsistent shapes: {first_shape} "
|
||||
f"(index 0) vs {shape} (index {i})")
|
||||
f"(index 0) vs {shape} (index {i})"
|
||||
)
|
||||
|
||||
# Treat the list as a stacked tensor:
|
||||
# shape = (len(list), *tensor.shape)
|
||||
return (len(value), ) + first_shape
|
||||
return (len(value),) + first_shape
|
||||
|
||||
def _validate_tensor_shape_expected(
|
||||
self,
|
||||
@@ -163,31 +162,38 @@ class TensorSchema:
|
||||
"""Validate that the actual tensor shape matches the expected shape."""
|
||||
|
||||
if len(actual_shape) != len(expected_shape):
|
||||
raise ValueError(f"{field_name} has rank {len(actual_shape)} "
|
||||
f"but expected {len(expected_shape)}. "
|
||||
f"Expected shape: {expected_shape}, "
|
||||
f"but got {actual_shape}")
|
||||
raise ValueError(
|
||||
f"{field_name} has rank {len(actual_shape)} "
|
||||
f"but expected {len(expected_shape)}. "
|
||||
f"Expected shape: {expected_shape}, "
|
||||
f"but got {actual_shape}"
|
||||
)
|
||||
|
||||
for i, dim in enumerate(expected_shape):
|
||||
if dim in dynamic_dims:
|
||||
continue
|
||||
elif isinstance(dim, int):
|
||||
if actual_shape[i] != dim:
|
||||
raise ValueError(f"{field_name} dim[{i}] expected "
|
||||
f"{dim}, got {actual_shape[i]}. "
|
||||
f"Expected shape: {expected_shape}, "
|
||||
f"but got {actual_shape}")
|
||||
raise ValueError(
|
||||
f"{field_name} dim[{i}] expected "
|
||||
f"{dim}, got {actual_shape[i]}. "
|
||||
f"Expected shape: {expected_shape}, "
|
||||
f"but got {actual_shape}"
|
||||
)
|
||||
elif isinstance(dim, str):
|
||||
if dim in shape_env:
|
||||
if actual_shape[i] != shape_env[dim]:
|
||||
raise ValueError(f"{field_name} dim[{i}] expected "
|
||||
f"'{dim}'={shape_env[dim]}, got "
|
||||
f"{actual_shape[i]}")
|
||||
raise ValueError(
|
||||
f"{field_name} dim[{i}] expected "
|
||||
f"'{dim}'={shape_env[dim]}, got "
|
||||
f"{actual_shape[i]}"
|
||||
)
|
||||
else:
|
||||
shape_env[dim] = actual_shape[i]
|
||||
else:
|
||||
raise TypeError(f"{field_name} dim[{i}] has unsupported "
|
||||
f"type: {type(dim)}")
|
||||
raise TypeError(
|
||||
f"{field_name} dim[{i}] has unsupported type: {type(dim)}"
|
||||
)
|
||||
|
||||
def validate(self) -> None:
|
||||
type_hints = get_type_hints(self.__class__, include_extras=True)
|
||||
@@ -195,8 +201,7 @@ class TensorSchema:
|
||||
|
||||
for field_name, field_type in type_hints.items():
|
||||
# Check if field is missing
|
||||
if (not hasattr(self, field_name)
|
||||
or getattr(self, field_name) is None):
|
||||
if not hasattr(self, field_name) or getattr(self, field_name) is None:
|
||||
# Check if field is marked as optional
|
||||
actual_type = field_type
|
||||
if get_origin(field_type) is Annotated:
|
||||
@@ -228,8 +233,12 @@ class TensorSchema:
|
||||
)
|
||||
|
||||
self._validate_tensor_shape_expected(
|
||||
actual_shape, expected_shape, field_name,
|
||||
shape_env, arg.dynamic_dims)
|
||||
actual_shape,
|
||||
expected_shape,
|
||||
field_name,
|
||||
shape_env,
|
||||
arg.dynamic_dims,
|
||||
)
|
||||
|
||||
def print_shapes(self) -> None:
|
||||
"""Print TensorShape annotations for debugging."""
|
||||
|
||||
Reference in New Issue
Block a user