Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM: a high-throughput and memory-efficient inference engine for LLMs"""
"""vLLM: a high-throughput and memory-efficient inference engine for LLMs."""
# The version.py should be independent library, and we always import the
# version library first. Such assumption is critical for some customization.
+21 -22
View File
@@ -1835,6 +1835,7 @@ class rocm_aiter_ops:
- MLA decode: mla_decode_fwd
- Quantization: per_tensor_quant, per_token_quant, group_fp8_quant
- Triton ops: triton_rotary_embed, triton_fp8_bmm, triton_gemm_a8w8_blockscale
"""
_MOE_DISPATCH_POLICY: int | None = None
@@ -1874,8 +1875,7 @@ class rocm_aiter_ops:
@classmethod
def refresh_env_variables(cls):
"""
Since the environment variables are assigned when the module is imported,
"""Since the environment variables are assigned when the module is imported,
This is a helper function to reload all the env variables from
the environment variables.
for example, after monkey patching the env variables in the unit test,
@@ -1900,8 +1900,7 @@ class rocm_aiter_ops:
@staticmethod
def get_aiter_activation_type(activation_str: str) -> "ActivationType | None":
"""
Given an activation type as a string, returns the corresponding aiter ActivationType enum.
"""Given an activation type as a string, returns the corresponding aiter ActivationType enum.
Supported activation types: "no", "none", "silu", "gelu", "swiglu".
Returns None if the mapping fails.
@@ -1910,6 +1909,7 @@ class rocm_aiter_ops:
Returns:
Aiter ActivationType enum value, or None if not found.
"""
# Import only locally, since aiter may not always be available.
try:
@@ -1933,8 +1933,7 @@ class rocm_aiter_ops:
@staticmethod
def get_aiter_quant_type(quant_type_str: str) -> "QuantType | None":
"""
Given a quantization type as a string, returns the corresponding aiter QuantType enum.
"""Given a quantization type as a string, returns the corresponding aiter QuantType enum.
Supported quantization types: "no", "per_tensor", "per_token", "per_1x32", "per_1x128", "per_128x128".
Returns None if the mapping fails.
@@ -1943,6 +1942,7 @@ class rocm_aiter_ops:
Returns:
Aiter QuantType enum value, or None if not found.
"""
try:
from aiter import QuantType
@@ -1974,7 +1974,8 @@ class rocm_aiter_ops:
enabled aiter. Only aiter's Triton kernels exist on gfx12 (no CK build),
so this deliberately stays off the gfx9/CK `@if_aiter_supported` umbrella
and gates only the Triton paths rdna4 uses. The gfx12 analog of
`is_enabled()`."""
`is_enabled()`.
"""
if not current_platform.is_rocm() or not IS_AITER_FOUND:
return False
from vllm.platforms.rocm import on_rdna4
@@ -3357,8 +3358,7 @@ class rocm_aiter_ops:
nLane: int,
gate_up: bool,
) -> "torch.Tensor":
"""
Shuffles the weight tensor into (A16W4) layout for AITER kernels.
"""Shuffles the weight tensor into (A16W4) layout for AITER kernels.
Args:
tensor: The input weight tensor to be shuffled.
@@ -3367,6 +3367,7 @@ class rocm_aiter_ops:
Returns:
torch.Tensor: The shuffled tensor.
"""
from aiter.ops.shuffle import shuffle_weight_a16w4
@@ -3378,8 +3379,7 @@ class rocm_aiter_ops:
num_experts: int,
gate_up: bool,
) -> "torch.Tensor":
"""
Shuffles the scale tensor into (A16W4) layout for AITER kernels.
"""Shuffles the scale tensor into (A16W4) layout for AITER kernels.
Args:
tensor: The input scale tensor to be shuffled.
@@ -3388,6 +3388,7 @@ class rocm_aiter_ops:
Returns:
torch.Tensor: The shuffled scale tensor.
"""
from aiter.ops.shuffle import shuffle_scale_a16w4
@@ -3404,8 +3405,7 @@ class rocm_aiter_ops:
def shuffle_weights(
*tensors: torch.Tensor, layout: tuple[int, int] = (16, 16)
) -> tuple[torch.Tensor, ...]:
"""
Applies shuffle_weight function from AITER to each
"""Applies shuffle_weight function from AITER to each
input tensor and returns them.
Rearranges (shuffles) the input tensor/s
@@ -3418,6 +3418,7 @@ class rocm_aiter_ops:
Returns:
A Tuple of shuffled tensors.
"""
from aiter.ops.shuffle import shuffle_weight
@@ -3467,8 +3468,7 @@ class rocm_aiter_ops:
out: torch.Tensor | None = None,
sink_ptr: torch.Tensor | None = None,
):
"""
Flash attention with variable length sequences.
"""Flash attention with variable length sequences.
This function is NOT wrapped with @is_aiter_supported decorator
to allow explicit backend selection via attention_config to work
@@ -3537,8 +3537,7 @@ class rocm_aiter_ops:
V_QScale: torch.Tensor,
out_: torch.Tensor,
):
"""
Paged attention forward pass using assembly kernel.
"""Paged attention forward pass using assembly kernel.
This function is NOT wrapped with @is_aiter_supported decorator
to allow explicit backend selection via attention_config to work
@@ -3580,8 +3579,7 @@ class rocm_aiter_ops:
out_: torch.Tensor,
kv_cache_dtype: str,
):
"""
Paged attention common function.
"""Paged attention common function.
This function is NOT wrapped with @is_aiter_supported decorator
to allow explicit backend selection via attention_config to work
@@ -3625,8 +3623,7 @@ class rocm_aiter_ops:
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Forward pass for mHC pre block.
"""Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
@@ -3645,6 +3642,7 @@ class rocm_aiter_ops:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
from aiter.ops.mhc import mhc_pre
@@ -3735,7 +3733,7 @@ class rocm_aiter_ops:
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""mHC pre using the pre-mix carried from the previous sublayer.
"""MHC pre using the pre-mix carried from the previous sublayer.
Same gates as :meth:`mhc_pre`, but the stream collapse applies the
caller's ``pre_mix`` instead of the one computed here, and the one
@@ -3762,6 +3760,7 @@ class rocm_aiter_ops:
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
next_pre_mix: shape (..., hc_mult), dtype torch.float32
"""
from aiter.ops.mhc import (
get_mhc_fused_post_pre_config,
+57 -53
View File
@@ -39,8 +39,7 @@ def create_fp4_scale_tensor(
device: torch.device,
is_sf_swizzled_layout: bool,
) -> torch.Tensor:
"""
Allocate the output scale tensor for scaled_fp4_quant.
"""Allocate the output scale tensor for scaled_fp4_quant.
When is_sf_swizzled_layout=True, we use rounded values to store the
swizzled scales. Due to the requirement of the Tensor Core, the minimum
@@ -73,8 +72,7 @@ def create_fp4_output_tensors(
is_sf_swizzled_layout: bool,
padded_n: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Allocate both output tensors for scaled_fp4_quant:
"""Allocate both output tensors for scaled_fp4_quant:
(quantized_output, output_scale).
Must match the C++ scaled_fp4_quant_func allocation exactly when
@@ -265,9 +263,18 @@ def vocab_parallel_embedding(
Args:
input_ids: ``[num_tokens]`` int32 or int64 token ids.
weight: ``[num_embeddings_per_partition, embedding_dim]`` shard.
org_vocab_start_index: First original vocab id owned by this rank.
org_vocab_end_index: One past the last original vocab id owned by
this rank.
num_org_vocab_padding: Padding rows between this rank's original and
added vocab shards.
added_vocab_start_index: First added vocab id owned by this rank.
added_vocab_end_index: One past the last added vocab id owned by
this rank.
Returns:
``[num_tokens, embedding_dim]`` partial embeddings.
"""
out = torch.empty(
input_ids.shape[0],
@@ -399,6 +406,7 @@ def apply_repetition_penalties(
prompt_mask: A boolean tensor indicating which tokens appear in the prompt.
output_mask: A boolean tensor indicating which tokens appear in the output.
repetition_penalties: The repetition penalties of shape (num_seqs, ).
"""
if logits.is_cuda and logits.is_contiguous():
apply_repetition_penalties_cuda(
@@ -787,8 +795,7 @@ def cutlass_scaled_mm(
out_dtype: torch.dtype,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""
`cutlass_scaled_mm` implements a fused version of
"""`cutlass_scaled_mm` implements a fused version of
`output = torch.mm((scale_a * a), (scale_b * b)).to(out_dtype)`
where scale_a * a and scale_b * b are implemented using numpy-style
broadcasting.
@@ -840,11 +847,11 @@ def cutlass_scaled_mm_azp(
azp: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Args:
azp_adj: In the per-tensor case, this should include the azp.
Always per-channel.
azp: Only set in the per-token case. Per-token if set.
"""Args:
azp_adj: In the per-tensor case, this should include the azp.
Always per-channel.
azp: Only set in the per-token case. Per-token if set.
"""
assert b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0
assert out_dtype is torch.bfloat16 or out_dtype is torch.float16
@@ -883,8 +890,7 @@ def get_cutlass_moe_mm_data(
blockscale_offsets: torch.Tensor | None = None,
is_gated: bool = True,
):
"""
Prepare data necessary to perform CUTLASS grouped matrix multiplications
"""Prepare data necessary to perform CUTLASS grouped matrix multiplications
used in CUTLASS-based fused MoE.
The function takes in topk_ids (token-expert mapping) and uses it to
@@ -943,8 +949,7 @@ def get_cutlass_moe_mm_problem_sizes_from_expert_offsets(
def shuffle_rows(input_tensor: torch.Tensor, dst2src_map: torch.Tensor):
"""
Shuffle and expand the input tensor according to the dst2src_map and store the result in output_tensor.
"""Shuffle and expand the input tensor according to the dst2src_map and store the result in output_tensor.
This is used in MoE to permute the input tensor before performing grouped matrix multiplications.
"""
num_tokens_permuted = dst2src_map.shape[0]
@@ -967,8 +972,7 @@ def get_cutlass_batched_moe_mm_data(
n: int,
k: int,
):
"""
Prepare data necessary to perform CUTLASS grouped matrix multiplications
"""Prepare data necessary to perform CUTLASS grouped matrix multiplications
used in CUTLASS-based fused MoE.
The function takes in expert_num_tokens (token count per expert) and
@@ -1006,8 +1010,7 @@ def cutlass_moe_mm(
per_act_token: bool,
per_out_ch: bool,
):
"""
A single grouped matrix multiplication used in CUTLASS-based fused MoE.
"""A single grouped matrix multiplication used in CUTLASS-based fused MoE.
The function executes fp8-quantized OUT = AB matrix multiplication.
- expert_offsets: Indices that mark at which token index each expert begins
@@ -1044,8 +1047,7 @@ def cutlass_fp4_moe_mm(
expert_offsets: torch.Tensor,
sf_offsets: torch.Tensor,
):
"""
An FP4 Blockscaled Group Gemm that takes in a_tensors, b_tensors and runs
"""An FP4 Blockscaled Group Gemm that takes in a_tensors, b_tensors and runs
the gemms for each combination based on the specified problem sizes.
This is used as the MoE gemm during NVFP4 Quantized MoERunner forward.
@@ -1083,8 +1085,7 @@ def cutlass_mxfp4_moe_mm(
expert_offsets: torch.Tensor,
sf_offsets: torch.Tensor,
):
"""
An MXFP4 Blockscaled Group Gemm for MoE (MXFP4 x MXFP4).
"""An MXFP4 Blockscaled Group Gemm for MoE (MXFP4 x MXFP4).
Uses mx_float4_t types with E8M0 scale factors and 32-element blocks.
- a/b_tensors: MXFP4 packed activations/weights (uint8, 2 E2M1 per byte)
@@ -1458,8 +1459,7 @@ def cutlass_w4a8_moe_mm(
group_scale_strides: torch.Tensor,
maybe_schedule: str | None = None,
) -> None:
"""
Executes the CUTLASS-based fused-MoE grouped matrix multiplication for the
"""Executes the CUTLASS-based fused-MoE grouped matrix multiplication for the
W4A8 quantization scheme. Uses group-wise quantization (INT4 -> FP8)
and both per-channel + per-token scaling in the epilogue.
@@ -1495,6 +1495,7 @@ def cutlass_w4a8_moe_mm(
Returns:
out_tensors updated in-place with the dequantized INT4xFP8 grouped GEMM result.
"""
return torch.ops._C.cutlass_w4a8_moe_mm(
out_tensors,
@@ -1535,8 +1536,7 @@ def scaled_fp4_quant(
backend: str = "none",
padded_n: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Quantize input tensor to FP4 and return quantized tensor and scale.
"""Quantize input tensor to FP4 and return quantized tensor and scale.
This function quantizes the last dimension of the given tensor `input`. For
every 16 consecutive elements, a single dynamically computed scaling factor
@@ -1559,6 +1559,7 @@ def scaled_fp4_quant(
tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every
two values are packed into a uint8 and float8_e4m3 scaling factors
in the sizzled layout.
"""
assert not current_platform.is_rocm()
assert input.ndim >= 1, f"input.ndim needs to be >= 1, but got {input.ndim}."
@@ -1613,17 +1614,19 @@ def scaled_fp4_experts_quant(
blockscale_offsets: torch.Tensor,
topk: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Quantize input tensor to NVFP4 and return quantized tensor and scale, for
"""Quantize input tensor to NVFP4 and return quantized tensor and scale, for
packed MoE Inputs.
Args:
input_tensor: The input tensor to be quantized to NVFP4
input_global_scale: A scalar scaling factor for the entire tensor.
expert_offsets: The expert offsets tensor
blockscale_offsets: The blockscale offsets tensor
topk: The number of experts each token is routed to
Outputs:
output: The quantized tensor in NVFP4
output_scales: The blockscale tensor in FP8-E4M3
"""
assert not current_platform.is_rocm()
assert input_tensor.ndim == 2, (
@@ -1675,8 +1678,7 @@ def silu_and_mul_scaled_fp4_experts_quant(
blockscale_offsets: torch.Tensor,
topk: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Fused SiLU+Mul+NVFP4 quantization for MoE intermediate activations.
"""Fused SiLU+Mul+NVFP4 quantization for MoE intermediate activations.
Args:
input_tensor: The input tensor with gate || up layout [m_topk, k*2]
@@ -1687,6 +1689,7 @@ def silu_and_mul_scaled_fp4_experts_quant(
Outputs:
output: The quantized tensor in NVFP4 [m_topk, k/2]
output_scales: The blockscale tensor in FP8-E4M3
"""
assert not current_platform.is_rocm()
assert input_tensor.ndim == 2, (
@@ -1740,8 +1743,7 @@ def mxfp4_experts_quant(
n_experts: int,
topk: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Quantize input tensor to MXFP4 for packed MoE inputs.
"""Quantize input tensor to MXFP4 for packed MoE inputs.
Uses 32-element blocks with E8M0 (power-of-two) scale factors.
MXFP4 has no global scale - only block-level E8M0 scale factors.
@@ -1754,6 +1756,7 @@ def mxfp4_experts_quant(
Returns:
output: [m_topk, k//2] packed E2M1 values (uint8)
output_scales: E8M0 blockscales in swizzled layout (uint8 view)
"""
assert not current_platform.is_rocm()
assert input_tensor.ndim == 2
@@ -1799,8 +1802,7 @@ def silu_and_mul_mxfp4_experts_quant(
n_experts: int,
topk: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Fused SiLU+Mul+MXFP4 quantization for MoE intermediate activations.
"""Fused SiLU+Mul+MXFP4 quantization for MoE intermediate activations.
MXFP4 has no global scale - only block-level E8M0 scale factors.
"""
assert not current_platform.is_rocm()
@@ -1846,8 +1848,7 @@ def scaled_fp8_quant(
output: torch.Tensor | None = None,
group_shape: tuple[int, int] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Quantize input tensor to FP8 and return quantized tensor and scale.
"""Quantize input tensor to FP8 and return quantized tensor and scale.
This function supports both static and dynamic quantization: If you
provide the scale, it will use static scaling and if you omit it,
@@ -1869,6 +1870,8 @@ def scaled_fp8_quant(
of the output to at least this value.
use_per_token_if_dynamic: Whether to do per_tensor or per_token
in the dynamic quantization case.
output: Optional tensor to write the quantized result into. A new
tensor is allocated when omitted.
group_shape: Optional tuple (group_m, group_n) specifying the group
shape for static quantization. Use -1 for "full extent" (e.g.,
(-1, -1) for per-tensor, (-1, 1) for per-channel, etc.)
@@ -1877,6 +1880,7 @@ def scaled_fp8_quant(
Returns:
tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and
scaling factor.
"""
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
@@ -1913,8 +1917,7 @@ def allspark_repack_weight(
zero_point: torch.Tensor | None = None,
has_zp: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Rearrange qweight, scale, and zero_point(if asymmetric) to n32k16 format
"""Rearrange qweight, scale, and zero_point(if asymmetric) to n32k16 format
for Ampere W8A16 Fused Gemm kernel
Args:
@@ -1928,6 +1931,7 @@ def allspark_repack_weight(
Returns:
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None] :
rearranged weight, scale, and optionally zero_point.
"""
K = qweight.shape[0]
N = qweight.shape[1]
@@ -1997,8 +2001,7 @@ def scaled_int8_quant(
azp: torch.Tensor | None = None,
symmetric: bool = True,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""
Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
"""Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
Args:
input: The input tensor to be quantized to int8.
@@ -2010,6 +2013,7 @@ def scaled_int8_quant(
Returns:
Output int8 tensor, scales, and optionally azp.
"""
if current_platform.is_xpu():
# XPU has no _C int8 quant op; use the torch.compile reference.
@@ -2463,8 +2467,7 @@ def grouped_topk(
bias: torch.Tensor,
scoring_func: int = 0,
):
"""
Perform grouped top-k routing for mixture of experts.
"""Perform grouped top-k routing for mixture of experts.
Args:
scores: Raw inputs (logits if scoring_func=1, scores if scoring_func=0)
@@ -2475,6 +2478,7 @@ def grouped_topk(
routed_scaling_factor: Scaling factor for routing weights
bias: Bias tensor (e_score_correction_bias). Always fused in kernel.
scoring_func: 0=none (no activation), 1=sigmoid
"""
if not (current_platform.is_cuda() or current_platform.is_xpu()):
raise NotImplementedError(
@@ -2911,8 +2915,7 @@ def swap_blocks(
block_size_in_bytes: int,
block_mapping: torch.Tensor,
) -> None:
"""
Copy specific blocks from one tensor to another.
"""Copy specific blocks from one tensor to another.
This method assumes each of the two input tensors is composed of
consecutive contiguous blocks, of size block_size_in_bytes.
@@ -2940,8 +2943,7 @@ def swap_blocks_batch(
sizes: torch.Tensor,
is_src_access_order_any: bool = False,
) -> None:
"""
Batch version of swap_blocks: submit all copies in a single driver call.
"""Batch version of swap_blocks: submit all copies in a single driver call.
Each entry specifies a raw pointer copy: src_ptrs[i] -> dst_ptrs[i]
of sizes[i] bytes. All three tensors must be CPU tensors with the
@@ -3030,6 +3032,7 @@ def cp_gather_and_upconvert_fp8_kv_cache(
host_cache: Optional pinned host rows used for non-resident entries
host_row_ids: Host source row for each remapped cache row
device_row_ids: Device source row, or -1, for each remapped cache row
"""
torch.ops._C_cache_ops.cp_gather_and_upconvert_fp8_kv_cache(
src_cache,
@@ -3059,6 +3062,7 @@ def cp_gather_and_upconvert_nvfp4_kv_cache(
block_table: Block indices [num_reqs, max_blocks]
workspace_starts: Workspace start offsets [num_reqs]
batch_size: Number of requests
"""
torch.ops._C_cache_ops.cp_gather_and_upconvert_nvfp4_kv_cache(
src_cache,
@@ -3080,6 +3084,7 @@ def concat_mla_q(
ql_nope: Query nope component [num_tokens, num_heads, nope_dim]
q_pe: Query rope component [num_tokens, num_heads, rope_dim]
q_out: Output tensor [num_tokens, num_heads, nope_dim + rope_dim]
"""
torch.ops._C_cache_ops.concat_mla_q(ql_nope, q_pe, q_out)
@@ -3802,8 +3807,7 @@ def onednn_scaled_int8_quant(
azp: torch.Tensor | None = None,
symmetric: bool = True,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""
Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
"""Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
Args:
input: The input tensor to be quantized to int8.
@@ -3815,6 +3819,7 @@ def onednn_scaled_int8_quant(
Returns:
Output int8 tensor, scales, and optionally azp.
"""
output = torch.empty_like(input, dtype=torch.int8)
token_num = input.numel() // input.shape[-1]
@@ -4674,8 +4679,7 @@ def safeFusedQuantizeNv(
xh_e4m3: torch.Tensor,
global_scale: torch.Tensor,
) -> None:
"""
Wrapper for QUTLASS fusedQuantizeNv method that operates on tensors in-place
"""Wrapper for QUTLASS fusedQuantizeNv method that operates on tensors in-place
rather than returning them, to prevent torch 2.12+ errors that outputs of custom
operators may not alias any inputs to the custom operator.
"""
@@ -4684,8 +4688,7 @@ def safeFusedQuantizeNv(
def hadacore_transform(x: torch.Tensor, inplace: bool = True) -> torch.Tensor:
"""
Perform Hadamard transforms using [Hadacore](https://arxiv.org/abs/2412.08832)
"""Perform Hadamard transforms using [Hadacore](https://arxiv.org/abs/2412.08832)
kernels. Note that these kernels exploit the recursive properties of
Sylvester Hadamards, and therefore do not require transform weight data
@@ -4698,6 +4701,7 @@ def hadacore_transform(x: torch.Tensor, inplace: bool = True) -> torch.Tensor:
Returns:
value after transformation
"""
return torch.ops._C.hadacore_transform(x, inplace)
+3
View File
@@ -286,6 +286,7 @@ def _xpu_fp8_bmm_impl(
This implementation centralizes access to
``torch.ops._xpu_C.fp8_bmm``. Both scales must be contiguous, while
``a`` and ``b`` may be non-contiguous views.
"""
return torch.ops._xpu_C.fp8_bmm(a, b, out_dtype, a_scale, b_scale, bias)
@@ -430,6 +431,7 @@ def _xpu_deepseek_fused_indexer_q_rope_fp8_impl(
index_weights_out: (T, H) float32 output, preallocated;
= index_weights * q_scale * softmax_scale * head_scale (the
per-(token, head) q_scale is folded in here). [written]
"""
torch.ops._xpu_C.deepseek_fused_indexer_q_rope_fp8(
index_q,
@@ -473,6 +475,7 @@ def _xpu_deepseek_fused_indexer_q_rope_mxfp4_impl(
index_weights_out: (T, H) float32 output, preallocated;
= index_weights * softmax_scale * head_scale (no q_scale folded;
per-block scales live in index_q_scale). [written]
"""
torch.ops._xpu_C.deepseek_fused_indexer_q_rope_mxfp4(
index_q,
+1 -2
View File
@@ -20,8 +20,7 @@ def get_cache_dir() -> Path:
@lru_cache
def get_vllm_public_assets(filename: str, s3_prefix: str | None = None) -> Path:
"""
Download an asset file from `s3://vllm-public-assets`
"""Download an asset file from `s3://vllm-public-assets`
and return the path to the downloaded file.
"""
asset_directory = get_cache_dir() / "vllm_public_assets"
+2 -6
View File
@@ -34,9 +34,7 @@ class ImageAsset:
name: ImageAssetName
def get_path(self, ext: str) -> Path:
"""
Return s3 path for given image.
"""
"""Return s3 path for given image."""
return get_vllm_public_assets(
filename=f"{self.name}.{ext}", s3_prefix=VLM_IMAGES_DIR
)
@@ -51,9 +49,7 @@ class ImageAsset:
@property
def image_embeds(self) -> torch.Tensor:
"""
Image embeddings, only used for testing purposes with llava 1.5.
"""
"""Image embeddings, only used for testing purposes with llava 1.5."""
image_path = self.get_path("pt")
return torch.load(image_path, map_location="cpu", weights_only=True)
+2 -4
View File
@@ -21,8 +21,7 @@ def _sample_frame_indices(total_frames: int, num_frames: int) -> npt.NDArray:
@lru_cache
def download_video_asset(filename: str) -> str:
"""
Download and open an image from huggingface
"""Download and open an image from huggingface
repo: raushan-testing-hf/videos-test
"""
video_directory = get_cache_dir() / "video-example-data"
@@ -141,8 +140,7 @@ class VideoAsset:
return ret
def get_audio(self, sampling_rate: float | None = None) -> npt.NDArray:
"""
Read audio data from the video asset, used in Qwen2.5-Omni examples.
"""Read audio data from the video asset, used in Qwen2.5-Omni examples.
See also: examples/generate/multimodal/qwen2_5_omni/only_thinker.py
"""
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Convert a plain-text file (local path or URL) into a JSONL dataset
"""Convert a plain-text file (local path or URL) into a JSONL dataset
compatible with ``CustomDataset`` (``--dataset-name custom``), by
randomly slicing the tokenized text into prompts.
@@ -66,8 +65,8 @@ def create_txt_slices_jsonl(
trust_remote_code: bool = False,
) -> None:
"""Read *input_path*, slice it into prompts, and write JSONL to
*output_path*."""
*output_path*.
"""
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name, trust_remote_code=trust_remote_code
)
+65 -98
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This module defines a framework for sampling benchmark requests from various
"""This module defines a framework for sampling benchmark requests from various
datasets. Each dataset subclass of BenchmarkDataset must implement sample
generation. Supported dataset types include:
- ShareGPT
@@ -75,9 +74,7 @@ DEFAULT_NUM_PROMPTS = 1000
@dataclass
class SampleRequest:
"""
Represents a single inference request for benchmarking.
"""
"""Represents a single inference request for benchmarking."""
prompt: str | list[str] | list[int] | list[list[int]] | list[dict]
prompt_len: int
@@ -112,8 +109,7 @@ class BenchmarkDataset(ABC):
disable_shuffle: bool = False,
**kwargs,
) -> None:
"""
Initialize the BenchmarkDataset with an optional dataset path and random
"""Initialize the BenchmarkDataset with an optional dataset path and random
seed.
Args:
@@ -121,6 +117,9 @@ class BenchmarkDataset(ABC):
indicates that a default or random dataset might be used.
random_seed (int): Seed value for reproducible shuffling or
sampling. Defaults to DEFAULT_SEED.
disable_shuffle (bool): If True, keep the dataset in its original
order instead of shuffling it.
"""
self.dataset_path = dataset_path
# Set the random seed, ensuring that a None value is replaced with the
@@ -134,8 +133,7 @@ class BenchmarkDataset(ABC):
prompt: str,
mm_content: MultiModalDataDict | dict | list[dict] | None = None,
) -> list[dict]:
"""
Transform a prompt and optional multimodal content into a chat format.
"""Transform a prompt and optional multimodal content into a chat format.
This method is used for chat models that expect a specific conversation
format.
"""
@@ -152,14 +150,14 @@ class BenchmarkDataset(ABC):
return [{"role": "user", "content": content}]
def load_data(self) -> None:
"""
Load data from the dataset path into self.data.
"""Load data from the dataset path into self.data.
This method must be overridden by subclasses since the method to load
data will vary depending on the dataset format and source.
Raises:
NotImplementedError: If a subclass does not implement this method.
"""
# TODO (jenniferzhao): add support for downloading data
raise NotImplementedError("load_data must be implemented in subclasses.")
@@ -169,8 +167,7 @@ class BenchmarkDataset(ABC):
max_loras: int | None = None,
lora_path: str | None = None,
) -> LoRARequest | None:
"""
Optionally select a random LoRA request.
"""Optionally select a random LoRA request.
This method is used when LoRA parameters are provided. It randomly
selects a LoRA based on max_loras.
@@ -184,6 +181,7 @@ class BenchmarkDataset(ABC):
Returns:
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
(or `None` if not applicable).
"""
if max_loras is None or lora_path is None:
return None
@@ -203,8 +201,7 @@ class BenchmarkDataset(ABC):
max_loras: int | None = None,
lora_path: str | None = None,
) -> LoRARequest | None:
"""
Optionally select a LoRA request using deterministic round-robin.
"""Optionally select a LoRA request using deterministic round-robin.
This method cycles through LoRA IDs in order based on the request
index, providing reproducible LoRA assignment.
@@ -219,6 +216,7 @@ class BenchmarkDataset(ABC):
Returns:
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
(or `None` if not applicable).
"""
if max_loras is None or lora_path is None:
return None
@@ -239,8 +237,7 @@ class BenchmarkDataset(ABC):
lora_path: str | None = None,
lora_assignment: str = "random",
) -> LoRARequest | None:
"""
Select a LoRA request using the specified assignment strategy.
"""Select a LoRA request using the specified assignment strategy.
Args:
index (int): The request index (used for round-robin).
@@ -252,6 +249,7 @@ class BenchmarkDataset(ABC):
Returns:
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
(or `None` if not applicable).
"""
if lora_assignment == "round-robin":
return BenchmarkDataset.get_round_robin_lora_request(
@@ -270,8 +268,7 @@ class BenchmarkDataset(ABC):
no_oversample: bool = False,
**kwargs,
) -> list[SampleRequest]:
"""
Abstract method to generate sample requests from the dataset.
"""Abstract method to generate sample requests from the dataset.
Subclasses must override this method to implement dataset-specific logic
for generating a list of SampleRequest objects.
@@ -281,10 +278,13 @@ class BenchmarkDataset(ABC):
for processing the dataset's text.
num_requests (int): The number of sample requests to generate.
request_id_prefix (str): The prefix of request_id.
no_oversample (bool): If True, do not oversample the dataset to
reach num_requests.
Returns:
list[SampleRequest]: A list of sample requests generated from the
dataset.
"""
raise NotImplementedError("sample must be implemented in subclasses.")
@@ -295,8 +295,7 @@ class BenchmarkDataset(ABC):
request_id_prefix: str = "",
no_oversample: bool = False,
) -> None:
"""
Oversamples the list of requests if its size is less than the desired
"""Oversamples the list of requests if its size is less than the desired
number.
Args:
@@ -305,6 +304,8 @@ class BenchmarkDataset(ABC):
num_requests (int): The target number of requests.
request_id_prefix (str): The prefix applied to generated request
identifiers.
no_oversample (bool): If True, return the requests unchanged
instead of oversampling up to num_requests.
"""
if no_oversample:
@@ -346,8 +347,7 @@ def is_valid_sequence(
max_total_len: int = 2048,
skip_min_output_len_check: bool = False,
) -> bool:
"""
Validate a sequence based on prompt and output lengths.
"""Validate a sequence based on prompt and output lengths.
Default pruning criteria are copied from the original `sample_hf_requests`
and `sample_sharegpt_requests` functions in benchmark_serving.py, as well as
@@ -379,8 +379,7 @@ def process_image(
*,
ensure_client_side_data: bool = False,
) -> Mapping[str, Any]:
"""
Process a single image input and return a multimedia content dictionary.
"""Process a single image input and return a multimedia content dictionary.
Supports the following input types:
@@ -402,6 +401,7 @@ def process_image(
Raises:
ValueError: If the input is not a supported type.
"""
if isinstance(image, dict) and "bytes" in image:
image = Image.open(BytesIO(image["bytes"]))
@@ -437,8 +437,7 @@ def process_image(
def process_video(video: Any) -> Mapping[str, Any]:
"""
Process a single video input and return a multimedia content dictionary.
"""Process a single video input and return a multimedia content dictionary.
Supports the following input types:
@@ -451,6 +450,7 @@ def process_video(video: Any) -> Mapping[str, Any]:
Raises:
ValueError: If the input is not a supported type.
"""
if isinstance(video, dict) and "bytes" in video:
video_bytes = video["bytes"]
@@ -474,8 +474,7 @@ def process_video(video: Any) -> Mapping[str, Any]:
def process_audio(audio: Any) -> tuple:
"""
Process a single audio input and return a (array, sample_rate) tuple.
"""Process a single audio input and return a (array, sample_rate) tuple.
Supports:
1. String: treated as a file path, loaded with soundfile.
@@ -502,8 +501,7 @@ def gen_prompt_decode_to_target_len(
add_special_tokens: bool = False,
rng: np.random.Generator | None = None,
) -> tuple[str, list[int], int]:
"""
Ensure decoded-then-encoded prompt length matches the target token length.
"""Ensure decoded-then-encoded prompt length matches the target token length.
This function decodes an initial token sequence to text and re-encodes it
, iteratively adjusting the token sequence length to match a target.
@@ -557,8 +555,7 @@ def gen_prompt_decode_to_target_len(
class RandomDataset(BenchmarkDataset):
"""
Synthetic text-only dataset for serving/throughput benchmarks.
"""Synthetic text-only dataset for serving/throughput benchmarks.
Strategy:
- Sample input/output token lengths per request from integer-uniform ranges
@@ -699,9 +696,7 @@ class RandomDataset(BenchmarkDataset):
allowed_tokens: np.ndarray,
prefix_len: int,
) -> list[int]:
"""
Get the prefix for the dataset.
"""
"""Get the prefix for the dataset."""
if prefix_len <= 0:
return []
@@ -738,8 +733,7 @@ class RandomDataset(BenchmarkDataset):
index: int,
allowed_tokens: np.ndarray,
) -> tuple[str, int, int]:
"""
Returns (prompt, total_input_len).
"""Returns (prompt, total_input_len).
NOTE: After decoding the prompt we have to encode and decode it again.
This is done because in some cases N consecutive tokens
@@ -778,8 +772,7 @@ class RandomDataset(BenchmarkDataset):
class RandomDatasetForReranking(RandomDataset):
"""
Random dataset specialized for the needs of scoring:
"""Random dataset specialized for the needs of scoring:
- Batches of inputs
- Inputs composed of pairs
"""
@@ -906,8 +899,7 @@ class RandomDatasetForReranking(RandomDataset):
class RandomMultiModalDataset(RandomDataset):
"""
Synthetic multimodal dataset (text + images) that extends RandomDataset.
"""Synthetic multimodal dataset (text + images) that extends RandomDataset.
Status:
- Images: supported via synthetic RGB data.
@@ -1022,8 +1014,7 @@ class RandomMultiModalDataset(RandomDataset):
def normalize_bucket_config(
self, bucket_config: dict[tuple[int, int, int], float]
) -> dict[tuple[int, int, int], float]:
"""
Remove zero probability entries
"""Remove zero probability entries
and normalize the bucket config to sum to 1.
"""
# Raise error if value is negative
@@ -1044,13 +1035,11 @@ class RandomMultiModalDataset(RandomDataset):
self,
mm_item_config: tuple[int, int, int],
) -> Mapping[str, Any]:
"""
Create synthetic images and videos and
"""Create synthetic images and videos and
apply process_image/process_video respectively.
This follows the OpenAI API chat completions
https://github.com/openai/openai-python
"""
if self.map_config_to_modality(mm_item_config) == "image":
return process_image(
self.generate_synthetic_image(mm_item_config[1], mm_item_config[0])
@@ -1071,9 +1060,7 @@ class RandomMultiModalDataset(RandomDataset):
limit_mm_per_prompt: dict[str, int],
bucket_config: dict[tuple[int, int, int], float],
) -> tuple[int, int, dict[str, int], dict[tuple[int, int, int], float]]:
"""
Get the sampling parameters for the multimodal items.
"""
"""Get the sampling parameters for the multimodal items."""
# Enforce num_mm_items_range_ratio <= 1
if not (0.0 <= num_mm_items_range_ratio <= 1.0):
raise ValueError("num_mm_items_range_ratio must be in [0, 1].")
@@ -1146,8 +1133,7 @@ class RandomMultiModalDataset(RandomDataset):
bucket_config: dict[tuple[int, int, int], float],
limit_mm_per_prompt: dict[str, int],
) -> Iterator[tuple[int, int, int]]:
"""
Iterator over the multimodal items for each request
"""Iterator over the multimodal items for each request
whose size is between min_num_mm_items and max_num_mm_items.
Loop over the bucket config and sample a multimodal item.
@@ -1160,6 +1146,7 @@ class RandomMultiModalDataset(RandomDataset):
`bucket_config` (tuple->float). The original dict passed to
`sample` is not mutated. If this ever changes, a test
is implemented and will fail.
"""
# Get the number of multimodal items to sample
request_num_mm_items = int(
@@ -1346,8 +1333,7 @@ class RandomMultiModalDataset(RandomDataset):
class ShareGPTDataset(BenchmarkDataset):
"""
Implements the ShareGPT dataset. Loads data from a JSON file and generates
"""Implements the ShareGPT dataset. Loads data from a JSON file and generates
sample requests based on conversation turns.
"""
@@ -1437,8 +1423,7 @@ class ShareGPTDataset(BenchmarkDataset):
class TimedTrace(BenchmarkDataset):
"""
Implements a base class to replay various timed traces.
"""Implements a base class to replay various timed traces.
Loads data from a JSON file and generates sample requests
based on the timing information in the traces.
"""
@@ -1927,6 +1912,7 @@ def add_random_dataset_base_args(
Args:
parser_or_group: Either a parser or an argument group to add arguments to.
"""
parser_or_group.add_argument(
"--random-input-len",
@@ -1988,6 +1974,7 @@ def add_random_multimodal_dataset_args(
Args:
parser_or_group: Either a parser or an argument group to add arguments to.
"""
parser_or_group.add_argument(
"--random-mm-base-items-per-request",
@@ -2503,8 +2490,7 @@ def get_samples(
class CustomDataset(BenchmarkDataset):
"""
Implements the Custom dataset. Loads data from a JSONL file and generates
"""Implements the Custom dataset. Loads data from a JSONL file and generates
sample requests based on conversation turns. E.g.,
```
{"prompt": "What is the capital of India?", "output_tokens": 10}
@@ -2632,8 +2618,7 @@ class CustomDataset(BenchmarkDataset):
class CustomImageDataset(CustomDataset):
"""
Implements the Custom image dataset. Loads data from a JSONL file and generates
"""Implements the Custom image dataset. Loads data from a JSONL file and generates
sample requests based on conversation turns. E.g.,
```
{
@@ -2917,8 +2902,7 @@ class CustomImageDataset(CustomDataset):
class CustomAudioDataset(CustomDataset):
"""
Custom dataset for audio benchmarking. Loads data from a JSONL file. E.g.,
"""Custom dataset for audio benchmarking. Loads data from a JSONL file. E.g.,
{"prompt": "Transcribe the audio.", "audio": "/path/to/audio.wav"}
Supports both:
@@ -3027,8 +3011,7 @@ class CustomAudioDataset(CustomDataset):
class SpecBench(CustomDataset):
"""
Implements the SpecBench dataset: https://github.com/hemingkx/Spec-Bench
"""Implements the SpecBench dataset: https://github.com/hemingkx/Spec-Bench
Download the dataset using:
wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl
""" # noqa: E501
@@ -3100,8 +3083,7 @@ class SpecBench(CustomDataset):
"SonnetDataset is deprecated and will be removed in a future version.",
)
class SonnetDataset(BenchmarkDataset):
"""
Simplified implementation of the Sonnet dataset. Loads poem lines from a
"""Simplified implementation of the Sonnet dataset. Loads poem lines from a
text file and generates sample requests. Default values here copied from
`benchmark_serving.py` for the sonnet dataset.
"""
@@ -3198,8 +3180,7 @@ class SonnetDataset(BenchmarkDataset):
class BurstGPTDataset(BenchmarkDataset):
"""
Implements the BurstGPT dataset. Loads data from a CSV file and generates
"""Implements the BurstGPT dataset. Loads data from a CSV file and generates
sample requests based on synthetic prompt generation. Only rows with Model
"GPT-4" and positive response tokens are used.
"""
@@ -3447,9 +3428,7 @@ class MultiModalConversationDataset(HuggingFaceDataset):
class VisionArenaDataset(HuggingFaceDataset):
"""
Vision Arena Dataset.
"""
"""Vision Arena Dataset."""
DEFAULT_OUTPUT_LEN = 128
SUPPORTED_DATASET_PATHS = {
@@ -3506,8 +3485,7 @@ class VisionArenaDataset(HuggingFaceDataset):
class MMVUDataset(HuggingFaceDataset):
"""
MMVU Dataset.
"""MMVU Dataset.
https://huggingface.co/datasets/yale-nlp/MMVU
"""
@@ -3586,8 +3564,7 @@ class MMVUDataset(HuggingFaceDataset):
class InstructCoderDataset(HuggingFaceDataset):
"""
InstructCoder Dataset.
"""InstructCoder Dataset.
https://huggingface.co/datasets/likaixin/InstructCoder
InstructCoder is the dataset designed for general code editing. It consists
@@ -3655,8 +3632,7 @@ class InstructCoderDataset(HuggingFaceDataset):
class MTBenchDataset(HuggingFaceDataset):
"""
MT-Bench Dataset.
"""MT-Bench Dataset.
https://huggingface.co/datasets/philschmid/mt-bench
We create a single turn dataset for MT-Bench.
@@ -3718,8 +3694,7 @@ class MTBenchDataset(HuggingFaceDataset):
class HumanEvalDataset(HuggingFaceDataset):
"""
HumanEvalDataset Dataset.
"""HumanEvalDataset Dataset.
https://huggingface.co/datasets/openai/openai_humaneval
We create a single turn dataset for HumanEval.
@@ -3779,8 +3754,7 @@ class HumanEvalDataset(HuggingFaceDataset):
class GSM8KDataset(HuggingFaceDataset):
"""
GSM8K Dataset.
"""GSM8K Dataset.
https://huggingface.co/datasets/openai/gsm8k
We create a single turn dataset for GSM8K.
@@ -3840,8 +3814,7 @@ class GSM8KDataset(HuggingFaceDataset):
class BlazeditDataset(HuggingFaceDataset):
"""
Blazedit Dataset.
"""Blazedit Dataset.
https://github.com/ise-uiuc/blazedit
5k char version: vdaita/edit_5k_char
@@ -3932,9 +3905,7 @@ Please generate the new code file in the "New file" section below.""" # noqa: E
class AIMODataset(HuggingFaceDataset):
"""
Dataset class for processing a AIMO dataset with reasoning questions.
"""
"""Dataset class for processing a AIMO dataset with reasoning questions."""
SUPPORTED_DATASET_PATHS = {
"AI-MO/aimo-validation-aime",
@@ -4026,6 +3997,7 @@ def _format_zeta_prompt(
Returns:
A dictionary with the formatted prompts and expected outputs.
"""
events = sample["events"]
input = sample["input"]
@@ -4042,9 +4014,7 @@ def _format_zeta_prompt(
class NextEditPredictionDataset(HuggingFaceDataset):
"""
Dataset class for processing a Next Edit Prediction dataset.
"""
"""Dataset class for processing a Next Edit Prediction dataset."""
SUPPORTED_DATASET_PATHS = {
"zed-industries/zeta",
@@ -4092,8 +4062,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):
class ASRDataset(HuggingFaceDataset):
"""
Dataset class for processing a ASR dataset for transcription.
"""Dataset class for processing a ASR dataset for transcription.
Tested on the following set:
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
@@ -4290,8 +4259,7 @@ class ASRDataset(HuggingFaceDataset):
class MLPerfDataset(HuggingFaceDataset):
"""
MLPerf Inference Dataset.
"""MLPerf Inference Dataset.
Dataset on HF:
https://huggingface.co/datasets/mgoin/mlperf-inference-llama2-data
@@ -4415,7 +4383,8 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):
def _generate_exact_length_tokens(target_length: int) -> tuple[list[int], int]:
"""Generate tokens that decode and re-encode to exactly
target_length."""
target_length.
"""
# Generate random tokens
tokens = np.random.randint(0, vocab_size, size=target_length).tolist()
@@ -4470,8 +4439,7 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):
class MMStarDataset(HuggingFaceDataset):
"""
Lin-Chen/MMStar: https://huggingface.co/datasets/Lin-Chen/MMStar
"""Lin-Chen/MMStar: https://huggingface.co/datasets/Lin-Chen/MMStar
refer to: https://github.com/sgl-project/SpecForge/pull/106
"""
@@ -4760,8 +4728,7 @@ class BFCLDataset(HuggingFaceDataset):
class SpeedBench(CustomDataset):
"""
SPEED-Bench dataset: https://huggingface.co/datasets/nvidia/SPEED-Bench
"""SPEED-Bench dataset: https://huggingface.co/datasets/nvidia/SPEED-Bench.
Download the dataset using:
+3 -5
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Shared utilities for benchmark dataset sampling.
"""
"""Shared utilities for benchmark dataset sampling."""
import logging
import math
@@ -46,8 +44,7 @@ def get_sampling_params(
output_len: int,
tokenizer: TokenizerLike,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
Sample per-request input/output token lengths and vocab offsets.
"""Sample per-request input/output token lengths and vocab offsets.
Lengths are drawn uniformly from integer ranges around the configured
means, controlled by *range_ratio*. It may be a single ``float``
@@ -60,6 +57,7 @@ def get_sampling_params(
Returns:
(input_lens, output_lens, offsets) – three 1-D ``np.ndarray`` of
shape ``(num_requests,)``.
"""
input_range_ratio, output_range_ratio = _resolve_range_ratios(range_ratio)
+6 -2
View File
@@ -22,7 +22,8 @@ AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
class StreamedResponseHandler:
"""Handles streaming HTTP responses by accumulating chunks until complete
messages are available."""
messages are available.
"""
def __init__(self):
self.buffer = ""
@@ -30,7 +31,8 @@ class StreamedResponseHandler:
def add_chunk(self, chunk_bytes: bytes) -> list[str]:
"""Add a chunk of bytes to the buffer and return any complete
messages."""
messages.
"""
chunk_str = self._decoder.decode(chunk_bytes)
self.buffer += chunk_str
@@ -167,10 +169,12 @@ async def async_request_openai_completions(
Args:
request_func_input: The input for the request function.
session: The aiohttp session used to issue the request.
pbar: The progress bar to display the progress.
Returns:
The output of the request function.
"""
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Completions API", "completions")
+3 -2
View File
@@ -22,12 +22,12 @@ async def wait_for_endpoint(
timeout_seconds: int = 600,
retry_interval: int = 5,
) -> RequestFuncOutput:
"""
Wait for an endpoint to become available before starting benchmarks.
"""Wait for an endpoint to become available before starting benchmarks.
Args:
request_func: The async request function to call
test_input: The RequestFuncInput to test with
session: The aiohttp session used to issue the probe requests
timeout_seconds: Maximum time to wait in seconds (default: 10 minutes)
retry_interval: Time between retries in seconds (default: 5 seconds)
@@ -36,6 +36,7 @@ async def wait_for_endpoint(
Raises:
ValueError: If the endpoint doesn't become available within the timeout
"""
deadline = time.perf_counter() + timeout_seconds
output = RequestFuncOutput(success=False)
+2 -5
View File
@@ -36,9 +36,7 @@ def extract_field(
def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
"""
Check if the benchmark is run with torch.compile
"""
"""Check if the benchmark is run with torch.compile."""
return not (
extract_field(args, extra_info, "compilation_config.mode") == "0"
or "eager" in getattr(args, "output_json", "")
@@ -49,8 +47,7 @@ def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
def convert_to_pytorch_benchmark_format(
args: argparse.Namespace, metrics: dict[str, list], extra_info: dict[str, Any]
) -> list:
"""
Save the benchmark results in the format used by PyTorch OSS benchmark with
"""Save the benchmark results in the format used by PyTorch OSS benchmark with
on metric per record
https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
"""
+6 -14
View File
@@ -44,8 +44,7 @@ else:
def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, float]]:
"""
Get all multimodal timing stats from the LLM engine.
"""Get all multimodal timing stats from the LLM engine.
Collects both preprocessing stats (HF processor, hashing, cache lookup,
prompt update) and encoder forward pass timing, merged by request_id.
@@ -70,6 +69,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
'num_encoder_calls': 1
}
}
"""
observability_config = llm_engine.vllm_config.observability_config
if not observability_config or not observability_config.enable_mm_processor_stats:
@@ -126,8 +126,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
def collect_mm_processor_stats(llm_engine: LLMEngine) -> dict[str, list[float]]:
"""
Collect multimodal processor timing stats.
"""Collect multimodal processor timing stats.
Returns a dictionary mapping stage names to lists of timing values (in seconds).
"""
all_stats = get_timing_stats_from_engine(llm_engine)
@@ -147,9 +146,7 @@ def calculate_mm_processor_metrics(
*,
unit: Literal["us", "ms", "s"] = "ms",
) -> dict[str, dict[str, float]]:
"""
Calculate aggregate metrics from stats by stage.
"""
"""Calculate aggregate metrics from stats by stage."""
unit2mult = {"us": 1000000, "ms": 1000, "s": 1}
unit_mult = unit2mult[unit]
@@ -181,9 +178,7 @@ def calculate_mm_processor_metrics(
def validate_args(args):
"""
Validate command-line arguments for mm_processor benchmark.
"""
"""Validate command-line arguments for mm_processor benchmark."""
if not getattr(args, "tokenizer", None):
args.tokenizer = args.model
if not hasattr(args, "dataset_path"):
@@ -214,9 +209,7 @@ def validate_args(args):
def benchmark_multimodal_processor(
args: argparse.Namespace,
) -> dict[str, Any]:
"""
Run the multimodal processor benchmark.
"""
"""Run the multimodal processor benchmark."""
from vllm import LLM, SamplingParams
validate_args(args)
@@ -455,7 +448,6 @@ def add_cli_args(parser: FlexibleArgumentParser) -> None:
def main(args: argparse.Namespace) -> None:
"""Main entry point for the multimodal processor benchmark."""
print("Starting multimodal processor benchmark...")
result = benchmark_multimodal_processor(args)
+6 -7
View File
@@ -29,8 +29,7 @@ def generate_timeline_plot(
itl_thresholds: list[float] | None = None,
labels: list[str] | None = None,
) -> None:
"""
Generate an HTML timeline plot from benchmark results.
"""Generate an HTML timeline plot from benchmark results.
Args:
results: List of per-request result dictionaries containing:
@@ -44,8 +43,8 @@ def generate_timeline_plot(
colors: List of colors for ITL categories (default: green, orange, red, black)
itl_thresholds: ITL thresholds in seconds (default: [1.0, 4.0, 6.0])
labels: Labels for ITL categories (default based on thresholds)
"""
"""
# Set defaults
if colors is None:
colors = ["#109618", "#FF7F0E", "#D62728"]
@@ -118,8 +117,7 @@ def construct_timeline_data(
itl_thresholds: list[float],
labels: list[str],
) -> list[dict[str, Any]]:
"""
Construct timeline data from request results.
"""Construct timeline data from request results.
Args:
requests_data: List of per-request result dictionaries
@@ -128,6 +126,7 @@ def construct_timeline_data(
Returns:
List of timeline segments for plotting
"""
def tostr(sec_time: float) -> str:
@@ -229,8 +228,7 @@ def generate_dataset_stats_plot(
results: list[dict[str, Any]],
output_path: Path,
) -> None:
"""
Generate a matplotlib figure with dataset statistics.
"""Generate a matplotlib figure with dataset statistics.
Creates a figure with 4 subplots:
- Top-left: Prompt tokens distribution (histogram)
@@ -243,6 +241,7 @@ def generate_dataset_stats_plot(
- prompt_len: Number of prompt tokens
- output_tokens: Number of output tokens
output_path: Path where the figure will be saved
"""
# Extract data
prompt_tokens = []
+8 -2
View File
@@ -407,13 +407,15 @@ async def get_request(
ramp_up_end_rps: int | None = None,
self_timed: bool = False,
) -> AsyncGenerator[tuple[SampleRequest, float], None]:
"""
Asynchronously generates requests at a specified rate
"""Asynchronously generates requests at a specified rate
with OPTIONAL burstiness and OPTIONAL ramp-up strategy.
Args:
input_requests:
A list of input requests, each represented as a SampleRequest.
self_timed:
If True, the requests carry their own arrival timing and no
request rate, burstiness or ramp-up is applied.
request_rate:
The rate at which requests are generated (requests/s).
burstiness (optional):
@@ -431,6 +433,7 @@ async def get_request(
The starting request rate for ramp-up.
ramp_up_end_rps (optional):
The ending request rate for ramp-up.
"""
assert burstiness > 0, (
f"A positive burstiness factor is expected, but given {burstiness}."
@@ -526,6 +529,7 @@ def calculate_metrics_for_embeddings(
Returns:
The calculated benchmark metrics.
"""
total_input = 0
total_input_sequences = 0
@@ -585,6 +589,7 @@ def calculate_metrics(
Returns:
A tuple of the benchmark metrics and the actual output lengths.
"""
actual_output_lens: list[int] = []
total_input = 0
@@ -1571,6 +1576,7 @@ def compute_result_filename(
Returns:
The computed filename path or None if no result saving is requested
"""
if not (args.plot_timeline or args.save_result or args.append_result):
return None
+3 -7
View File
@@ -107,8 +107,7 @@ def _metric_to_json(m: MetricStats) -> dict[str, Any]:
@contextmanager
def cold_startup():
"""
Context manager to measure cold startup time:
"""Context manager to measure cold startup time:
1. Uses a temporary directory for vLLM cache to avoid any pollution
between cold startup iterations.
2. Uses inductor's fresh_cache to clear torch.compile caches.
@@ -132,8 +131,7 @@ def cold_startup():
def run_startup_in_subprocess(engine_args, result_queue):
"""
Run LLM startup in a subprocess and return timing metrics via a queue.
"""Run LLM startup in a subprocess and return timing metrics via a queue.
This ensures complete isolation between iterations.
"""
try:
@@ -229,11 +227,9 @@ def main(args: argparse.Namespace):
engine_args = EngineArgs.from_cli_args(args)
def create_llm_and_measure_startup():
"""
Create LLM instance in a subprocess and measure startup time.
"""Create LLM instance in a subprocess and measure startup time.
Returns timing metrics, using subprocess for complete isolation.
"""
# Create a queue for inter-process communication
result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
process = multiprocessing.Process(
+4 -6
View File
@@ -18,14 +18,14 @@ class ParameterSweep(list["ParameterSweepItem"]):
@classmethod
def read_from_dict(cls, data: dict[str, dict[str, object]]):
"""
Read parameter sweep from a dict format where keys are names.
"""Read parameter sweep from a dict format where keys are names.
Example:
{
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9}
}
"""
records = [{"_benchmark_name": name, **params} for name, params in data.items()]
return cls.from_records(records)
@@ -66,8 +66,7 @@ class ParameterSweepItem(dict[str, object]):
@property
def name(self) -> str:
"""
Get the name for this parameter sweep item.
"""Get the name for this parameter sweep item.
Returns the '_benchmark_name' field if present, otherwise returns a text
representation of all parameters.
@@ -103,8 +102,7 @@ class ParameterSweepItem(dict[str, object]):
return any(k in self for k in self._iter_param_key_candidates(param_key))
def _normalize_cmd_kv_pair(self, k: str, v: object) -> list[str]:
"""
Normalize a key-value pair into command-line arguments.
"""Normalize a key-value pair into command-line arguments.
Returns a list containing either:
- A single element for boolean flags (e.g., ['--flag'] or ['--flag=true'])
+1 -2
View File
@@ -178,8 +178,7 @@ def _json_load_bytes(path: Path) -> list[dict[str, object]]:
def _convert_inf_nan_strings(data: list[dict[str, object]]) -> list[dict[str, object]]:
"""
Convert string values "inf", "-inf", and "nan" to their float equivalents.
"""Convert string values "inf", "-inf", and "nan" to their float equivalents.
This handles the case where JSON serialization represents inf/nan as strings.
"""
+3 -6
View File
@@ -195,8 +195,7 @@ def run_vllm_chat(
warmup_requests: list[SampleRequest] | None = None,
prequeue_requests: bool = False,
) -> tuple[float, list[RequestOutput]]:
"""
Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
"""Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
multimodal models as it properly handles multimodal inputs and chat
formatting. For non-multimodal models, use run_vllm() instead.
"""
@@ -536,6 +535,7 @@ def _to_serve_args(args: argparse.Namespace) -> argparse.Namespace:
Returns:
A namespace satisfying get_samples's attribute reads.
"""
d = vars(args).copy()
# random_*: prefer --random-* over legacy --input/output/prefix-len.
@@ -627,10 +627,7 @@ def assign_loras(requests, args):
def validate_args(args):
"""
Validate command-line arguments.
"""
"""Validate command-line arguments."""
# === Deprecation and Defaulting ===
if args.dataset is not None:
warnings.warn(
+9 -12
View File
@@ -73,6 +73,7 @@ def make_copy_and_call(
Returns:
A wrapper function that copies inputs and calls the compiled function
"""
def copy_and_call(*args: Any) -> Any:
@@ -122,8 +123,7 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
class CompilerManager:
"""
A manager to manage the compilation process, including
"""A manager to manage the compilation process, including
caching the compiled graph, loading the compiled graph,
and compiling the graph.
@@ -150,7 +150,8 @@ class CompilerManager:
def compile_context(self, compile_range: Range) -> Generator[None, None, None]:
"""Provide compilation context for the duration of compilation to set
any torch global properties we want to scope to a single Inductor
compilation (e.g. partition rules, pass context)."""
compilation (e.g. partition rules, pass context).
"""
with pass_context(compile_range):
if self.compilation_config.use_inductor_graph_partition:
with inductor_partition_rule_context(
@@ -163,8 +164,7 @@ class CompilerManager:
def initialize_cache(
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
) -> None:
"""
Initialize the cache directory for the compiler.
"""Initialize the cache directory for the compiler.
The organization of the cache directory is as follows:
cache_dir=/path/to/hash_str/rank_i_j/prefix/
@@ -177,7 +177,6 @@ class CompilerManager:
base cache dir of /path/to/hash_str/rank_i_j/ ,
to store some common compilation artifacts.
"""
self.disable_cache = disable_cache
self.cache_dir = cache_dir
self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py")
@@ -438,12 +437,10 @@ def _merge_empty_only_subgraphs(
node_to_subgraph_id: dict[fx.Node, int],
split_op_graphs: list[int],
) -> None:
"""
Merge a partition that only contains an empty allocation op into the
"""Merge a partition that only contains an empty allocation op into the
previous partition. This avoids generating standalone empty submodules,
which can lead to empty cudagraph captures.
"""
nodes_by_subgraph_id: dict[int, list[fx.Node]] = defaultdict(list)
for node, subgraph_id in node_to_subgraph_id.items():
nodes_by_subgraph_id[subgraph_id].append(node)
@@ -637,8 +634,7 @@ def wrap_with_cudagraph_if_needed(
is_first_graph: bool,
is_last_graph: bool,
) -> Any:
"""
Wrap a piecewise backend with CUDA graph wrapper if needed.
"""Wrap a piecewise backend with CUDA graph wrapper if needed.
This function is shared between VllmBackend and
construct_serializable_fn_from_inductor_cache.
@@ -651,6 +647,7 @@ def wrap_with_cudagraph_if_needed(
Returns:
The wrapped backend if CUDA graphs are enabled, otherwise the original backend
"""
if (
not compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
@@ -883,8 +880,8 @@ class VllmBackend:
sym_shape_indices
- returns_tuple_map: dict mapping submod_name to
returns_tuple
"""
"""
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT:
return None, None, None
+6 -6
View File
@@ -8,8 +8,7 @@ from vllm.config import CUDAGraphMode, VllmConfig
class AbstractStaticGraphWrapper(Protocol):
"""
StaticGraphWrapper interface that allows platforms to wrap a callable
"""StaticGraphWrapper interface that allows platforms to wrap a callable
to be captured as a static graph.
"""
@@ -20,8 +19,7 @@ class AbstractStaticGraphWrapper(Protocol):
runtime_mode: CUDAGraphMode,
**kwargs: Any,
) -> None:
"""
Initializes the StaticGraphWrapper class with graph capturing and
"""Initializes the StaticGraphWrapper class with graph capturing and
execution-related configurations.
Args:
@@ -31,15 +29,16 @@ class AbstractStaticGraphWrapper(Protocol):
graph runtime. See CUDAGraphMode in vllm/config.py.
Note that only the subset enum `NONE`, `PIECEWISE` and `FULL`
are used as concrete runtime mode for cudagraph dispatching.
Keyword Args:
kwargs: Additional keyword arguments for platform-specific
configurations.
"""
raise NotImplementedError
def __call__(self, *args: Any, **kwargs: Any) -> Any:
"""
Executes the wrapped callable.
"""Executes the wrapped callable.
If the current runtime mode in the ForwardContext matches the runtime
mode of this instance, it replays the CUDAGraph or captures it using
@@ -53,5 +52,6 @@ class AbstractStaticGraphWrapper(Protocol):
Returns:
Any: Output of the executed callable.
"""
raise NotImplementedError
+4 -5
View File
@@ -172,8 +172,7 @@ def patch_pytree_map_over_slice():
class VllmSerializableFunction(SerializableCallable): # type: ignore[misc]
"""
A wrapper around a compiled function by vllm. It will forward the tensor
"""A wrapper around a compiled function by vllm. It will forward the tensor
inputs to the compiled function and return the result.
It also implements a serialization interface to support PyTorch's precompile
with custom backend, so that we can save and load the compiled function on
@@ -410,9 +409,7 @@ class VllmSerializableFunction(SerializableCallable): # type: ignore[misc]
@property
def co_name(self) -> Literal["VllmSerializableFunction"]:
"""
Used for depyf debugging.
"""
"""Used for depyf debugging."""
return "VllmSerializableFunction"
@@ -445,6 +442,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
If modifying the backend creation/wrapping logic, consider updating both.
Args:
fake_mode: FakeTensorMode used to unpickle the cached graph.
state: Deserialized state dict containing graph_module, example_inputs,
prefix, sym_tensor_indices, is_encoder, etc.
standalone_compile_artifacts: The StandaloneCompiledArtifacts containing
@@ -455,6 +453,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
Returns:
A VllmSerializableFunction that can be called directly.
"""
from vllm.compilation.backends import (
VllmBackend,
+2
View File
@@ -154,6 +154,7 @@ def generate_execution_code(
consts is a list of non-primitive constant objects referenced
by the generated code via __vllm_consts__. These objects are
kept alive for the lifetime of the compiled function.
"""
code, submod_names, consts = generate_execution_code_with_name(
split_gm, "execution_fn", with_submod=True
@@ -181,6 +182,7 @@ def compile_execution_fn(
Returns:
A callable that executes the stitching logic.
"""
trace_structured(
"artifact",
+11 -21
View File
@@ -25,9 +25,7 @@ logger = init_logger(__name__)
class CompilerInterface:
"""
The interface for a compiler that can be used by vLLM.
"""
"""The interface for a compiler that can be used by vLLM."""
# The name of the compiler, e.g. inductor.
# This is a class-level attribute.
@@ -36,8 +34,7 @@ class CompilerInterface:
def initialize_cache(
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
) -> None:
"""
when the vLLM process uses `cache_dir` as the cache directory,
"""When the vLLM process uses `cache_dir` as the cache directory,
the compiler should initialize itself with the cache directory,
e.g. by re-directing its own cache directory to a sub-directory.
@@ -52,8 +49,7 @@ class CompilerInterface:
pass
def compute_hash(self, vllm_config: VllmConfig) -> str:
"""
Gather all the relevant information from the vLLM config,
"""Gather all the relevant information from the vLLM config,
to compute a hash so that we can cache the compiled model.
See [`VllmConfig.compute_hash`][vllm.config.VllmConfig.compute_hash]
@@ -71,8 +67,7 @@ class CompilerInterface:
compile_range: Range,
key: str | None = None,
) -> tuple[Callable[..., Any] | None, Any | None]:
"""
Compile the graph with the given example inputs and compiler config,
"""Compile the graph with the given example inputs and compiler config,
with a range. The `compile_range` specifies the range of the inputs,
it could be concrete size (if compile_sizes is provided), e.g. [4, 4]
or a range [5, 8].
@@ -105,8 +100,7 @@ class CompilerInterface:
graph_index: int,
compile_range: Range,
) -> Callable[..., Any]:
"""
Load the compiled function from the handle.
"""Load the compiled function from the handle.
Raises an error if the handle is invalid.
The handle is the second return value of the `compile` function.
@@ -115,8 +109,7 @@ class CompilerInterface:
class AlwaysHitShapeEnv:
"""
Why do we need this class:
"""Why do we need this class:
For normal `torch.compile` usage, every compilation will have
one Dynamo bytecode compilation and one Inductor compilation.
@@ -160,7 +153,8 @@ def _get_vllm_functorch_config() -> dict[str, Any]:
"""Return the functorch config overrides that vLLM applies at compile time.
Used by both set_functorch_config() and get_inductor_factors() to ensure
the compile-time config and cache key are always consistent."""
the compile-time config and cache key are always consistent.
"""
cfg: dict[str, Any] = {}
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT:
cfg["bundled_autograd_cache"] = False
@@ -249,8 +243,7 @@ def _patch_standalone_compile_atomic_save() -> None:
class InductorStandaloneAdaptor(CompilerInterface):
"""
The adaptor for the Inductor compiler.
"""The adaptor for the Inductor compiler.
Requires PyTorch 2.8+.
This is not on by default yet, but we plan to turn it on by default for
PyTorch 2.8.
@@ -447,9 +440,7 @@ class InductorStandaloneAdaptor(CompilerInterface):
class InductorAdaptor(CompilerInterface):
"""
The adaptor for the Inductor compiler, version 2.5, 2.6, 2.7.
"""
"""The adaptor for the Inductor compiler, version 2.5, 2.6, 2.7."""
name = "inductor"
@@ -727,8 +718,7 @@ class InductorAdaptor(CompilerInterface):
return compiled_graph
def metrics_context(self) -> contextlib.AbstractContextManager[Any]:
"""
This method returns the Dynamo metrics context (if it exists,
"""This method returns the Dynamo metrics context (if it exists,
otherwise a null context). It is used by various compile components.
Present in torch>=2.6, it's used inside FxGraphCache in
torch==2.6 (but not after). It might also be used in various other
+1 -1
View File
@@ -38,7 +38,7 @@ class CUDAGraphStat:
class CUDAGraphLogging:
"""Aggregate and log cudagraph metrics"""
"""Aggregate and log cudagraph metrics."""
COLUMN_HEADERS = [
"Unpadded Tokens",
+4 -10
View File
@@ -56,8 +56,7 @@ def should_torch_compile_mm_encoder(vllm_config: VllmConfig) -> bool:
def ignore_torch_compile(cls: type[_T]) -> type[_T]:
"""
A decorator to ignore support_torch_compile decorator
"""A decorator to ignore support_torch_compile decorator
on the class. This is useful when a parent class has
a support_torch_compile decorator, but we don't want to
compile the class `cls` that inherits the parent class.
@@ -76,9 +75,7 @@ def ignore_torch_compile(cls: type[_T]) -> type[_T]:
def _should_ignore_torch_compile(cls: type[_T]) -> bool:
"""
Check if the class should be ignored for torch.compile.
"""
"""Check if the class should be ignored for torch.compile."""
return getattr(cls, IGNORE_COMPILE_KEY, False)
@@ -125,8 +122,7 @@ def support_torch_compile(
enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False,
) -> Callable[[type[_T]], type[_T]] | type[_T]:
"""
A decorator to add support for compiling the forward method of a class.
"""A decorator to add support for compiling the forward method of a class.
Usage 1: use directly as a decorator without arguments:
@@ -338,7 +334,6 @@ def _support_torch_compile(
is_encoder: bool = False,
) -> type[_T]:
"""Internal implementation of support_torch_compile decorator."""
if TorchCompileWithNoGuardsWrapper in cls.__bases__:
# support decorating multiple times
return cls
@@ -736,8 +731,7 @@ def _support_torch_compile(
def maybe_use_cudagraph_partition_wrapper(
vllm_config: VllmConfig,
) -> Generator[None, None, None]:
"""
Context manager to set/unset customized cudagraph partition wrappers.
"""Context manager to set/unset customized cudagraph partition wrappers.
If we're using Inductor-based graph partitioning, we currently have the
whole `fx.Graph` before Inductor lowering and the piecewise
+2 -3
View File
@@ -12,12 +12,10 @@ logger = init_logger(__name__)
def should_split(node: torch.fx.Node, splitting_ops: list[str]) -> bool:
"""
Check if a node should be split for dynamo graph partition.
"""Check if a node should be split for dynamo graph partition.
It operates on dynamo graph, so the node.target can be anything.
We need to check and split only on OpOverload and OpOverloadPacket.
"""
if node.op != "call_function":
return False
@@ -50,6 +48,7 @@ def inductor_partition_rule_context(
Args:
splitting_ops: List of operator names to partition on.
"""
if not splitting_ops:
logger.debug("No partition ops provided; skipping rule registration.")
@@ -45,8 +45,7 @@ if current_platform.is_cuda_alike():
class ActivationQuantPattern(VllmPatternReplacement):
"""
Base class for Activation+Quant fusions.
"""Base class for Activation+Quant fusions.
Should not be used directly.
"""
@@ -79,9 +78,7 @@ class ActivationQuantPattern(VllmPatternReplacement):
class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
"""
Fusion for SiluMul+Fp8StaticQuant Pattern
"""
"""Fusion for SiluMul+Fp8StaticQuant Pattern."""
def __init__(self) -> None:
super().__init__(kFp8StaticTensorSym)
@@ -126,9 +123,7 @@ class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
"""
Fusion for SiluMul+Nvfp4Quant Pattern
"""
"""Fusion for SiluMul+Nvfp4Quant Pattern."""
def __init__(self) -> None:
super().__init__(kNvfp4Dynamic)
@@ -182,8 +177,7 @@ class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
class SiluMulBlockQuantPattern(ActivationQuantPattern):
"""
Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
"""Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
Supports group_size 128 and 64 via QuantKey.
Parameterized on is_scale_transposed for different scale layouts.
"""
@@ -281,8 +275,7 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern):
class ActivationQuantFusionPass(VllmFusionPatternMatcherPass):
"""
This pass fuses a pre-defined set of custom ops into fused ops.
"""This pass fuses a pre-defined set of custom ops into fused ops.
It uses the torch pattern matcher to find the patterns and replace them.
Because patterns can only be registered once, the pass is a singleton.
@@ -66,7 +66,8 @@ def _view_flashinfer_nvfp4_scale_out_as_int32(
def _norm_input_weight_dtype_match(match: pm.Match) -> bool:
"""Prevent fusion when the norm input and weight dtypes differ (e.g. a Gemma
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm."""
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm.
"""
for node in match.nodes:
if node.target == _IR_RMS_NORM_OP:
x, weight = node.args[0], node.args[1]
@@ -347,8 +348,7 @@ class BasePattern:
class AllReduceRMSNormPattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (without residual)
"""This pattern replaces the allreduce + rms norm (without residual)
with fused flashinfer implementation.
Applies to allreduce + rmsnorm before attn in the first Transformer block.
"""
@@ -409,8 +409,7 @@ class AllReduceRMSNormPattern(BasePattern):
class AllReduceFusedAddRMSNormPattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (with residual)
"""This pattern replaces the allreduce + rms norm (with residual)
with fused flashinfer implementation.
Applies to o_proj + rmsnorm after attn and mlp + rmsnorm before attn.
"""
@@ -609,8 +608,7 @@ class AllReduceFusedAddGemmaRMSNormPattern(BasePattern):
class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (without residual)
"""This pattern replaces the allreduce + rms norm (without residual)
+ static fp8 quant with fused flashinfer implementation.
Applies to allreduce + rmsnorm + quant before attn
in the first Transformer block.
@@ -684,8 +682,7 @@ class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (with residual)
"""This pattern replaces the allreduce + rms norm (with residual)
+ static fp8 quant with fused flashinfer implementation.
Applies to o_proj + rmsnorm after attn + quant and
mlp + rmsnorm + quant before attn.
@@ -767,8 +764,7 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (without residual)
"""This pattern replaces the allreduce + rms norm (without residual)
+ static nvfp4 quant with fused flashinfer implementation.
Applies to allreduce + rmsnorm + quant before attn
in the first Transformer block.
@@ -864,8 +860,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (with residual)
"""This pattern replaces the allreduce + rms norm (with residual)
+ static nvfp4 quant with fused flashinfer implementation.
Applies to o_proj + rmsnorm after attn + quant and
mlp + rmsnorm + quant before attn.
@@ -36,8 +36,7 @@ _FP8_QUANT_KEY = QuantKey(dtype=FP8_DTYPE, scale=kStaticTensorScale, symmetric=T
class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
"""
Fusion for Attention+Fp8StaticQuant.
"""Fusion for Attention+Fp8StaticQuant.
Only triggers when the attention implementation returns True in
`fused_output_quant_supported()`. If the pattern is found, the
@@ -172,8 +171,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
class AttnNvfp4QuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
"""
Fusion for Attention+Nvfp4Quant.
"""Fusion for Attention+Nvfp4Quant.
Only triggers when the attention implementation returns True in
`fused_output_quant_supported()`. If the pattern is found, the
@@ -360,8 +358,7 @@ class AttnNvfp4QuantPattern(
class AttnQuantFusionPass(VllmFusionPatternMatcherPass):
"""
This pass fuses post-attention quantization onto attention if supported.
"""This pass fuses post-attention quantization onto attention if supported.
It uses the pattern matcher and matches each layer manually, as strings
cannot be wildcarded. This also lets us check support on attention layers
@@ -79,7 +79,7 @@ class MatcherCustomOp(ABC):
return torch.empty(*args, dtype=torch.float32, device=self.device, **kwargs)
def inputs(self) -> list[torch.Tensor]:
"""Utility for inputs to the pattern"""
"""Utility for inputs to the pattern."""
raise NotImplementedError
@@ -33,8 +33,7 @@ MLA_ATTN_OP = torch.ops.vllm.unified_mla_attention_with_output.default
class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
"""
Fusion for MLA Attention+Fp8StaticQuant.
"""Fusion for MLA Attention+Fp8StaticQuant.
Matches the pattern: MLA attention -> static FP8 quant, and replaces
it with MLA attention(output_scale=scale, output=fp8_buffer).
@@ -174,8 +173,7 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
class MLAAttnNvfp4QuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
"""
Fusion for MLA Attention+Nvfp4Quant.
"""Fusion for MLA Attention+Nvfp4Quant.
Matches the pattern: MLA attention -> NVFP4 quant, and replaces
it with MLA attention(output_scale=scale, output_block_scale=block_scale,
@@ -362,8 +360,7 @@ class MLAAttnNvfp4QuantPattern(
class MLAAttnFp8GroupQuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
"""
Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
"""Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
Matches the pattern: MLA attention -> per_token_group_fp8_quant, and
replaces it with MLA attention(output_block_scale=group_scale_buffer).
@@ -572,8 +569,7 @@ class MLAAttnFp8GroupQuantPattern(
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
"""
This pass fuses post-attention quantization onto MLA attention if supported.
"""This pass fuses post-attention quantization onto MLA attention if supported.
It uses the pattern matcher and matches each MLA layer manually, as strings
cannot be wildcarded. This also lets us check support on attention layers
@@ -34,8 +34,7 @@ P = ParamSpec("P")
class QkNormRopePattern:
"""
Match the unfused sequence in attention blocks and replace with the fused op.
"""Match the unfused sequence in attention blocks and replace with the fused op.
Unfused (conceptually):
q, k, v = split(qkv, [qsz, kvsz, kvsz], -1)
@@ -106,8 +106,7 @@ direct_register_custom_op(
class QkNormRopeKvCachePattern:
"""
Match the unfused sequence:
"""Match the unfused sequence:
q, k, v = split(qkv, ...)
q = rms_norm(q.view(heads), q_weight).view(flat)
k = rms_norm(k.view(heads), k_weight).view(flat)
@@ -413,8 +412,7 @@ class QkNormRopeKvCachePattern:
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
"""
Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
"""Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
attention layers that support the combined operation, eliminating two
@@ -168,8 +168,7 @@ if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
class FusedRMSQuantKey(NamedTuple):
"""
Named tuple for identifying the type of RMSNorm + quant fusion.
"""Named tuple for identifying the type of RMSNorm + quant fusion.
quant: type of quantization
fused_add: does the op also perform the residual add
"""
@@ -771,8 +770,7 @@ class FusedAddRMSNormNvfp4QuantPattern:
class RMSNormQuantFusionPass(VllmPatternMatcherPass):
"""
This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
"""This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
It also supports fused_add_rms_norm.
"""
@@ -180,8 +180,7 @@ class AiterFusedAddRMSNormDynamicQuantPattern(AiterRMSNormQuantPattern):
class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""
This pattern fuses aiter rms_norm & group fp8 quant custom
"""This pattern fuses aiter rms_norm & group fp8 quant custom
ops into an aiter rms_norm_group_fp8_quant op.
"""
@@ -236,8 +235,7 @@ class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""
This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
"""This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
into a aiter rms_norm_with_add_group_fp8_quant op.
"""
@@ -298,8 +296,7 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""
Pattern matching ``rms_norm`` whose output feeds *two* distinct
"""Pattern matching ``rms_norm`` whose output feeds *two* distinct
``rocm_aiter_group_fp8_quant`` consumers, replacing it with two
independent fused ``rms_norm_group_fp8_quant`` ops.
@@ -367,8 +364,7 @@ class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
"""
View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
"""View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
Matches the same 1-to-2 fan-out, but with a ``view``/``reshape`` between
the ``rms_norm`` output and the two ``rocm_aiter_group_fp8_quant``
@@ -453,8 +449,7 @@ class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""
Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
"""Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
with rocm_aiter_fused_rms_gated_fp8_group_quant.
The norm operates per-head on (N*H, D) tensors. The compiler folds the
@@ -544,8 +539,7 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
"""
This pass fuses aiter rms_norm & vllm/aiter quant custom ops
"""This pass fuses aiter rms_norm & vllm/aiter quant custom ops
into a fused rms_norm_quant op.
It also supports fused_add_rms_norm.
"""
@@ -681,8 +675,7 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
"""
This pattern fuses aiter silu_and_mul & group fp8 quant custom
"""This pattern fuses aiter silu_and_mul & group fp8 quant custom
ops into an aiter silu_and_mul_group_fp8_quant op.
"""
@@ -722,8 +715,7 @@ class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
"""
This pass fuses a pre-defined set of custom ops into fused ops.
"""This pass fuses a pre-defined set of custom ops into fused ops.
It uses the torch pattern matcher to find the patterns and replace them.
Because patterns can only be registered once, the pass is a singleton.
@@ -744,8 +736,7 @@ class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
class AddAiterRMSNormPadPattern:
"""
This pattern replaces an aiter_rmsnorm_with_add & a pad op
"""This pattern replaces an aiter_rmsnorm_with_add & a pad op
with a custom triton_add_rmsnorm_pad op from AITER.
"""
@@ -820,8 +811,7 @@ class AddAiterRMSNormPadPattern:
class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
"""
This pass replaces an AITER CK RMSNorm + residual add and a pad op
"""This pass replaces an AITER CK RMSNorm + residual add and a pad op
with an triton_add_rmsnorm_pad op from AITER.
"""
@@ -854,8 +844,7 @@ class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
class MLADualRMSNormPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]
):
"""
Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
"""Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
AITER's ``fused_qk_rmsnorm`` HIP kernel.
Target FX-graph pattern (unfused, ``vllm_ir`` stage)::
@@ -946,8 +935,7 @@ class MLADualRMSPerTokenQuantPattern(
],
]
):
"""
Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
"""Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
plus kv-latent RMSNorm -- into AITER's ``fused_qk_rmsnorm_per_token_quant``.
With a per-token FP8 ``q_b_proj`` (Quark / ModelOpt), the earlier
@@ -1064,8 +1052,7 @@ class MLADualRMSPerTokenQuantPattern(
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
"""
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
"""Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
``fused_qk_rmsnorm`` HIP kernel.
@@ -52,8 +52,7 @@ def fused_rope_and_unified_kv_cache_update_impl(
is_neox: bool,
layer_name: LayerNameType,
) -> torch.Tensor:
"""
This impl fetches the KV cache and slot mapping from the forward context,
"""This impl fetches the KV cache and slot mapping from the forward context,
then calls the layer impl's `AttentionImpl.do_rope_and_kv_cache_update` method.
It also returns a dummy tensor, similar to `Attention.unified_kv_cache_update`,
that is passed to unified_attention to signal a side effect and
@@ -98,8 +97,7 @@ direct_register_custom_op(
class RopeStaticQQuantKVCachePattern:
"""
Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
"""Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
dependency ordering.
"""
@@ -277,8 +275,7 @@ class RopeStaticQQuantKVCachePattern:
class RopeReshapeKVCachePattern:
"""
This pattern matches the following unfused inplace ops:
"""This pattern matches the following unfused inplace ops:
q, k = rotary_embedding(positions, q, k, head_size, cos_sin_cache, is_neox)
kv_cache_dummy = unified_kv_cache_update(k, v, layer_name)
@@ -410,8 +407,7 @@ class RopeReshapeKVCachePattern:
class RopeKVCacheFusionPass(VllmPatternMatcherPass):
"""
This pass fuses the rotary embedding and KV cache update operations
"""This pass fuses the rotary embedding and KV cache update operations
into a single fused kernel if available.
It uses the pattern matcher and matches each layer manually, as strings
@@ -57,8 +57,7 @@ def get_sequence_parallelism_threshold(
tp_size: int,
element_size: int,
) -> int | None:
"""
Calculate the minimum token threshold for applying sequence parallelism.
"""Calculate the minimum token threshold for applying sequence parallelism.
Returns None if sequence parallelism should not be applied based on model size.
@@ -496,8 +495,7 @@ class MiddleAllReduceRMSNormStaticNVFP4Pattern(_SequenceParallelPatternHelper):
class SequenceParallelismPass(VllmPatternMatcherPass):
"""
This pass enables sequence parallelism for models.
"""This pass enables sequence parallelism for models.
It identifies patterns where an AllReduce operation is followed by
an RMSNorm (or RMSNorm and then Quantization) operation.
These patterns are replaced with a ReduceScatter operation, followed by
@@ -590,8 +588,7 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
self.dump_patterns(config, self.patterns)
def is_applicable_for_range(self, compile_range: Range) -> bool:
"""
Determines if sequence parallelism should be applied for the given
"""Determines if sequence parallelism should be applied for the given
compile range.
SP is only beneficial for larger batch sizes where the communication
+8 -12
View File
@@ -64,14 +64,12 @@ def _hash_source_cached(*srcs: str | type | types.FunctionType) -> str:
class InductorPass(CustomGraphPass): # type: ignore[misc]
"""
A custom graph pass that uses a hash of its source as the UUID.
"""A custom graph pass that uses a hash of its source as the UUID.
This is defined as a convenience and should work in most cases.
"""
def uuid(self) -> str:
"""
Provide a unique identifier for the pass, used in Inductor code cache.
"""Provide a unique identifier for the pass, used in Inductor code cache.
This should depend on the pass implementation, so that changes to the
pass result in recompilation.
By default, the object source is hashed.
@@ -80,14 +78,14 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
@staticmethod
def hash_source(*srcs: str | Any) -> str:
"""
Utility method to hash the sources of functions or objects.
"""Utility method to hash the sources of functions or objects.
Args:
srcs: strings or objects to add to the hash.
Objects and functions have their source inspected.
Results are cached by resolved types to avoid repeated
inspect.getsource() calls.
"""
# Resolve instances to their class for a hashable cache key.
cache_key = tuple(
@@ -98,11 +96,11 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
@staticmethod
def hash_dict(dict_: dict[Any, Any]) -> str:
"""
Utility method to hash a dictionary, can alternatively be used for uuid.
"""Utility method to hash a dictionary, can alternatively be used for uuid.
Returns:
A sha256 hash of the json rep of the dictionary.
"""
encoded = json.dumps(dict_, sort_keys=True).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
@@ -112,8 +110,7 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
class CallableInductorPass(InductorPass):
"""
This class is a wrapper for a callable that automatically provides an
"""This class is a wrapper for a callable that automatically provides an
implementation of the UUID.
"""
@@ -131,8 +128,7 @@ class CallableInductorPass(InductorPass):
def enable_fake_mode(fn: Callable[P, R]) -> Callable[P, R]:
"""
Applies a FakeTensorMode context. This is useful when you don't want to
"""Applies a FakeTensorMode context. This is useful when you don't want to
create or run things with real tensors.
"""
@@ -70,8 +70,7 @@ def user_writes_to_node(user: fx.Node, node: fx.Node) -> bool:
class UnsafeCloneEliminationPass(VllmInductorPass):
"""
This pass removes clone nodes that are no longer needed after vLLM IR lowering.
"""This pass removes clone nodes that are no longer needed after vLLM IR lowering.
It uses donated_input_ids to eliminate clones of donated graph inputs, preserving
contents of non-donated graph inputs.
@@ -19,8 +19,7 @@ logger = init_logger(__name__)
class VllmIRInplaceFunctionalizationPass(VllmInductorPass):
"""
This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
"""This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
The maybe_inplace overloads have the same signature as the default overload
so the pass simply replaces the called overload.
That makes the graph properly functional.
+2 -5
View File
@@ -23,9 +23,7 @@ logger = init_logger(__name__)
class VllmIRLoweringPass(VllmInductorPass):
"""
This pass lowers vLLM IR ops to their implementations the priority list.
"""
"""This pass lowers vLLM IR ops to their implementations the priority list."""
def __init__(self, vllm_config: VllmConfig) -> None:
super().__init__(vllm_config)
@@ -113,8 +111,7 @@ class VllmIRLoweringPass(VllmInductorPass):
logger.warning("Full node list: %s", ",".join(str(n) for n in failed_nodes))
def uuid(self) -> str:
"""
IR op priority & impl sources affect lowering pass output,
"""IR op priority & impl sources affect lowering pass output,
so we include them in the cache key.
"""
priorities = {name: op.get_priority() for name, op in IrOp.registry.items()}
+3 -6
View File
@@ -71,8 +71,7 @@ R = TypeVar("R")
def with_pattern_match_debug(fn: Callable[P, R]) -> Callable[P, R]:
"""
Function decorator that turns on inductor pattern match debug
"""Function decorator that turns on inductor pattern match debug
for the duration of the call.
Used to avoid logging builtin Inductor pattern matching.
"""
@@ -89,8 +88,7 @@ def with_pattern_match_debug(fn: Callable[P, R]) -> Callable[P, R]:
class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
"""
The pass manager for post-grad passes.
"""The pass manager for post-grad passes.
It handles configuration, adding custom passes, and running passes.
It supports uuid for the Inductor code cache. That includes torch<2.6
support using pickling (in .inductor_pass.CustomGraphPass).
@@ -236,8 +234,7 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
self.passes.append(pass_)
def uuid(self) -> str:
"""
The PostGradPassManager is set as a custom pass in the Inductor and
"""The PostGradPassManager is set as a custom pass in the Inductor and
affects compilation caching. Its uuid depends on the UUIDs of all
dependent passes and the pass config. See InductorPass for more info.
"""
@@ -17,8 +17,7 @@ logger = init_logger(__name__)
class FixFunctionalizationPass(VllmInductorPass):
"""
This pass defunctionalizes certain nodes to avoid redundant tensor copies.
"""This pass defunctionalizes certain nodes to avoid redundant tensor copies.
After this pass, DCE (dead-code elimination) should never be run,
as de-functionalized nodes may appear as dead code.
@@ -260,9 +259,7 @@ class FixFunctionalizationPass(VllmInductorPass):
self.nodes_to_remove.clear()
def _remove(self, node_or_nodes: torch.fx.Node | Iterable[torch.fx.Node]) -> None:
"""
Stage a node (or nodes) for removal at the end of the pass.
"""
"""Stage a node (or nodes) for removal at the end of the pass."""
if isinstance(node_or_nodes, torch.fx.Node):
self.nodes_to_remove.append(node_or_nodes)
else:
@@ -275,8 +272,7 @@ class FixFunctionalizationPass(VllmInductorPass):
mutated_args: dict[int, torch.fx.Node | str],
args: tuple[torch.fx.Node | str, ...] | None = None,
) -> None:
"""
De-functionalize a node by replacing it with a call to the original.
"""De-functionalize a node by replacing it with a call to the original.
It also replaces the getitem users with the mutated arguments.
See replace_users_with_mutated_args and insert_defunctionalized.
"""
@@ -287,14 +283,14 @@ class FixFunctionalizationPass(VllmInductorPass):
def replace_users_with_mutated_args(
self, node: torch.fx.Node, mutated_args: dict[int, torch.fx.Node | str]
) -> None:
"""
Replace mutated getitem users of the auto-functionalized node with the
"""Replace mutated getitem users of the auto-functionalized node with the
mutated arguments.
Args:
node: The auto-functionalized node
mutated_args: The mutated arguments, indexed by getitem index.
If the value of an arg is a string, `node.kwargs[arg]` is used.
"""
for idx, user in self.getitem_users(node).items():
# Some functionalized nodes may return both a result at getitem[0]
@@ -310,8 +306,7 @@ class FixFunctionalizationPass(VllmInductorPass):
self._remove(user)
def getitem_users(self, node: torch.fx.Node) -> dict[int, torch.fx.Node]:
"""
Returns the operator.getitem users of the auto-functionalized node,
"""Returns the operator.getitem users of the auto-functionalized node,
indexed by the index they are getting.
"""
users = {}
@@ -327,8 +322,7 @@ class FixFunctionalizationPass(VllmInductorPass):
node: torch.fx.Node,
args: tuple[torch.fx.Node | str, ...] | None = None,
) -> None:
"""
Insert a new defunctionalized node into the graph before node.
"""Insert a new defunctionalized node into the graph before node.
If one of the kwargs is 'out', provide args directly,
as node.kwargs cannot be used.
See https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351
@@ -338,6 +332,7 @@ class FixFunctionalizationPass(VllmInductorPass):
node: The auto-functionalized node to defunctionalize
args: If we cannot use kwargs, specify args directly.
If an arg is a string, `node.kwargs[arg]` is used.
""" # noqa: E501
assert is_func(node, auto_functionalized), (
f"node must be auto-functionalized, is {node} instead"
@@ -16,8 +16,7 @@ logger = init_logger(__name__)
class NoOpEliminationPass(VllmInductorPass):
"""
This is an inductor pass that removes redundant reshape/slice operations.
"""This is an inductor pass that removes redundant reshape/slice operations.
It is required for RMSNorm-quant fusion to work properly.
That's because apply_fp8_linear adds a reshape, which is redundant
in the 2D-case. Additionally, torch internal no-op elimination pass does
@@ -106,8 +105,7 @@ class NoOpEliminationPass(VllmInductorPass):
# ---------------------- Shape comparison helpers ----------------------
def dims_equivalent(self, dim: int | SymInt, i_dim: int | SymInt) -> bool:
"""
This function checks if two dimensions are equivalent.
"""This function checks if two dimensions are equivalent.
Args:
dim: The dimension arg to reshape/slice
@@ -119,6 +117,7 @@ class NoOpEliminationPass(VllmInductorPass):
There are two cases in which the dimensions are equivalent:
1. The dimensions are equal (both integers)
2. The dimensions both correspond to the same SymInt
"""
# Case 1
return statically_known_true(dim == i_dim) # type: ignore[no-any-return]
@@ -6,8 +6,7 @@ from ..vllm_inductor_pass import VllmInductorPass
class PostCleanupPass(VllmInductorPass):
"""
This pass performs cleanup after custom passes.
"""This pass performs cleanup after custom passes.
It topologically sorts the graph and removes unused nodes.
This is needed because the pattern matcher does not guarantee producing
a topologically sorted graph, and there may be unused nodes left around.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Replace ``slice_scatter`` and ``split_with_sizes`` nodes with a single
"""Replace ``slice_scatter`` and ``split_with_sizes`` nodes with a single
assignment if there are no users for the inplace tensor written to by
the slice_scatter call.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
input tensor with the same split sizes.
On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor
@@ -10,9 +9,10 @@ that CSE fails to merge. This pass detects and replaces the duplicates
so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion)
see a single split node with all users attached.
See also:
See Also:
- vLLM #33295 (original issue)
- PyTorch #174472 (upstream CSE gap)
"""
import operator
@@ -30,7 +30,8 @@ logger = init_logger(__name__)
class SplitCoalescingPass(VllmInductorPass):
"""Replace duplicate ``split_with_sizes`` nodes with a single canonical
node when they share the same input tensor and split sizes."""
node when they share the same input tensor and split sizes.
"""
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
+7 -13
View File
@@ -35,8 +35,7 @@ class InductorCompilationConfig:
class VllmInductorPass(InductorPass):
"""
An inductor pass with access to vLLM PassConfig.
"""An inductor pass with access to vLLM PassConfig.
It provides timing, logging, and dumping utilities.
"""
@@ -93,8 +92,7 @@ def get_match_table() -> dict[str, int]:
class VllmPatternMatcherPass(VllmInductorPass):
"""
A VllmInductorPass that uses the Inductor pattern matcher.
"""A VllmInductorPass that uses the Inductor pattern matcher.
Provides pattern registration with match counting, debug dumping, and logging.
"""
@@ -109,7 +107,7 @@ class VllmPatternMatcherPass(VllmInductorPass):
)
def _replace_op_overloads(self, string: str) -> str:
"""Replace <OpOverload(..., ...)> with nicer formulations"""
"""Replace <OpOverload(..., ...)> with nicer formulations."""
return str(
self._OP_OVERLOAD_PATTERN.sub(
lambda m: f"torch.ops.{m.group(1)}.{m.group(2)}",
@@ -123,8 +121,7 @@ class VllmPatternMatcherPass(VllmInductorPass):
logger.debug("fusion pass matches: %s", dict(cls.match_table))
def dump_patterns(self, config: VllmConfig, pm_pass: PatternMatcherPass) -> None:
"""
If debug dumping is enabled, dump the Inductor pattern-matcher patterns
"""If debug dumping is enabled, dump the Inductor pattern-matcher patterns
into the debug_dump_path folder next to the dumped fx graphs.
This method does its best to print something that looks like Python code
@@ -195,8 +192,7 @@ R = TypeVar("R")
class VllmPatternReplacement(ABC, Generic[P, R]):
"""
A pattern/replacement pair for FX graph fusion.
"""A pattern/replacement pair for FX graph fusion.
Implement the three abstract members below, then pass
instances to VllmFusionPatternMatcherPass.register(). The pass will
@@ -216,8 +212,7 @@ class VllmPatternReplacement(ABC, Generic[P, R]):
@property
@abstractmethod
def replacement(self) -> Callable[P, R]:
"""
Returns a closure defining the FX subgraph to
"""Returns a closure defining the FX subgraph to
substitute in place of each match.
"""
...
@@ -294,8 +289,7 @@ def _remove_noop_permutes(gm: fx.GraphModule) -> None:
class VllmFusionPatternMatcherPass(VllmPatternMatcherPass):
"""
A VllmPatternMatcherPass for passes that use VllmPatternReplacement objects.
"""A VllmPatternMatcherPass for passes that use VllmPatternReplacement objects.
Subclasses register patterns via self.register() in their own __init__.
"""
+1 -2
View File
@@ -96,8 +96,7 @@ class PiecewiseBackend:
compiled_runnables: dict[str, Callable[..., Any]] | None = None,
submod_name: str = "",
):
"""
The backend for piecewise compilation.
"""The backend for piecewise compilation.
It mainly handles the compilation of static shapes and
dispatching based on runtime shape.
+3 -7
View File
@@ -45,8 +45,7 @@ def _compilation_context() -> Generator[None, None, None]:
class TorchCompileWithNoGuardsWrapper:
"""
A wrapper class for torch.compile, it ensures that all guards are dropped
"""A wrapper class for torch.compile, it ensures that all guards are dropped
when CompilationMode is not CompilationMode.STOCK_TORCH_COMPILE.
When guards are dropped, the first time __call__ is invoked, a single
compilation is triggered. Dynamo should never be traced again after that
@@ -272,8 +271,7 @@ class TorchCompileWithNoGuardsWrapper:
@contextmanager
def _dispatch_to_compiled_code(self) -> Generator[None, None, None]:
# noqa: E501
"""
Context manager to dispatch to internally compiled code for torch<2.8.
"""Context manager to dispatch to internally compiled code for torch<2.8.
Why does this work? Because Dynamo guarantees that the compiled
bytecode has exactly the same arguments, cell variables, and free
variables as the original code. Therefore we can directly switch
@@ -291,9 +289,7 @@ class TorchCompileWithNoGuardsWrapper:
def reset_compile_wrapper(model: torch.nn.Module) -> None:
"""
Clean up compiled model and captured CUDA graphs for elastic EP.
"""
"""Clean up compiled model and captured CUDA graphs for elastic EP."""
if not isinstance(model, TorchCompileWithNoGuardsWrapper) and hasattr(
model, "model"
):
+1 -2
View File
@@ -150,8 +150,7 @@ class AttentionConfig:
return self.indexer_kv_dtype
def compute_hash(self) -> str:
"""
Provide a hash that uniquely identifies all the configs
"""Provide a hash that uniquely identifies all the configs
that affect the structure of the computation
graph from input ids/embeddings to the final hidden states,
excluding anything before input ids/embeddings and after
+1 -2
View File
@@ -248,8 +248,7 @@ class CacheConfig:
KV offloading is only activated when kv_offloading_size is set."""
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
+15 -24
View File
@@ -36,7 +36,8 @@ logger = init_logger(__name__)
class CompilationMode(enum.IntEnum):
"""The compilation approach used for torch.compile-based compilation of the
model."""
model.
"""
NONE = 0
"""No torch.compile compilation is applied, model runs in fully eager pytorch mode.
@@ -192,12 +193,10 @@ class PassConfig:
# TODO(luka) better pass enabling system.
def flashinfer_max_size(self, world_size: int) -> int | None:
"""
Returns the max communication size in bytes for flashinfer
"""Returns the max communication size in bytes for flashinfer
allreduce fusion for the given world size. Returns None if world size
is not supported by configs as it's not supported by flashinfer.
"""
MiB = 1024 * 1024
FI_SUPPORTED_WORLD_SIZES = [2, 4, 8, 16]
if world_size not in FI_SUPPORTED_WORLD_SIZES:
@@ -223,12 +222,10 @@ class PassConfig:
return FI_ALLREDUCE_FUSION_MAX_SIZE_MB.get(capability.to_int(), {})
def compute_hash(self) -> str:
"""
Produces a hash unique to the pass configuration.
"""Produces a hash unique to the pass configuration.
Any new fields that affect compilation should be added to the hash.
Any future fields that don't affect compilation should be excluded.
"""
return hash_factors(get_hash_factors(self, set()))
@field_validator(
@@ -317,8 +314,7 @@ class PassConfig:
self.fuse_rope_kvcache_cat_mla = False
def log_enabled_passes(self) -> None:
"""
Log the enabled custom fusion passes.
"""Log the enabled custom fusion passes.
This is called at the end of VLLMConfig post_init,
after all defaults are finalized.
TODO also log the compile ranges for which this is enabled.
@@ -341,7 +337,8 @@ class PassConfig:
class DynamicShapesType(str, enum.Enum):
"""Types of dynamic shapes handling in torch.compile().
see Dynamic shapes and vllm guard dropping in torch_compile.md
for more details."""
for more details.
"""
BACKED = "backed"
"""Use backed dynamic shapes. torch.compile() guards on backed dynamic
@@ -391,10 +388,7 @@ class DynamicShapesConfig:
"""
def compute_hash(self) -> str:
"""
Provide a hash for DynamicShapesConfig
"""
"""Provide a hash for DynamicShapesConfig."""
from vllm.config.utils import get_hash_factors, hash_factors
factors = get_hash_factors(self, set())
@@ -790,8 +784,7 @@ class CompilationConfig:
]
def compute_hash(self) -> str:
"""
Provide a hash that uniquely identifies all the configs
"""Provide a hash that uniquely identifies all the configs
that affect the structure of the computation
graph from input ids/embeddings to the final hidden states,
excluding anything before input ids/embeddings and after
@@ -856,8 +849,7 @@ class CompilationConfig:
@field_validator("mode", mode="before")
@classmethod
def validate_mode_before(cls, value: Any) -> Any:
"""
Enable parsing the `mode` field from string mode names.
"""Enable parsing the `mode` field from string mode names.
Accepts both integers (0-3) and string names, like NONE, STOCK_TORCH_COMPILE,
DYNAMO_TRACE_ONCE, VLLM_COMPILE.
"""
@@ -1079,15 +1071,17 @@ class CompilationConfig:
prefix: str = "",
is_encoder: bool = False,
) -> str | Callable:
"""
Initialize the backend for the compilation config from a vllm config.
"""Initialize the backend for the compilation config from a vllm config.
Arguments:
vllm_config: The vllm config to initialize the backend from.
prefix: Cache directory prefix for this compiled module.
is_encoder: Whether this module is used in an encoder (as
opposed to a text backbone).
Returns:
The backend for the compilation config.
"""
if self.mode is None:
raise ValueError(
@@ -1122,7 +1116,6 @@ class CompilationConfig:
configs are set. This includes:
- initialize compile_sizes
"""
computed_compile_sizes: list[int] = []
if self.compile_sizes is not None:
# de-duplicate the sizes provided by the config
@@ -1321,13 +1314,11 @@ class CompilationConfig:
return self.backend == "inductor" and self.mode != CompilationMode.NONE
def custom_op_log_check(self):
"""
This method logs the enabled/disabled custom ops and checks that the
"""This method logs the enabled/disabled custom ops and checks that the
passed custom_ops field only contains relevant ops.
It is called at the end of set_current_vllm_config,
after the custom ops have been instantiated.
"""
if len(self.enabled_custom_ops) + len(self.disabled_custom_ops) == 0:
logger.debug("No custom ops found in model.")
return
+1 -2
View File
@@ -28,8 +28,7 @@ class DeviceConfig:
`__post_init__`."""
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
+1 -2
View File
@@ -31,8 +31,7 @@ class EncoderCacheManagerConfig:
class EncoderCacheManagerMetadata(ABC): # noqa: B024
"""
Abstract Metadata used to communicate between the
"""Abstract Metadata used to communicate between the
Scheduler EncoderCacheManager and Worker EncoderCacheManager.
"""
+1 -2
View File
@@ -63,8 +63,7 @@ class ECTransferConfig:
Only supported in V1."""
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
+7 -15
View File
@@ -18,8 +18,7 @@ logger = init_logger(__name__)
@config
class IrOpPriorityConfig:
"""
Configuration for vLLM IR op priority for dispatching/lowering during the
"""Configuration for vLLM IR op priority for dispatching/lowering during the
forward pass. Each member is a list of strings, which will be installed
in worker init via vllm.ir.ops.<op_name>.set_default().
A single comma-separated string is accepted as well,
@@ -38,8 +37,7 @@ class IrOpPriorityConfig:
"""Priority list for vllm.ir.ops.gelu_and_mul_sparse"""
def compute_hash(self) -> str:
"""
Produces a hash unique to the pass configuration.
"""Produces a hash unique to the pass configuration.
Any new fields that affect compilation should be added to the hash.
Any future fields that don't affect compilation should be excluded.
@@ -71,8 +69,7 @@ class IrOpPriorityConfig:
return value
def _iter_op_priorities(self):
"""
Yield (IrOp, priority_list) for each field, after importing platform
"""Yield (IrOp, priority_list) for each field, after importing platform
kernels and validating each entry.
"""
from vllm.ir.op import IrOp
@@ -89,16 +86,13 @@ class IrOpPriorityConfig:
yield IrOp.registry[field.name], op_priority
def set_default(self) -> None:
"""
Permanently set the IR op priority for all op members.
"""
"""Permanently set the IR op priority for all op members."""
for ir_op, op_priority in self._iter_op_priorities():
ir_op.set_default(op_priority)
@contextlib.contextmanager
def set_priority(self):
"""
Context manager to set the IR op priority for all op members.
"""Context manager to set the IR op priority for all op members.
It also imports IR kernel implementations for the current platform
to ensure all implementations are made available.
"""
@@ -111,8 +105,7 @@ class IrOpPriorityConfig:
def with_default(
cls, default: list[str], /, **kwargs: list[str]
) -> "IrOpPriorityConfig":
"""
A helper to create an IrOpPriorityConfig where fields not specified in kwargs
"""A helper to create an IrOpPriorityConfig where fields not specified in kwargs
use the given default list.
"""
for field in fields(cls): # type: ignore[arg-type]
@@ -355,8 +348,7 @@ class KernelConfig:
return value
def compute_hash(self) -> str:
"""
Produces a hash unique to the pass configuration.
"""Produces a hash unique to the pass configuration.
Any new fields that affect compilation should be added to the hash.
Any future fields that don't affect compilation should be excluded.
"""
+1 -2
View File
@@ -106,8 +106,7 @@ class KVTransferConfig:
'fail': immediately fail the request with an error finish reason (default)"""
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
+1 -2
View File
@@ -118,8 +118,7 @@ class LoadConfig:
"""
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
+1 -2
View File
@@ -86,8 +86,7 @@ class LoRAConfig:
MoE models whose adapters use this layout; ignored otherwise."""
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
+15 -21
View File
@@ -418,8 +418,7 @@ class ModelConfig:
mm_processor_device: InitVar[MMProcessorDevice | None] = None
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
@@ -938,7 +937,7 @@ class ModelConfig:
@model_validator(mode="after")
def validate_model_config_after(self: "ModelConfig") -> "ModelConfig":
"""Called after __post_init__"""
"""Called after __post_init__."""
if not isinstance(self.tokenizer, str):
raise ValueError(
f"tokenizer must be a string, got "
@@ -1006,7 +1005,8 @@ class ModelConfig:
def _get_transformers_backend_cls(self) -> str:
"""Determine which Transformers modeling backend class will be used if
`model_impl` is set to `transformers` or `auto`."""
`model_impl` is set to `transformers` or `auto`.
"""
cls = "Transformers"
# If 'hf_config is not hf_text_config' it's a nested config, i.e. multimodal
cls += "MultiModal" if self.hf_config is not self.hf_text_config else ""
@@ -1083,8 +1083,8 @@ class ModelConfig:
Args:
model: Model name or path
tokenizer: Tokenizer name or path
"""
"""
# Skip if model_weights is already set (model already pulled)
if self.model_weights:
return
@@ -1662,9 +1662,7 @@ class ModelConfig:
raise AssertionError(f"Unsupported block type: {block_type}")
def get_mamba_chunk_size(self) -> int:
"""
Returns the mamba chunk size if it exists
"""
"""Returns the mamba chunk size if it exists."""
# used by e.g. Bamba, FalconH1, Granite
chunk_size = getattr(self.hf_text_config, "mamba_chunk_size", None)
if chunk_size is None:
@@ -1679,11 +1677,11 @@ class ModelConfig:
return chunk_size
def get_multimodal_config(self) -> MultiModalConfig:
"""
Get the multimodal configuration of the model.
"""Get the multimodal configuration of the model.
Raises:
ValueError: If the model is not multimodal.
"""
if self.multimodal_config is None:
raise ValueError("The model is not multimodal.")
@@ -1691,8 +1689,7 @@ class ModelConfig:
return self.multimodal_config
def try_get_generation_config(self) -> dict[str, Any]:
"""
This method attempts to retrieve the non-default values of the
"""This method attempts to retrieve the non-default values of the
generation config for this model.
The generation config can contain information about special tokens, as
@@ -1701,6 +1698,7 @@ class ModelConfig:
Returns:
A dictionary containing the non-default generation config.
"""
if self.generation_config in {"auto", "vllm"}:
config = try_get_generation_config(
@@ -1726,8 +1724,7 @@ class ModelConfig:
return config.to_diff_dict()
def get_diff_sampling_param(self) -> dict[str, Any]:
"""
This method returns a dictionary containing the non-default sampling
"""This method returns a dictionary containing the non-default sampling
parameters with `override_generation_config` applied.
The default sampling parameters are:
@@ -1739,6 +1736,7 @@ class ModelConfig:
Returns:
A dictionary containing the non-default sampling parameters.
"""
src = self.generation_config
@@ -1870,8 +1868,7 @@ class ModelConfig:
@property
def score_type(self) -> ScoreType:
"""
Scoring API handles score/rerank for:
"""Scoring API handles score/rerank for:
- "classify" task (score_type: cross-encoder models)
- "embed" task (score_type: bi-encoder models)
@@ -1964,8 +1961,7 @@ class ModelConfig:
@property
def head_dtype(self) -> torch.dtype:
"""
"head" refers to the last Linear layer(s) of an LLM,
"""The "head" refers to the last Linear layer(s) of an LLM,
such as the lm_head in a generation model,
or the score or classifier in a classification model.
@@ -1976,7 +1972,6 @@ class ModelConfig:
fp32, which is required for RL training-inference consistency
(the trainer computes logits in fp32).
"""
head_dtype = _get_head_dtype(
config=self.hf_config, dtype=self.dtype, runner_type=self.runner_type
)
@@ -2174,8 +2169,7 @@ class ModelConfig:
def get_served_model_name(model: str, served_model_name: str | list[str] | None):
"""
If the input is a non-empty list, the first model_name in
"""If the input is a non-empty list, the first model_name in
`served_model_name` is taken.
If the input is a non-empty string, it is used directly.
For cases where the input is either an empty string or an
+1 -3
View File
@@ -14,9 +14,7 @@ logger = init_logger(__name__)
@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
class ModelArchitectureConfig:
"""
Configuration for model architecture that required by vLLM runtime
"""
"""Configuration for model architecture that required by vLLM runtime."""
architectures: list[str]
"""List of model architecture class names (e.g., ['LlamaForCausalLM']).
+8 -7
View File
@@ -357,6 +357,7 @@ class MultiModalConfig:
Returns:
The kwargs to build the config with, unchanged unless the flag adds
a `device`.
"""
if mm_processor_device in (None, "auto"):
return mm_processor_kwargs
@@ -390,6 +391,7 @@ class MultiModalConfig:
ValueError: If `device` is not something `torch.device` accepts.
`validate_mm_processor_device` is what surfaces this during
startup, so the value is only parsed once.
"""
device = (self.mm_processor_kwargs or {}).get("device")
if device is None:
@@ -418,6 +420,7 @@ class MultiModalConfig:
ValueError: If the requested device is not a torch device, or if it
is the accelerator on an instance that also runs the language
model.
"""
from vllm.platforms import current_platform
@@ -452,8 +455,7 @@ class MultiModalConfig:
)
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
@@ -476,8 +478,7 @@ class MultiModalConfig:
return hash_str
def get_limit_per_prompt(self, modality: str) -> int:
"""
Get the maximum number of input items allowed per prompt
"""Get the maximum number of input items allowed per prompt
for the given modality (backward compatible).
"""
if self.language_model_only:
@@ -495,8 +496,7 @@ class MultiModalConfig:
self,
inference_kwargs: Mapping[str, object],
) -> dict[str, object]:
"""
Get the keyword arguments to pass to the multi-modal processor
"""Get the keyword arguments to pass to the multi-modal processor
according to the extra arguments passed during inference.
"""
kwargs = self.mm_processor_kwargs or {}
@@ -524,7 +524,8 @@ class MultiModalConfig:
def get_video_pruning_spec(self) -> tuple[VideoPruningMethod, float] | None:
"""Return `(method, rate)` when video pruning is enabled, else None.
`rate` is the fraction of video tokens to prune."""
`rate` is the fraction of video tokens to prune.
"""
if self.video_pruning_rate is not None and self.video_pruning_rate > 0:
return (self.video_pruning_method, float(self.video_pruning_rate))
return None
+3 -3
View File
@@ -112,8 +112,7 @@ class ObservabilityConfig:
)
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
@@ -157,7 +156,8 @@ class ObservabilityConfig:
cls, value: list[DetailedTraceModules] | None
) -> list[DetailedTraceModules] | None:
"""Handle the legacy case where users might provide a comma-separated
string instead of a list of strings."""
string instead of a list of strings.
"""
if value is not None and len(value) == 1 and "," in value[0]:
value = cast(list[DetailedTraceModules], value[0].split(","))
return value
+1 -2
View File
@@ -137,8 +137,7 @@ class OffloadConfig:
return self
def compute_hash(self) -> str:
"""
Provide a hash that uniquely identifies all the offload configs.
"""Provide a hash that uniquely identifies all the offload configs.
All fields are included because PrefetchOffloader patches module
forwards and inserts custom ops (wait_prefetch, start_prefetch)
+4 -6
View File
@@ -602,15 +602,13 @@ class ParallelConfig:
@property
def local_engines_only(self) -> bool:
"""
Client manages local+remote EngineCores in pure internal LB case.
"""Client manages local+remote EngineCores in pure internal LB case.
Client manages local EngineCores in hybrid and external LB case.
"""
return self.data_parallel_external_lb or self.data_parallel_hybrid_lb
def get_next_dp_init_port(self) -> int:
"""
We might need to initialize process groups in multiple
"""We might need to initialize process groups in multiple
processes that is related to data parallelism,
e.g. both in the worker and in the engine, which
can live in different processes. To avoid port conflicts, we
@@ -794,6 +792,7 @@ class ParallelConfig:
Returns:
(has_unfinished_global, pause_consensus)
"""
tensor = torch.tensor(
[int(has_unfinished), int(pending_pause)], dtype=torch.int32, device="cpu"
@@ -815,8 +814,7 @@ class ParallelConfig:
return tensor.item()
def compute_hash(self):
"""
Provide a hash that uniquely identifies all the configs
"""Provide a hash that uniquely identifies all the configs
that affect the structure of the computation
graph from input ids/embeddings to the final hidden states,
excluding anything before input ids/embeddings and after
+1 -2
View File
@@ -176,8 +176,7 @@ class PoolerConfig:
return self.tok_pooling_type
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
+1 -2
View File
@@ -151,8 +151,7 @@ class ProfilerConfig:
"""
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
+4 -2
View File
@@ -47,13 +47,15 @@ class ReasoningConfig:
@property
def enabled(self) -> bool:
"""Returns True if reasoning is enabled (i.e. if token IDs have been
initialized), False otherwise."""
initialized), False otherwise.
"""
return self._enabled
@property
def reasoning_start_token_ids(self) -> list[int] | None:
"""Token IDs derived from `reasoning_start_str`. Set automatically by
`initialize_token_ids`. Not intended to be configured directly."""
`initialize_token_ids`. Not intended to be configured directly.
"""
return self._reasoning_start_token_ids
@property
+2 -5
View File
@@ -200,9 +200,7 @@ class SchedulerConfig:
@staticmethod
def default_factory(**kwargs):
"""
Factory method to create `SchedulerConfig` with default values for `InitVar`s.
"""
"""Create a `SchedulerConfig` with default values for its `InitVar`s."""
if "max_model_len" not in kwargs:
kwargs["max_model_len"] = 8192
if "is_encoder_decoder" not in kwargs:
@@ -233,8 +231,7 @@ class SchedulerConfig:
return resolve_obj_by_qualname(self.scheduler_cls)
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
+8 -9
View File
@@ -543,7 +543,8 @@ class SpeculativeConfig:
@staticmethod
def _acceptance_length_to_rates(length: float, n: int) -> list[float]:
"""Mean acceptance length to unconditional per-position rates, using
the minimum-variance schedule."""
the minimum-variance schedule.
"""
num_drafts = length - 1 # expected number of accepted draft tokens
num_full = int(num_drafts)
return (
@@ -557,7 +558,8 @@ class SpeculativeConfig:
length: float | None,
) -> list[float]:
"""Return per-position unconditional acceptance rates from exactly one
of `rates` or `length` (validates range, length, and monotonicity)."""
of `rates` or `length` (validates range, length, and monotonicity).
"""
if (rates is None) == (length is None):
raise ValueError(
"rejection_sample_method='synthetic' requires exactly one of "
@@ -598,8 +600,7 @@ class SpeculativeConfig:
top-k base-logit candidates. Requires draft tensor parallel size 1."""
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
@@ -1627,7 +1628,6 @@ class SpeculativeConfig:
speculative_max_model_len is mainly used for testing that sequences can
skip speculation.
"""
if speculative_max_model_len is not None:
if speculative_max_model_len > draft_max_model_len:
raise ValueError(
@@ -1671,6 +1671,7 @@ class SpeculativeConfig:
Args:
draft_hf_config: The draft model's HF config, mutated in place.
target_max_model_len: The target model's max_model_len.
"""
draft_max_position_embeddings = getattr(
draft_hf_config, "max_position_embeddings", None
@@ -1695,8 +1696,7 @@ class SpeculativeConfig:
speculative_draft_tensor_parallel_size: int | None,
draft_hf_config: PretrainedConfig,
) -> int:
"""
Verifies and adjusts the tensor parallel size for a draft model
"""Verifies and adjusts the tensor parallel size for a draft model
specified using speculative_draft_tensor_parallel_size.
"""
# If speculative_draft_tensor_parallel_size is unset then set it
@@ -1725,8 +1725,7 @@ class SpeculativeConfig:
return speculative_draft_tensor_parallel_size
def update_arch_(self):
"""
EagleConfig and ExtractHiddenStatesConfig update architectures, so update all
"""EagleConfig and ExtractHiddenStatesConfig update architectures, so update all
architectures-related fields in self.draft_model_config
"""
self.draft_model_config.hf_text_config = get_hf_text_config(
+1 -2
View File
@@ -42,8 +42,7 @@ class StructuredOutputsConfig:
"""Whether to use structured input for reasoning."""
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
+14 -14
View File
@@ -64,7 +64,9 @@ def config(
cls: The class to decorate
config: The pydantic ConfigDict to use. If provided, it will be merged with
the default config.
**kwargs: Additional arguments to pass to pydantic.dataclass."""
**kwargs: Additional arguments to pass to pydantic.dataclass.
"""
# Extra fields are forbidden by default
merged_config = ConfigDict(extra="forbid")
if config is not None:
@@ -82,7 +84,8 @@ def config(
def get_field(cls: ConfigType, name: str) -> Any:
"""Get the default factory field of a dataclass by name. Used for getting
default factory fields in `EngineArgs`."""
default factory fields in `EngineArgs`.
"""
if not is_dataclass(cls):
raise TypeError("The given class is not a dataclass.")
try:
@@ -119,7 +122,8 @@ def is_init_field(cls: ConfigType, name: str) -> bool:
def replace(dataclass_instance: ConfigT, /, **kwargs) -> ConfigT:
"""Like [`dataclasses.replace`](https://docs.python.org/3/library/dataclasses.html#dataclasses.replace),
but compatible with Pydantic dataclasses which use `pydantic.fields.Field` instead
of `dataclasses.field`"""
of `dataclasses.field`
"""
cls = type(dataclass_instance)
dataclass_dict = dataclass_instance.__dict__
dataclass_dict = {k: v for k, v in dataclass_dict.items() if is_init_field(cls, k)}
@@ -134,8 +138,7 @@ def getattr_iter(
default_factory: Callable[[], Any] | None = None,
warn: bool = False,
) -> Any:
"""
A helper function that retrieves an attribute from an object which may
"""A helper function that retrieves an attribute from an object which may
have multiple possible names. This is useful when fetching attributes from
arbitrary `transformers.PretrainedConfig` instances.
@@ -158,12 +161,10 @@ def getattr_iter(
def get_attr_docs(cls: type[Any]) -> dict[str, str]:
"""
Get any docstrings placed after attribute assignments in a class body.
"""Get any docstrings placed after attribute assignments in a class body.
https://davidism.com/mit-license/
"""
cls_node = ast.parse(textwrap.dedent(inspect.getsource(cls))).body[0]
if not isinstance(cls_node, ast.ClassDef):
@@ -385,8 +386,7 @@ def hash_factors(items: dict[str, object]) -> str:
@dataclass
class Range:
"""
A range of numbers.
"""A range of numbers.
Inclusive of start, inclusive of end.
"""
@@ -445,8 +445,7 @@ def get_from_deprecated_env_if_set(
removal_version: str,
field_name: str | None = None,
) -> str | None:
"""
Get value from deprecated environment variable with warning.
"""Get value from deprecated environment variable with warning.
Args:
env_name: Name of the deprecated environment variable
@@ -455,6 +454,7 @@ def get_from_deprecated_env_if_set(
Returns:
The environment variable value if set, None otherwise
"""
if envs.is_set(env_name):
value = os.environ.get(env_name)
@@ -477,8 +477,7 @@ def set_from_deprecated_env_if_set(
to_bool: bool = False,
to_int: bool = False,
) -> None:
"""
Set object field from deprecated environment variable with warning.
"""Set object field from deprecated environment variable with warning.
Args:
config: Config object to set the field on
@@ -489,6 +488,7 @@ def set_from_deprecated_env_if_set(
to_int: Whether to convert the environment variable value to integer
Returns:
None
"""
if to_bool and to_int:
raise ValueError("Cannot convert to both boolean and integer.")
+19 -28
View File
@@ -153,8 +153,8 @@ IS_DENSE = False
def enable_norm_fusion(cfg: "VllmConfig") -> bool:
"""Enable if either RMS norm or quant FP8 custom op is active;
otherwise Inductor handles fusion."""
otherwise Inductor handles fusion.
"""
return (
cfg.compilation_config.is_custom_op_enabled("rms_norm")
or cfg.compilation_config.is_custom_op_enabled("quant_fp8")
@@ -163,8 +163,7 @@ def enable_norm_fusion(cfg: "VllmConfig") -> bool:
def enable_act_fusion(cfg: "VllmConfig") -> bool:
"""
Enable if either SiLU+Mul or quant FP8 custom op is active;
"""Enable if either SiLU+Mul or quant FP8 custom op is active;
otherwise Inductor handles fusion.
Also enable for FP4 models as FP4 quant is always custom so Inductor cannot fuse it.
"""
@@ -220,7 +219,6 @@ def enable_rope_kvcache_fusion(cfg: "VllmConfig") -> bool:
def enable_rope_kvcache_mla_fusion(cfg: "VllmConfig") -> bool:
"""Enable if use_inductor_graph_partition is enabled."""
return (
cfg.compilation_config.use_inductor_graph_partition
or not cfg.compilation_config.splitting_ops_contain_kv_cache_update()
@@ -229,7 +227,6 @@ def enable_rope_kvcache_mla_fusion(cfg: "VllmConfig") -> bool:
def enable_norm_pad_fusion(cfg: "VllmConfig") -> bool:
"""Enable if using AITER RMSNorm and hidden size is 2880 i.e. gpt-oss."""
return (
cfg.kernel_config.ir_op_priority.fused_add_rms_norm[0] == "aiter"
and cfg.model_config is not None
@@ -460,8 +457,7 @@ class VllmConfig:
"""
def compute_hash(self, include_version: bool = True) -> str:
"""
WARNING: Whenever a new field is added to this config,
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
@@ -473,6 +469,7 @@ class VllmConfig:
Args:
include_version: Include the vLLM version in the hash.
"""
factors: list[Any] = []
@@ -747,7 +744,8 @@ class VllmConfig:
def _is_dflash2_draft(self) -> bool:
"""Whether the DFlash draft is a DFlash2 one, by the architecture the
speculator selects on (v1/worker/gpu/spec_decode/__init__.py)."""
speculator selects on (v1/worker/gpu/spec_decode/__init__.py).
"""
spec = self.speculative_config
if spec is None or spec.method != "dflash":
return False
@@ -798,8 +796,7 @@ class VllmConfig:
@property
def needs_dp_coordinator(self) -> bool:
"""
Determine if the DPCoordinator process is needed.
"""Determine if the DPCoordinator process is needed.
The DPCoordinator is needed in two cases:
1. For MoE models with DP > 1: to handle wave coordination
@@ -809,8 +806,8 @@ class VllmConfig:
Returns:
True if DPCoordinator process is needed, False otherwise.
"""
"""
# For non-MoE models, only need coordinator in internal/hybrid LB mode
# (for stats collection).
return self.parallel_config.data_parallel_size > 1 and (
@@ -820,8 +817,7 @@ class VllmConfig:
)
def enable_trace_function_call_for_thread(self) -> None:
"""
Set up function tracing for the current thread,
"""Set up function tracing for the current thread,
if enabled via the `VLLM_TRACE_FUNCTION` environment variable.
"""
if envs.VLLM_TRACE_FUNCTION:
@@ -955,6 +951,7 @@ class VllmConfig:
config_obj: Configuration object to update.
key: Attribute name.
value: Default value (static or callable).
"""
if getattr(config_obj, key) is None:
# Some config values are known before initialization and are
@@ -976,6 +973,7 @@ class VllmConfig:
Args:
defaults: Dictionary of default values to apply.
"""
def apply_recursive(config_obj: Any, config_defaults: dict[str, Any]) -> None:
@@ -1276,7 +1274,6 @@ class VllmConfig:
def __post_init__(self):
"""Verify configs are valid & consistent with each other."""
# To give each torch profile run a unique instance name.
self.instance_id = f"{time.time_ns()}"
@@ -2133,8 +2130,7 @@ class VllmConfig:
]
def _set_max_num_scheduled_tokens(self):
"""
In most cases, the scheduler may schedule a batch with as many tokens as the
"""In most cases, the scheduler may schedule a batch with as many tokens as the
worker is configured to handle.
"""
if self.speculative_config is not None:
@@ -2172,8 +2168,7 @@ class VllmConfig:
)
def _set_cudagraph_sizes(self):
"""
vLLM defines the default candidate list of batch sizes for CUDA graph
"""VLLM defines the default candidate list of batch sizes for CUDA graph
capture as:
```python
@@ -2219,8 +2214,8 @@ class VllmConfig:
padded CUDA graph will be used.
- If batch size > largest `cudagraph_capture_sizes`, cudagraph will
not be used.
"""
"""
if (
self.model_config is not None
and not self.model_config.enforce_eager
@@ -2441,9 +2436,7 @@ class VllmConfig:
self.compilation_config.post_init_cudagraph_sizes()
def _set_compile_ranges(self):
"""
Set the compile ranges for the compilation config.
"""
"""Set the compile ranges for the compilation config."""
compilation_config = self.compilation_config
computed_compile_ranges_endpoints = []
@@ -3310,8 +3303,7 @@ _current_prefix: str | None = None
def set_current_vllm_config(
vllm_config: VllmConfig, check_compile=False, prefix: str | None = None
):
"""
Temporarily set the current vLLM config.
"""Temporarily set the current vLLM config.
Used during model initialization.
We save the current vLLM config in a global variable,
so that all modules can access it, e.g. custom ops
@@ -3392,15 +3384,14 @@ def get_layers_from_vllm_config(
layer_type: type[T],
layer_names: Iterable[str] | None = None,
) -> dict[str, T]:
"""
Get layers from the vLLM config.
"""Get layers from the vLLM config.
Args:
vllm_config: The vLLM config.
layer_type: The type of the layer to get.
layer_names: The names of the layers to get. If None, return all layers.
"""
"""
forward_context = vllm_config.compilation_config.static_forward_context
if layer_names is None:
layer_names = forward_context.keys()
+13 -17
View File
@@ -80,8 +80,7 @@ def use_memory_pool_with_allocator(
class CuMemAllocator:
"""
A singleton class that manages a memory pool for CUDA tensors.
"""A singleton class that manages a memory pool for CUDA tensors.
The memory in this pool can be offloaded or discarded when the
allocator sleeps.
@@ -109,8 +108,7 @@ class CuMemAllocator:
@staticmethod
def get_instance() -> "CuMemAllocator":
"""
CuMemAllocator is a singleton class.
"""CuMemAllocator is a singleton class.
We cannot call the constructor directly.
Call this method to get the instance.
"""
@@ -182,9 +180,9 @@ class CuMemAllocator:
self.release_pools()
def _python_malloc_callback(self, allocation_handle: HandleType) -> None:
"""Internal method to store the allocation data
when memory is allocated in the memory pool.
"""
Internal method to store the allocation data
when memory is allocated in the memory pool."""
py_d_mem = allocation_handle[2]
self.pointer_to_data[py_d_mem] = AllocationData(
allocation_handle, self.current_tag
@@ -198,9 +196,9 @@ class CuMemAllocator:
return
def _python_free_callback(self, ptr: int) -> HandleType:
"""Internal method to look up the allocation data
when memory is freed in the memory pool.
"""
Internal method to look up the allocation data
when memory is freed in the memory pool."""
data = self.pointer_to_data.pop(ptr)
if data.cpu_backup_tensor is not None:
data.cpu_backup_tensor = None
@@ -227,14 +225,14 @@ class CuMemAllocator:
return data.handle
def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None:
"""
Put the allocator in sleep mode.
"""Put the allocator in sleep mode.
All data in the memory allocation with the specified tag will be
offloaded to CPU memory, and others will be discarded.
Args:
offload_tags: The tags of the memory allocation that will be
offloaded. The rest of the memory allocation will be discarded.
"""
if offload_tags is None:
# by default, allocated tensors are offloaded
@@ -325,8 +323,7 @@ class CuMemAllocator:
)
def wake_up(self, tags: list[str] | None = None) -> None:
"""
Wake up the allocator from sleep mode.
"""Wake up the allocator from sleep mode.
All data that is previously offloaded will be loaded back to GPU
memory, and the rest of the data will have empty memory.
@@ -334,6 +331,7 @@ class CuMemAllocator:
tags: The tags of the memory allocation that will be loaded
back to GPU memory. If None, all memory allocation will be loaded
back to GPU memory.
"""
gc.collect()
torch.accelerator.empty_cache()
@@ -357,14 +355,14 @@ class CuMemAllocator:
@contextmanager
def use_memory_pool(self, tag: str | None = None):
"""
A context manager to use the memory pool.
"""A context manager to use the memory pool.
All memory allocation created inside the context will be allocated
in the memory pool, and has the specified tag.
Args:
tag: The tag of the memory allocation. If None, the default tag
will be used.
"""
if tag is None:
tag = CuMemAllocator.default_tag
@@ -416,9 +414,7 @@ class CuMemAllocator:
torch.cuda.memory._set_allocator_settings("expandable_segments:True")
def get_current_usage(self) -> int:
"""
Get the total number of bytes allocated in the memory pool.
"""
"""Get the total number of bytes allocated in the memory pool."""
sum_bytes: int = 0
for ptr, data in self.pointer_to_data.items():
handle = data.handle
+10 -5
View File
@@ -70,7 +70,8 @@ class SleepModeBackend(ABC):
def state(self) -> SleepModeState:
"""Current lifecycle state. Lets ``/health`` distinguish a healthy-idle
(suspended) engine from a healthy-serving one (see RFC #34303)."""
(suspended) engine from a healthy-serving one (see RFC #34303).
"""
return self._state
# -- Capability introspection (no instance required) --
@@ -83,26 +84,30 @@ class SleepModeBackend(ABC):
@classmethod
def preserves_communicators(cls) -> bool:
"""If False, collective communicators (e.g. NCCL) are destroyed by
``suspend`` and the executor must re-initialize them on ``resume``."""
``suspend`` and the executor must re-initialize them on ``resume``.
"""
return False
@classmethod
def preserves_compiled_artifacts(cls) -> bool:
"""If True, torch.compile / JIT kernels survive suspend/resume and need
not be recompiled on resume."""
not be recompiled on resume.
"""
return False
@classmethod
def preserves_graphs_with_communicators(cls) -> bool:
"""If True, CUDA graphs containing collective communicators (e.g. NCCL)
stay valid after resume. False when communicators are rebuilt (embedded
comm handles go stale)."""
comm handles go stale).
"""
return False
@classmethod
def supports_durable_storage(cls) -> bool:
"""If True, suspended state can be persisted beyond the process
lifetime (disk or object storage) and restored in a new process."""
lifetime (disk or object storage) and restored in a new process.
"""
return False
+1
View File
@@ -116,6 +116,7 @@ class XpuMemAllocator:
Sleep will offload selected payloads to CPU or discard and unmap XPU
physical memory. Wake-up remaps physical memory back to the same
reserved virtual address and restores payload.
"""
instance: "XpuMemAllocator | None" = None
@@ -22,9 +22,7 @@ class AiterCustomAllreduce:
@classmethod
def effective_max_size(cls) -> int:
"""
Max input byte size eligible for AITER custom allreduce.
"""
"""Max input byte size eligible for AITER custom allreduce."""
return cls.MAX_SIZE // 2
def __init__(
@@ -44,8 +44,7 @@ logger = init_logger(__name__)
class AgRsAll2AllManager(All2AllManagerBase):
"""
An implementation of all2all communication based on
"""An implementation of all2all communication based on
all-gather (dispatch) and reduce-scatter (combine).
"""
@@ -79,9 +78,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Gather hidden_states and router_logits from all dp ranks.
"""
"""Gather hidden_states and router_logits from all dp ranks."""
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
@@ -111,9 +108,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Gather hidden_states and router_logits from all dp ranks.
"""
"""Gather hidden_states and router_logits from all dp ranks."""
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
@@ -140,9 +135,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Reduce-scatter hidden_states across all dp ranks.
"""
"""Reduce-scatter hidden_states across all dp ranks."""
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(
hidden_states.shape[0] // dist_group.world_size,
@@ -156,9 +149,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
class DeepEPAll2AllManagerBase(All2AllManagerBase):
"""
All2All communication based on DeepEP High-Throughput kernels.
"""
"""All2All communication based on DeepEP High-Throughput kernels."""
def __init__(self, cpu_group, tcp_store_group=None):
assert has_deep_ep(), (
@@ -210,9 +201,7 @@ class DeepEPAll2AllManagerBase(All2AllManagerBase):
class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
"""
All2All communication based on DeepEP High-Throughput kernels.
"""
"""All2All communication based on DeepEP High-Throughput kernels."""
def __init__(self, cpu_group, tcp_store_group=None):
super().__init__(cpu_group, tcp_store_group)
@@ -271,9 +260,7 @@ class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
"""
All2All communication based on DeepEP Low-Latency kernels.
"""
"""All2All communication based on DeepEP Low-Latency kernels."""
_buffer: Any = None
_mask: torch.Tensor | None = None
@@ -293,8 +280,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
num_global_experts: int,
num_local_experts: int,
) -> dict[Any, Any]:
"""
max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
"""max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
can dispatch all the ranks must hold the same value.
token_hidden_size: the hidden dimension of each token.
num_ep_ranks: the number of EP group ranks.
@@ -330,8 +316,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
return kwargs
def get_handle(self, kwargs):
"""
The kwargs for DeepEPLLAll2AllManager is dictated by
"""The kwargs for DeepEPLLAll2AllManager is dictated by
_make_all2all_kwargs.
"""
import deep_ep # type: ignore[import-not-found]
@@ -385,8 +370,7 @@ class _NixlEPBufferState:
class NixlEPAll2AllManager(All2AllManagerBase):
"""
All2All communication based on NIXL EP kernels.
"""All2All communication based on NIXL EP kernels.
This backend supports elastic EP with dynamic rank connection/disconnection.
"""
@@ -612,9 +596,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
"""
All2All communication based on flashinfer all2allv/two-sided NVLink kernels.
"""
"""All2All communication based on flashinfer all2allv/two-sided NVLink kernels."""
# This type lint could be removed after all of the work in
# https://github.com/vllm-project/vllm/issues/26533 done.
@@ -640,7 +622,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
rank: int,
gpus_per_node: int,
):
"""Initialize workspace"""
"""Initialize workspace."""
if self.initialized:
return
@@ -681,7 +663,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
)
def ensure_alltoall_workspace_initialized(self):
"""Ensure workspace is initialized"""
"""Ensure workspace is initialized."""
if not has_flashinfer_nvlink_two_sided():
return False
@@ -700,7 +682,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
return self
def cleanup(self):
"""Clean up workspace"""
"""Clean up workspace."""
if (
self.initialized
and self.workspace_tensor is not None
@@ -719,8 +701,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
"""
All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
"""All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
This is a newer kernel from trtllm that should perform better than the kernel
used by flashinfer_nvlink_two_sided.
"""
@@ -1023,8 +1004,7 @@ class MoriAll2AllManager(All2AllManagerBase):
class DeepEPV2All2AllManager(All2AllManagerBase):
"""
All2All communication based on DeepEP v2 ElasticBuffer (unified API).
"""All2All communication based on DeepEP v2 ElasticBuffer (unified API).
Uses NCCL Gin backend with analytical SM calculation.
"""
@@ -119,8 +119,7 @@ NCCL_SYMM_MEM_ALL_REDUCE_CONFIG: dict[str, Any] = {
def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor) -> bool:
"""
Determine if NCCL symmetric memory allreduce should be used.
"""Determine if NCCL symmetric memory allreduce should be used.
Based on H100 and GB200 benchmarks, NCCL symm_mem is preferred for:
- Small tensors (≤16K): Lower latency than custom_AR
@@ -157,7 +156,8 @@ def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor)
def should_nccl_symm_mem_ag_rs() -> bool:
"""Check whether NCCL symmetric memory should be used for
AllGather / ReduceScatter collectives."""
AllGather / ReduceScatter collectives.
"""
from vllm.distributed.device_communicators.pynccl_allocator import (
is_symmetric_memory_enabled,
)
@@ -344,7 +344,6 @@ _gpu_p2p_access_cache: dict[str, bool] | None = None
def gpu_p2p_access_check(src: int, tgt: int) -> bool:
"""Check if GPU src can access GPU tgt."""
# if the cache variable is already calculated,
# read from the cache instead of checking it again
global _gpu_p2p_access_cache
@@ -114,6 +114,7 @@ class All2AllManagerBase:
Returns:
An int32 device tensor where 0 marks a live rank and 1 marks a
masked (dead/unreachable) rank.
"""
raise NotImplementedError
@@ -171,8 +172,7 @@ class All2AllManagerBase:
class DeviceCommunicatorBase:
"""
Base class for device-specific communicator.
"""Base class for device-specific communicator.
It can use the `cpu_group` to initialize the communicator.
If the device has PyTorch integration (PyTorch can recognize its
communication backend), the `device_group` will also be given.
@@ -318,8 +318,7 @@ class DeviceCommunicatorBase:
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
"""NOTE: We assume that the input tensor is on the same device across
all the ranks.
NOTE: `dst` is the local rank of the destination rank.
"""
@@ -347,7 +346,7 @@ class DeviceCommunicatorBase:
return output_tensor
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""Sends a tensor to the destination rank in a blocking way."""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
dst = (self.rank_in_group + 1) % self.world_size
@@ -385,8 +384,7 @@ class DeviceCommunicatorBase:
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
if extra_tensors is not None:
@@ -404,8 +402,7 @@ class DeviceCommunicatorBase:
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
if extra_tensors is not None:
@@ -415,8 +412,7 @@ class DeviceCommunicatorBase:
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
return hidden_states
@@ -69,8 +69,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
logger.info("Using allgather_reducescatter all2all manager.")
def _all_group_ranks_share_shm_group_name(self) -> bool:
"""
CPUSHM requires all ranks in this group to agree on one SHM group name.
"""CPUSHM requires all ranks in this group to agree on one SHM group name.
This is a lightweight consistency check for VLLM_DIST_IDENT/name inputs.
"""
local_name = _CPUSHMDistributed.make_group_name(self)
@@ -89,8 +88,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
"""NOTE: We assume that the input tensor is on the same device across
all the ranks.
NOTE: `dst` is the local rank of the destination rank.
"""
@@ -180,11 +178,9 @@ class CpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
@@ -204,8 +200,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -220,8 +215,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -611,7 +611,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
return symm_output
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""Sends a tensor to the destination rank in a blocking way."""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
dst = (self.rank_in_group + 1) % self.world_size
@@ -851,11 +851,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
@@ -875,8 +873,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -891,8 +888,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -129,8 +129,7 @@ class CustomAllreduce:
max_mnnvl_reduce_scatter_size=_DEFAULT_MNNVL_REDUCE_SCATTER_MAX_SIZE,
symm_mem_enabled=False,
) -> None:
"""
Args:
"""Args:
group: the process group to work on. If None, it will use the
default process group.
device: the device to bind the CustomAllreduce to. If None,
@@ -138,6 +137,7 @@ class CustomAllreduce:
It is the caller's responsibility to make sure each communicator
is bind to a unique device, and all communicators in this group
are in the same node.
"""
self._IS_CAPTURING = False
self._ptr = 0
@@ -370,8 +370,7 @@ class CustomAllreduce:
@contextmanager
def capture(self):
"""
The main responsibility of this context manager is the
"""The main responsibility of this context manager is the
`register_graph_buffers` call at the end of the context.
It records all the buffer addresses used in the CUDA graph.
"""
@@ -140,6 +140,7 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]:
is True only when ``auto`` selects mnnvl for a single node, so that
workspace creation can fall back to trtllm on single-node topologies
without NVSwitch multicast support (where mnnvl is unavailable).
"""
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
if backend != "auto":
@@ -168,8 +169,7 @@ def get_fi_ar_workspace(
dtype: torch.dtype,
group: ProcessGroup,
):
"""
Return the allreduce workspace for non-quant patterns, initializing if needed.
"""Return the allreduce workspace for non-quant patterns, initializing if needed.
Used by AllReduceFusionPass (non-quant patterns) and FlashInferAllReduce
for standalone allreduce. Backend is controlled by
@@ -234,8 +234,7 @@ def get_fi_ar_quant_workspace(
dtype: torch.dtype,
group: ProcessGroup,
):
"""
Return the allreduce workspace for quant patterns, initializing if needed.
"""Return the allreduce workspace for quant patterns, initializing if needed.
Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var, matching
non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm
@@ -98,8 +98,7 @@ class PyNcclCommunicator:
device: int | str | torch.device,
library_path: str | None = None,
):
"""
Args:
"""Args:
group: the process group to work on. If None, it will use the
default process group.
device: the device to bind the PyNcclCommunicator to. If None,
@@ -108,6 +107,7 @@ class PyNcclCommunicator:
use the default library path.
It is the caller's responsibility to make sure each communicator
is bind to a unique device.
"""
if not isinstance(group, StatelessProcessGroup):
assert dist.is_initialized()
@@ -62,8 +62,7 @@ class QuickAllReduce:
}
def __init__(self, group: ProcessGroup, device: int | str | torch.device) -> None:
"""
Custom allreduce provides non-destructive acceleration and is
"""Custom allreduce provides non-destructive acceleration and is
available for CUDA and ROCm MI300 series.
Custom quick allreduce leverages quantization for further
@@ -86,6 +85,7 @@ class QuickAllReduce:
It is the caller's responsibility to make sure each communicator
is bind to a unique device, and all communicators in this group
are in the same node.
"""
self.disabled = True
if not self._rocm_arch_available():
@@ -301,8 +301,7 @@ class QuickAllReduce:
return False
def create_shared_buffer(self):
"""
Creates a shared buffer for quickreduce.
"""Creates a shared buffer for quickreduce.
Has to be called after init_custom_qr
"""
handle = ops.qr_get_handle(self._ptr)
@@ -312,9 +311,7 @@ class QuickAllReduce:
ops.qr_open_handles(self._ptr, handles)
def should_quick_allreduce(self, inp: torch.Tensor):
"""
Check if quickreduce is available
"""
"""Check if quickreduce is available."""
if self.disabled:
return False
if inp.dtype not in self._SUPPORTED_DTYPES:
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
class RayPPCommunicator(Communicator):
"""
Communicator to be used for pipeline parallelism in Ray Compiled Graph.
"""Communicator to be used for pipeline parallelism in Ray Compiled Graph.
This is wraps around the vLLM _PP GroupCoordinator.
This class is not thread-safe.
@@ -38,8 +37,7 @@ class RayPPCommunicator(Communicator):
cuda_stream: torch.cuda.Stream | None,
use_communication_streams: bool = False,
):
"""
Initialize a RayPPCommunicator that can be used to communicate with
"""Initialize a RayPPCommunicator that can be used to communicate with
other Ray Compiled Graph actors for pipeline parallelism.
Args:
@@ -54,6 +52,7 @@ class RayPPCommunicator(Communicator):
is not supported.
use_communication_streams: Whether to use communication streams.
This is not supported.
"""
self._world_size = world_size
self._rank: int | None = None
@@ -87,8 +86,7 @@ class RayPPCommunicator(Communicator):
self._closed = False
def _build_actor_rank_mapping(self):
"""
Use collective communication to build a mapping from actor IDs to ranks.
"""Use collective communication to build a mapping from actor IDs to ranks.
This should be called once during initialization.
"""
if self._comm is None:
@@ -128,9 +126,7 @@ class RayPPCommunicator(Communicator):
return self._actor_handles
def get_rank(self, actor: ray.actor.ActorHandle) -> int:
"""
Return the given actor's rank using device communicator collective ops.
"""
"""Return the given actor's rank using device communicator collective ops."""
assert hasattr(self, "_actor_id_to_rank"), (
"Actor rank mapping not built. "
"This should have been done during initialization."
@@ -144,20 +140,15 @@ class RayPPCommunicator(Communicator):
raise ValueError(f"Actor {actor} not found in communicator group")
def get_self_rank(self) -> int | None:
"""
Return this actor's rank.
"""
"""Return this actor's rank."""
return self._rank
def get_world_size(self) -> int:
"""
Return the number of ranks in the RayPPCommunicator group.
"""
"""Return the number of ranks in the RayPPCommunicator group."""
return self._world_size
def send(self, buf: "torch.Tensor", peer_rank: int) -> None:
"""
Send a torch.Tensor to a peer.
"""Send a torch.Tensor to a peer.
This returns when the send kernel has been queued, but the kernel may
not have completed. Therefore, the caller should ensure that there are
@@ -170,6 +161,7 @@ class RayPPCommunicator(Communicator):
buf: The torch.Tensor to send. It should already be on this
actor's default device.
peer_rank: The rank of the actor to send to.
"""
if self._closed:
raise RayChannelError("RayPPCommunicator has been destroyed.")
@@ -184,8 +176,7 @@ class RayPPCommunicator(Communicator):
peer_rank: int,
allocator: TorchTensorAllocator,
) -> "torch.Tensor":
"""
Receive a torch.Tensor from a peer and synchronize the current stream.
"""Receive a torch.Tensor from a peer and synchronize the current stream.
After this call returns, the receive buffer is safe to read from
any stream. An RayChannelError will be raised if an error occurred
@@ -197,6 +188,7 @@ class RayPPCommunicator(Communicator):
peer_rank: The rank of the actor to receive from.
allocator: The allocator to use to create the received tensor.
This is ignored for this implementation.
"""
if self._closed:
raise RayChannelError("RayPPCommunicator has been destroyed.")
@@ -77,8 +77,7 @@ _memory_fence_lock = threading.Lock()
def memory_fence():
"""
Full memory barrier for shared memory synchronization.
"""Full memory barrier for shared memory synchronization.
Ensures all prior memory writes are visible to other processes before
any subsequent reads. This is critical for lock-free producer-consumer
@@ -111,8 +110,7 @@ LONG_WAIT_TIME_LOG_MSG = (
class SpinCondition:
"""
This class implements an interface similar to a threading.Condition. It
"""This class implements an interface similar to a threading.Condition. It
allows a writer to notify readers to wake up and read from the shared memory
buffer. This notification is done over a zmq socket.
@@ -241,6 +239,7 @@ def check_shm_free_space(
Raises:
RuntimeError: If the SHM filesystem has insufficient space.
"""
if os.path.isdir(shm_path):
free_bytes = shutil.disk_usage(shm_path).free
@@ -651,7 +650,8 @@ class MessageQueue:
def shutdown(self):
"""If this is an idle reader, wakes it up so it can clean up and shut
down"""
down
"""
self.shutting_down = True
if self._spin_condition is not None:
self._spin_condition.cancel()
@@ -761,7 +761,8 @@ class MessageQueue:
def should_warn(self) -> bool:
"""Returns true if it's time to log a warning for a timeout that is not
indefinite"""
indefinite
"""
if self._should_warn:
elapsed = time.monotonic() - self.started
if elapsed >= VLLM_RINGBUFFER_WARNING_INTERVAL * self.n_warning:
@@ -941,8 +942,7 @@ class MessageQueue:
reader_rank: int = 0,
blocking: bool = False,
) -> tuple["MessageQueue", list[Handle]]:
"""
Creates a MessageQueue for a process group with a single reader.
"""Creates a MessageQueue for a process group with a single reader.
This method is designed for scenarios where only one process (the reader)
will consume messages, and all other processes are writers. It sets up
@@ -962,6 +962,7 @@ class MessageQueue:
tuple[MessageQueue, list[Handle]]:
The MessageQueue instance for the calling process,
and a list of handles (only non-empty for the reader process).
"""
from vllm.platforms.interface import get_assigned_physical_gpu_ids
@@ -994,8 +995,7 @@ class MessageQueue:
external_writer_handle=None,
blocking: bool = True,
) -> "MessageQueue":
"""
Creates a MessageQueue for a distributed process group with one writer and
"""Creates a MessageQueue for a distributed process group with one writer and
multiple readers.
This method is designed for scenarios where one process (the writer) sends
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
class SingleWriterShmRingBuffer:
"""
A single-writer, multiple-reader ring buffer implementation using shared
"""A single-writer, multiple-reader ring buffer implementation using shared
memory. This class provides a thread-safe ring buffer where one process
can write data while multiple processes/threads can read from it.
@@ -190,8 +189,7 @@ class SingleWriterShmRingBuffer:
return int.from_bytes(byte_data, "little", signed=True)
def allocate_buf(self, size: int) -> tuple[int, int]:
"""
Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
"""Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
Memory layout:
`[4-byte monotonic_id][4-byte size][buffer data...]`
"""
@@ -260,8 +258,7 @@ class SingleWriterShmRingBuffer:
is_free_fn: Callable[[int, memoryview], bool],
nbytes: int | None = None,
) -> Iterable[int]:
"""
Free a buffer of the given size. This is a no-op in shared memory,
"""Free a buffer of the given size. This is a no-op in shared memory,
but we need to keep track of the metadata.
If freed memory spreads across the end and start of the ring buffer,
@@ -269,10 +266,13 @@ class SingleWriterShmRingBuffer:
still might not be a contiguous space of `nbytes` available.
Args:
is_free_fn (Callable[[int, memoryview], bool]): Predicate called with
a monotonic id and the buffer, returning True when that buffer
can be reclaimed.
nbytes (int, optional): The size of the buffer to free. If None,
frees the maximum size of the ring buffer.
"""
"""
assert self.is_writer, "Only the writer can free buffers."
logger.debug(
"Freeing up space in the ring buffer, "
@@ -412,8 +412,7 @@ class ShmObjectStorageHandle:
class SingleWriterShmObjectStorage:
"""
A single-writer, multiple-reader object storage system built on top of a
"""A single-writer, multiple-reader object storage system built on top of a
shared memory ring buffer. Provides key-value storage with automatic memory
management and cross-process serialization support.
@@ -458,8 +457,7 @@ class SingleWriterShmObjectStorage:
serde_class: type[ObjectSerde] = MsgpackSerde,
reader_lock: LockType | None = None,
):
"""
Initialize the object storage.
"""Initialize the object storage.
Args:
max_object_size: Maximum size for a single object in bytes.
@@ -467,10 +465,11 @@ class SingleWriterShmObjectStorage:
ring_buffer: The shared memory ring buffer for storing objects.
serde_class: Serializer/deserializer for objects.
reader_lock: Optional lock for synchronizing reader access.
Raises:
ValueError: If reader_lock is None for readers.
"""
"""
self.max_object_size = max_object_size
self.n_readers = n_readers
self.serde_class = serde_class
@@ -547,22 +546,17 @@ class SingleWriterShmObjectStorage:
del self.writer_flag[freed_id]
def is_cached(self, key: str) -> bool:
"""
Check if the object with the given key is cached.
"""
"""Check if the object with the given key is cached."""
return key in self.key_index
def get_cached(self, key: str) -> tuple[int, int]:
"""
Get the cached object by key if it exists.
"""
"""Get the cached object by key if it exists."""
address, monotonic_id = self.key_index[key]
self.increment_writer_flag(monotonic_id)
return address, monotonic_id
def put(self, key: str, value: Any) -> tuple[int, int]:
"""
Store a key-value pair in the object storage.
"""Store a key-value pair in the object storage.
Attempts to free max_object_size bytes using FIFO order
when the ring buffer runs out of space during a put() operation.
@@ -574,6 +568,7 @@ class SingleWriterShmObjectStorage:
MemoryError: If there's not enough space in the buffer
ValueError: If the serialized object is too large
ValueError: If the key already exists in the storage
"""
if key in self.key_index:
raise ValueError(f"Key '{key}' already exists in the storage.")
@@ -639,8 +634,7 @@ class SingleWriterShmObjectStorage:
address: int = 0,
monotonic_id: int = 0,
) -> None:
"""
Touch an existing cached item to update its eviction status.
"""Touch an existing cached item to update its eviction status.
For writers (ShmObjectStoreSenderCache): Increment writer_flag
For readers (ShmObjectStoreReceiverCache): Increment reader_count
@@ -700,8 +694,7 @@ class SingleWriterShmObjectStorage:
)
def default_is_free_check(self, id: int, buf: memoryview) -> bool:
"""
Default is_free function that checks if the first 4 bytes are zero.
"""Default is_free function that checks if the first 4 bytes are zero.
This indicates that the buffer is free.
"""
reader_count = int.from_bytes(buf[0:4], "little", signed=True)
@@ -205,11 +205,9 @@ class XpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
@@ -229,8 +227,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -245,8 +242,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
ECConnectorBase Class for Distributed Encoder Cache &
"""ECConnectorBase Class for Distributed Encoder Cache &
P2P Encoder cache communication in V1
The class provides the following primitives:
@@ -50,8 +49,7 @@ class ECConnectorRole(enum.Enum):
class ECConnectorMetadata(ABC): # noqa: B024
"""
Abstract Metadata used to communicate between the
"""Abstract Metadata used to communicate between the
Scheduler ECConnector and Worker ECConnector.
"""
@@ -59,8 +57,7 @@ class ECConnectorMetadata(ABC): # noqa: B024
class ECConnectorWorkerMetadata(ABC):
"""
Abstract Metadata used to communicate back
"""Abstract Metadata used to communicate back
Worker ECConnector -> Scheduler ECConnector.
Each worker can output its own metadata.
@@ -73,9 +70,7 @@ class ECConnectorWorkerMetadata(ABC):
def aggregate(
self, other: "ECConnectorWorkerMetadata"
) -> "ECConnectorWorkerMetadata":
"""
Aggregate metadata with another `ECConnectorWorkerMetadata` object.
"""
"""Aggregate metadata with another `ECConnectorWorkerMetadata` object."""
pass
@@ -103,8 +98,7 @@ class ECConnectorBase(ABC):
return self._is_consumer
def shutdown(self) -> None:
"""
Shutdown the connector. This is called when the process
"""Shutdown the connector. This is called when the process
is shutting down to ensure that all the async operations are
completed and the connector is cleaned up properly.
"""
@@ -123,6 +117,7 @@ class ECConnectorBase(ABC):
Args:
connector_metadata (dict): the connector metadata.
"""
self._connector_metadata = connector_metadata
@@ -141,8 +136,8 @@ class ECConnectorBase(ABC):
Returns:
ConnectorMetadata: the connector metadata.
"""
"""
# Should only be called while set to valid metadata.
assert self._connector_metadata is not None
return self._connector_metadata
@@ -151,10 +146,11 @@ class ECConnectorBase(ABC):
self,
ec_caches: dict[str, torch.Tensor],
):
"""
Initialize with the EC caches.
"""Initialize with the EC caches.
Args:
ec_caches: dictionary of encoder cache
"""
# TODO: Implement this later for P2P feature
return
@@ -163,8 +159,7 @@ class ECConnectorBase(ABC):
def start_load_caches(
self, encoder_cache: dict[str, torch.Tensor], **kwargs
) -> None:
"""
Start loading the cache from the connector into vLLM's encoder cache.
"""Start loading the cache from the connector into vLLM's encoder cache.
This method loads the encoder cache based on metadata provided by the scheduler.
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
@@ -174,6 +169,7 @@ class ECConnectorBase(ABC):
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
data hashes (`mm_hash`) to encoder cache tensors.
kwargs (dict): Additional keyword arguments for the connector.
"""
pass
@@ -181,8 +177,7 @@ class ECConnectorBase(ABC):
def save_caches(
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
) -> None:
"""
Save the encoder cache to the connector.
"""Save the encoder cache to the connector.
This method saves the encoder cache from the worker's local storage
to shared storage or another external connector.
@@ -192,14 +187,14 @@ class ECConnectorBase(ABC):
data hashes (`mm_hash`) to encoder cache tensors.
mm_hash (str): The hash of the multimodal data whose cache is being saved.
kwargs (dict): Additional keyword arguments for the connector.
"""
pass
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
"""Notifies worker-side connector ids of requests that have
finished generating tokens on the worker.
The scheduler process (via the Executors) will use this output
to track which workers are done.
@@ -210,16 +205,17 @@ class ECConnectorBase(ABC):
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
return None, None
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
"""
Build the ECConnector worker metadata for this engine step.
"""Build the ECConnector worker metadata for this engine step.
Returns:
ECConnectorWorkerMetadata: the worker metadata.
None if no worker metadata is available.
"""
return None
@@ -232,8 +228,7 @@ class ECConnectorBase(ABC):
self,
identifier: str,
) -> bool:
"""
Check if a single encoder cache exists
"""Check if a single encoder cache exists.
Args:
identifier (str): the identifier of the media.
@@ -241,14 +236,14 @@ class ECConnectorBase(ABC):
Returns:
A bool where value is True if cache exist for
the media
"""
pass
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
"""
Ensure encoder cache items are available for the given request.
"""Ensure encoder cache items are available for the given request.
May initiate asynchronous transfers for items not yet local.
Args:
@@ -258,6 +253,7 @@ class ECConnectorBase(ABC):
Returns:
True if all items are ready or no transfer is needed.
False if any items are still in transit (request should be deferred).
"""
return True
@@ -282,11 +278,12 @@ class ECConnectorBase(ABC):
@abstractmethod
def update_state_after_alloc(self, request: "Request", index: int):
"""
Update ECConnector state to decide allocate cache for requests
"""Update ECConnector state to decide allocate cache for requests.
Args:
request (Request): the request object.
index (int): index of the encoder cache entry being allocated.
"""
pass
@@ -294,37 +291,37 @@ class ECConnectorBase(ABC):
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> ECConnectorMetadata:
"""
Build the connector metadata for this step.
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
pass
def update_connector_output(self, connector_output: ECConnectorOutput):
"""
Update ECConnector state from worker-side connectors output.
"""Update ECConnector state from worker-side connectors output.
Args:
connector_output (ECConnectorOutput): the worker-side
connectors output.
"""
return
def request_finished(
self, request: "Request"
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its encoder cache is freed.
"""Called when a request has finished, before its encoder cache is freed.
Returns:
True if the request is being saved/sent asynchronously and cached
should not be freed until the request_id is returned from
get_finished().
"""
return False, None
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Lightweight mmap-backed shared memory region for encoder cache (EC) data.
"""Lightweight mmap-backed shared memory region for encoder cache (EC) data.
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
@@ -229,6 +229,7 @@ class ECCPUScheduler:
Returns:
True if no item needs to wait for a remote encoding. False if any
item is pending or the request must be aborted.
"""
if not request.ec_transfer_params:
return True
@@ -259,6 +260,7 @@ class ECCPUScheduler:
Raises:
_RemoteUnavailable: The remote source is invalid, the read failed,
or the item's total wait budget expired.
"""
pos = feature.mm_position
mm_hash = feature.identifier
@@ -334,6 +336,7 @@ class ECCPUScheduler:
item's remote announcement so subsequent steps do not retry it.
False if only placeholder metadata (or no input) is available,
recording the request for take_unavailable_requests() to drain.
"""
mm_hash = feature.identifier
self._deferred_since.pop((request.request_id, mm_hash), None)
@@ -15,7 +15,8 @@ from collections import OrderedDict
class CacheEntry:
"""A single cache entry. Read `.block_ids` and `.ready` freely;
mutations only through EmbeddingCache methods."""
mutations only through EmbeddingCache methods.
"""
__slots__ = ("block_ids", "_pin_count")
@@ -67,8 +67,7 @@ class ECExampleConnector(ECConnectorBase):
raise ValueError("ec_transfer_config must be set for ECConnectorBase")
def start_load_caches(self, encoder_cache, **kwargs) -> None:
"""
Start loading the cache from the connector into vLLM's encoder cache.
"""Start loading the cache from the connector into vLLM's encoder cache.
This method loads the encoder cache based on metadata provided by the scheduler.
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
@@ -78,6 +77,7 @@ class ECExampleConnector(ECConnectorBase):
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
data hashes (`mm_hash`) to encoder cache tensors.
kwargs (dict): Additional keyword arguments for the connector.
"""
from vllm.platforms import current_platform
@@ -108,8 +108,7 @@ class ECExampleConnector(ECConnectorBase):
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)
def save_caches(self, encoder_cache, mm_hash, **kwargs) -> None:
"""
Save the encoder cache to the connector.
"""Save the encoder cache to the connector.
This method saves the encoder cache from the worker's local storage
to shared storage or another external connector.
@@ -119,6 +118,7 @@ class ECExampleConnector(ECConnectorBase):
data hashes (`mm_hash`) to encoder cache tensors.
mm_hash (str): The hash of the multimodal data whose cache is being saved.
kwargs (dict): Additional keyword arguments for the connector.
"""
# Return if it is PD Instance
if not self.is_producer:
@@ -134,14 +134,14 @@ class ECExampleConnector(ECConnectorBase):
self,
identifier: str,
) -> bool:
"""
Check if cache exist externally for the media
"""Check if cache exist externally for the media.
Args:
identifier (str): the identifier of the media.
Returns:
Bool indicate that media exists in cache or not
"""
return self._found_match_for_mm_data(identifier)
@@ -150,9 +150,7 @@ class ECExampleConnector(ECConnectorBase):
request: "Request",
index: int,
) -> None:
"""
Update ECConnector state after encoder cache allocation.
"""
"""Update ECConnector state after encoder cache allocation."""
mm_hash = request.mm_features[index].identifier
# Only load cache if it is consumer and cache exists
if not self.is_consumer or not self.has_cache_item(mm_hash):
@@ -212,8 +210,7 @@ class ECExampleConnector(ECConnectorBase):
mm_hash: str,
create_folder: bool = True, # <- now defaults to True
) -> str:
"""
Return the folder in which the cache for this mm_hash lives.
"""Return the folder in which the cache for this mm_hash lives.
If `create_folder` is True (default) the directory is created
recursively the first time it is needed.
"""
@@ -223,8 +220,7 @@ class ECExampleConnector(ECConnectorBase):
return foldername
def _generate_filename_debug(self, mm_hash: str) -> str:
"""
Return the full path of the safetensors file for this mm_hash.
"""Return the full path of the safetensors file for this mm_hash.
Ensures the parent directory exists because
`_generate_foldername_debug` is called with its default
(`create_folder=True`).
@@ -24,10 +24,7 @@ def has_ec_transfer() -> bool:
def ensure_ec_transfer_initialized(vllm_config: "VllmConfig") -> None:
"""
Initialize EC cache connector.
"""
"""Initialize EC cache connector."""
global _EC_CONNECTOR_AGENT
if vllm_config.ec_transfer_config is None:
+1 -3
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
The async worker that transfers experts in the background.
"""
"""The async worker that transfers experts in the background."""
import threading
from typing import TYPE_CHECKING

Some files were not shown because too many files have changed in this diff Show More